From 38329e80c82301bba2cec38fe13d7c68c42bc29d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 08:09:48 -0700 Subject: [PATCH 001/117] feat(onboard): add managed startup profile schema Signed-off-by: Aaron Erickson --- ci/source-shape-test-budget.json | 5 + .../onboard/managed-startup-profile.test.ts | 1027 ++++++++++ src/lib/onboard/managed-startup/profile.ts | 1682 +++++++++++++++++ src/lib/onboard/managed-startup/transport.ts | 6 + test/helpers/vitest-watch-triggers.ts | 4 + test/vitest-watch-triggers.test.ts | 10 + 6 files changed, 2734 insertions(+) create mode 100644 src/lib/onboard/managed-startup-profile.test.ts create mode 100644 src/lib/onboard/managed-startup/profile.ts create mode 100644 src/lib/onboard/managed-startup/transport.ts diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 08dbb6efa65..fc9c6e97ceb 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -16,6 +16,11 @@ "test": "accepts only the tracked published Hermes base digest", "category": "security" }, + { + "file": "src/lib/onboard/managed-startup-profile.test.ts", + "test": "classifies every stock Docker ARG as startup-affordance or deliberate exclusion", + "category": "compatibility" + }, { "file": "src/lib/readiness/host.test.ts", "test": "bounds and redacts successful probe text before schema validation", diff --git a/src/lib/onboard/managed-startup-profile.test.ts b/src/lib/onboard/managed-startup-profile.test.ts new file mode 100644 index 00000000000..fecb56b3994 --- /dev/null +++ b/src/lib/onboard/managed-startup-profile.test.ts @@ -0,0 +1,1027 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + decodeManagedStartupProfile, + encodeManagedStartupProfile, + fingerprintManagedStartupProfile, + MANAGED_STARTUP_AGENTS, + MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY, + MANAGED_STARTUP_PROFILE_CAPABILITIES, + MANAGED_STARTUP_PROFILE_EXCLUDED_DOCKER_INPUTS, + MANAGED_STARTUP_PROFILE_MAX_BYTES, + MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + type ManagedStartupAgent, + type ManagedStartupProfile, + serializeManagedStartupProfile, + validateManagedStartupProfile, +} from "./managed-startup/profile"; + +const CA_SHA256 = "a".repeat(64); + +const MESSAGING_PLAN = { + schemaVersion: 1, + sandboxName: "demo", + agent: "portable", + workflow: "onboard", + channels: [], + disabledChannels: [], + credentialBindings: [ + { + credentialId: "slackBotToken", + providerEnvKey: "SLACK_BOT_TOKEN", + placeholder: "openshell:resolve:env:SLACK_BOT_TOKEN", + credentialAvailable: true, + }, + ], + networkPolicy: { presets: [], entries: [] }, + agentRender: [ + { + channelId: "discord", + agent: "openclaw", + target: "openclaw.json", + kind: "json-fragment", + path: "channels.discord", + value: { token: "openshell:resolve:env:DISCORD_BOT_TOKEN" }, + templateRefs: ["credential.discordBotToken.placeholder"], + }, + { + channelId: "slack", + agent: "openclaw", + target: "openclaw.json", + kind: "json-fragment", + path: "channels.slack", + value: { token: "xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" }, + templateRefs: ["credential.slackBotToken.placeholder"], + }, + ], + buildSteps: [], + stateUpdates: [], + healthChecks: [], +} as const; + +const OPENCLAW_PROFILE = { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent: "openclaw", + agentConfig: { + agent: "openclaw", + webSearch: { enabled: true, provider: "brave" }, + otel: { + enabled: true, + endpointUrl: "http://host.openshell.internal:4318", + serviceName: "openclaw-gateway", + sampleRate: 0.75, + }, + agentTimeoutSeconds: 900, + heartbeatEvery: "30m", + extraAgents: { + agents: [ + { + id: "reviewer", + workspace: "/sandbox/reviewer", + model: "inference/nvidia/nemotron-3-ultra-550b-a55b", + }, + ], + defaults: { subagents: { maxSpawnDepth: 3 } }, + main: { tools: { profile: "coding" } }, + }, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + minimalBootstrap: true, + }, + inference: { + routeProvider: "inference", + upstreamProvider: "nvidia-prod", + model: "nvidia/nemotron-3-ultra-550b-a55b", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses", + primaryModelRef: "inference/nvidia/nemotron-3-ultra-550b-a55b", + compatibility: { supportsDeveloperRole: true, maxRetries: 2 }, + inputModalities: ["text", "image"], + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: "http://proxy.example.test:8080", + hostHttpsUrl: "http://connect-proxy.example.test:3128", + hostNoProxy: ["inference.local", "127.0.0.1", "localhost"], + }, + dashboard: { + agent: "openclaw", + mode: "remote", + url: "https://dashboard.example.test:18789", + port: 18_789, + bindAddress: "0.0.0.0", + wslExposure: true, + }, + tools: { + disclosure: "progressive", + enabledGateways: [], + }, + messaging: { plan: { ...MESSAGING_PLAN, agent: "openclaw" } }, + tuning: { + contextWindow: 131_072, + maxTokens: 8192, + reasoning: true, + reasoningEffort: "high", + }, + corporateCa: { bundleSha256: CA_SHA256 }, +} as const satisfies ManagedStartupProfile; + +const HERMES_PROFILE = { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent: "hermes", + agentConfig: { + agent: "hermes", + webSearch: { enabled: true, provider: "tavily" }, + }, + inference: { + routeProvider: "custom", + upstreamProvider: "anthropic-prod", + model: "claude-sonnet-4-5", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "anthropic-messages", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: "http://proxy.example.test:8080", + hostHttpsUrl: "https://proxy.example.test:8443", + hostNoProxy: ["localhost", "127.0.0.1"], + }, + dashboard: { + agent: "hermes", + mode: "loopback-forwarded", + url: "http://127.0.0.1:19189", + publicPort: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }, + tools: { + disclosure: "direct", + enabledGateways: ["nous-web", "nous-image", "nous-audio", "nous-browser", "nous-code"], + }, + messaging: { plan: { ...MESSAGING_PLAN, agent: "hermes" } }, + tuning: { + contextWindow: 65_536, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + corporateCa: { bundleSha256: CA_SHA256 }, +} as const satisfies ManagedStartupProfile; + +const DCODE_PROFILE = { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent: "langchain-deepagents-code", + agentConfig: { + agent: "langchain-deepagents-code", + autoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + }, + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + model: "openai/gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }, + dashboard: { + agent: "langchain-deepagents-code", + mode: "disabled", + }, + tools: { + disclosure: "progressive", + enabledGateways: [], + }, + messaging: { plan: null }, + tuning: { + contextWindow: null, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + corporateCa: { bundleSha256: CA_SHA256 }, +} as const satisfies ManagedStartupProfile; + +const VALID_PROFILES = [OPENCLAW_PROFILE, HERMES_PROFILE, DCODE_PROFILE] as const; + +function encodeUnknown(value: unknown): string { + return Buffer.from(JSON.stringify(value), "utf8").toString("base64url"); +} + +function dockerArgs(relativePath: string): Set { + const source = readFileSync(relativePath, "utf8"); + return new Set( + [...source.matchAll(/^ARG\s+([A-Z][A-Z0-9_]*)/gmu)].map((match) => match[1] as string), + ); +} + +const STOCK_DOCKER_ARGS = { + openclaw: dockerArgs(path.join(process.cwd(), "Dockerfile")), + hermes: dockerArgs(path.join(process.cwd(), "agents/hermes/Dockerfile")), + "langchain-deepagents-code": dockerArgs( + path.join(process.cwd(), "agents/langchain-deepagents-code/Dockerfile"), + ), +} satisfies Record>; + +describe("managed startup profile", () => { + it.each( + VALID_PROFILES, + )("round-trips each $agent profile through canonical encoding", (profile) => { + const validated = validateManagedStartupProfile(profile); + const encoded = encodeManagedStartupProfile(profile); + + expect(decodeManagedStartupProfile(encoded)).toEqual(validated); + expect(encoded).not.toContain(profile.inference.model); + expect(fingerprintManagedStartupProfile(profile)).toMatch(/^[a-f0-9]{64}$/); + }); + + it("round-trips all OpenClaw-only startup settings", () => { + const profile = decodeManagedStartupProfile(encodeManagedStartupProfile(OPENCLAW_PROFILE)); + expect(profile).toMatchObject({ + agentConfig: { + webSearch: { enabled: true, provider: "brave" }, + otel: { + enabled: true, + endpointUrl: "http://host.openshell.internal:4318", + serviceName: "openclaw-gateway", + sampleRate: 0.75, + }, + agentTimeoutSeconds: 900, + heartbeatEvery: "30m", + minimalBootstrap: true, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + }, + inference: { + api: "openai-responses", + primaryModelRef: "inference/nvidia/nemotron-3-ultra-550b-a55b", + compatibility: { supportsDeveloperRole: true, maxRetries: 2 }, + inputModalities: ["image", "text"], + }, + dashboard: { + mode: "remote", + bindAddress: "0.0.0.0", + wslExposure: true, + }, + tuning: { + contextWindow: 131_072, + maxTokens: 8192, + reasoning: true, + reasoningEffort: "high", + }, + }); + expect(profile.agentConfig.agent === "openclaw" && profile.agentConfig.extraAgents).toEqual( + OPENCLAW_PROFILE.agentConfig.extraAgents, + ); + }); + + it("round-trips Hermes forwarding, declared gateways, messaging, and context tuning", () => { + const profile = decodeManagedStartupProfile(encodeManagedStartupProfile(HERMES_PROFILE)); + expect(profile).toMatchObject({ + dashboard: { + mode: "loopback-forwarded", + publicPort: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }, + agentConfig: { webSearch: { enabled: true, provider: "tavily" } }, + tuning: { + contextWindow: 65_536, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + }); + expect(profile.tools.enabledGateways).toEqual([ + "nous-audio", + "nous-browser", + "nous-code", + "nous-image", + "nous-web", + ]); + expect(profile.messaging.plan).not.toBeNull(); + }); + + it("round-trips langchain-deepagents-code upstream metadata, managed proxy, approval, and observability", () => { + const profile = decodeManagedStartupProfile(encodeManagedStartupProfile(DCODE_PROFILE)); + expect(profile).toMatchObject({ + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + routedBaseUrl: "https://inference.local/v1", + api: "openai-completions", + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }, + agentConfig: { + autoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + }, + dashboard: { mode: "disabled" }, + }); + }); + + it("canonicalizes object keys and set-like lists before fingerprinting", () => { + const reordered = { + ...HERMES_PROFILE, + proxy: { + ...HERMES_PROFILE.proxy, + hostNoProxy: [...HERMES_PROFILE.proxy.hostNoProxy].reverse(), + }, + tools: { + ...HERMES_PROFILE.tools, + enabledGateways: [...HERMES_PROFILE.tools.enabledGateways].reverse(), + }, + }; + const serialized = serializeManagedStartupProfile(HERMES_PROFILE); + + expect(serializeManagedStartupProfile(reordered)).toBe(serialized); + expect(fingerprintManagedStartupProfile(reordered)).toBe( + createHash("sha256").update(serialized, "utf8").digest("hex"), + ); + }); + + it("exports complete, fail-closed capabilities for every supported agent", () => { + expect(Object.keys(MANAGED_STARTUP_PROFILE_CAPABILITIES).sort()).toEqual( + [...MANAGED_STARTUP_AGENTS].sort(), + ); + expect(MANAGED_STARTUP_PROFILE_CAPABILITIES.openclaw.dashboardModes).toEqual([ + "loopback", + "remote", + ]); + expect(MANAGED_STARTUP_PROFILE_CAPABILITIES.hermes.dashboardModes).toEqual([ + "disabled", + "loopback-forwarded", + ]); + expect(MANAGED_STARTUP_PROFILE_CAPABILITIES.hermes.inputModalities).toEqual([]); + expect(MANAGED_STARTUP_PROFILE_CAPABILITIES["langchain-deepagents-code"].inferenceApis).toEqual( + ["openai-completions"], + ); + expect( + MANAGED_STARTUP_PROFILE_CAPABILITIES["langchain-deepagents-code"].inputModalities, + ).toEqual([]); + }); + + // source-shape-contract: compatibility -- Every shipped Docker build input must map to versioned startup intent or a declared build-only exclusion + it("classifies every stock Docker ARG as startup-affordance or deliberate exclusion", () => { + for (const agent of MANAGED_STARTUP_AGENTS) { + const classified = new Set([ + ...MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[agent].map(({ input }) => input), + ...MANAGED_STARTUP_PROFILE_EXCLUDED_DOCKER_INPUTS[agent].map(({ input }) => input), + ]); + expect([...STOCK_DOCKER_ARGS[agent]].filter((input) => !classified.has(input))).toEqual([]); + } + }); + + it.each(MANAGED_STARTUP_AGENTS)("keeps the %s affordance inventory unambiguous", (agent) => { + const inventory = MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[agent]; + expect(new Set(inventory.map(({ input }) => input)).size).toBe(inventory.length); + expect(inventory.every(({ profilePath }) => !profilePath.startsWith("env."))).toBe(true); + }); + + it.each( + VALID_PROFILES, + )("maps every $agent inventory entry to an explicit profile field", (profile) => { + for (const { profilePath } of MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[profile.agent]) { + let current: unknown = profile; + for (const segment of profilePath.split(".")) { + expect(current).not.toBeNull(); + expect(typeof current).toBe("object"); + expect(Object.hasOwn(current as object, segment)).toBe(true); + current = (current as Record)[segment]; + } + } + }); + + it("rejects non-canonical transports instead of accepting ambiguous fingerprints", () => { + const raw = JSON.stringify(OPENCLAW_PROFILE); + expect(() => decodeManagedStartupProfile(Buffer.from(raw).toString("base64url"))).toThrow( + /canonical form/, + ); + }); + + it.each([ + { + label: "top level", + mutate: (profile: ManagedStartupProfile) => ({ ...profile, extension: true }), + }, + { + label: "inference", + mutate: (profile: ManagedStartupProfile) => ({ + ...profile, + inference: { ...profile.inference, extension: true }, + }), + }, + { + label: "proxy", + mutate: (profile: ManagedStartupProfile) => ({ + ...profile, + proxy: { ...profile.proxy, extension: true }, + }), + }, + { + label: "dashboard", + mutate: (profile: ManagedStartupProfile) => ({ + ...profile, + dashboard: { ...profile.dashboard, extension: true }, + }), + }, + { + label: "messaging wrapper", + mutate: (profile: ManagedStartupProfile) => ({ + ...profile, + messaging: { ...profile.messaging, extension: true }, + }), + }, + { + label: "agent config", + mutate: (profile: ManagedStartupProfile) => ({ + ...profile, + agentConfig: { ...profile.agentConfig, extension: true }, + }), + }, + { + label: "CA digest", + mutate: (profile: ManagedStartupProfile) => ({ + ...profile, + corporateCa: { ...profile.corporateCa, extension: true }, + }), + }, + ])("rejects recursively unknown keys at $label", ({ mutate }) => { + expect(() => validateManagedStartupProfile(mutate(OPENCLAW_PROFILE))).toThrow( + /unsupported fields/, + ); + }); + + it.each([ + ["agentConfig", { ...OPENCLAW_PROFILE, agentConfig: HERMES_PROFILE.agentConfig }], + ["dashboard", { ...OPENCLAW_PROFILE, dashboard: HERMES_PROFILE.dashboard }], + ])("rejects a mismatched %s agent discriminator", (_label, profile) => { + expect(() => validateManagedStartupProfile(profile)).toThrow(/must match agent/); + }); + + it.each([ + ["schema version", { ...OPENCLAW_PROFILE.messaging.plan, schemaVersion: 2 }], + ["agent", { ...OPENCLAW_PROFILE.messaging.plan, agent: "hermes" }], + ])("rejects a messaging plan with a mismatched %s", (_label, plan) => { + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + messaging: { plan }, + }), + ).toThrow(/version 1 plan for the selected agent/); + }); + + it.each([ + ["credential-named compatibility field", { accessToken: "not-even-a-real-token" }], + ["token-shaped compatibility field name", { [`nvapi-${"a".repeat(32)}`]: true }], + ["credential-prefixed public key field", { secret_public_key: "opaque-secret" }], + ["provider-prefixed token field", { slackBotToken: "opaque-secret" }], + ["provider-prefixed API key field", { customApiKey: "opaque-secret" }], + ["camel-case secret public key field", { secretPublicKey: "opaque-secret" }], + ["generic provider token field", { githubToken: "opaque-secret" }], + ["generic provider key field", { openaiKey: "opaque-secret" }], + ["generic provider secret field", { matrixSecret: "opaque-secret" }], + ["generic provider PAT field", { githubPat: "opaque-secret" }], + [ + "credential-named field hidden under a non-messaging plan", + { plan: { accessToken: "not-even-a-real-token" } }, + ], + ["provider token", { note: `nvapi-${"a".repeat(32)}` }], + ["bearer value", { note: `Bearer ${"a".repeat(32)}` }], + [ + "private key", + { + note: `-----BEGIN ${"PRIVATE"} KEY-----\nabc\n-----END ${"PRIVATE"} KEY-----`, + }, + ], + ])("rejects %s anywhere in the profile", (_label, compatibility) => { + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + inference: { ...OPENCLAW_PROFILE.inference, compatibility }, + }), + ).toThrow(/credential-shaped/); + }); + + it.each([ + "publicKey", + "public_key", + "public-key", + ])("does not classify exact %s metadata as credential material", (field) => { + expect( + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + inference: { + ...OPENCLAW_PROFILE.inference, + compatibility: { [field]: "non-secret metadata" }, + }, + }).inference.compatibility, + ).toEqual({ [field]: "non-secret metadata" }); + }); + + it("rejects raw credentials nested inside an otherwise opaque messaging plan", () => { + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + messaging: { + plan: { + ...OPENCLAW_PROFILE.messaging.plan, + credentialBindings: [ + { + providerEnvKey: "SLACK_BOT_TOKEN", + value: `xoxb-${"a".repeat(32)}`, + }, + ], + }, + }, + }), + ).toThrow(/credential-shaped string data/); + }); + + it("rejects credential fields in a messaging plan unless they contain provider placeholders", () => { + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + messaging: { + plan: { + ...OPENCLAW_PROFILE.messaging.plan, + password: "hunter2", + }, + }, + }), + ).toThrow(/credential-shaped field name/); + }); + + it.each([ + ["routed inference", "inference", "routedBaseUrl"], + ["upstream inference", "inference", "upstreamEndpointUrl"], + ["OTEL", "otel", "endpointUrl"], + ["dashboard", "dashboard", "url"], + ] as const)("rejects credentials embedded in the %s URL", (_label, scope, field) => { + const profile = + scope === "inference" + ? { + ...DCODE_PROFILE, + inference: { + ...DCODE_PROFILE.inference, + [field]: "https://user:password@example.test/v1", + }, + } + : scope === "otel" + ? { + ...OPENCLAW_PROFILE, + agentConfig: { + ...OPENCLAW_PROFILE.agentConfig, + otel: { + ...OPENCLAW_PROFILE.agentConfig.otel, + [field]: "https://user:password@example.test/v1", + }, + }, + } + : { + ...OPENCLAW_PROFILE, + dashboard: { + ...OPENCLAW_PROFILE.dashboard, + [field]: "https://user:password@example.test/v1", + }, + }; + expect(() => validateManagedStartupProfile(profile)).toThrow( + /embedded credentials|credential-free/, + ); + }); + + it.each([ + "token", + "api_key", + ])("rejects credential-shaped query parameter %s in an opaque URL", (credentialField) => { + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + inference: { + ...OPENCLAW_PROFILE.inference, + compatibility: { + note: `https://example.test/hook?${credentialField}=opaque-secret`, + }, + }, + }), + ).toThrow(/URL with embedded credentials/); + }); + + it.each([ + "#access_token=opaque-secret", + "#token=opaque-secret", + "#/route?api_key=opaque-secret", + ])("rejects credential-shaped parameters in an opaque URL fragment", (fragment) => { + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + inference: { + ...OPENCLAW_PROFILE.inference, + compatibility: { + note: `https://example.test/callback${fragment}`, + }, + }, + }), + ).toThrow(/URL with embedded credentials/); + }); + + it("accepts an HTTP CONNECT origin for host HTTPS proxy intent", () => { + expect(validateManagedStartupProfile(OPENCLAW_PROFILE).proxy.hostHttpsUrl).toBe( + "http://connect-proxy.example.test:3128", + ); + }); + + it("keeps langchain-deepagents-code messaging, dashboards, and tuning fail-closed while retaining host proxy intent", () => { + expect(() => + validateManagedStartupProfile({ + ...DCODE_PROFILE, + messaging: { plan: { schemaVersion: 1 } }, + }), + ).toThrow(/messaging\.plan must be null/); + expect( + validateManagedStartupProfile({ + ...DCODE_PROFILE, + proxy: { ...DCODE_PROFILE.proxy, hostHttpUrl: "http://proxy.example.test:8080" }, + }).proxy.hostHttpUrl, + ).toBe("http://proxy.example.test:8080"); + expect(() => + validateManagedStartupProfile({ + ...DCODE_PROFILE, + tuning: { ...DCODE_PROFILE.tuning, contextWindow: 65_536 }, + }), + ).toThrow(/does not support startup tuning/); + expect(() => + validateManagedStartupProfile({ + ...DCODE_PROFILE, + dashboard: { agent: "langchain-deepagents-code", mode: "remote" }, + }), + ).toThrow(/dashboard\.mode must be disabled/); + }); + + it("rejects unsupported langchain-deepagents-code inference APIs and OpenClaw-only inference fields", () => { + expect(() => + validateManagedStartupProfile({ + ...DCODE_PROFILE, + inference: { ...DCODE_PROFILE.inference, api: "openai-responses" }, + }), + ).toThrow(/not supported/); + expect(() => + validateManagedStartupProfile({ + ...HERMES_PROFILE, + inference: { + ...HERMES_PROFILE.inference, + compatibility: { supportsDeveloperRole: true }, + }, + }), + ).toThrow(/does not support/); + }); + + it("accepts only declared Hermes gateway IDs and rejects gateways for other agents", () => { + expect(validateManagedStartupProfile(HERMES_PROFILE).tools.enabledGateways).toHaveLength(5); + expect(() => + validateManagedStartupProfile({ + ...HERMES_PROFILE, + tools: { ...HERMES_PROFILE.tools, enabledGateways: ["filesystem"] }, + }), + ).toThrow(/unsupported value/); + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + tools: { ...OPENCLAW_PROFILE.tools, enabledGateways: ["nous-web"] }, + }), + ).toThrow(/supported only by hermes/); + }); + + it("enforces adapter-specific web-search providers", () => { + expect(() => + validateManagedStartupProfile({ + ...HERMES_PROFILE, + agentConfig: { + ...HERMES_PROFILE.agentConfig, + webSearch: { enabled: true, provider: "brave" }, + }, + }), + ).toThrow(/not supported/); + }); + + it("enforces resolved OpenClaw dashboard exposure and device-auth semantics", () => { + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + dashboard: { ...OPENCLAW_PROFILE.dashboard, mode: "loopback" }, + }), + ).toThrow(/must reflect/); + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + agentConfig: { + ...OPENCLAW_PROFILE.agentConfig, + deviceAuth: { disabled: false, optOutSource: "operator" }, + }, + }), + ).toThrow(/requires device auth to be disabled/); + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + dashboard: { ...OPENCLAW_PROFILE.dashboard, port: 18_790 }, + }), + ).toThrow(/must match dashboard\.url/); + }); + + it("supports disabled or loopback-forwarded Hermes dashboards only", () => { + const disabled = validateManagedStartupProfile({ + ...HERMES_PROFILE, + dashboard: { + agent: "hermes", + mode: "disabled", + url: "http://127.0.0.1:18789", + publicPort: null, + internalPort: null, + tuiEnabled: false, + }, + }); + expect(disabled.dashboard.mode).toBe("disabled"); + expect(() => + validateManagedStartupProfile({ + ...HERMES_PROFILE, + dashboard: { ...HERMES_PROFILE.dashboard, url: "https://dashboard.example.test" }, + }), + ).toThrow(/must remain loopback/); + expect(() => + validateManagedStartupProfile({ + ...HERMES_PROFILE, + dashboard: { ...HERMES_PROFILE.dashboard, publicPort: 19_190 }, + }), + ).toThrow(/must match dashboard\.url/); + }); + + it.each([ + ["publicPort", 8642], + ["publicPort", 18_642], + ["internalPort", 8642], + ["internalPort", 18_642], + ] as const)("rejects Hermes dashboard %s collisions with reserved API port %i", (field, port) => { + expect(() => + validateManagedStartupProfile({ + ...HERMES_PROFILE, + dashboard: { ...HERMES_PROFILE.dashboard, [field]: port }, + }), + ).toThrow(/reserved API ports 8642 or 18642/); + }); + + it.each([ + "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----", + `MII${"A".repeat(300)}`, + `data:application/x-x509-ca-cert;base64,MII${"A".repeat(300)}`, + ])("rejects raw CA material while accepting only its digest", (rawCa) => { + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + inference: { ...OPENCLAW_PROFILE.inference, model: rawCa }, + }), + ).toThrow(/raw certificate data/); + }); + + it.each([ + ["bad schema", { ...OPENCLAW_PROFILE, schemaVersion: 2 }], + [ + "invalid langchain-deepagents-code approval mode", + { + ...DCODE_PROFILE, + agentConfig: { + ...DCODE_PROFILE.agentConfig, + autoApprovalMode: "always", + }, + }, + ], + [ + "bad heartbeat", + { + ...OPENCLAW_PROFILE, + agentConfig: { ...OPENCLAW_PROFILE.agentConfig, heartbeatEvery: "every hour" }, + }, + ], + [ + "invalid CA digest", + { + ...OPENCLAW_PROFILE, + corporateCa: { bundleSha256: "not-a-digest" }, + }, + ], + ])("rejects malformed profile: %s", (_label, profile) => { + expect(() => validateManagedStartupProfile(profile)).toThrow(/Invalid managed startup profile/); + }); + + it.each([ + ["empty", ""], + ["not base64url", "%%%"], + ["invalid base64url quantum", "a"], + ["invalid JSON", Buffer.from("{", "utf8").toString("base64url")], + ["invalid UTF-8", Buffer.from([0xc3, 0x28]).toString("base64url")], + ])("rejects malformed encoded payload: %s", (_label, encoded) => { + expect(() => decodeManagedStartupProfile(encoded)).toThrow(/Invalid managed startup profile/); + }); + + it("rejects decoded payloads over the bounded profile size", () => { + const encoded = Buffer.alloc(MANAGED_STARTUP_PROFILE_MAX_BYTES + 1, 0x61).toString("base64url"); + expect(() => decodeManagedStartupProfile(encoded)).toThrow(/size limit/); + }); + + it("rejects structurally deep payloads within the byte cap", () => { + const deep = JSON.parse(`${'{"nested":'.repeat(40)}null${"}".repeat(40)}`) as Record< + string, + unknown + >; + expect(() => validateManagedStartupProfile(deep)).toThrow(/complexity limit/); + }); + + it("checks direct payload complexity before reading excess properties", () => { + const wide = Object.fromEntries( + Array.from({ length: 5000 }, (_, index) => [`field-${String(index)}`, null]), + ); + let excessPropertyRead = false; + Object.defineProperty(wide, "excess", { + enumerable: true, + get() { + excessPropertyRead = true; + throw new Error("excess property was read"); + }, + }); + + expect(() => validateManagedStartupProfile(wide)).toThrow(/complexity limit/); + expect(excessPropertyRead).toBe(false); + }); + + it("rejects a custom JSON serializer before invoking it", () => { + const inputModalities = [...OPENCLAW_PROFILE.inference.inputModalities]; + let serializerInvoked = false; + Object.defineProperty(inputModalities, "toJSON", { + value() { + serializerInvoked = true; + return ["text"]; + }, + }); + + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + inference: { ...OPENCLAW_PROFILE.inference, inputModalities }, + }), + ).toThrow(/custom JSON serializer/); + expect(serializerInvoked).toBe(false); + }); + + it("rejects an inherited array serializer before invoking it", () => { + const enabledGateways: string[] = []; + let serializerInvoked = false; + const prototype = Object.create(Array.prototype) as Record; + Object.defineProperty(prototype, "toJSON", { + value() { + serializerInvoked = true; + return [`nvapi-${"a".repeat(32)}`]; + }, + }); + Object.setPrototypeOf(enabledGateways, prototype); + + expect(() => + validateManagedStartupProfile({ + ...DCODE_PROFILE, + tools: { ...DCODE_PROFILE.tools, enabledGateways }, + }), + ).toThrow(/standard JSON prototype/); + expect(serializerInvoked).toBe(false); + }); + + it("rejects an array map override before invoking it", () => { + const enabledGateways: string[] = []; + let mapInvoked = false; + Object.defineProperty(enabledGateways, "map", { + value() { + mapInvoked = true; + return [`nvapi-${"a".repeat(32)}`]; + }, + }); + + expect(() => + validateManagedStartupProfile({ + ...DCODE_PROFILE, + tools: { ...DCODE_PROFILE.tools, enabledGateways }, + }), + ).toThrow(/only indexed JSON values/); + expect(mapInvoked).toBe(false); + }); + + it("rejects a polluted Object prototype serializer before invoking it", () => { + let serializerInvoked = false; + let caught: unknown; + Object.defineProperty(Object.prototype, "toJSON", { + configurable: true, + value() { + serializerInvoked = true; + return { note: `nvapi-${"a".repeat(32)}` }; + }, + }); + try { + validateManagedStartupProfile(OPENCLAW_PROFILE); + } catch (error) { + caught = error; + } finally { + Reflect.deleteProperty(Object.prototype, "toJSON"); + } + + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toMatch(/custom JSON serializer/); + expect(serializerInvoked).toBe(false); + }); + + it("does not invoke polluted Array prototype mapping or sorting methods", () => { + const mapDescriptor = Object.getOwnPropertyDescriptor(Array.prototype, "map"); + const sortDescriptor = Object.getOwnPropertyDescriptor(Array.prototype, "sort"); + let prototypeMethodInvoked = false; + let serialized: string | undefined; + const poison = { + configurable: true, + value() { + prototypeMethodInvoked = true; + return [`nvapi-${"a".repeat(32)}`]; + }, + writable: true, + }; + Object.defineProperty(Array.prototype, "map", poison); + Object.defineProperty(Array.prototype, "sort", poison); + try { + serialized = serializeManagedStartupProfile({ + ...OPENCLAW_PROFILE, + messaging: { + plan: { + ...OPENCLAW_PROFILE.messaging.plan, + payload: ["safe"], + }, + }, + }); + } finally { + if (mapDescriptor) Object.defineProperty(Array.prototype, "map", mapDescriptor); + if (sortDescriptor) Object.defineProperty(Array.prototype, "sort", sortDescriptor); + } + + expect(prototypeMethodInvoked).toBe(false); + expect(serialized).toContain('"payload":["safe"]'); + expect(serialized).not.toContain("nvapi-"); + }); + + it("screens non-enumerable profile fields before rebuilding them", () => { + const inference = { ...OPENCLAW_PROFILE.inference }; + Object.defineProperty(inference, "model", { + value: `nvapi-${"a".repeat(32)}`, + enumerable: false, + }); + + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + inference, + }), + ).toThrow(/credential-shaped string data/); + }); + + it("rejects a noncanonical payload even when its profile values are otherwise valid", () => { + const encoded = encodeUnknown({ + ...HERMES_PROFILE, + tools: { + ...HERMES_PROFILE.tools, + enabledGateways: [...HERMES_PROFILE.tools.enabledGateways].reverse(), + }, + }); + expect(() => decodeManagedStartupProfile(encoded)).toThrow(/canonical form/); + }); +}); diff --git a/src/lib/onboard/managed-startup/profile.ts b/src/lib/onboard/managed-startup/profile.ts new file mode 100644 index 00000000000..5dccdb3f236 --- /dev/null +++ b/src/lib/onboard/managed-startup/profile.ts @@ -0,0 +1,1682 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; +import { TextDecoder } from "node:util"; + +/** + * Versioned, bounded schema for managed-image startup intent. + * Runtime-specific construction and activation stay outside this module. + */ +export const MANAGED_STARTUP_PROFILE_SCHEMA_VERSION = 1 as const; + +/** Profiles are configuration, not a general-purpose transport. */ +export const MANAGED_STARTUP_PROFILE_MAX_BYTES = 64 * 1024; + +/** Maximum canonical base64url size for a profile at the decoded byte cap. */ +export const MANAGED_STARTUP_PROFILE_MAX_ENCODED_BYTES = + Math.ceil(MANAGED_STARTUP_PROFILE_MAX_BYTES / 3) * 4; + +const MAX_IDENTIFIER_BYTES = 256; +const MAX_MODEL_BYTES = 1024; +const MAX_URL_BYTES = 2048; +const MAX_LIST_ITEMS = 128; +const MAX_JSON_NODES = 4096; +const MAX_JSON_DEPTH = 32; +const MAX_TUNING_INTEGER = 1_000_000_000; +const SHA256_RE = /^[a-f0-9]{64}$/; +const CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f-\u009f]/u; +const BASE64URL_RE = /^[A-Za-z0-9_-]+$/; +const RAW_CA_PEM_RE = /-----BEGIN (?:TRUSTED )?CERTIFICATE-----/iu; +const RAW_CA_DER_BASE64_RE = /^MII[A-Za-z0-9+/=\r\n]{253,}$/u; +const RAW_CA_DATA_URI_RE = + /data:application\/(?:pkix-cert|x-x509-ca-cert);base64,MII[A-Za-z0-9+/=]{253,}/iu; +const URL_CANDIDATE_RE = /[A-Za-z][A-Za-z0-9+.-]*:\/\/[^\s"'<>]+/gu; +const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true }); +const CREDENTIAL_SHAPED_NAME_PATTERN = + /(?:^|[_-])(?:api[_-]?key|access[_-]?key|secret[_-]?key|auth[_-]?token|refresh[_-]?token|access[_-]?token|client[_-]?secret|private[_-]?key|pass[_-]?code|personal[_-]?access[_-]?token|connection[_-]?string|webhook(?:[_-]?url)?|key|secret|token|password|passwd|passcode|auth|authorization|credential|credentials|bearer|bearer[_-]?token|cookie|cookies|pat|private|privatekey|pin|webhookurl|dsn|connectionstring)(?:$|[_-])/iu; +const CREDENTIAL_COMPOUND_NAME_PATTERN = + /^(?:access|refresh|client|bearer|auth|api|private|signing|session|bot|app|resolved)(?:token|key|secret|password)$/iu; +const CREDENTIAL_CAMEL_SUFFIX_PATTERN = + /(?:apiKey|accessKey|secretKey|authToken|refreshToken|accessToken|clientSecret|privateKey|passcode|password|passwd|passphrase|bearerToken|botToken|appToken|sessionToken|signingKey|secretPublicKey|personalAccessToken|connectionString|webhookUrl)$/iu; +const CREDENTIAL_CAMEL_BOUNDARY_PATTERN = /[a-z0-9](?:Token|Key|Secret|Password|Passphrase|Pat)$/u; +const CREDENTIAL_ENV_NAME_PATTERN = + /^(?:[A-Z0-9]+_)*(?:TOKEN|KEY|SECRET|PASSWORD|PASSWD|PASS|PASSPHRASE|CREDENTIAL)S?$/u; +const CREDENTIAL_HEADER_NAME_PATTERN = + /^(?:authorization|proxy-authorization|cookie|set-cookie|.+-(?:key|token|secret|password|passphrase|credential|auth)s?)$/iu; +const PUBLIC_KEY_NAME_PATTERN = /^public[-_]?keys?$/iu; +const PASS_CREDENTIAL_NAME_PATTERN = /(?:^|[-_])pass(?:wd)?$/iu; +const NON_SECRET_KEY_METADATA_NAMES = new Set([ + "envKey", + "installCacheEnvKey", + "providerEnvKey", + "stateKey", +]); +const MESSAGING_CREDENTIAL_PLACEHOLDER_RE = + /^(?:openshell:resolve:env:|[A-Za-z0-9]+-OPENSHELL-RESOLVE-ENV-)(?:v[0-9]+_)?[A-Z][A-Z0-9_]*$/u; +const SECRET_VALUE_PATTERNS: readonly RegExp[] = [ + /nvapi-[A-Za-z0-9_-]{10,}/u, + /nvcf-[A-Za-z0-9_-]{10,}/u, + /ghp_[A-Za-z0-9_-]{10,}/u, + /github_pat_[A-Za-z0-9_]{30,}/u, + /sk-(?:proj-|ant-)?[A-Za-z0-9_-]{10,}/u, + /(?:xox[bpas]|xapp)-[A-Za-z0-9-]{10,}/u, + /A(?:K|S)IA[A-Z0-9]{16}/u, + /hf_[A-Za-z0-9]{10,}/u, + /glpat-[A-Za-z0-9_-]{10,}/u, + /gsk_[A-Za-z0-9]{10,}/u, + /pypi-[A-Za-z0-9_-]{10,}/u, + /tvly-[A-Za-z0-9_-]{10,}/u, + /lsv2_(?:pt|sk)_[A-Za-z0-9]{10,}(?:_[A-Za-z0-9]+)*/u, + /\bbot\d{8,10}:[A-Za-z0-9_-]{35}\b/u, + /\b\d{8,10}:[A-Za-z0-9_-]{35}\b/u, + /\b[A-Za-z0-9]{24}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,}\b/u, + /\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{2,}\.[A-Za-z0-9_-]{10,}\b/u, + /\bBearer\s+[A-Za-z0-9_.+/=-]{10,}/iu, + /-----BEGIN (?:[A-Z0-9]+ )?PRIVATE KEY-----/u, +]; + +export const MANAGED_STARTUP_INFERENCE_APIS = [ + "openai-completions", + "openai-responses", + "anthropic-messages", +] as const; +export type ManagedStartupInferenceApi = (typeof MANAGED_STARTUP_INFERENCE_APIS)[number]; +export type ManagedStartupToolDisclosure = "progressive" | "direct"; +export const MANAGED_STARTUP_REASONING_EFFORTS = ["default", "low", "medium", "high"] as const; +export type ManagedStartupReasoningEffort = (typeof MANAGED_STARTUP_REASONING_EFFORTS)[number]; +export const MANAGED_STARTUP_DCODE_AUTO_APPROVAL_MODES = ["disabled", "thread-opt-in"] as const; +export type ManagedStartupDcodeAutoApprovalMode = + (typeof MANAGED_STARTUP_DCODE_AUTO_APPROVAL_MODES)[number]; +export const MANAGED_STARTUP_HERMES_TOOL_GATEWAYS = [ + "nous-web", + "nous-image", + "nous-audio", + "nous-browser", + "nous-code", +] as const; +export type ManagedStartupHermesToolGateway = (typeof MANAGED_STARTUP_HERMES_TOOL_GATEWAYS)[number]; +export type ManagedStartupInputModality = "text" | "image"; +export type ManagedStartupWebSearchProvider = "brave" | "tavily"; +export type ManagedStartupDeviceAuthOptOutSource = "operator" | "managed-onboard"; + +export const MANAGED_STARTUP_AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; + +export type ManagedStartupAgent = (typeof MANAGED_STARTUP_AGENTS)[number]; + +export type ManagedStartupJsonScalar = string | number | boolean | null; +export type ManagedStartupJsonValue = + | ManagedStartupJsonScalar + | ManagedStartupJsonObject + | readonly ManagedStartupJsonValue[]; +export interface ManagedStartupJsonObject { + readonly [key: string]: ManagedStartupJsonValue; +} + +export interface ManagedStartupInference { + /** Stable route name installed in the sandbox-facing inference config. */ + readonly routeProvider: string; + /** User-selected provider upstream of the managed inference route. */ + readonly upstreamProvider: string; + readonly model: string; + /** Sandbox-facing managed inference route (normally inference.local). */ + readonly routedBaseUrl: string; + /** Direct upstream metadata for langchain-deepagents-code; never the sandbox route. */ + readonly upstreamEndpointUrl: string | null; + readonly api: ManagedStartupInferenceApi; + /** OpenClaw's provider/model reference. Other adapters require null. */ + readonly primaryModelRef: string | null; + /** OpenClaw provider compatibility options. Other adapters require null. */ + readonly compatibility: ManagedStartupJsonObject | null; + /** OpenClaw model inputs. Other adapters require null. */ + readonly inputModalities: readonly ManagedStartupInputModality[] | null; +} + +export interface ManagedStartupProxy { + /** Managed OpenShell policy-proxy route declared for every adapter. */ + readonly managedHost: string; + readonly managedPort: number; + /** + * Optional host-proxy intent is separate from the root-owned managed route. + * The langchain-deepagents-code contract preserves this intent for its create + * boundary only. + */ + readonly hostHttpUrl: string | null; + readonly hostHttpsUrl: string | null; + readonly hostNoProxy: readonly string[]; +} + +export interface ManagedStartupOpenClawDashboard { + readonly agent: "openclaw"; + readonly mode: "loopback" | "remote"; + readonly url: string; + readonly port: number; + readonly bindAddress: "127.0.0.1" | "0.0.0.0"; + readonly wslExposure: boolean; +} + +export interface ManagedStartupHermesDashboardDisabled { + readonly agent: "hermes"; + readonly mode: "disabled"; + /** CHAT_UI_URL remains a stock image input even when host forwarding is off. */ + readonly url: string; + readonly publicPort: null; + readonly internalPort: null; + readonly tuiEnabled: false; +} + +export interface ManagedStartupHermesDashboardForwarded { + readonly agent: "hermes"; + readonly mode: "loopback-forwarded"; + readonly url: string; + readonly publicPort: number; + readonly internalPort: number; + readonly tuiEnabled: boolean; +} + +export type ManagedStartupHermesDashboard = + | ManagedStartupHermesDashboardDisabled + | ManagedStartupHermesDashboardForwarded; + +export interface ManagedStartupDcodeDashboard { + readonly agent: "langchain-deepagents-code"; + readonly mode: "disabled"; +} + +export type ManagedStartupDashboard = + | ManagedStartupOpenClawDashboard + | ManagedStartupHermesDashboard + | ManagedStartupDcodeDashboard; + +export interface ManagedStartupWebSearch { + readonly enabled: boolean; + /** + * The selected provider is retained even when disabled because the stock + * Dockerfiles currently materialize both build inputs independently. + */ + readonly provider: ManagedStartupWebSearchProvider; +} + +export interface ManagedStartupTools { + readonly disclosure: ManagedStartupToolDisclosure; + /** Declared Hermes gateway preset IDs; required empty for other adapters. */ + readonly enabledGateways: readonly ManagedStartupHermesToolGateway[]; +} + +export interface ManagedStartupMessaging { + /** + * A host-prevalidated SandboxMessagingPlan. The existing messaging + * validator owns its nested schema. This boundary rechecks its version and + * agent discriminator, validates JSON shape and size, and rejects defined + * credential shapes. + */ + readonly plan: ManagedStartupJsonObject | null; +} + +export interface ManagedStartupTuning { + readonly contextWindow: number | null; + readonly maxTokens: number | null; + readonly reasoning: boolean | null; + readonly reasoningEffort: ManagedStartupReasoningEffort | null; +} + +/** + * Only the digest crosses the startup-profile boundary. The host-side CA + * applicator owns the actual certificate bytes and verifies this digest before + * making them available to the sandbox. + */ +export interface ManagedStartupCorporateCa { + readonly bundleSha256: string | null; +} + +export interface ManagedStartupExtraAgents { + /** + * Canonical form of NEMOCLAW_EXTRA_AGENTS_JSON. The existing OpenClaw + * validator owns the nested agent schema; this boundary preserves all + * prevalidated values without accepting the legacy array/object ambiguity. + */ + readonly agents: readonly ManagedStartupJsonObject[]; + readonly defaults: ManagedStartupJsonObject; + readonly main: ManagedStartupJsonObject; +} + +export interface ManagedStartupOpenClawOtel { + readonly enabled: boolean; + readonly endpointUrl: string; + readonly serviceName: string; + readonly sampleRate: number; +} + +export interface ManagedStartupDeviceAuth { + readonly disabled: boolean; + readonly optOutSource: ManagedStartupDeviceAuthOptOutSource; +} + +export interface ManagedStartupOpenClawConfig { + readonly agent: "openclaw"; + readonly webSearch: ManagedStartupWebSearch; + readonly otel: ManagedStartupOpenClawOtel; + readonly agentTimeoutSeconds: number; + readonly heartbeatEvery: string | null; + readonly extraAgents: ManagedStartupExtraAgents; + readonly deviceAuth: ManagedStartupDeviceAuth; + readonly minimalBootstrap: boolean; +} + +export interface ManagedStartupHermesConfig { + readonly agent: "hermes"; + readonly webSearch: ManagedStartupWebSearch; +} + +export interface ManagedStartupDcodeConfig { + readonly agent: "langchain-deepagents-code"; + readonly autoApprovalMode: ManagedStartupDcodeAutoApprovalMode; + readonly observabilityEnabled: boolean; +} + +export type ManagedStartupAgentConfig = + | ManagedStartupOpenClawConfig + | ManagedStartupHermesConfig + | ManagedStartupDcodeConfig; + +export interface ManagedStartupProfile { + readonly schemaVersion: typeof MANAGED_STARTUP_PROFILE_SCHEMA_VERSION; + readonly agent: ManagedStartupAgent; + readonly agentConfig: ManagedStartupAgentConfig; + readonly inference: ManagedStartupInference; + readonly proxy: ManagedStartupProxy; + readonly dashboard: ManagedStartupDashboard; + readonly tools: ManagedStartupTools; + readonly messaging: ManagedStartupMessaging; + readonly tuning: ManagedStartupTuning; + readonly corporateCa: ManagedStartupCorporateCa; +} + +export type ManagedStartupDashboardMode = "disabled" | "loopback" | "remote" | "loopback-forwarded"; + +export interface ManagedStartupAgentCapabilities { + readonly inferenceApis: readonly ManagedStartupInferenceApi[]; + readonly dashboardModes: readonly ManagedStartupDashboardMode[]; + readonly inputModalities: readonly ManagedStartupInputModality[]; + readonly webSearchProviders: readonly ManagedStartupWebSearchProvider[]; + readonly toolGateways: readonly ManagedStartupHermesToolGateway[]; + readonly tuningFields: readonly ( + | "contextWindow" + | "maxTokens" + | "reasoning" + | "reasoningEffort" + )[]; + readonly supportsMessaging: boolean; + readonly supportsInferenceCompatibility: boolean; + readonly supportsUpstreamEndpoint: boolean; + readonly supportsHostProxyIntent: boolean; + readonly supportsPrimaryModelRef: boolean; + readonly supportsAgentTimeout: boolean; + readonly supportsHeartbeat: boolean; + readonly supportsExtraAgents: boolean; + readonly supportsDeviceAuth: boolean; + readonly observability: "openclaw-otel" | "dcode-marker" | "none"; + readonly supportsMinimalBootstrap: boolean; +} + +/** + * Host-side negotiation must use this table before dispatch. A runtime that + * does not advertise the requested semantic capability is rejected instead of + * silently dropping a field. + */ +export const MANAGED_STARTUP_PROFILE_CAPABILITIES = { + openclaw: { + inferenceApis: MANAGED_STARTUP_INFERENCE_APIS, + dashboardModes: ["loopback", "remote"], + inputModalities: ["text", "image"], + webSearchProviders: ["brave", "tavily"], + toolGateways: [], + tuningFields: ["contextWindow", "maxTokens", "reasoning", "reasoningEffort"], + supportsMessaging: true, + supportsInferenceCompatibility: true, + supportsUpstreamEndpoint: false, + supportsHostProxyIntent: true, + supportsPrimaryModelRef: true, + supportsAgentTimeout: true, + supportsHeartbeat: true, + supportsExtraAgents: true, + supportsDeviceAuth: true, + observability: "openclaw-otel", + supportsMinimalBootstrap: true, + }, + hermes: { + inferenceApis: MANAGED_STARTUP_INFERENCE_APIS, + dashboardModes: ["disabled", "loopback-forwarded"], + inputModalities: [], + webSearchProviders: ["tavily"], + toolGateways: MANAGED_STARTUP_HERMES_TOOL_GATEWAYS, + tuningFields: ["contextWindow"], + supportsMessaging: true, + supportsInferenceCompatibility: false, + supportsUpstreamEndpoint: false, + supportsHostProxyIntent: true, + supportsPrimaryModelRef: false, + supportsAgentTimeout: false, + supportsHeartbeat: false, + supportsExtraAgents: false, + supportsDeviceAuth: false, + observability: "none", + supportsMinimalBootstrap: false, + }, + "langchain-deepagents-code": { + inferenceApis: ["openai-completions"], + dashboardModes: ["disabled"], + inputModalities: [], + webSearchProviders: [], + toolGateways: [], + tuningFields: [], + supportsMessaging: false, + supportsInferenceCompatibility: false, + supportsUpstreamEndpoint: true, + supportsHostProxyIntent: true, + supportsPrimaryModelRef: false, + supportsAgentTimeout: false, + supportsHeartbeat: false, + supportsExtraAgents: false, + supportsDeviceAuth: false, + observability: "dcode-marker", + supportsMinimalBootstrap: false, + }, +} as const satisfies Record; + +export type ManagedStartupAffordanceSource = "docker-arg" | "runtime-env" | "host-material"; +export type ManagedStartupAffordanceRepresentation = "value" | "derived" | "digest-handoff"; + +export interface ManagedStartupAffordance { + readonly input: string; + readonly profilePath: string; + readonly source: ManagedStartupAffordanceSource; + readonly representation: ManagedStartupAffordanceRepresentation; +} + +function affordance( + input: string, + profilePath: string, + source: ManagedStartupAffordanceSource = "docker-arg", + representation: ManagedStartupAffordanceRepresentation = "value", +): ManagedStartupAffordance { + return { input, profilePath, source, representation }; +} + +const HOST_PROXY_AFFORDANCES = [ + affordance("HTTP_PROXY", "proxy.hostHttpUrl", "runtime-env"), + affordance("http_proxy", "proxy.hostHttpUrl", "runtime-env", "derived"), + affordance("HTTPS_PROXY", "proxy.hostHttpsUrl", "runtime-env"), + affordance("https_proxy", "proxy.hostHttpsUrl", "runtime-env", "derived"), + affordance("NO_PROXY", "proxy.hostNoProxy", "runtime-env"), + affordance("no_proxy", "proxy.hostNoProxy", "runtime-env", "derived"), +] as const; + +/** + * Complete v1 mapping from Docker/start inputs to typed profile fields. + * Data form lets the Dockerfile drift test reject an unclassified input. + */ +export const MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY = { + openclaw: [ + affordance("NEMOCLAW_MODEL", "inference.model"), + affordance("NEMOCLAW_INFERENCE_PROVIDER_ID", "inference.routeProvider"), + affordance("NEMOCLAW_UPSTREAM_PROVIDER", "inference.upstreamProvider"), + affordance("NEMOCLAW_PRIMARY_MODEL_REF", "inference.primaryModelRef"), + affordance("NEMOCLAW_INFERENCE_BASE_URL", "inference.routedBaseUrl"), + affordance("NEMOCLAW_INFERENCE_API", "inference.api"), + affordance("NEMOCLAW_INFERENCE_COMPAT_B64", "inference.compatibility"), + affordance("NEMOCLAW_INFERENCE_INPUTS", "inference.inputModalities"), + affordance("NEMOCLAW_CONTEXT_WINDOW", "tuning.contextWindow"), + affordance("NEMOCLAW_MAX_TOKENS", "tuning.maxTokens"), + affordance("NEMOCLAW_REASONING", "tuning.reasoning"), + affordance("NEMOCLAW_REASONING_EFFORT", "tuning.reasoningEffort"), + affordance("NEMOCLAW_TOOL_DISCLOSURE", "tools.disclosure"), + affordance("NEMOCLAW_AGENT_TIMEOUT", "agentConfig.agentTimeoutSeconds"), + affordance("NEMOCLAW_AGENT_HEARTBEAT_EVERY", "agentConfig.heartbeatEvery"), + affordance("NEMOCLAW_EXTRA_AGENTS_JSON_B64", "agentConfig.extraAgents"), + affordance("NEMOCLAW_DISABLE_DEVICE_AUTH", "agentConfig.deviceAuth.disabled"), + affordance("NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE", "agentConfig.deviceAuth.optOutSource"), + affordance("NEMOCLAW_WEB_SEARCH_ENABLED", "agentConfig.webSearch.enabled"), + affordance("NEMOCLAW_WEB_SEARCH_PROVIDER", "agentConfig.webSearch.provider"), + affordance("NEMOCLAW_OPENCLAW_OTEL", "agentConfig.otel.enabled"), + affordance("NEMOCLAW_OPENCLAW_OTEL_ENDPOINT", "agentConfig.otel.endpointUrl"), + affordance("NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME", "agentConfig.otel.serviceName"), + affordance("NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE", "agentConfig.otel.sampleRate"), + affordance("CHAT_UI_URL", "dashboard.url"), + affordance("NEMOCLAW_DASHBOARD_BIND", "dashboard.bindAddress"), + affordance("NEMOCLAW_WSL_DASHBOARD_EXPOSURE", "dashboard.wslExposure"), + affordance("NEMOCLAW_DASHBOARD_PORT", "dashboard.port", "runtime-env"), + affordance("NEMOCLAW_PROXY_HOST", "proxy.managedHost"), + affordance("NEMOCLAW_PROXY_PORT", "proxy.managedPort"), + affordance("NEMOCLAW_MESSAGING_PLAN_B64", "messaging.plan"), + affordance("NEMOCLAW_MINIMAL_BOOTSTRAP", "agentConfig.minimalBootstrap", "runtime-env"), + affordance( + "NEMOCLAW_CORPORATE_CA_B64", + "corporateCa.bundleSha256", + "host-material", + "digest-handoff", + ), + ...HOST_PROXY_AFFORDANCES, + ], + hermes: [ + affordance("NEMOCLAW_MODEL", "inference.model"), + affordance("NEMOCLAW_INFERENCE_PROVIDER_ID", "inference.routeProvider"), + affordance("NEMOCLAW_UPSTREAM_PROVIDER", "inference.upstreamProvider"), + affordance("NEMOCLAW_INFERENCE_BASE_URL", "inference.routedBaseUrl"), + affordance("NEMOCLAW_INFERENCE_API", "inference.api"), + affordance("NEMOCLAW_CONTEXT_WINDOW", "tuning.contextWindow"), + affordance("NEMOCLAW_TOOL_DISCLOSURE", "tools.disclosure"), + affordance( + "NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER", + "tools.enabledGateways", + "docker-arg", + "derived", + ), + affordance("NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64", "tools.enabledGateways"), + affordance("NEMOCLAW_WEB_SEARCH_ENABLED", "agentConfig.webSearch.enabled"), + affordance("NEMOCLAW_WEB_SEARCH_PROVIDER", "agentConfig.webSearch.provider"), + affordance("NEMOCLAW_MESSAGING_PLAN_B64", "messaging.plan"), + affordance("CHAT_UI_URL", "dashboard.url"), + affordance("NEMOCLAW_HERMES_DASHBOARD", "dashboard.mode", "runtime-env"), + affordance("NEMOCLAW_HERMES_DASHBOARD_PORT", "dashboard.publicPort", "runtime-env"), + affordance("NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT", "dashboard.internalPort", "runtime-env"), + affordance("NEMOCLAW_HERMES_DASHBOARD_TUI", "dashboard.tuiEnabled", "runtime-env"), + affordance("NEMOCLAW_PROXY_HOST", "proxy.managedHost", "runtime-env"), + affordance("NEMOCLAW_PROXY_PORT", "proxy.managedPort", "runtime-env"), + affordance( + "NEMOCLAW_CORPORATE_CA_B64", + "corporateCa.bundleSha256", + "host-material", + "digest-handoff", + ), + ...HOST_PROXY_AFFORDANCES, + ], + "langchain-deepagents-code": [ + affordance("NEMOCLAW_MODEL", "inference.model"), + affordance("NEMOCLAW_INFERENCE_PROVIDER_ID", "inference.routeProvider"), + affordance("NEMOCLAW_UPSTREAM_PROVIDER", "inference.upstreamProvider"), + affordance("NEMOCLAW_UPSTREAM_ENDPOINT_URL", "inference.upstreamEndpointUrl"), + affordance("NEMOCLAW_INFERENCE_BASE_URL", "inference.routedBaseUrl"), + affordance("NEMOCLAW_INFERENCE_API", "inference.api"), + affordance("NEMOCLAW_TOOL_DISCLOSURE", "tools.disclosure"), + affordance("NEMOCLAW_DCODE_AUTO_APPROVAL", "agentConfig.autoApprovalMode"), + affordance("NEMOCLAW_PROXY_HOST", "proxy.managedHost"), + affordance("NEMOCLAW_PROXY_PORT", "proxy.managedPort"), + affordance("NEMOCLAW_OBSERVABILITY", "agentConfig.observabilityEnabled", "runtime-env"), + affordance( + "NEMOCLAW_CORPORATE_CA_B64", + "corporateCa.bundleSha256", + "host-material", + "digest-handoff", + ), + ...HOST_PROXY_AFFORDANCES, + ], +} as const satisfies Record; + +export interface ManagedStartupExcludedDockerInput { + readonly input: string; + readonly reason: + | "release-composition" + | "integrity-pin" + | "build-provenance" + | "platform-build" + | "fixed-image-contract"; +} + +/** + * Docker inputs intentionally outside a startup profile. None are resolved + * deployment behavior: they select/pin release artifacts, invalidate build + * caches, or perform a platform-specific image ownership rewrite. + */ +export const MANAGED_STARTUP_PROFILE_EXCLUDED_DOCKER_INPUTS = { + openclaw: [ + { input: "BASE_IMAGE", reason: "release-composition" }, + { input: "OPENCLAW_VERSION", reason: "release-composition" }, + { input: "OPENCLAW_2026_7_1_INTEGRITY", reason: "integrity-pin" }, + { input: "OPENCLAW_2026_7_1_TARBALL", reason: "release-composition" }, + { input: "OPENCLAW_DIAGNOSTICS_OTEL_2026_7_1_INTEGRITY", reason: "integrity-pin" }, + { input: "OPENCLAW_BRAVE_PLUGIN_2026_7_1_INTEGRITY", reason: "integrity-pin" }, + { input: "NEMOCLAW_E2E_FIXTURE_LEGACY_OPENCLAW", reason: "release-composition" }, + { input: "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION", reason: "release-composition" }, + { input: "OPENCLAW_2026_3_11_INTEGRITY", reason: "integrity-pin" }, + { input: "OPENCLAW_2026_3_11_TARBALL", reason: "release-composition" }, + { input: "OPENCLAW_2026_4_24_INTEGRITY", reason: "integrity-pin" }, + { input: "OPENCLAW_2026_4_24_TARBALL", reason: "release-composition" }, + { input: "CODEX_ACP_0_11_1_INTEGRITY", reason: "integrity-pin" }, + { input: "MCPORTER_VERSION", reason: "release-composition" }, + { input: "MCPORTER_0_7_3_INTEGRITY", reason: "integrity-pin" }, + { input: "MCPORTER_0_7_3_TARBALL", reason: "release-composition" }, + { input: "NEMOCLAW_BUILD_ID", reason: "build-provenance" }, + { input: "NEMOCLAW_DARWIN_VM_COMPAT", reason: "platform-build" }, + { input: "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER", reason: "fixed-image-contract" }, + ], + hermes: [ + { input: "BASE_IMAGE", reason: "release-composition" }, + { input: "SSL_CERT_FILE", reason: "fixed-image-contract" }, + { input: "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION", reason: "release-composition" }, + { input: "NEMOCLAW_HERMES_PROFILE_POLICY_PATCHER_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_GATEWAY_RUNTIME_METADATA_PATCHER_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_CRON_RUNTIME_PATCHER_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_CRON_EXECUTIONS_SOURCE_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_BACKUP_SOURCE_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_DISCORD_RECOVERY_PATCHER_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_LANGFUSE_PATCHER_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_WRAPPER_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_VALIDATOR_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_TIRITH_FINALIZER_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_BUILD_ID", reason: "build-provenance" }, + { input: "NEMOCLAW_DARWIN_VM_COMPAT", reason: "platform-build" }, + { input: "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER", reason: "fixed-image-contract" }, + ], + "langchain-deepagents-code": [ + { input: "BASE_IMAGE", reason: "release-composition" }, + { input: "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION", reason: "release-composition" }, + { input: "NEMOCLAW_BUILD_ID", reason: "build-provenance" }, + { input: "NEMOCLAW_DARWIN_VM_COMPAT", reason: "platform-build" }, + { input: "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER", reason: "fixed-image-contract" }, + ], +} as const satisfies Record; + +export class ManagedStartupProfileError extends Error { + constructor(message: string) { + super(`Invalid managed startup profile: ${message}`); + this.name = "ManagedStartupProfileError"; + } +} + +const PROFILE_KEYS = new Set([ + "schemaVersion", + "agent", + "agentConfig", + "inference", + "proxy", + "dashboard", + "tools", + "messaging", + "tuning", + "corporateCa", +]); +const INFERENCE_KEYS = new Set([ + "routeProvider", + "upstreamProvider", + "model", + "routedBaseUrl", + "upstreamEndpointUrl", + "api", + "primaryModelRef", + "compatibility", + "inputModalities", +]); +const PROXY_KEYS = new Set([ + "managedHost", + "managedPort", + "hostHttpUrl", + "hostHttpsUrl", + "hostNoProxy", +]); +const OPENCLAW_DASHBOARD_KEYS = new Set([ + "agent", + "mode", + "url", + "port", + "bindAddress", + "wslExposure", +]); +const HERMES_DASHBOARD_KEYS = new Set([ + "agent", + "mode", + "url", + "publicPort", + "internalPort", + "tuiEnabled", +]); +const DCODE_DASHBOARD_KEYS = new Set(["agent", "mode"]); +const TOOLS_KEYS = new Set(["disclosure", "enabledGateways"]); +const MESSAGING_KEYS = new Set(["plan"]); +const TUNING_KEYS = new Set(["contextWindow", "maxTokens", "reasoning", "reasoningEffort"]); +const CORPORATE_CA_KEYS = new Set(["bundleSha256"]); +const OPENCLAW_CONFIG_KEYS = new Set([ + "agent", + "webSearch", + "otel", + "agentTimeoutSeconds", + "heartbeatEvery", + "extraAgents", + "deviceAuth", + "minimalBootstrap", +]); +const HERMES_CONFIG_KEYS = new Set(["agent", "webSearch"]); +const DCODE_CONFIG_KEYS = new Set(["agent", "autoApprovalMode", "observabilityEnabled"]); +const WEB_SEARCH_KEYS = new Set(["enabled", "provider"]); +const OTEL_KEYS = new Set(["enabled", "endpointUrl", "serviceName", "sampleRate"]); +const DEVICE_AUTH_KEYS = new Set(["disabled", "optOutSource"]); +const EXTRA_AGENTS_KEYS = new Set(["agents", "defaults", "main"]); +const MANAGED_STARTUP_AGENT_SET = new Set(MANAGED_STARTUP_AGENTS); +const DCODE_AUTO_APPROVAL_MODE_SET = new Set(MANAGED_STARTUP_DCODE_AUTO_APPROVAL_MODES); +const INFERENCE_API_SET = new Set(MANAGED_STARTUP_INFERENCE_APIS); +const REASONING_EFFORT_SET = new Set(MANAGED_STARTUP_REASONING_EFFORTS); +const HERMES_GATEWAY_SET = new Set(MANAGED_STARTUP_HERMES_TOOL_GATEWAYS); +const HERMES_RESERVED_API_PORTS = new Set([8642, 18_642]); + +function isPlainObject(value: unknown): value is Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function isCredentialShapedName(name: string): boolean { + if (PUBLIC_KEY_NAME_PATTERN.test(name) || NON_SECRET_KEY_METADATA_NAMES.has(name)) return false; + return ( + CREDENTIAL_SHAPED_NAME_PATTERN.test(name) || + CREDENTIAL_COMPOUND_NAME_PATTERN.test(name) || + CREDENTIAL_CAMEL_SUFFIX_PATTERN.test(name) || + CREDENTIAL_CAMEL_BOUNDARY_PATTERN.test(name) || + CREDENTIAL_ENV_NAME_PATTERN.test(name) || + CREDENTIAL_HEADER_NAME_PATTERN.test(name) || + PASS_CREDENTIAL_NAME_PATTERN.test(name) + ); +} + +function valueLooksLikeSecret(value: string): boolean { + for (let index = 0; index < SECRET_VALUE_PATTERNS.length; index += 1) { + const descriptor = Object.getOwnPropertyDescriptor(SECRET_VALUE_PATTERNS, String(index)); + if (descriptor && "value" in descriptor && descriptor.value.test(value)) return true; + } + return false; +} + +function isMessagingCredentialPlaceholder(path: readonly string[], value: unknown): boolean { + return ( + path.length >= 2 && + path[0] === "messaging" && + path[1] === "plan" && + typeof value === "string" && + MESSAGING_CREDENTIAL_PLACEHOLDER_RE.test(value) + ); +} + +function containsUrlWithCredentialMaterial(value: string): boolean { + const candidates = value.match(URL_CANDIDATE_RE) ?? []; + for (let index = 0; index < candidates.length; index += 1) { + const descriptor = Object.getOwnPropertyDescriptor(candidates, String(index)); + if (!descriptor || !("value" in descriptor)) continue; + try { + const url = new URL(descriptor.value); + let credentialQuery = false; + url.searchParams.forEach((_queryValue, key) => { + if (isCredentialShapedName(key)) credentialQuery = true; + }); + const fragment = url.hash.startsWith("#") ? url.hash.slice(1) : url.hash; + const queryStart = fragment.indexOf("?"); + const fragmentParameters = new URLSearchParams( + queryStart >= 0 ? fragment.slice(queryStart + 1) : fragment, + ); + let credentialFragment = false; + fragmentParameters.forEach((_fragmentValue, key) => { + if (isCredentialShapedName(key)) credentialFragment = true; + }); + if (url.username || url.password || credentialQuery || credentialFragment) return true; + } catch { + // A field-level URL validator owns malformed strings where a URL is expected. + } + } + return false; +} + +function invalid(reason: string): never { + throw new ManagedStartupProfileError(reason); +} + +function mapArrayByIndex(values: readonly T[], mapper: (value: T, index: number) => U): U[] { + const mapped: U[] = []; + for (let index = 0; index < values.length; index += 1) { + const descriptor = Object.getOwnPropertyDescriptor(values, String(index)); + if (!descriptor || !("value" in descriptor)) { + invalid("payload arrays must contain only indexed JSON values"); + } + Object.defineProperty(mapped, String(index), { + configurable: true, + enumerable: true, + value: mapper(descriptor.value as T, index), + writable: true, + }); + } + return mapped; +} + +function sortStrings(values: string[]): string[] { + for (let index = 1; index < values.length; index += 1) { + const selected = values[index] as string; + let insertion = index; + while (insertion > 0 && (values[insertion - 1] as string) > selected) { + Object.defineProperty(values, String(insertion), { + configurable: true, + enumerable: true, + value: values[insertion - 1], + writable: true, + }); + insertion -= 1; + } + Object.defineProperty(values, String(insertion), { + configurable: true, + enumerable: true, + value: selected, + writable: true, + }); + } + return values; +} + +function requireRecord(value: unknown, where: string): Record { + if (!isPlainObject(value)) invalid(`${where} must be an object`); + return value; +} + +function rejectUnknownKeys( + value: Record, + allowed: ReadonlySet, + where: string, +): void { + const keys = Object.keys(value); + for (let index = 0; index < keys.length; index += 1) { + if (!allowed.has(keys[index] as string)) { + invalid(`${where} contains unsupported fields`); + } + } +} + +function requireBoolean(value: unknown, where: string): boolean { + if (typeof value !== "boolean") invalid(`${where} must be a boolean`); + return value; +} + +function requireNullableBoolean(value: unknown, where: string): boolean | null { + if (value === null) return null; + return requireBoolean(value, where); +} + +function requireBoundedString( + value: unknown, + where: string, + maxBytes = MAX_IDENTIFIER_BYTES, +): string { + if ( + typeof value !== "string" || + value.length === 0 || + value !== value.trim() || + Buffer.byteLength(value, "utf8") > maxBytes || + CONTROL_CHARACTER_RE.test(value) + ) { + invalid(`${where} must be a bounded, non-empty string without control characters`); + } + return value; +} + +function requireStringEnum( + value: unknown, + allowed: ReadonlySet, + where: string, +): T { + const normalized = requireBoundedString(value, where); + if (!allowed.has(normalized)) invalid(`${where} is not supported`); + return normalized as T; +} + +function requireNullablePositiveInteger(value: unknown, where: string): number | null { + if (value === null) return null; + if ( + typeof value !== "number" || + !Number.isSafeInteger(value) || + value < 1 || + value > MAX_TUNING_INTEGER + ) { + invalid(`${where} must be null or a bounded positive integer`); + } + return value; +} + +function requirePositiveInteger( + value: unknown, + where: string, + maximum = MAX_TUNING_INTEGER, +): number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 1 || value > maximum) { + invalid(`${where} must be a bounded positive integer`); + } + return value; +} + +function requirePort(value: unknown, where: string, minimum = 1): number { + if (typeof value !== "number" || !Number.isInteger(value) || value < 1 || value > 65_535) { + invalid(`${where} must be a valid TCP port`); + } + if (value < minimum) invalid(`${where} must be at least ${String(minimum)}`); + return value; +} + +function requireStringList(value: unknown, where: string): readonly string[] { + if (!Array.isArray(value) || value.length > MAX_LIST_ITEMS) { + invalid(`${where} must be a bounded string list`); + } + const items = mapArrayByIndex(value, (item) => requireBoundedString(item, `${where} item`)); + const unique = new Set(); + for (let index = 0; index < items.length; index += 1) { + unique.add(items[index] as string); + } + if (unique.size !== items.length) invalid(`${where} must not contain duplicates`); + return sortStrings(items); +} + +function requireEnumList( + value: unknown, + allowed: ReadonlySet, + where: string, + options: { readonly allowEmpty: boolean }, +): readonly T[] { + const items = requireStringList(value, where); + if (!options.allowEmpty && items.length === 0) invalid(`${where} must not be empty`); + for (let index = 0; index < items.length; index += 1) { + const item = items[index] as string; + if (!allowed.has(item)) invalid(`${where} contains an unsupported value`); + } + return items as readonly T[]; +} + +function cloneJsonValue(value: unknown, where: string): ManagedStartupJsonValue { + const clone = (current: unknown, depth: number): ManagedStartupJsonValue => { + if (depth > MAX_JSON_DEPTH) invalid(`${where} exceeds the JSON depth limit`); + if (current === null || typeof current === "string" || typeof current === "boolean") { + return current; + } + if (typeof current === "number") { + if (!Number.isFinite(current)) invalid(`${where} contains a non-finite number`); + return current; + } + if (Array.isArray(current)) { + return mapArrayByIndex(current, (item) => clone(item, depth + 1)); + } + if (!isPlainObject(current)) invalid(`${where} contains a non-JSON value`); + const result: ManagedStartupJsonObject = {}; + const keys = Object.getOwnPropertyNames(current); + for (let index = 0; index < keys.length; index += 1) { + const key = keys[index] as string; + if ( + key.length === 0 || + Buffer.byteLength(key, "utf8") > MAX_IDENTIFIER_BYTES || + CONTROL_CHARACTER_RE.test(key) + ) { + invalid(`${where} contains an invalid object key`); + } + const descriptor = Object.getOwnPropertyDescriptor(current, key); + if (!descriptor || !("value" in descriptor)) { + invalid(`${where} contains a non-JSON value`); + } + Object.defineProperty(result, key, { + configurable: true, + enumerable: true, + value: clone(descriptor.value, depth + 1), + writable: true, + }); + } + return result; + }; + return clone(value, 0); +} + +function requireJsonObjectOrNull(value: unknown, where: string): ManagedStartupJsonObject | null { + if (value === null) return null; + if (!isPlainObject(value)) invalid(`${where} must be null or a plain JSON object`); + return cloneJsonValue(value, where) as ManagedStartupJsonObject; +} + +function requireJsonObject(value: unknown, where: string): ManagedStartupJsonObject { + const object = requireJsonObjectOrNull(value, where); + if (object === null) invalid(`${where} must be a plain JSON object`); + return object; +} + +function requireHttpUrl(value: unknown, where: string): string { + const raw = requireBoundedString(value, where, MAX_URL_BYTES); + let parsed: URL; + try { + parsed = new URL(raw); + } catch { + invalid(`${where} must be a valid HTTP(S) URL`); + } + if ( + (parsed.protocol !== "http:" && parsed.protocol !== "https:") || + parsed.username || + parsed.password || + parsed.search || + parsed.hash + ) { + invalid(`${where} must be a credential-free HTTP(S) URL without query or fragment data`); + } + const pathname = parsed.pathname.replace(/\/+$/u, ""); + return pathname === "" ? parsed.origin : `${parsed.origin}${pathname}`; +} + +function requireProxyUrl( + value: unknown, + allowedSchemes: ReadonlySet, + where: string, +): string | null { + if (value === null) return null; + const raw = requireBoundedString(value, where, MAX_URL_BYTES); + let parsed: URL; + try { + parsed = new URL(raw); + } catch { + invalid(`${where} must be a valid HTTP(S) proxy URL`); + } + if ( + !allowedSchemes.has(parsed.protocol) || + parsed.username || + parsed.password || + parsed.pathname !== "/" || + parsed.search || + parsed.hash + ) { + invalid(`${where} must be a credential-free HTTP(S) proxy origin`); + } + return parsed.origin; +} + +function requireManagedProxyHost(value: unknown, where: string): string { + const host = requireBoundedString(value, where); + if (!/^[A-Za-z0-9._-]+$/u.test(host)) { + invalid(`${where} must be a hostname or IPv4 address without a scheme or separators`); + } + return host; +} + +function isLoopbackUrl(value: string): boolean { + const hostname = new URL(value).hostname.toLowerCase(); + return ( + hostname === "localhost" || + hostname === "127.0.0.1" || + hostname === "::1" || + hostname === "[::1]" + ); +} + +function configuredDashboardPort(value: string): number { + const explicit = new URL(value).port; + return explicit === "" ? 18_789 : Number(explicit); +} + +function requireSampleRate(value: unknown, where: string): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) { + invalid(`${where} must be a number between 0 and 1`); + } + return value; +} + +function assertPayloadStructureAndCredentialShapes(root: unknown): void { + const pending: Array<{ + value: unknown; + depth: number; + path: readonly string[]; + }> = [{ value: root, depth: 0, path: [] }]; + let discoveredNodes = 1; + let observedBytes = 0; + + const observeText = (value: string): void => { + observedBytes += Buffer.byteLength(value, "utf8"); + if (observedBytes > MANAGED_STARTUP_PROFILE_MAX_BYTES) { + invalid(`payload exceeds ${String(MANAGED_STARTUP_PROFILE_MAX_BYTES)} bytes`); + } + }; + const reserveNode = (depth: number): void => { + discoveredNodes += 1; + if (discoveredNodes > MAX_JSON_NODES || depth > MAX_JSON_DEPTH) { + invalid("payload structure exceeds the complexity limit"); + } + observedBytes += 1; + }; + + while (pending.length > 0) { + const current = pending.pop(); + if (!current) break; + if (current.depth > MAX_JSON_DEPTH) { + invalid("payload structure exceeds the complexity limit"); + } + + if (typeof current.value === "string") { + observeText(current.value); + if ( + !isMessagingCredentialPlaceholder(current.path, current.value) && + valueLooksLikeSecret(current.value) + ) { + invalid("payload contains credential-shaped string data"); + } + if ( + RAW_CA_PEM_RE.test(current.value) || + RAW_CA_DER_BASE64_RE.test(current.value) || + RAW_CA_DATA_URI_RE.test(current.value) + ) { + invalid("payload contains raw certificate data; provide only the CA SHA-256 digest"); + } + if (containsUrlWithCredentialMaterial(current.value)) { + invalid("payload contains a URL with embedded credentials"); + } + continue; + } + if (Array.isArray(current.value)) { + if (Object.getPrototypeOf(current.value) !== Array.prototype) { + invalid("payload arrays must use the standard JSON prototype"); + } + if ("toJSON" in current.value) { + invalid("payload must not define a custom JSON serializer"); + } + if ( + Object.getOwnPropertySymbols(current.value).length > 0 || + Object.getOwnPropertyNames(current.value).length !== current.value.length + 1 + ) { + invalid("payload arrays must contain only indexed JSON values"); + } + for (let index = 0; index < current.value.length; index += 1) { + const depth = current.depth + 1; + reserveNode(depth); + const descriptor = Object.getOwnPropertyDescriptor(current.value, String(index)); + if (descriptor && !("value" in descriptor)) { + invalid("payload must contain only JSON data properties"); + } + pending.push({ + value: descriptor?.value, + depth, + path: current.path, + }); + } + continue; + } + if (current.value !== null && typeof current.value === "object") { + if (!isPlainObject(current.value)) invalid("payload must contain only plain JSON objects"); + if ("toJSON" in current.value) { + invalid("payload must not define a custom JSON serializer"); + } + const keys = Object.getOwnPropertyNames(current.value); + if ( + Object.getOwnPropertySymbols(current.value).length > 0 || + discoveredNodes + keys.length > MAX_JSON_NODES + ) { + invalid("payload structure exceeds the complexity limit"); + } + for (let index = 0; index < keys.length; index += 1) { + const key = keys[index] as string; + const depth = current.depth + 1; + reserveNode(depth); + observeText(key); + if ( + valueLooksLikeSecret(key) || + RAW_CA_PEM_RE.test(key) || + RAW_CA_DER_BASE64_RE.test(key) || + RAW_CA_DATA_URI_RE.test(key) || + containsUrlWithCredentialMaterial(key) + ) { + invalid("payload contains credential-shaped data in a field name"); + } + const descriptor = Object.getOwnPropertyDescriptor(current.value, key); + if (!descriptor || !("value" in descriptor)) { + invalid("payload must contain only JSON data properties"); + } + const child = descriptor.value; + if (isCredentialShapedName(key) && !isMessagingCredentialPlaceholder(current.path, child)) { + invalid("payload contains a credential-shaped field name"); + } + pending.push({ + value: child, + depth, + path: [...current.path, key], + }); + } + continue; + } + if ( + current.value !== null && + typeof current.value !== "number" && + typeof current.value !== "boolean" + ) { + invalid("payload must contain only JSON values"); + } + } +} + +function assertPayloadWithinByteLimit(value: unknown): void { + let serialized: string | undefined; + try { + serialized = JSON.stringify(value); + } catch { + invalid("payload is not serializable JSON"); + } + if ( + serialized === undefined || + Buffer.byteLength(serialized, "utf8") > MANAGED_STARTUP_PROFILE_MAX_BYTES + ) { + invalid(`payload exceeds ${String(MANAGED_STARTUP_PROFILE_MAX_BYTES)} bytes`); + } +} + +function validateWebSearch(value: unknown, agent: "openclaw" | "hermes"): ManagedStartupWebSearch { + const webSearch = requireRecord(value, "agentConfig.webSearch"); + rejectUnknownKeys(webSearch, WEB_SEARCH_KEYS, "agentConfig.webSearch"); + const provider = requireStringEnum( + webSearch.provider, + new Set(agent === "openclaw" ? ["brave", "tavily"] : ["tavily"]), + "agentConfig.webSearch.provider", + ); + return { + enabled: requireBoolean(webSearch.enabled, "agentConfig.webSearch.enabled"), + provider, + }; +} + +function validateOpenClawOtel(value: unknown): ManagedStartupOpenClawOtel { + const otel = requireRecord(value, "agentConfig.otel"); + rejectUnknownKeys(otel, OTEL_KEYS, "agentConfig.otel"); + return { + enabled: requireBoolean(otel.enabled, "agentConfig.otel.enabled"), + endpointUrl: requireHttpUrl(otel.endpointUrl, "agentConfig.otel.endpointUrl"), + serviceName: requireBoundedString( + otel.serviceName, + "agentConfig.otel.serviceName", + MAX_IDENTIFIER_BYTES, + ), + sampleRate: requireSampleRate(otel.sampleRate, "agentConfig.otel.sampleRate"), + }; +} + +function validateExtraAgents(value: unknown): ManagedStartupExtraAgents { + const extraAgents = requireRecord(value, "agentConfig.extraAgents"); + rejectUnknownKeys(extraAgents, EXTRA_AGENTS_KEYS, "agentConfig.extraAgents"); + if (!Array.isArray(extraAgents.agents) || extraAgents.agents.length > MAX_LIST_ITEMS) { + invalid("agentConfig.extraAgents.agents must be a bounded JSON object list"); + } + return { + agents: mapArrayByIndex(extraAgents.agents, (agent, index) => + requireJsonObject(agent, `agentConfig.extraAgents.agents[${String(index)}]`), + ), + defaults: requireJsonObject(extraAgents.defaults, "agentConfig.extraAgents.defaults"), + main: requireJsonObject(extraAgents.main, "agentConfig.extraAgents.main"), + }; +} + +function validateDeviceAuth(value: unknown): ManagedStartupDeviceAuth { + const deviceAuth = requireRecord(value, "agentConfig.deviceAuth"); + rejectUnknownKeys(deviceAuth, DEVICE_AUTH_KEYS, "agentConfig.deviceAuth"); + return { + disabled: requireBoolean(deviceAuth.disabled, "agentConfig.deviceAuth.disabled"), + optOutSource: requireStringEnum( + deviceAuth.optOutSource, + new Set(["operator", "managed-onboard"]), + "agentConfig.deviceAuth.optOutSource", + ), + }; +} + +function validateAgentConfig( + value: unknown, + expectedAgent: ManagedStartupAgent, +): ManagedStartupAgentConfig { + const config = requireRecord(value, "agentConfig"); + const agent = requireStringEnum( + config.agent, + MANAGED_STARTUP_AGENT_SET, + "agentConfig.agent", + ); + if (agent !== expectedAgent) invalid("agentConfig.agent must match agent"); + + if (agent === "openclaw") { + rejectUnknownKeys(config, OPENCLAW_CONFIG_KEYS, "agentConfig"); + const heartbeatEvery = + config.heartbeatEvery === null + ? null + : requireBoundedString( + config.heartbeatEvery, + "agentConfig.heartbeatEvery", + MAX_IDENTIFIER_BYTES, + ); + if (heartbeatEvery !== null && !/^\d+(?:s|m|h)$/u.test(heartbeatEvery)) { + invalid("agentConfig.heartbeatEvery must be null or a duration ending in s, m, or h"); + } + return { + agent, + webSearch: validateWebSearch(config.webSearch, agent), + otel: validateOpenClawOtel(config.otel), + agentTimeoutSeconds: requirePositiveInteger( + config.agentTimeoutSeconds, + "agentConfig.agentTimeoutSeconds", + ), + heartbeatEvery, + extraAgents: validateExtraAgents(config.extraAgents), + deviceAuth: validateDeviceAuth(config.deviceAuth), + minimalBootstrap: requireBoolean(config.minimalBootstrap, "agentConfig.minimalBootstrap"), + }; + } + if (agent === "hermes") { + rejectUnknownKeys(config, HERMES_CONFIG_KEYS, "agentConfig"); + return { agent, webSearch: validateWebSearch(config.webSearch, agent) }; + } + + rejectUnknownKeys(config, DCODE_CONFIG_KEYS, "agentConfig"); + return { + agent, + autoApprovalMode: requireStringEnum( + config.autoApprovalMode, + DCODE_AUTO_APPROVAL_MODE_SET, + "agentConfig.autoApprovalMode", + ), + observabilityEnabled: requireBoolean( + config.observabilityEnabled, + "agentConfig.observabilityEnabled", + ), + }; +} + +function validateDashboard( + value: unknown, + expectedAgent: ManagedStartupAgent, +): ManagedStartupDashboard { + const dashboard = requireRecord(value, "dashboard"); + const agent = requireStringEnum( + dashboard.agent, + MANAGED_STARTUP_AGENT_SET, + "dashboard.agent", + ); + if (agent !== expectedAgent) invalid("dashboard.agent must match agent"); + + if (agent === "openclaw") { + rejectUnknownKeys(dashboard, OPENCLAW_DASHBOARD_KEYS, "dashboard"); + const mode = requireStringEnum<"loopback" | "remote">( + dashboard.mode, + new Set(["loopback", "remote"]), + "dashboard.mode", + ); + const url = requireHttpUrl(dashboard.url, "dashboard.url"); + const bindAddress = requireStringEnum<"127.0.0.1" | "0.0.0.0">( + dashboard.bindAddress, + new Set(["127.0.0.1", "0.0.0.0"]), + "dashboard.bindAddress", + ); + const wslExposure = requireBoolean(dashboard.wslExposure, "dashboard.wslExposure"); + const hasRemoteExposure = !isLoopbackUrl(url) || bindAddress === "0.0.0.0" || wslExposure; + if ((mode === "remote") !== hasRemoteExposure) { + invalid("OpenClaw dashboard.mode must reflect its URL, bind address, and WSL exposure"); + } + const port = requirePort(dashboard.port, "dashboard.port", 1024); + if (port === 8642) + invalid("OpenClaw dashboard.port must not use reserved Hermes API port 8642"); + if (configuredDashboardPort(url) !== port) { + invalid("OpenClaw dashboard.port must match dashboard.url"); + } + return { + agent, + mode, + url, + port, + bindAddress, + wslExposure, + }; + } + if (agent === "hermes") { + rejectUnknownKeys(dashboard, HERMES_DASHBOARD_KEYS, "dashboard"); + const mode = requireStringEnum<"disabled" | "loopback-forwarded">( + dashboard.mode, + new Set(["disabled", "loopback-forwarded"]), + "dashboard.mode", + ); + const url = requireHttpUrl(dashboard.url, "dashboard.url"); + if (!isLoopbackUrl(url)) { + invalid("Hermes dashboard.url must remain loopback; OpenShell owns the host forward"); + } + if (mode === "disabled") { + if ( + dashboard.publicPort !== null || + dashboard.internalPort !== null || + dashboard.tuiEnabled !== false + ) { + invalid("disabled Hermes dashboard must not configure ports or TUI"); + } + return { + agent, + mode, + url, + publicPort: null, + internalPort: null, + tuiEnabled: false, + }; + } + const publicPort = requirePort(dashboard.publicPort, "dashboard.publicPort", 1024); + const internalPort = requirePort(dashboard.internalPort, "dashboard.internalPort", 1024); + if (publicPort === internalPort) { + invalid("Hermes dashboard publicPort and internalPort must differ"); + } + if (HERMES_RESERVED_API_PORTS.has(publicPort) || HERMES_RESERVED_API_PORTS.has(internalPort)) { + invalid("Hermes dashboard ports must not use reserved API ports 8642 or 18642"); + } + if (configuredDashboardPort(url) !== publicPort) { + invalid("Hermes dashboard.publicPort must match dashboard.url"); + } + return { + agent, + mode, + url, + publicPort, + internalPort, + tuiEnabled: requireBoolean(dashboard.tuiEnabled, "dashboard.tuiEnabled"), + }; + } + + rejectUnknownKeys(dashboard, DCODE_DASHBOARD_KEYS, "dashboard"); + if (dashboard.mode !== "disabled") { + invalid("langchain-deepagents-code dashboard.mode must be disabled"); + } + return { agent, mode: "disabled" }; +} + +function validateInference(value: unknown, agent: ManagedStartupAgent): ManagedStartupInference { + const inference = requireRecord(value, "inference"); + rejectUnknownKeys(inference, INFERENCE_KEYS, "inference"); + const api = requireStringEnum( + inference.api, + INFERENCE_API_SET, + "inference.api", + ); + const supportedInferenceApis: readonly string[] = + MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].inferenceApis; + let apiSupported = false; + for (let index = 0; index < supportedInferenceApis.length; index += 1) { + if (supportedInferenceApis[index] === api) apiSupported = true; + } + if (!apiSupported) { + invalid(`inference.api is not supported by ${agent}`); + } + const upstreamEndpointUrl = + inference.upstreamEndpointUrl === null + ? null + : requireHttpUrl(inference.upstreamEndpointUrl, "inference.upstreamEndpointUrl"); + const primaryModelRef = + inference.primaryModelRef === null + ? null + : requireBoundedString( + inference.primaryModelRef, + "inference.primaryModelRef", + MAX_MODEL_BYTES, + ); + const compatibility = requireJsonObjectOrNull(inference.compatibility, "inference.compatibility"); + const inputModalities = + inference.inputModalities === null + ? null + : requireEnumList( + inference.inputModalities, + new Set(["text", "image"]), + "inference.inputModalities", + { allowEmpty: false }, + ); + + if (agent === "openclaw") { + if (upstreamEndpointUrl !== null) { + invalid("inference.upstreamEndpointUrl must be null for openclaw"); + } + if (primaryModelRef === null || inputModalities === null) { + invalid("openclaw requires primaryModelRef and inputModalities"); + } + } else { + if (primaryModelRef !== null || compatibility !== null || inputModalities !== null) { + invalid(`${agent} does not support primaryModelRef, compatibility, or inputModalities`); + } + if (agent === "hermes" && upstreamEndpointUrl !== null) { + invalid("inference.upstreamEndpointUrl must be null for hermes"); + } + } + + return { + routeProvider: requireBoundedString(inference.routeProvider, "inference.routeProvider"), + upstreamProvider: requireBoundedString( + inference.upstreamProvider, + "inference.upstreamProvider", + ), + model: requireBoundedString(inference.model, "inference.model", MAX_MODEL_BYTES), + routedBaseUrl: requireHttpUrl(inference.routedBaseUrl, "inference.routedBaseUrl"), + upstreamEndpointUrl, + api, + primaryModelRef, + compatibility, + inputModalities, + }; +} + +function validateProxy(value: unknown, agent: ManagedStartupAgent): ManagedStartupProxy { + const proxy = requireRecord(value, "proxy"); + rejectUnknownKeys(proxy, PROXY_KEYS, "proxy"); + const hostHttpUrl = requireProxyUrl(proxy.hostHttpUrl, new Set(["http:"]), "proxy.hostHttpUrl"); + // HTTPS_PROXY conventionally names an HTTP CONNECT proxy, so either scheme + // is valid while credentials and non-origin paths remain forbidden. + const hostHttpsUrl = requireProxyUrl( + proxy.hostHttpsUrl, + new Set(["http:", "https:"]), + "proxy.hostHttpsUrl", + ); + const hostNoProxy = requireStringList(proxy.hostNoProxy, "proxy.hostNoProxy"); + if ( + !MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].supportsHostProxyIntent && + (hostHttpUrl !== null || hostHttpsUrl !== null || hostNoProxy.length > 0) + ) { + invalid(`${agent} rejects host proxy intent and accepts only its root-owned managed route`); + } + return { + managedHost: requireManagedProxyHost(proxy.managedHost, "proxy.managedHost"), + managedPort: requirePort(proxy.managedPort, "proxy.managedPort"), + hostHttpUrl, + hostHttpsUrl, + hostNoProxy, + }; +} + +function validateTools(value: unknown, agent: ManagedStartupAgent): ManagedStartupTools { + const tools = requireRecord(value, "tools"); + rejectUnknownKeys(tools, TOOLS_KEYS, "tools"); + const enabledGateways = requireEnumList( + tools.enabledGateways, + HERMES_GATEWAY_SET, + "tools.enabledGateways", + { allowEmpty: true }, + ); + if (agent !== "hermes" && enabledGateways.length > 0) { + invalid("tools.enabledGateways is supported only by hermes"); + } + return { + disclosure: requireStringEnum( + tools.disclosure, + new Set(["progressive", "direct"]), + "tools.disclosure", + ), + enabledGateways, + }; +} + +function validateTuning(value: unknown, agent: ManagedStartupAgent): ManagedStartupTuning { + const tuning = requireRecord(value, "tuning"); + rejectUnknownKeys(tuning, TUNING_KEYS, "tuning"); + const result: ManagedStartupTuning = { + contextWindow: requireNullablePositiveInteger(tuning.contextWindow, "tuning.contextWindow"), + maxTokens: requireNullablePositiveInteger(tuning.maxTokens, "tuning.maxTokens"), + reasoning: requireNullableBoolean(tuning.reasoning, "tuning.reasoning"), + reasoningEffort: + tuning.reasoningEffort === null + ? null + : requireStringEnum( + tuning.reasoningEffort, + REASONING_EFFORT_SET, + "tuning.reasoningEffort", + ), + }; + if (agent === "openclaw") { + if ( + result.contextWindow === null || + result.maxTokens === null || + result.reasoning === null || + result.reasoningEffort === null + ) { + invalid("openclaw requires contextWindow, maxTokens, reasoning, and reasoningEffort tuning"); + } + } else if (agent === "hermes") { + if (result.maxTokens !== null || result.reasoning !== null || result.reasoningEffort !== null) { + invalid("hermes supports only contextWindow tuning"); + } + } else if ( + result.contextWindow !== null || + result.maxTokens !== null || + result.reasoning !== null || + result.reasoningEffort !== null + ) { + invalid("langchain-deepagents-code does not support startup tuning fields"); + } + return result; +} + +/** + * Validate unknown input and return a canonical, deeply rebuilt profile. + * Unknown keys are rejected at every object boundary, and unordered set-like + * lists are sorted so all producers fingerprint the same resolved intent. + */ +export function validateManagedStartupProfile(value: unknown): ManagedStartupProfile { + assertPayloadStructureAndCredentialShapes(value); + assertPayloadWithinByteLimit(value); + const profile = requireRecord(value, "profile"); + rejectUnknownKeys(profile, PROFILE_KEYS, "profile"); + if (profile.schemaVersion !== MANAGED_STARTUP_PROFILE_SCHEMA_VERSION) { + invalid(`schemaVersion must be ${String(MANAGED_STARTUP_PROFILE_SCHEMA_VERSION)}`); + } + const agent = requireStringEnum( + profile.agent, + MANAGED_STARTUP_AGENT_SET, + "agent", + ); + + const messaging = requireRecord(profile.messaging, "messaging"); + rejectUnknownKeys(messaging, MESSAGING_KEYS, "messaging"); + const messagingPlan = requireJsonObjectOrNull(messaging.plan, "messaging.plan"); + if (agent === "langchain-deepagents-code" && messagingPlan !== null) { + invalid("messaging.plan must be null for langchain-deepagents-code"); + } + if ( + messagingPlan !== null && + (messagingPlan.schemaVersion !== 1 || messagingPlan.agent !== agent) + ) { + invalid("messaging.plan must be a version 1 plan for the selected agent"); + } + + const corporateCa = requireRecord(profile.corporateCa, "corporateCa"); + rejectUnknownKeys(corporateCa, CORPORATE_CA_KEYS, "corporateCa"); + const bundleSha256 = corporateCa.bundleSha256; + if ( + bundleSha256 !== null && + (typeof bundleSha256 !== "string" || !SHA256_RE.test(bundleSha256)) + ) { + invalid("corporateCa.bundleSha256 must be null or a lowercase SHA-256 digest"); + } + + const agentConfig = validateAgentConfig(profile.agentConfig, agent); + const dashboard = validateDashboard(profile.dashboard, agent); + if ( + agentConfig.agent === "openclaw" && + dashboard.agent === "openclaw" && + dashboard.mode === "remote" && + !agentConfig.deviceAuth.disabled + ) { + invalid("remote OpenClaw dashboard exposure requires device auth to be disabled"); + } + + return { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent, + agentConfig, + inference: validateInference(profile.inference, agent), + proxy: validateProxy(profile.proxy, agent), + dashboard, + tools: validateTools(profile.tools, agent), + messaging: { + plan: messagingPlan, + }, + tuning: validateTuning(profile.tuning, agent), + corporateCa: { bundleSha256 }, + }; +} + +function canonicalizeJson(value: unknown): unknown { + if (Array.isArray(value)) return mapArrayByIndex(value, (item) => canonicalizeJson(item)); + if (!isPlainObject(value)) return value; + const result: Record = {}; + const keys = sortStrings(Object.keys(value)); + for (let index = 0; index < keys.length; index += 1) { + const key = keys[index] as string; + Object.defineProperty(result, key, { + configurable: true, + enumerable: true, + value: canonicalizeJson(value[key]), + writable: true, + }); + } + return result; +} + +/** Canonical JSON used by both the transport and fingerprint. */ +export function serializeManagedStartupProfile(profile: ManagedStartupProfile): string { + const validated = validateManagedStartupProfile(profile); + const serialized = JSON.stringify(canonicalizeJson(validated)); + if (Buffer.byteLength(serialized, "utf8") > MANAGED_STARTUP_PROFILE_MAX_BYTES) { + invalid(`canonical payload exceeds ${String(MANAGED_STARTUP_PROFILE_MAX_BYTES)} bytes`); + } + return serialized; +} + +/** Encode canonical JSON as unpadded base64url for an argv/env-safe handoff. */ +export function encodeManagedStartupProfile(profile: ManagedStartupProfile): string { + return Buffer.from(serializeManagedStartupProfile(profile), "utf8").toString("base64url"); +} + +/** Decode only the canonical representation produced by encodeManagedStartupProfile. */ +export function decodeManagedStartupProfile(encoded: string): ManagedStartupProfile { + if ( + typeof encoded !== "string" || + encoded.length === 0 || + Buffer.byteLength(encoded, "ascii") > MANAGED_STARTUP_PROFILE_MAX_ENCODED_BYTES || + !BASE64URL_RE.test(encoded) || + encoded.length % 4 === 1 + ) { + invalid("encoded payload is malformed or exceeds the size limit"); + } + const bytes = Buffer.from(encoded, "base64url"); + if ( + bytes.length === 0 || + bytes.length > MANAGED_STARTUP_PROFILE_MAX_BYTES || + bytes.toString("base64url") !== encoded + ) { + invalid("encoded payload is malformed or exceeds the size limit"); + } + + let raw: string; + try { + raw = UTF8_DECODER.decode(bytes); + } catch { + invalid("payload is not valid UTF-8"); + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + invalid("payload is not valid JSON"); + } + const profile = validateManagedStartupProfile(parsed); + if (serializeManagedStartupProfile(profile) !== raw) { + invalid("payload is not in canonical form"); + } + return profile; +} + +/** SHA-256 over canonical decoded JSON, independent of object key insertion order. */ +export function fingerprintManagedStartupProfile(profile: ManagedStartupProfile): string { + return createHash("sha256").update(serializeManagedStartupProfile(profile), "utf8").digest("hex"); +} diff --git a/src/lib/onboard/managed-startup/transport.ts b/src/lib/onboard/managed-startup/transport.ts new file mode 100644 index 00000000000..41d489ac47e --- /dev/null +++ b/src/lib/onboard/managed-startup/transport.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Environment variable names for the startup profile and CA bundle transports. */ +export const MANAGED_STARTUP_PROFILE_ENV = "NEMOCLAW_STARTUP_PROFILE_B64"; +export const MANAGED_STARTUP_CA_ENV = "NEMOCLAW_CORPORATE_CA_B64"; diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index fae85644d77..86390a961a1 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -46,6 +46,10 @@ function runTests(...tests: string[]): () => string[] { } export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ + { + pattern: /(?:^|\/)(?:Dockerfile|agents\/(?:hermes|langchain-deepagents-code)\/Dockerfile)$/, + testsToRun: runTests("src/lib/onboard/managed-startup-profile.test.ts"), + }, { pattern: /(?:^|\/)agents\/hermes\/policy-additions\.yaml$/, testsToRun: runTests( diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index 5d9234cbb2c..599426fe744 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -48,6 +48,9 @@ const E2E_WORKFLOW_CONTRACTS = [ ] as const; const OPAQUE_INPUTS = [ + "Dockerfile", + "agents/hermes/Dockerfile", + "agents/langchain-deepagents-code/Dockerfile", "agents/hermes/policy-additions.yaml", "src/lib/messaging/channels/telegram/policy/openclaw.yaml", "nemoclaw-blueprint/policies/presets/local-inference.yaml", @@ -83,6 +86,13 @@ describe("Vitest opaque-input watch triggers", () => { }); it("maps current opaque inputs to their direct contract tests (#6692)", () => { + expect(triggeredBy("Dockerfile")).toEqual(["src/lib/onboard/managed-startup-profile.test.ts"]); + expect(triggeredBy("agents/hermes/Dockerfile")).toEqual([ + "src/lib/onboard/managed-startup-profile.test.ts", + ]); + expect(triggeredBy("agents/langchain-deepagents-code/Dockerfile")).toEqual([ + "src/lib/onboard/managed-startup-profile.test.ts", + ]); expect(triggeredBy("agents/hermes/policy-additions.yaml")).toEqual([ "src/lib/onboard/initial-policy-real-policy.test.ts", "src/lib/onboard/initial-policy.test.ts", From 7f3859a32e9791ec40337785d72cad8674b37c96 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 10:50:56 -0700 Subject: [PATCH 002/117] docs(onboard): clarify startup profile transport encoding Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-startup/profile.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/onboard/managed-startup/profile.ts b/src/lib/onboard/managed-startup/profile.ts index 5dccdb3f236..729ada6491b 100644 --- a/src/lib/onboard/managed-startup/profile.ts +++ b/src/lib/onboard/managed-startup/profile.ts @@ -1632,7 +1632,7 @@ export function serializeManagedStartupProfile(profile: ManagedStartupProfile): return serialized; } -/** Encode canonical JSON as unpadded base64url for an argv/env-safe handoff. */ +/** Encode canonical JSON as unpadded base64url for transport through argv or an environment variable. */ export function encodeManagedStartupProfile(profile: ManagedStartupProfile): string { return Buffer.from(serializeManagedStartupProfile(profile), "utf8").toString("base64url"); } From 32561a916f0b60336761a460120259adf7281f6e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 14:25:08 -0700 Subject: [PATCH 003/117] fix(onboard): harden managed startup profile Signed-off-by: Aaron Erickson --- .../onboard/managed-startup-profile.test.ts | 66 ++++++++++++++++++- src/lib/onboard/managed-startup/profile.ts | 24 +++++-- 2 files changed, 84 insertions(+), 6 deletions(-) diff --git a/src/lib/onboard/managed-startup-profile.test.ts b/src/lib/onboard/managed-startup-profile.test.ts index fecb56b3994..1fa30c15d1a 100644 --- a/src/lib/onboard/managed-startup-profile.test.ts +++ b/src/lib/onboard/managed-startup-profile.test.ts @@ -408,6 +408,15 @@ describe("managed startup profile", () => { expect(inventory.every(({ profilePath }) => !profilePath.startsWith("env."))).toBe(true); }); + it("classifies the shared Hermes dashboard port as runtime startup intent", () => { + expect(MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY.hermes).toContainEqual({ + input: "NEMOCLAW_DASHBOARD_PORT", + profilePath: "dashboard.publicPort", + source: "runtime-env", + representation: "value", + }); + }); + it.each( VALID_PROFILES, )("maps every $agent inventory entry to an explicit profile field", (profile) => { @@ -785,6 +794,15 @@ describe("managed startup profile", () => { ).toThrow(/must match dashboard\.url/); }); + it("rejects an explicit Hermes context window below the image contract minimum", () => { + expect(() => + validateManagedStartupProfile({ + ...HERMES_PROFILE, + tuning: { ...HERMES_PROFILE.tuning, contextWindow: 63_999 }, + }), + ).toThrow(/contextWindow must be at least 64000 tokens/); + }); + it.each([ ["publicPort", 8642], ["publicPort", 18_642], @@ -801,6 +819,10 @@ describe("managed startup profile", () => { it.each([ "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----", + Buffer.from( + `-----BEGIN CERTIFICATE-----\n${"A".repeat(300)}\n-----END CERTIFICATE-----`, + "utf8", + ).toString("base64"), `MII${"A".repeat(300)}`, `data:application/x-x509-ca-cert;base64,MII${"A".repeat(300)}`, ])("rejects raw CA material while accepting only its digest", (rawCa) => { @@ -964,6 +986,46 @@ describe("managed startup profile", () => { expect(serializerInvoked).toBe(false); }); + it("does not invoke an inherited getter to supply a missing schema field", () => { + const corporateCa: Record = {}; + let getterInvoked = false; + Object.defineProperty(Object.prototype, "bundleSha256", { + configurable: true, + get() { + getterInvoked = true; + return null; + }, + }); + + let caught: unknown; + try { + validateManagedStartupProfile({ ...DCODE_PROFILE, corporateCa }); + } catch (error) { + caught = error; + } finally { + Reflect.deleteProperty(Object.prototype, "bundleSha256"); + } + + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toMatch(/bundleSha256/); + expect(getterInvoked).toBe(false); + }); + + it("rejects non-enumerable unknown fields instead of silently stripping them", () => { + const inference = { ...OPENCLAW_PROFILE.inference }; + Object.defineProperty(inference, "hiddenExtension", { + value: "safe", + enumerable: false, + }); + + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + inference, + }), + ).toThrow(/unsupported fields/); + }); + it("does not invoke polluted Array prototype mapping or sorting methods", () => { const mapDescriptor = Object.getOwnPropertyDescriptor(Array.prototype, "map"); const sortDescriptor = Object.getOwnPropertyDescriptor(Array.prototype, "sort"); @@ -990,8 +1052,8 @@ describe("managed startup profile", () => { }, }); } finally { - if (mapDescriptor) Object.defineProperty(Array.prototype, "map", mapDescriptor); - if (sortDescriptor) Object.defineProperty(Array.prototype, "sort", sortDescriptor); + Object.defineProperty(Array.prototype, "map", mapDescriptor as PropertyDescriptor); + Object.defineProperty(Array.prototype, "sort", sortDescriptor as PropertyDescriptor); } expect(prototypeMethodInvoked).toBe(false); diff --git a/src/lib/onboard/managed-startup/profile.ts b/src/lib/onboard/managed-startup/profile.ts index 729ada6491b..1647afcbfd2 100644 --- a/src/lib/onboard/managed-startup/profile.ts +++ b/src/lib/onboard/managed-startup/profile.ts @@ -25,10 +25,13 @@ const MAX_LIST_ITEMS = 128; const MAX_JSON_NODES = 4096; const MAX_JSON_DEPTH = 32; const MAX_TUNING_INTEGER = 1_000_000_000; +const MIN_HERMES_CONTEXT_WINDOW = 64_000; const SHA256_RE = /^[a-f0-9]{64}$/; const CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f-\u009f]/u; const BASE64URL_RE = /^[A-Za-z0-9_-]+$/; const RAW_CA_PEM_RE = /-----BEGIN (?:TRUSTED )?CERTIFICATE-----/iu; +const RAW_CA_PEM_BASE64_RE = + /^LS0tLS1CRUdJTi(?:BDRVJUSUZJQ0FURS0tLS0t|BUlVTVEVEIENFUlRJRklDQVRFLS0tLS0)/u; const RAW_CA_DER_BASE64_RE = /^MII[A-Za-z0-9+/=\r\n]{253,}$/u; const RAW_CA_DATA_URI_RE = /data:application\/(?:pkix-cert|x-x509-ca-cert);base64,MII[A-Za-z0-9+/=]{253,}/iu; @@ -478,6 +481,7 @@ export const MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY = { affordance("NEMOCLAW_WEB_SEARCH_PROVIDER", "agentConfig.webSearch.provider"), affordance("NEMOCLAW_MESSAGING_PLAN_B64", "messaging.plan"), affordance("CHAT_UI_URL", "dashboard.url"), + affordance("NEMOCLAW_DASHBOARD_PORT", "dashboard.publicPort", "runtime-env"), affordance("NEMOCLAW_HERMES_DASHBOARD", "dashboard.mode", "runtime-env"), affordance("NEMOCLAW_HERMES_DASHBOARD_PORT", "dashboard.publicPort", "runtime-env"), affordance("NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT", "dashboard.internalPort", "runtime-env"), @@ -883,7 +887,11 @@ function requireEnumList( return items as readonly T[]; } -function cloneJsonValue(value: unknown, where: string): ManagedStartupJsonValue { +function cloneJsonValue( + value: unknown, + where: string, + options: { readonly nullPrototypeObjects?: boolean } = {}, +): ManagedStartupJsonValue { const clone = (current: unknown, depth: number): ManagedStartupJsonValue => { if (depth > MAX_JSON_DEPTH) invalid(`${where} exceeds the JSON depth limit`); if (current === null || typeof current === "string" || typeof current === "boolean") { @@ -897,7 +905,9 @@ function cloneJsonValue(value: unknown, where: string): ManagedStartupJsonValue return mapArrayByIndex(current, (item) => clone(item, depth + 1)); } if (!isPlainObject(current)) invalid(`${where} contains a non-JSON value`); - const result: ManagedStartupJsonObject = {}; + const result: ManagedStartupJsonObject = options.nullPrototypeObjects + ? (Object.create(null) as ManagedStartupJsonObject) + : {}; const keys = Object.getOwnPropertyNames(current); for (let index = 0; index < keys.length; index += 1) { const key = keys[index] as string; @@ -1053,6 +1063,7 @@ function assertPayloadStructureAndCredentialShapes(root: unknown): void { } if ( RAW_CA_PEM_RE.test(current.value) || + RAW_CA_PEM_BASE64_RE.test(current.value) || RAW_CA_DER_BASE64_RE.test(current.value) || RAW_CA_DATA_URI_RE.test(current.value) ) { @@ -1111,6 +1122,7 @@ function assertPayloadStructureAndCredentialShapes(root: unknown): void { if ( valueLooksLikeSecret(key) || RAW_CA_PEM_RE.test(key) || + RAW_CA_PEM_BASE64_RE.test(key) || RAW_CA_DER_BASE64_RE.test(key) || RAW_CA_DATA_URI_RE.test(key) || containsUrlWithCredentialMaterial(key) @@ -1522,6 +1534,9 @@ function validateTuning(value: unknown, agent: ManagedStartupAgent): ManagedStar invalid("openclaw requires contextWindow, maxTokens, reasoning, and reasoningEffort tuning"); } } else if (agent === "hermes") { + if (result.contextWindow !== null && result.contextWindow < MIN_HERMES_CONTEXT_WINDOW) { + invalid(`hermes contextWindow must be at least ${String(MIN_HERMES_CONTEXT_WINDOW)} tokens`); + } if (result.maxTokens !== null || result.reasoning !== null || result.reasoningEffort !== null) { invalid("hermes supports only contextWindow tuning"); } @@ -1543,8 +1558,9 @@ function validateTuning(value: unknown, agent: ManagedStartupAgent): ManagedStar */ export function validateManagedStartupProfile(value: unknown): ManagedStartupProfile { assertPayloadStructureAndCredentialShapes(value); - assertPayloadWithinByteLimit(value); - const profile = requireRecord(value, "profile"); + const ownedValue = cloneJsonValue(value, "profile", { nullPrototypeObjects: true }); + assertPayloadWithinByteLimit(ownedValue); + const profile = requireRecord(ownedValue, "profile"); rejectUnknownKeys(profile, PROFILE_KEYS, "profile"); if (profile.schemaVersion !== MANAGED_STARTUP_PROFILE_SCHEMA_VERSION) { invalid(`schemaVersion must be ${String(MANAGED_STARTUP_PROFILE_SCHEMA_VERSION)}`); From e1845dd7a4ea3f1dca67aaa91544d234bbcc8bcf Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 16:10:45 -0700 Subject: [PATCH 004/117] fix(onboard): close startup profile contract gaps Signed-off-by: Aaron Erickson --- ci/source-shape-test-budget.json | 5 + .../onboard/managed-startup-profile.test.ts | 197 ++++++++++++++++ src/lib/onboard/managed-startup/profile.ts | 213 +++++++++++++++++- 3 files changed, 404 insertions(+), 11 deletions(-) diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index fc9c6e97ceb..3dd45f3afe5 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -21,6 +21,11 @@ "test": "classifies every stock Docker ARG as startup-affordance or deliberate exclusion", "category": "compatibility" }, + { + "file": "src/lib/onboard/managed-startup-profile.test.ts", + "test": "classifies every centralized runtime input as profile intent or an explicit deferral", + "category": "compatibility" + }, { "file": "src/lib/readiness/host.test.ts", "test": "bounds and redacts successful probe text before schema validation", diff --git a/src/lib/onboard/managed-startup-profile.test.ts b/src/lib/onboard/managed-startup-profile.test.ts index 1fa30c15d1a..256c77bb0b2 100644 --- a/src/lib/onboard/managed-startup-profile.test.ts +++ b/src/lib/onboard/managed-startup-profile.test.ts @@ -15,9 +15,11 @@ import { MANAGED_STARTUP_AGENTS, MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY, MANAGED_STARTUP_PROFILE_CAPABILITIES, + MANAGED_STARTUP_PROFILE_DEFERRED_RUNTIME_INPUTS, MANAGED_STARTUP_PROFILE_EXCLUDED_DOCKER_INPUTS, MANAGED_STARTUP_PROFILE_MAX_BYTES, MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + MANAGED_STARTUP_RUNTIME_CLEANUP_OBLIGATIONS, type ManagedStartupAgent, type ManagedStartupProfile, serializeManagedStartupProfile, @@ -247,6 +249,57 @@ const STOCK_DOCKER_ARGS = { ), } satisfies Record>; +const RUNTIME_INPUT_SOURCE_FILES = [ + "src/lib/onboard/sandbox-create-launch.ts", + "src/lib/onboard/openclaw-runtime-env.ts", + "src/lib/onboard/extra-placeholder-keys.ts", + "src/lib/onboard/host-proxy-env.ts", + "src/lib/onboard/hermes-dashboard.ts", + "src/lib/hermes-dashboard.ts", +] as const; +const QUOTED_RUNTIME_INPUT_RE = + /["']((?:(?:NEMOCLAW|OPENCLAW)_[A-Z0-9_]+)|CHAT_UI_URL|HTTP_PROXY|HTTPS_PROXY|NO_PROXY|http_proxy|https_proxy|no_proxy)["']/gu; +const STOCK_RUNTIME_INPUTS = new Set( + RUNTIME_INPUT_SOURCE_FILES.flatMap((relativePath) => [ + ...readFileSync(path.join(process.cwd(), relativePath), "utf8").matchAll( + QUOTED_RUNTIME_INPUT_RE, + ), + ]).map((match) => match[1] as string), +); +const OPENCLAW_AUTO_PAIR_CONSUMER_INPUTS = new Set( + readFileSync(path.join(process.cwd(), "scripts/nemoclaw-start.sh"), "utf8").match( + /\bNEMOCLAW_AUTO_PAIR_[A-Z0-9_]+\b/gu, + ) ?? [], +); +const STOCK_RUNTIME_INPUT_AGENTS = { + CHAT_UI_URL: ["openclaw", "hermes"], + HTTPS_PROXY: MANAGED_STARTUP_AGENTS, + HTTP_PROXY: MANAGED_STARTUP_AGENTS, + NEMOCLAW_AUTO_PAIR_DEADLINE_SECS: ["openclaw"], + NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS: ["openclaw"], + NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS: ["openclaw"], + NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS: ["openclaw"], + NEMOCLAW_DASHBOARD_BIND: ["openclaw"], + NEMOCLAW_DASHBOARD_PORT: ["openclaw", "hermes"], + NEMOCLAW_EXTRA_PLACEHOLDER_KEYS: MANAGED_STARTUP_AGENTS, + NEMOCLAW_HERMES_DASHBOARD: ["hermes"], + NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT: ["hermes"], + NEMOCLAW_HERMES_DASHBOARD_PORT: ["hermes"], + NEMOCLAW_HERMES_DASHBOARD_TUI: ["hermes"], + NEMOCLAW_MINIMAL_BOOTSTRAP: ["openclaw"], + NEMOCLAW_OBSERVABILITY: ["langchain-deepagents-code"], + NEMOCLAW_PROXY_HOST: MANAGED_STARTUP_AGENTS, + NEMOCLAW_PROXY_PORT: MANAGED_STARTUP_AGENTS, + NEMOCLAW_SANDBOX_NAME: ["langchain-deepagents-code"], + NO_PROXY: MANAGED_STARTUP_AGENTS, + OPENCLAW_HOME: ["openclaw"], + OPENCLAW_STATE_DIR: ["openclaw"], + OPENCLAW_WORKSPACE_DIR: ["openclaw"], + http_proxy: MANAGED_STARTUP_AGENTS, + https_proxy: MANAGED_STARTUP_AGENTS, + no_proxy: MANAGED_STARTUP_AGENTS, +} as const satisfies Record; + describe("managed startup profile", () => { it.each( VALID_PROFILES, @@ -391,6 +444,22 @@ describe("managed startup profile", () => { ).toEqual([]); }); + it("keeps exported capabilities deeply frozen and validation authority private", () => { + const capabilities = MANAGED_STARTUP_PROFILE_CAPABILITIES["langchain-deepagents-code"]; + expect(Object.isFrozen(MANAGED_STARTUP_PROFILE_CAPABILITIES)).toBe(true); + expect(Object.isFrozen(capabilities)).toBe(true); + expect(Object.isFrozen(capabilities.inferenceApis)).toBe(true); + expect(() => + (capabilities.inferenceApis as unknown as string[]).push("openai-responses"), + ).toThrow(TypeError); + expect(() => + validateManagedStartupProfile({ + ...DCODE_PROFILE, + inference: { ...DCODE_PROFILE.inference, api: "openai-responses" }, + }), + ).toThrow(/not supported/); + }); + // source-shape-contract: compatibility -- Every shipped Docker build input must map to versioned startup intent or a declared build-only exclusion it("classifies every stock Docker ARG as startup-affordance or deliberate exclusion", () => { for (const agent of MANAGED_STARTUP_AGENTS) { @@ -402,6 +471,77 @@ describe("managed startup profile", () => { } }); + // source-shape-contract: compatibility -- Every centralized agent runtime input must map to versioned startup intent or a typed downstream owner + it("classifies every centralized runtime input as profile intent or an explicit deferral", () => { + expect([...STOCK_RUNTIME_INPUTS].sort()).toEqual( + Object.keys(STOCK_RUNTIME_INPUT_AGENTS).sort(), + ); + const missing = Object.entries(STOCK_RUNTIME_INPUT_AGENTS).flatMap(([input, agents]) => + agents + .filter( + (agent) => + !new Set([ + ...MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[agent].map( + ({ input: profileInput }) => profileInput, + ), + ...MANAGED_STARTUP_PROFILE_DEFERRED_RUNTIME_INPUTS[agent].map( + ({ input: deferredInput }) => deferredInput, + ), + ]).has(input), + ) + .map((agent) => `${agent}:${input}`), + ); + expect(missing).toEqual([]); + + const openClawAutoPairInputs = MANAGED_STARTUP_PROFILE_DEFERRED_RUNTIME_INPUTS.openclaw.filter( + ({ input }) => input.startsWith("NEMOCLAW_AUTO_PAIR_"), + ); + expect([...OPENCLAW_AUTO_PAIR_CONSUMER_INPUTS].sort()).toEqual( + openClawAutoPairInputs.map(({ input }) => input).sort(), + ); + expect( + Object.fromEntries(openClawAutoPairInputs.map(({ admission, input }) => [input, admission])), + ).toEqual({ + NEMOCLAW_AUTO_PAIR_DEADLINE_SECS: "managed-launch-forwarded", + NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS: "managed-launch-forwarded", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: "image-consumed-not-forwarded", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "image-consumed-not-forwarded", + NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS: "managed-launch-forwarded", + NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS: "managed-launch-forwarded", + }); + }); + + it("records generic cross-agent emissions as cleanup obligations, not supported semantics", () => { + expect(MANAGED_STARTUP_RUNTIME_CLEANUP_OBLIGATIONS).toHaveLength(2); + for (const obligation of MANAGED_STARTUP_RUNTIME_CLEANUP_OBLIGATIONS) { + const supportedAgents = + STOCK_RUNTIME_INPUT_AGENTS[obligation.input as keyof typeof STOCK_RUNTIME_INPUT_AGENTS]; + expect(obligation.owner).toBe("application-environment"); + for (const agent of obligation.emittedFor) { + expect(supportedAgents).not.toContain(agent); + expect( + MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[agent].map(({ input }) => input), + ).not.toContain(obligation.input); + } + for (const agent of obligation.supportedFor) { + expect(supportedAgents).toContain(agent); + } + } + }); + + it.each( + MANAGED_STARTUP_AGENTS, + )("keeps deferred %s runtime inputs separate from typed profile intent", (agent) => { + const profileInputs = new Set( + MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[agent].map(({ input }) => input), + ); + expect( + MANAGED_STARTUP_PROFILE_DEFERRED_RUNTIME_INPUTS[agent].filter(({ input }) => + profileInputs.has(input), + ), + ).toEqual([]); + }); + it.each(MANAGED_STARTUP_AGENTS)("keeps the %s affordance inventory unambiguous", (agent) => { const inventory = MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[agent]; expect(new Set(inventory.map(({ input }) => input)).size).toBe(inventory.length); @@ -715,6 +855,18 @@ describe("managed startup profile", () => { ).toThrow(/does not support/); }); + it("rejects an OpenClaw primary model reference that disagrees with its provider and model", () => { + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + inference: { + ...OPENCLAW_PROFILE.inference, + primaryModelRef: "different-provider/different-model", + }, + }), + ).toThrow(/primaryModelRef must match routeProvider and model/); + }); + it("accepts only declared Hermes gateway IDs and rejects gateways for other agents", () => { expect(validateManagedStartupProfile(HERMES_PROFILE).tools.enabledGateways).toHaveLength(5); expect(() => @@ -1011,6 +1163,51 @@ describe("managed startup profile", () => { expect(getterInvoked).toBe(false); }); + it("requires own messaging discriminators without invoking inherited getters", () => { + let getterInvoked = false; + Object.defineProperties(Object.prototype, { + schemaVersion: { + configurable: true, + get() { + getterInvoked = true; + return 1; + }, + }, + agent: { + configurable: true, + get() { + getterInvoked = true; + return "openclaw"; + }, + }, + }); + + try { + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + messaging: { plan: {} }, + }), + ).toThrow(/version 1 plan for the selected agent/); + } finally { + Reflect.deleteProperty(Object.prototype, "schemaVersion"); + Reflect.deleteProperty(Object.prototype, "agent"); + } + + expect(getterInvoked).toBe(false); + }); + + it("returns opaque JSON objects without inherited prototype data", () => { + const profile = validateManagedStartupProfile(OPENCLAW_PROFILE); + expect(Object.getPrototypeOf(profile.inference.compatibility as object)).toBeNull(); + expect(Object.getPrototypeOf(profile.messaging.plan as object)).toBeNull(); + expect( + Object.getPrototypeOf( + (profile.messaging.plan as Record>).networkPolicy, + ), + ).toBeNull(); + }); + it("rejects non-enumerable unknown fields instead of silently stripping them", () => { const inference = { ...OPENCLAW_PROFILE.inference }; Object.defineProperty(inference, "hiddenExtension", { diff --git a/src/lib/onboard/managed-startup/profile.ts b/src/lib/onboard/managed-startup/profile.ts index 1647afcbfd2..bacf4be6f3a 100644 --- a/src/lib/onboard/managed-startup/profile.ts +++ b/src/lib/onboard/managed-startup/profile.ts @@ -328,9 +328,29 @@ export interface ManagedStartupAgentCapabilities { * does not advertise the requested semantic capability is rejected instead of * silently dropping a field. */ -export const MANAGED_STARTUP_PROFILE_CAPABILITIES = { +const VALIDATED_INFERENCE_APIS_BY_AGENT = Object.freeze({ + openclaw: Object.freeze([...MANAGED_STARTUP_INFERENCE_APIS]), + hermes: Object.freeze([...MANAGED_STARTUP_INFERENCE_APIS]), + "langchain-deepagents-code": Object.freeze(["openai-completions"] as const), +}) satisfies Readonly>; + +function freezeAgentCapabilities( + capabilities: ManagedStartupAgentCapabilities, +): Readonly { + return Object.freeze({ + ...capabilities, + inferenceApis: Object.freeze([...capabilities.inferenceApis]), + dashboardModes: Object.freeze([...capabilities.dashboardModes]), + inputModalities: Object.freeze([...capabilities.inputModalities]), + webSearchProviders: Object.freeze([...capabilities.webSearchProviders]), + toolGateways: Object.freeze([...capabilities.toolGateways]), + tuningFields: Object.freeze([...capabilities.tuningFields]), + }); +} + +const PROFILE_CAPABILITIES = { openclaw: { - inferenceApis: MANAGED_STARTUP_INFERENCE_APIS, + inferenceApis: [...VALIDATED_INFERENCE_APIS_BY_AGENT.openclaw], dashboardModes: ["loopback", "remote"], inputModalities: ["text", "image"], webSearchProviders: ["brave", "tavily"], @@ -349,11 +369,11 @@ export const MANAGED_STARTUP_PROFILE_CAPABILITIES = { supportsMinimalBootstrap: true, }, hermes: { - inferenceApis: MANAGED_STARTUP_INFERENCE_APIS, + inferenceApis: [...VALIDATED_INFERENCE_APIS_BY_AGENT.hermes], dashboardModes: ["disabled", "loopback-forwarded"], inputModalities: [], webSearchProviders: ["tavily"], - toolGateways: MANAGED_STARTUP_HERMES_TOOL_GATEWAYS, + toolGateways: [...MANAGED_STARTUP_HERMES_TOOL_GATEWAYS], tuningFields: ["contextWindow"], supportsMessaging: true, supportsInferenceCompatibility: false, @@ -386,7 +406,18 @@ export const MANAGED_STARTUP_PROFILE_CAPABILITIES = { observability: "dcode-marker", supportsMinimalBootstrap: false, }, -} as const satisfies Record; +} satisfies Record; + +for (const agent of MANAGED_STARTUP_AGENTS) { + Object.defineProperty(PROFILE_CAPABILITIES, agent, { + configurable: false, + enumerable: true, + value: freezeAgentCapabilities(PROFILE_CAPABILITIES[agent]), + writable: false, + }); +} + +export const MANAGED_STARTUP_PROFILE_CAPABILITIES = Object.freeze(PROFILE_CAPABILITIES); export type ManagedStartupAffordanceSource = "docker-arg" | "runtime-env" | "host-material"; export type ManagedStartupAffordanceRepresentation = "value" | "derived" | "digest-handoff"; @@ -518,6 +549,159 @@ export const MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY = { ], } as const satisfies Record; +export type ManagedStartupDeferredRuntimeOwner = + | "application-environment" + | "credential-plumbing" + | "engine-identity" + | "fixed-image-contract"; + +export type ManagedStartupDeferredRuntimeAdmission = + | "managed-launch-forwarded" + | "image-consumed-not-forwarded"; + +export interface ManagedStartupDeferredRuntimeInput { + readonly input: string; + readonly owner: ManagedStartupDeferredRuntimeOwner; + readonly admission: ManagedStartupDeferredRuntimeAdmission; + readonly reason: string; +} + +function deferredRuntimeInput( + input: string, + owner: ManagedStartupDeferredRuntimeOwner, + reason: string, + admission: ManagedStartupDeferredRuntimeAdmission = "managed-launch-forwarded", +): ManagedStartupDeferredRuntimeInput { + return Object.freeze({ input, owner, admission, reason }); +} + +/** + * Runtime inputs intentionally deferred from the secret-free v1 profile. + * Every other input emitted by the sandbox-create environment must resolve to + * an affordance above. These remain owned by the application-environment, + * credential-plumbing, engine-identity, or fixed-image-contract surfaces + * instead of becoming implicit runtime-specific profile fields. + */ +export const MANAGED_STARTUP_PROFILE_DEFERRED_RUNTIME_INPUTS = Object.freeze({ + openclaw: Object.freeze([ + deferredRuntimeInput( + "NEMOCLAW_AUTO_PAIR_DEADLINE_SECS", + "application-environment", + "operator scheduler tuning is applied by the application environment transaction", + ), + deferredRuntimeInput( + "NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS", + "application-environment", + "operator scheduler tuning is applied by the application environment transaction", + ), + deferredRuntimeInput( + "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS", + "application-environment", + "the image consumes this documented scheduler control but managed launch must admit it in the application environment transaction", + "image-consumed-not-forwarded", + ), + deferredRuntimeInput( + "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS", + "application-environment", + "the image consumes this documented scheduler control but managed launch must admit it in the application environment transaction", + "image-consumed-not-forwarded", + ), + deferredRuntimeInput( + "NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS", + "application-environment", + "operator scheduler tuning is applied by the application environment transaction", + ), + deferredRuntimeInput( + "NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS", + "application-environment", + "operator scheduler tuning is applied by the application environment transaction", + ), + deferredRuntimeInput( + "OPENCLAW_HOME", + "fixed-image-contract", + "the managed image and agent definition own this fixed runtime layout path", + ), + deferredRuntimeInput( + "OPENCLAW_STATE_DIR", + "fixed-image-contract", + "the managed image and agent definition own this fixed runtime layout path", + ), + deferredRuntimeInput( + "OPENCLAW_WORKSPACE_DIR", + "fixed-image-contract", + "the managed image and agent definition own this fixed runtime layout path", + ), + deferredRuntimeInput( + "NEMOCLAW_EXTRA_PLACEHOLDER_KEYS", + "credential-plumbing", + "credential provider construction owns key metadata outside the secret-free profile", + ), + ]), + hermes: Object.freeze([ + deferredRuntimeInput( + "NEMOCLAW_EXTRA_PLACEHOLDER_KEYS", + "credential-plumbing", + "credential provider construction owns key metadata outside the secret-free profile", + ), + ]), + "langchain-deepagents-code": Object.freeze([ + deferredRuntimeInput( + "NEMOCLAW_SANDBOX_NAME", + "engine-identity", + "the lifecycle engine owns instance identity outside reusable startup intent", + ), + deferredRuntimeInput( + "NEMOCLAW_EXTRA_PLACEHOLDER_KEYS", + "credential-plumbing", + "credential provider construction owns key metadata outside the secret-free profile", + ), + ]), +}) satisfies Readonly>; + +export interface ManagedStartupRuntimeCleanupObligation { + readonly input: string; + readonly emittedFor: readonly ManagedStartupAgent[]; + readonly supportedFor: readonly ManagedStartupAgent[]; + readonly owner: "application-environment"; + readonly reason: string; +} + +function runtimeCleanupObligation( + input: string, + emittedFor: readonly ManagedStartupAgent[], + supportedFor: readonly ManagedStartupAgent[], + reason: string, +): ManagedStartupRuntimeCleanupObligation { + return Object.freeze({ + input, + emittedFor: Object.freeze([...emittedFor]), + supportedFor: Object.freeze([...supportedFor]), + owner: "application-environment" as const, + reason, + }); +} + +/** + * Known generic launch emissions that are not supported cross-agent semantics. + * The application environment transaction must remove these leaks before + * buildless activation; listing them here prevents profile construction from + * accidentally blessing the current implementation detail. + */ +export const MANAGED_STARTUP_RUNTIME_CLEANUP_OBLIGATIONS = Object.freeze([ + runtimeCleanupObligation( + "NEMOCLAW_DASHBOARD_BIND", + ["hermes"], + ["openclaw"], + "generic managed-dashboard construction currently emits the OpenClaw-only bind control for Hermes", + ), + runtimeCleanupObligation( + "NEMOCLAW_MINIMAL_BOOTSTRAP", + ["hermes", "langchain-deepagents-code"], + ["openclaw"], + "generic host-proxy construction currently emits the OpenClaw-only bootstrap control for other agents", + ), +]) satisfies readonly ManagedStartupRuntimeCleanupObligation[]; + export interface ManagedStartupExcludedDockerInput { readonly input: string; readonly reason: @@ -937,7 +1121,7 @@ function cloneJsonValue( function requireJsonObjectOrNull(value: unknown, where: string): ManagedStartupJsonObject | null { if (value === null) return null; if (!isPlainObject(value)) invalid(`${where} must be null or a plain JSON object`); - return cloneJsonValue(value, where) as ManagedStartupJsonObject; + return cloneJsonValue(value, where, { nullPrototypeObjects: true }) as ManagedStartupJsonObject; } function requireJsonObject(value: unknown, where: string): ManagedStartupJsonObject { @@ -1390,13 +1574,14 @@ function validateDashboard( function validateInference(value: unknown, agent: ManagedStartupAgent): ManagedStartupInference { const inference = requireRecord(value, "inference"); rejectUnknownKeys(inference, INFERENCE_KEYS, "inference"); + const routeProvider = requireBoundedString(inference.routeProvider, "inference.routeProvider"); + const model = requireBoundedString(inference.model, "inference.model", MAX_MODEL_BYTES); const api = requireStringEnum( inference.api, INFERENCE_API_SET, "inference.api", ); - const supportedInferenceApis: readonly string[] = - MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].inferenceApis; + const supportedInferenceApis = VALIDATED_INFERENCE_APIS_BY_AGENT[agent]; let apiSupported = false; for (let index = 0; index < supportedInferenceApis.length; index += 1) { if (supportedInferenceApis[index] === api) apiSupported = true; @@ -1434,6 +1619,9 @@ function validateInference(value: unknown, agent: ManagedStartupAgent): ManagedS if (primaryModelRef === null || inputModalities === null) { invalid("openclaw requires primaryModelRef and inputModalities"); } + if (primaryModelRef !== `${routeProvider}/${model}`) { + invalid("openclaw primaryModelRef must match routeProvider and model"); + } } else { if (primaryModelRef !== null || compatibility !== null || inputModalities !== null) { invalid(`${agent} does not support primaryModelRef, compatibility, or inputModalities`); @@ -1444,12 +1632,12 @@ function validateInference(value: unknown, agent: ManagedStartupAgent): ManagedS } return { - routeProvider: requireBoundedString(inference.routeProvider, "inference.routeProvider"), + routeProvider, upstreamProvider: requireBoundedString( inference.upstreamProvider, "inference.upstreamProvider", ), - model: requireBoundedString(inference.model, "inference.model", MAX_MODEL_BYTES), + model, routedBaseUrl: requireHttpUrl(inference.routedBaseUrl, "inference.routedBaseUrl"), upstreamEndpointUrl, api, @@ -1579,7 +1767,10 @@ export function validateManagedStartupProfile(value: unknown): ManagedStartupPro } if ( messagingPlan !== null && - (messagingPlan.schemaVersion !== 1 || messagingPlan.agent !== agent) + (!Object.hasOwn(messagingPlan, "schemaVersion") || + !Object.hasOwn(messagingPlan, "agent") || + messagingPlan.schemaVersion !== 1 || + messagingPlan.agent !== agent) ) { invalid("messaging.plan must be a version 1 plan for the selected agent"); } From 357081a6f1fecd00b73f2a6068e36724b1e27e84 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 18:07:59 -0700 Subject: [PATCH 005/117] feat(onboard): construct managed startup profiles Signed-off-by: Aaron Erickson --- src/lib/onboard/host-proxy-env.ts | 17 +- .../managed-startup-image-runtime.test.ts | 231 +++++ .../managed-startup-onboard-profile.test.ts | 439 ++++++++ .../managed-startup-profile-builder.test.ts | 616 ++++++++++++ src/lib/onboard/managed-startup/hold.ts | 4 + .../onboard/managed-startup/image-runtime.ts | 238 +++++ .../managed-startup/onboard-profile.ts | 219 ++++ .../managed-startup/profile-builder.ts | 951 ++++++++++++++++++ 8 files changed, 2714 insertions(+), 1 deletion(-) create mode 100644 src/lib/onboard/managed-startup-image-runtime.test.ts create mode 100644 src/lib/onboard/managed-startup-onboard-profile.test.ts create mode 100644 src/lib/onboard/managed-startup-profile-builder.test.ts create mode 100644 src/lib/onboard/managed-startup/hold.ts create mode 100644 src/lib/onboard/managed-startup/image-runtime.ts create mode 100644 src/lib/onboard/managed-startup/onboard-profile.ts create mode 100644 src/lib/onboard/managed-startup/profile-builder.ts diff --git a/src/lib/onboard/host-proxy-env.ts b/src/lib/onboard/host-proxy-env.ts index 99bca5190ee..506847e22b4 100644 --- a/src/lib/onboard/host-proxy-env.ts +++ b/src/lib/onboard/host-proxy-env.ts @@ -12,12 +12,18 @@ const HOST_PROXY_ENV_NAMES = [ "https_proxy", "no_proxy", ] as const; +const HOST_PROXY_URL_ENV_NAMES = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "http_proxy", + "https_proxy", +] as const; type HostProxyEnvOptions = { dropCredentialBearingProxyUrls?: boolean; }; -function isCredentialBearingProxyUrl(value: string): boolean { +export function isCredentialBearingProxyUrl(value: string): boolean { try { const parsed = new URL(value.includes("://") ? value : `http://${value}`); return parsed.username !== "" || parsed.password !== ""; @@ -26,6 +32,15 @@ function isCredentialBearingProxyUrl(value: string): boolean { } } +export function hasCredentialBearingHostProxyEnvironment( + env: NodeJS.ProcessEnv = process.env, +): boolean { + return HOST_PROXY_URL_ENV_NAMES.some((name) => { + const value = env[name]?.trim(); + return value ? isCredentialBearingProxyUrl(value) : false; + }); +} + export function appendHostProxyEnvArgs( envArgs: string[], env: NodeJS.ProcessEnv = process.env, diff --git a/src/lib/onboard/managed-startup-image-runtime.test.ts b/src/lib/onboard/managed-startup-image-runtime.test.ts new file mode 100644 index 00000000000..f643e15bf80 --- /dev/null +++ b/src/lib/onboard/managed-startup-image-runtime.test.ts @@ -0,0 +1,231 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildManagedStartupImageActionPlan, + type ManagedStartupImageActionPlanInput, +} from "./managed-startup/image-runtime"; +import type { ManagedStartupAgent, ManagedStartupDashboard } from "./managed-startup/profile"; + +function dashboard(agent: ManagedStartupAgent): ManagedStartupDashboard { + switch (agent) { + case "openclaw": + return { + agent, + mode: "loopback", + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }; + case "hermes": + return { + agent, + mode: "disabled", + url: "http://127.0.0.1:18789", + publicPort: null, + internalPort: null, + tuiEnabled: false, + }; + case "langchain-deepagents-code": + return { agent, mode: "disabled" }; + } +} + +function actionInput( + agent: ManagedStartupAgent, + mode: "apply" | "clear" = "apply", +): ManagedStartupImageActionPlanInput { + const messagingActions = + agent === "langchain-deepagents-code" + ? [] + : [ + { + kind: "apply-messaging-plan" as const, + agent, + mode, + phase: "runtime-setup" as const, + runAs: "root" as const, + }, + { + kind: "apply-messaging-plan" as const, + agent, + mode, + phase: "post-agent-install" as const, + runAs: "sandbox" as const, + }, + ]; + return { + agent, + actions: [ + ...messagingActions.slice(0, 1), + { kind: "generate-agent-config", agent, runAs: "sandbox" }, + ...messagingActions.slice(1), + { kind: "configure-dashboard", dashboard: dashboard(agent) }, + ], + }; +} + +describe("buildManagedStartupImageActionPlan", () => { + it.each([ + "openclaw", + "hermes", + ] as const)("constructs the complete offline %s messaging and config plan", (agent) => { + const plan = buildManagedStartupImageActionPlan(actionInput(agent)); + + expect(plan.map(({ action, runAs }) => ({ action, runAs }))).toEqual([ + { action: "messaging-runtime-setup", runAs: "root" }, + { action: "generate-agent-config", runAs: "sandbox" }, + { action: "messaging-post-agent-install", runAs: "sandbox" }, + ]); + expect(plan[0]?.argv).toContain("runtime-setup"); + expect(plan[0]?.argv).not.toContain("--managed-startup-runtime"); + expect(plan[2]?.argv).toContain("post-agent-install"); + expect(plan[2]?.argv).toContain("--managed-startup-runtime"); + expect(plan.some((command) => command.argv.includes("agent-install"))).toBe(false); + expect( + plan.some((command) => + command.argv.some((argument) => /^(?:npm|npx|pip|pip3|uv)$/u.test(argument)), + ), + ).toBe(false); + expect(Object.isFrozen(plan)).toBe(true); + expect(plan.every((command) => Object.isFrozen(command) && Object.isFrozen(command.argv))).toBe( + true, + ); + }); + + it("constructs DCode's complete offline config plan without messaging actions", () => { + expect(buildManagedStartupImageActionPlan(actionInput("langchain-deepagents-code"))).toEqual([ + { + action: "generate-agent-config", + runAs: "sandbox", + argv: [ + "/usr/local/bin/node", + "--experimental-strip-types", + "/opt/nemoclaw-deepagents-code/generate-config.ts", + ], + }, + ]); + }); + + it.each([ + ["openclaw", "/scripts/generate-openclaw-config.mts"], + ["hermes", "/opt/nemoclaw-hermes-config/generate-config.ts"], + ["langchain-deepagents-code", "/opt/nemoclaw-deepagents-code/generate-config.ts"], + ] as const)("selects the reviewed %s generator asset", (agent, generator) => { + const command = buildManagedStartupImageActionPlan(actionInput(agent)).find( + ({ action }) => action === "generate-agent-config", + ); + expect(command?.argv.at(-1)).toBe(generator); + }); + + it("constructs the same reviewed commands for apply and clear messaging intent", () => { + expect(buildManagedStartupImageActionPlan(actionInput("openclaw", "clear"))).toEqual( + buildManagedStartupImageActionPlan(actionInput("openclaw", "apply")), + ); + }); + + it.each([ + [ + "cross-agent action", + { + ...actionInput("openclaw"), + actions: [ + ...actionInput("openclaw").actions.slice(0, 1), + { kind: "generate-agent-config", agent: "hermes", runAs: "sandbox" }, + ...actionInput("openclaw").actions.slice(2), + ], + }, + /action for hermes cannot be used by openclaw/, + ], + [ + "partial messaging plan", + { + ...actionInput("hermes"), + actions: actionInput("hermes").actions.filter( + (action) => + action.kind !== "apply-messaging-plan" || action.phase !== "post-agent-install", + ), + }, + /requires 1 action for each messaging phase/, + ], + [ + "duplicate config action", + { + ...actionInput("langchain-deepagents-code"), + actions: [ + ...actionInput("langchain-deepagents-code").actions, + { + kind: "generate-agent-config", + agent: "langchain-deepagents-code", + runAs: "sandbox", + }, + ], + }, + /exactly one agent config/, + ], + [ + "out-of-order messaging", + { + ...actionInput("openclaw"), + actions: [ + ...actionInput("openclaw").actions.slice(1, 3), + actionInput("openclaw").actions[0], + actionInput("openclaw").actions[3], + ], + }, + /not in the required construction order/, + ], + [ + "root config generation", + { + ...actionInput("hermes"), + actions: actionInput("hermes").actions.map((action) => + action.kind === "generate-agent-config" ? { ...action, runAs: "root" } : action, + ), + }, + /configuration generation must run as sandbox/, + ], + [ + "sandbox messaging runtime setup", + { + ...actionInput("openclaw"), + actions: actionInput("openclaw").actions.map((action) => + action.kind === "apply-messaging-plan" && action.phase === "runtime-setup" + ? { ...action, runAs: "sandbox" } + : action, + ), + }, + /messaging runtime setup must run as root/, + ], + [ + "root messaging post-agent configuration", + { + ...actionInput("openclaw"), + actions: actionInput("openclaw").actions.map((action) => + action.kind === "apply-messaging-plan" && action.phase === "post-agent-install" + ? { ...action, runAs: "root" } + : action, + ), + }, + /messaging post-agent configuration must run as sandbox/, + ], + [ + "arbitrary command action", + { + ...actionInput("langchain-deepagents-code"), + actions: [ + ...actionInput("langchain-deepagents-code").actions, + { kind: "run-command", argv: ["npm", "install"] }, + ], + }, + /unsupported managed startup construction action/, + ], + ])("fails closed for an incomplete or mismatched construction contract: %s", (_name, input, message) => { + expect(() => + buildManagedStartupImageActionPlan(input as ManagedStartupImageActionPlanInput), + ).toThrow(message); + }); +}); diff --git a/src/lib/onboard/managed-startup-onboard-profile.test.ts b/src/lib/onboard/managed-startup-onboard-profile.test.ts new file mode 100644 index 00000000000..19400f40cef --- /dev/null +++ b/src/lib/onboard/managed-startup-onboard-profile.test.ts @@ -0,0 +1,439 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildManagedStartupOnboardProfile, + type ManagedStartupOnboardProfileInput, +} from "./managed-startup/onboard-profile"; + +const EMPTY_ENVIRONMENT: NodeJS.ProcessEnv = {}; + +function messagingPlan(agent: "openclaw" | "hermes") { + return { + schemaVersion: 1, + sandboxName: "adapter-test", + agent, + workflow: "onboard", + channels: [], + disabledChannels: [], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + stateUpdates: [], + healthChecks: [], + } as const; +} + +function openClawInput( + overrides: Partial = {}, +): ManagedStartupOnboardProfileInput { + return { + agentName: "openclaw", + inference: { + routeProvider: "openai", + upstreamProvider: "openai-api", + model: "gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses", + primaryModelRef: "openai/gpt-5.4", + compatibility: {}, + }, + chatUiUrl: "http://127.0.0.1:18789", + effectiveDashboardPort: 18_789, + manageDashboard: true, + dashboardBindAddress: undefined, + wslExposure: false, + hermesDashboardState: { config: null, enabled: false }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: "disabled", + observabilityEnabled: false, + environment: EMPTY_ENVIRONMENT, + corporateCa: null, + ...overrides, + }; +} + +function hermesInput( + overrides: Partial = {}, +): ManagedStartupOnboardProfileInput { + return { + agentName: "hermes", + inference: { + routeProvider: "inference", + upstreamProvider: "hermes-provider", + model: "moonshotai/kimi-k2.6", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + chatUiUrl: "http://127.0.0.1:18789", + effectiveDashboardPort: 18_789, + manageDashboard: true, + dashboardBindAddress: undefined, + wslExposure: false, + hermesDashboardState: { + config: { + enabled: false, + port: 9119, + internalPort: 19_119, + tuiEnabled: false, + }, + enabled: false, + }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: "disabled", + observabilityEnabled: false, + environment: EMPTY_ENVIRONMENT, + corporateCa: null, + ...overrides, + }; +} + +function dcodeInput( + overrides: Partial = {}, +): ManagedStartupOnboardProfileInput { + return { + agentName: "langchain-deepagents-code", + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + model: "openai/gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + chatUiUrl: "", + effectiveDashboardPort: 0, + manageDashboard: false, + dashboardBindAddress: undefined, + wslExposure: false, + hermesDashboardState: { config: null, enabled: false }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: "disabled", + observabilityEnabled: false, + environment: EMPTY_ENVIRONMENT, + corporateCa: null, + ...overrides, + }; +} + +describe("buildManagedStartupOnboardProfile", () => { + it("maps a remote OpenClaw dashboard and its complete agent-owned state", () => { + const plan = messagingPlan("openclaw"); + const built = buildManagedStartupOnboardProfile( + openClawInput({ + chatUiUrl: "https://dashboard.example.test:19443", + effectiveDashboardPort: 19_443, + dashboardBindAddress: "0.0.0.0", + wslExposure: true, + webSearch: { fetchEnabled: true, provider: "tavily" }, + toolDisclosure: "direct", + messagingPlan: plan, + }), + ); + + expect(built.profile).toMatchObject({ + agent: "openclaw", + dashboard: { + agent: "openclaw", + mode: "remote", + url: "https://dashboard.example.test:19443", + port: 19_443, + bindAddress: "0.0.0.0", + wslExposure: true, + }, + agentConfig: { + agent: "openclaw", + webSearch: { enabled: true, provider: "tavily" }, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + }, + tools: { disclosure: "direct", enabledGateways: [] }, + }); + expect(built.profile.messaging.plan).not.toBeNull(); + expect(built.profile.inference.upstreamEndpointUrl).toBeNull(); + }); + + it("keeps bracketed IPv6 loopback OpenClaw dashboards in loopback mode", () => { + const built = buildManagedStartupOnboardProfile( + openClawInput({ + chatUiUrl: "http://[::1]:18789", + }), + ); + + expect(built.profile.dashboard).toEqual({ + agent: "openclaw", + mode: "loopback", + url: "http://[::1]:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }); + }); + + it("rejects an OpenClaw bind value that would otherwise be silently downgraded", () => { + expect(() => + buildManagedStartupOnboardProfile( + openClawInput({ + dashboardBindAddress: "127.0.0.1", + }), + ), + ).toThrow(/dashboard bind address must be empty or 0\.0\.0\.0/); + }); + + it("maps Hermes with its dashboard disabled and Tavily retained", () => { + const built = buildManagedStartupOnboardProfile(hermesInput()); + + expect(built.profile).toMatchObject({ + agent: "hermes", + dashboard: { + agent: "hermes", + mode: "disabled", + url: "http://127.0.0.1:18789", + publicPort: null, + internalPort: null, + tuiEnabled: false, + }, + agentConfig: { + agent: "hermes", + webSearch: { enabled: false, provider: "tavily" }, + }, + inference: { + upstreamEndpointUrl: null, + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + }); + }); + + it("maps Hermes forwarding, tool gateways, messaging, and context independently", () => { + const plan = messagingPlan("hermes"); + const built = buildManagedStartupOnboardProfile( + hermesInput({ + chatUiUrl: "http://127.0.0.1:19189", + effectiveDashboardPort: 19_189, + hermesDashboardState: { + config: { + enabled: true, + port: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }, + enabled: true, + }, + webSearch: { fetchEnabled: true, provider: "tavily" }, + hermesToolGateways: ["nous-web", "nous-image"], + messagingPlan: plan, + environment: { + ...EMPTY_ENVIRONMENT, + NEMOCLAW_CONTEXT_WINDOW: "65536", + }, + }), + ); + + expect(built.profile.dashboard).toEqual({ + agent: "hermes", + mode: "loopback-forwarded", + url: "http://127.0.0.1:19189", + publicPort: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }); + expect(built.profile.tools.enabledGateways).toEqual(["nous-image", "nous-web"]); + expect(built.profile.messaging.plan).not.toBeNull(); + expect(built.profile.tuning).toEqual({ + contextWindow: 65_536, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }); + }); + + it("maps DCode without dashboard, messaging, web-search, gateway, or tuning state", () => { + const built = buildManagedStartupOnboardProfile( + dcodeInput({ + dcodeAutoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + }), + ); + + expect(built.profile).toMatchObject({ + agent: "langchain-deepagents-code", + agentConfig: { + agent: "langchain-deepagents-code", + autoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + }, + dashboard: { agent: "langchain-deepagents-code", mode: "disabled" }, + inference: { + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + tools: { disclosure: "progressive", enabledGateways: [] }, + messaging: { plan: null }, + tuning: { + contextWindow: null, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + }); + }); + + it("does not inspect host CA settings during profile construction", () => { + const built = buildManagedStartupOnboardProfile( + openClawInput({ + environment: { + NEMOCLAW_CORPORATE_CA_BUNDLE: "/host/path/that-does-not-exist.pem", + NEMOCLAW_CORPORATE_CA_IMPORT: "1", + }, + corporateCa: null, + }), + ); + + expect(built.profile.corporateCa.bundleSha256).toBeNull(); + expect(built.corporateCaB64).toBeUndefined(); + }); + + it("rejects a DCode dashboard while retaining credential-free host-proxy intent", () => { + expect(() => buildManagedStartupOnboardProfile(dcodeInput({ manageDashboard: true }))).toThrow( + /DCode must not enable a dashboard/, + ); + + const built = buildManagedStartupOnboardProfile( + dcodeInput({ + environment: { + ...EMPTY_ENVIRONMENT, + HTTP_PROXY: "http://proxy.example.test:8080", + }, + }), + ); + expect(built.profile.proxy.hostHttpUrl).toBe("http://proxy.example.test:8080"); + expect(built.profile.proxy.hostNoProxy).toContain("inference.local"); + }); + + it("preserves an explicitly resolved routed base URL for every agent", () => { + const route = "https://portable-route.example.test/v1"; + + expect( + buildManagedStartupOnboardProfile( + openClawInput({ + inference: { ...openClawInput().inference, routedBaseUrl: route }, + }), + ).profile.inference.routedBaseUrl, + ).toBe(route); + expect( + buildManagedStartupOnboardProfile( + hermesInput({ + inference: { ...hermesInput().inference, routedBaseUrl: route }, + }), + ).profile.inference.routedBaseUrl, + ).toBe(route); + expect( + buildManagedStartupOnboardProfile( + dcodeInput({ + inference: { ...dcodeInput().inference, routedBaseUrl: route }, + }), + ).profile.inference.routedBaseUrl, + ).toBe(route); + }); + + it.each([ + ["openclaw", openClawInput], + ["hermes", hermesInput], + ["langchain-deepagents-code", dcodeInput], + ] as const)("keeps credential-bearing proxy aliases out of the %s profile", (agent, input) => { + const built = buildManagedStartupOnboardProfile( + input({ + environment: { + ...EMPTY_ENVIRONMENT, + HTTP_PROXY: "http://upper:upper-secret@upper.example.test:8080", + HTTPS_PROXY: "http://upper-tls:upper-secret@upper-tls.example.test:8443", + NO_PROXY: "upper.internal", + http_proxy: "http://lower:lower-secret@lower.example.test:8081", + https_proxy: "http://lower-tls:lower-secret@lower-tls.example.test:8444", + no_proxy: "lower.internal", + }, + }), + ); + + expect(built.profile.proxy).toMatchObject({ + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }); + expect(built.credentialProxyReplayRequired).toBe(agent !== "langchain-deepagents-code"); + const serialized = JSON.stringify(built); + expect(serialized).not.toContain("upper-secret"); + expect(serialized).not.toContain("lower-secret"); + expect(serialized).not.toContain("upper.internal"); + expect(serialized).not.toContain("lower.internal"); + }); + + it("filters stale semantic env while preserving environment-owned tuning and proxy knobs", () => { + const built = buildManagedStartupOnboardProfile( + openClawInput({ + toolDisclosure: "direct", + webSearch: { fetchEnabled: true, provider: "tavily" }, + environment: { + ...EMPTY_ENVIRONMENT, + NEMOCLAW_MODEL: "stale-model", + NEMOCLAW_INFERENCE_API: "anthropic-messages", + NEMOCLAW_TOOL_DISCLOSURE: "progressive", + NEMOCLAW_WEB_SEARCH_ENABLED: "0", + NEMOCLAW_WEB_SEARCH_PROVIDER: "brave", + CHAT_UI_URL: "https://stale.example.test:19999", + NEMOCLAW_CONTEXT_WINDOW: "262144", + NEMOCLAW_MAX_TOKENS: "8192", + NEMOCLAW_PROXY_HOST: "host.containers.internal", + NEMOCLAW_PROXY_PORT: "3129", + HTTP_PROXY: "http://proxy.example.test:8080", + }, + }), + ); + + expect(built.profile.inference).toMatchObject({ + model: "gpt-5.4", + api: "openai-responses", + }); + expect(built.profile.tools.disclosure).toBe("direct"); + expect(built.profile.agentConfig).toMatchObject({ + agent: "openclaw", + webSearch: { enabled: true, provider: "tavily" }, + }); + expect(built.profile.dashboard).toMatchObject({ + url: "http://127.0.0.1:18789", + }); + expect(built.profile.tuning).toEqual({ + contextWindow: 262_144, + maxTokens: 8192, + reasoning: false, + reasoningEffort: "default", + }); + expect(built.profile.proxy).toMatchObject({ + managedHost: "host.containers.internal", + managedPort: 3129, + hostHttpUrl: "http://proxy.example.test:8080", + }); + }); +}); diff --git a/src/lib/onboard/managed-startup-profile-builder.test.ts b/src/lib/onboard/managed-startup-profile-builder.test.ts new file mode 100644 index 00000000000..64095af3a1c --- /dev/null +++ b/src/lib/onboard/managed-startup-profile-builder.test.ts @@ -0,0 +1,616 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; + +import { describe, expect, it } from "vitest"; + +import { PEM } from "./__test-helpers__/corporate-ca-fixtures"; +import { decodeManagedStartupProfile } from "./managed-startup/profile"; +import { + assertManagedStartupProfileBuilderInventoryCoverage, + buildManagedStartupProfile, + type ManagedStartupProfileBuilderInput, + type ValidatedManagedStartupProfileTransport, +} from "./managed-startup/profile-builder"; + +function messagingPlan(agent: "openclaw" | "hermes") { + return { + schemaVersion: 1, + sandboxName: "portable-agent", + agent, + workflow: "onboard", + channels: [], + disabledChannels: [], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + stateUpdates: [], + healthChecks: [], + } as const; +} + +function encodeJson(value: unknown): string { + return Buffer.from(JSON.stringify(value), "utf8").toString("base64"); +} + +function openClawInput( + overrides: Partial = {}, +): ManagedStartupProfileBuilderInput { + return { + agent: "openclaw", + inference: { + routeProvider: "openai", + upstreamProvider: "openai-api", + model: "gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses", + primaryModelRef: "openai/gpt-5.4", + compatibility: {}, + }, + dashboard: { + agent: "openclaw", + mode: "loopback", + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: null, + observabilityEnabled: null, + environment: {}, + corporateCa: null, + ...overrides, + }; +} + +function hermesInput( + overrides: Partial = {}, +): ManagedStartupProfileBuilderInput { + return { + agent: "hermes", + inference: { + routeProvider: "inference", + upstreamProvider: "compatible-anthropic-endpoint", + model: "claude-sonnet-4-6", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + dashboard: { + agent: "hermes", + mode: "disabled", + url: "http://127.0.0.1:18789", + publicPort: null, + internalPort: null, + tuiEnabled: false, + }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: null, + observabilityEnabled: null, + environment: {}, + corporateCa: null, + ...overrides, + }; +} + +function dcodeInput( + overrides: Partial = {}, +): ManagedStartupProfileBuilderInput { + return { + agent: "langchain-deepagents-code", + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + model: "openai/gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + dashboard: { + agent: "langchain-deepagents-code", + mode: "disabled", + }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: "disabled", + observabilityEnabled: false, + environment: {}, + corporateCa: null, + ...overrides, + }; +} + +describe("buildManagedStartupProfile", () => { + it("parses and hydrates messaging before exposing the validated transport handoff", () => { + const compactPlan = { + schemaVersion: 1, + sandboxName: "portable-agent", + agent: "openclaw", + workflow: "onboard", + channels: [], + disabledChannels: [], + } as const; + + const built = buildManagedStartupProfile( + openClawInput({ + messagingPlan: compactPlan, + }), + ); + const acceptsOnlyValidatedTransport = ( + transport: ValidatedManagedStartupProfileTransport, + ): string => transport; + + expect(acceptsOnlyValidatedTransport(built.encodedProfile)).toBe(built.encodedProfile); + expect(built.profile.messaging.plan).toMatchObject({ + schemaVersion: 1, + agent: "openclaw", + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + runtimeSetup: { nodePreloads: [], envAliases: [], secretScans: [] }, + stateUpdates: [], + healthChecks: [], + }); + }); + + it.each([ + ["wrong-agent discriminator", { ...messagingPlan("hermes"), sandboxName: "portable-agent" }], + [ + "malformed nested channel", + { + ...messagingPlan("openclaw"), + channels: [{ channelId: "discord", inputs: { invalid: true } }], + }, + ], + ])("rejects %s before encoding a profile transport", (_name, plan) => { + expect(() => buildManagedStartupProfile(openClawInput({ messagingPlan: plan }))).toThrow( + /valid openclaw SandboxMessagingPlan/, + ); + }); + + it("builds OpenClaw with every stock behavior knob and canonical transport", () => { + const plan = messagingPlan("openclaw"); + const extraAgents = { + agents: [{ id: "reviewer", workspace: "/sandbox/reviewer" }], + defaults: { subagents: { maxSpawnDepth: 2 } }, + main: { tools: { profile: "coding" } }, + }; + const input = openClawInput({ + dashboard: { + agent: "openclaw", + mode: "remote", + url: "https://dashboard.example.test:19443", + port: 19_443, + bindAddress: "0.0.0.0", + wslExposure: true, + }, + webSearch: { fetchEnabled: true, provider: "tavily" }, + toolDisclosure: "direct", + messagingPlan: plan, + environment: { + NEMOCLAW_MODEL: "gpt-5.4", + NEMOCLAW_INFERENCE_PROVIDER_ID: "openai", + NEMOCLAW_UPSTREAM_PROVIDER: "openai-api", + NEMOCLAW_PRIMARY_MODEL_REF: "openai/gpt-5.4", + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_INFERENCE_API: "openai-responses", + NEMOCLAW_INFERENCE_COMPAT_B64: encodeJson({}), + NEMOCLAW_INFERENCE_INPUTS: "text,image", + NEMOCLAW_CONTEXT_WINDOW: "262144", + NEMOCLAW_MAX_TOKENS: "8192", + NEMOCLAW_REASONING: "true", + NEMOCLAW_REASONING_EFFORT: " HIGH ", + NEMOCLAW_TOOL_DISCLOSURE: "direct", + NEMOCLAW_AGENT_TIMEOUT: "900", + NEMOCLAW_AGENT_HEARTBEAT_EVERY: "30m", + NEMOCLAW_EXTRA_AGENTS_JSON: JSON.stringify(extraAgents), + NEMOCLAW_DISABLE_DEVICE_AUTH: "1", + NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE: "managed-onboard", + NEMOCLAW_WEB_SEARCH_ENABLED: "1", + NEMOCLAW_WEB_SEARCH_PROVIDER: "tavily", + NEMOCLAW_OPENCLAW_OTEL: "yes", + NEMOCLAW_OPENCLAW_OTEL_ENDPOINT: "https://otel.example.test:4318", + NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME: "nemoclaw-openclaw", + NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE: "0.25", + CHAT_UI_URL: "https://dashboard.example.test:19443", + NEMOCLAW_DASHBOARD_BIND: "0.0.0.0", + NEMOCLAW_WSL_DASHBOARD_EXPOSURE: "1", + NEMOCLAW_DASHBOARD_PORT: "19443", + NEMOCLAW_PROXY_HOST: "host.containers.internal", + NEMOCLAW_PROXY_PORT: "3129", + NEMOCLAW_MESSAGING_PLAN_B64: encodeJson(plan), + NEMOCLAW_MINIMAL_BOOTSTRAP: "1", + HTTP_PROXY: "http://proxy.example.test:8080", + http_proxy: "http://proxy.example.test:8080", + HTTPS_PROXY: "https://connect.example.test:8443", + https_proxy: "https://connect.example.test:8443", + NO_PROXY: "metadata.example.test", + no_proxy: "metadata.example.test", + }, + }); + + const built = buildManagedStartupProfile(input); + + expect(built.profile.agent).toBe("openclaw"); + expect(built.profile.inference).toMatchObject({ + routeProvider: "openai", + upstreamProvider: "openai-api", + model: "gpt-5.4", + api: "openai-responses", + primaryModelRef: "openai/gpt-5.4", + compatibility: {}, + inputModalities: ["image", "text"], + }); + expect(built.profile.agentConfig).toMatchObject({ + agent: "openclaw", + webSearch: { enabled: true, provider: "tavily" }, + otel: { + enabled: true, + endpointUrl: "https://otel.example.test:4318", + serviceName: "nemoclaw-openclaw", + sampleRate: 0.25, + }, + agentTimeoutSeconds: 900, + heartbeatEvery: "30m", + extraAgents, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + minimalBootstrap: true, + }); + expect(built.profile.proxy).toMatchObject({ + managedHost: "host.containers.internal", + managedPort: 3129, + hostHttpUrl: "http://proxy.example.test:8080", + hostHttpsUrl: "https://connect.example.test:8443", + }); + expect(built.profile.proxy.hostNoProxy).toEqual( + expect.arrayContaining([ + "metadata.example.test", + "localhost", + "host.docker.internal", + "host.containers.internal", + "inference.local", + ]), + ); + expect(built.profile.messaging.plan).not.toBeNull(); + expect(built.profile.tuning).toEqual({ + contextWindow: 262_144, + maxTokens: 8192, + reasoning: true, + reasoningEffort: "high", + }); + expect(built.corporateCaB64).toBeUndefined(); + expect(decodeManagedStartupProfile(built.encodedProfile)).toEqual(built.profile); + expect(built.startupProfileSha256).toBe( + createHash("sha256").update(built.encodedProfile, "utf8").digest("hex"), + ); + }); + + it("builds Hermes with Tavily, gateway presets, messaging, context, and forwarding", () => { + const plan = messagingPlan("hermes"); + const gateways = ["nous-web", "nous-image"] as const; + const built = buildManagedStartupProfile( + hermesInput({ + dashboard: { + agent: "hermes", + mode: "loopback-forwarded", + url: "http://127.0.0.1:19189", + publicPort: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }, + webSearch: { fetchEnabled: true, provider: "tavily" }, + toolDisclosure: "direct", + hermesToolGateways: gateways, + messagingPlan: plan, + environment: { + NEMOCLAW_MODEL: "claude-sonnet-4-6", + NEMOCLAW_INFERENCE_PROVIDER_ID: "inference", + NEMOCLAW_UPSTREAM_PROVIDER: "compatible-anthropic-endpoint", + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_INFERENCE_API: "openai-completions", + NEMOCLAW_CONTEXT_WINDOW: "65536", + NEMOCLAW_TOOL_DISCLOSURE: "direct", + NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER: "1", + NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64: encodeJson(gateways), + NEMOCLAW_WEB_SEARCH_ENABLED: "1", + NEMOCLAW_WEB_SEARCH_PROVIDER: "tavily", + NEMOCLAW_MESSAGING_PLAN_B64: encodeJson(plan), + CHAT_UI_URL: "http://127.0.0.1:19189", + NEMOCLAW_DASHBOARD_PORT: "19189", + NEMOCLAW_HERMES_DASHBOARD: "true", + NEMOCLAW_HERMES_DASHBOARD_PORT: "19189", + NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT: "29189", + NEMOCLAW_HERMES_DASHBOARD_TUI: "yes", + NEMOCLAW_PROXY_HOST: "10.200.0.1", + NEMOCLAW_PROXY_PORT: "3128", + HTTP_PROXY: "http://proxy.example.test:8080", + NO_PROXY: "internal.example.test", + }, + }), + ); + + expect(built.profile.agentConfig).toEqual({ + agent: "hermes", + webSearch: { enabled: true, provider: "tavily" }, + }); + expect(built.profile.inference).toMatchObject({ + routeProvider: "inference", + upstreamProvider: "compatible-anthropic-endpoint", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }); + expect(built.profile.dashboard).toEqual({ + agent: "hermes", + mode: "loopback-forwarded", + url: "http://127.0.0.1:19189", + publicPort: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }); + expect(built.profile.tools).toEqual({ + disclosure: "direct", + enabledGateways: ["nous-image", "nous-web"], + }); + expect(built.profile.tuning).toEqual({ + contextWindow: 65_536, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }); + expect(decodeManagedStartupProfile(built.encodedProfile)).toEqual(built.profile); + }); + + it("builds DCode with its direct upstream, approval, and observability contract", () => { + const built = buildManagedStartupProfile( + dcodeInput({ + dcodeAutoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + toolDisclosure: "direct", + environment: { + NEMOCLAW_MODEL: "openai/gpt-5.4", + NEMOCLAW_INFERENCE_PROVIDER_ID: "inference", + NEMOCLAW_UPSTREAM_PROVIDER: "openrouter", + NEMOCLAW_UPSTREAM_ENDPOINT_URL: "https://openrouter.ai/api/v1", + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_INFERENCE_API: "openai-completions", + NEMOCLAW_TOOL_DISCLOSURE: "direct", + NEMOCLAW_DCODE_AUTO_APPROVAL: "thread-opt-in", + NEMOCLAW_PROXY_HOST: "10.200.0.1", + NEMOCLAW_PROXY_PORT: "3128", + NEMOCLAW_OBSERVABILITY: "1", + }, + }), + ); + + expect(built.profile).toMatchObject({ + agent: "langchain-deepagents-code", + agentConfig: { + agent: "langchain-deepagents-code", + autoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + }, + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }, + dashboard: { + agent: "langchain-deepagents-code", + mode: "disabled", + }, + messaging: { plan: null }, + tuning: { + contextWindow: null, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + }); + expect(decodeManagedStartupProfile(built.encodedProfile)).toEqual(built.profile); + }); + + it("keeps validated corporate CA bytes out of the profile and returns a digest-bound transport", () => { + const built = buildManagedStartupProfile( + openClawInput({ + corporateCa: { + pem: PEM, + sourcePath: "/public/corporate-ca.pem", + sourceEnv: "NEMOCLAW_CORPORATE_CA_BUNDLE", + }, + }), + ); + + const normalizedPem = PEM.endsWith("\n") ? PEM : `${PEM}\n`; + expect(built.corporateCaB64).toBe(Buffer.from(normalizedPem, "utf8").toString("base64")); + expect(built.profile.corporateCa.bundleSha256).toBe( + createHash("sha256").update(normalizedPem, "utf8").digest("hex"), + ); + const decodedProfile = Buffer.from(built.encodedProfile, "base64url").toString("utf8"); + expect(decodedProfile).not.toContain("BEGIN CERTIFICATE"); + expect(decodedProfile).not.toContain("/public/corporate-ca.pem"); + }); + + it("uses managed OpenClaw defaults without relying on Dockerfile defaults", () => { + const built = buildManagedStartupProfile(openClawInput()); + + expect(built.profile.agentConfig).toMatchObject({ + agent: "openclaw", + webSearch: { enabled: false, provider: "brave" }, + otel: { + enabled: false, + endpointUrl: "http://host.openshell.internal:4318", + serviceName: "openclaw-gateway", + sampleRate: 1, + }, + agentTimeoutSeconds: 600, + heartbeatEvery: null, + extraAgents: { + agents: [], + defaults: { subagents: {} }, + main: {}, + }, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + minimalBootstrap: false, + }); + expect(built.profile.tuning).toEqual({ + contextWindow: 131_072, + maxTokens: 4096, + reasoning: false, + reasoningEffort: "default", + }); + }); + + it("retains Hermes Tavily selection while web search is disabled", () => { + const built = buildManagedStartupProfile(hermesInput()); + expect(built.profile.agentConfig).toEqual({ + agent: "hermes", + webSearch: { enabled: false, provider: "tavily" }, + }); + }); + + it.each([ + ["DCode messaging", dcodeInput({ messagingPlan: messagingPlan("openclaw") }), /messagingPlan/], + [ + "DCode web search", + dcodeInput({ webSearch: { fetchEnabled: false, provider: "brave" } }), + /web-search/, + ], + [ + "OpenClaw Hermes gateways", + openClawInput({ hermesToolGateways: ["nous-web"] }), + /another agent/, + ], + [ + "Hermes Brave", + hermesInput({ webSearch: { fetchEnabled: true, provider: "brave" } }), + /only the Tavily/, + ], + [ + "Hermes OpenClaw OTEL knob", + hermesInput({ environment: { NEMOCLAW_OPENCLAW_OTEL: "1" } }), + /not supported by hermes/, + ], + ])("rejects unsupported cross-agent intent: %s", (_label, input, message) => { + expect(() => buildManagedStartupProfile(input)).toThrow(message); + }); + + it.each([ + [ + "oversized context", + openClawInput({ environment: { NEMOCLAW_CONTEXT_WINDOW: "9999999999" } }), + /NEMOCLAW_CONTEXT_WINDOW/, + ], + [ + "malformed reasoning", + openClawInput({ environment: { NEMOCLAW_REASONING: "sometimes" } }), + /NEMOCLAW_REASONING/, + ], + [ + "malformed reasoning effort", + openClawInput({ environment: { NEMOCLAW_REASONING_EFFORT: "maximum" } }), + /NEMOCLAW_REASONING_EFFORT/, + ], + [ + "conflicting proxy aliases", + openClawInput({ + environment: { + HTTP_PROXY: "http://one.example.test:8080", + http_proxy: "http://two.example.test:8080", + }, + }), + /conflicting values/, + ], + [ + "bare no-proxy intent", + openClawInput({ environment: { NO_PROXY: "localhost" } }), + /requires an HTTP_PROXY or HTTPS_PROXY/, + ], + [ + "raw CA transport", + openClawInput({ environment: { NEMOCLAW_CORPORATE_CA_B64: "ZmFrZQ==" } }), + /separate corporateCa input/, + ], + [ + "conflicting semantic model", + openClawInput({ environment: { NEMOCLAW_MODEL: "another-model" } }), + /conflicts with the resolved semantic/, + ], + ])("fails closed for malformed or conflicting stock input: %s", (_label, input, message) => { + expect(() => buildManagedStartupProfile(input)).toThrow(message); + }); + + it.each([ + openClawInput({ + environment: { HTTP_PROXY: "http://operator:password@proxy.example.test:8080" }, + }), + openClawInput({ + inference: { + ...openClawInput().inference, + model: "sk-proj-secret-material-1234567890", + }, + }), + openClawInput({ + environment: { + NEMOCLAW_EXTRA_AGENTS_JSON: JSON.stringify({ + agents: [ + { + id: "reviewer", + api_key: ["sk", "secret", "material", "1234567890"].join("-"), + }, + ], + }), + }, + }), + ])("rejects secret material instead of serializing it", (input) => { + expect(() => buildManagedStartupProfile(input)).toThrow(); + }); + + it("rejects malformed or non-CA certificate material", () => { + expect(() => + buildManagedStartupProfile( + openClawInput({ + corporateCa: { + pem: "-----BEGIN CERTIFICATE-----\nMIIBfake\n-----END CERTIFICATE-----\n", + sourcePath: "/public/bad.pem", + sourceEnv: "test", + }, + }), + ), + ).toThrow(/invalid X.509/); + }); + + it("audits the exact all-agent affordance inventory", () => { + expect(() => assertManagedStartupProfileBuilderInventoryCoverage()).not.toThrow(); + }); +}); diff --git a/src/lib/onboard/managed-startup/hold.ts b/src/lib/onboard/managed-startup/hold.ts new file mode 100644 index 00000000000..36015e8475e --- /dev/null +++ b/src/lib/onboard/managed-startup/hold.ts @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const MANAGED_STARTUP_HOLD_EXECUTABLE = "/usr/local/bin/nemoclaw-managed-startup-hold"; diff --git a/src/lib/onboard/managed-startup/image-runtime.ts b/src/lib/onboard/managed-startup/image-runtime.ts new file mode 100644 index 00000000000..1c2f0c8af10 --- /dev/null +++ b/src/lib/onboard/managed-startup/image-runtime.ts @@ -0,0 +1,238 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + MANAGED_STARTUP_AGENTS, + type ManagedStartupAgent, + type ManagedStartupDashboard, +} from "./profile"; + +/** + * This module owns only pure managed-image command construction. It does not + * execute commands, mutate sandbox state, or activate a compute driver. + */ +export type ManagedStartupImageIdentity = "root" | "sandbox"; +export type ManagedStartupMessagingAgent = "openclaw" | "hermes"; + +export interface ManagedStartupGenerateConfigConstructionAction { + readonly kind: "generate-agent-config"; + readonly agent: ManagedStartupAgent; + readonly runAs: "sandbox"; +} + +interface ManagedStartupApplyMessagingConstructionActionBase { + readonly kind: "apply-messaging-plan"; + readonly agent: ManagedStartupMessagingAgent; + readonly mode: "apply" | "clear"; +} + +export interface ManagedStartupApplyMessagingRuntimeConstructionAction + extends ManagedStartupApplyMessagingConstructionActionBase { + readonly phase: "runtime-setup"; + readonly runAs: "root"; +} + +export interface ManagedStartupApplyMessagingConfigConstructionAction + extends ManagedStartupApplyMessagingConstructionActionBase { + readonly phase: "post-agent-install"; + readonly runAs: "sandbox"; +} + +export type ManagedStartupApplyMessagingConstructionAction = + | ManagedStartupApplyMessagingRuntimeConstructionAction + | ManagedStartupApplyMessagingConfigConstructionAction; + +export interface ManagedStartupConfigureDashboardConstructionAction { + readonly kind: "configure-dashboard"; + readonly dashboard: ManagedStartupDashboard; +} + +export type ManagedStartupImageConstructionAction = + | ManagedStartupGenerateConfigConstructionAction + | ManagedStartupApplyMessagingConstructionAction + | ManagedStartupConfigureDashboardConstructionAction; + +/** + * The application mapper must produce this structural handoff only after it + * decodes and revalidates the profile and its nested messaging plan. + */ +export interface ManagedStartupImageActionPlanInput { + readonly agent: ManagedStartupAgent; + readonly actions: readonly ManagedStartupImageConstructionAction[]; +} + +export interface ManagedStartupImageActionCommand { + readonly action: + | "generate-agent-config" + | "messaging-runtime-setup" + | "messaging-post-agent-install"; + readonly runAs: ManagedStartupImageIdentity; + readonly argv: readonly string[]; +} + +export class ManagedStartupImageActionPlanError extends Error { + constructor(message: string) { + super(`Cannot build managed startup image action plan: ${message}`); + this.name = "ManagedStartupImageActionPlanError"; + } +} + +function fail(message: string): never { + throw new ManagedStartupImageActionPlanError(message); +} + +function exactAgent(value: string): ManagedStartupAgent { + if ((MANAGED_STARTUP_AGENTS as readonly string[]).includes(value)) { + return value as ManagedStartupAgent; + } + return fail(`unsupported agent ${JSON.stringify(value)}`); +} + +function generatorCommand(agent: ManagedStartupAgent): readonly string[] { + switch (agent) { + case "openclaw": + return [ + "/usr/local/bin/node", + "--experimental-strip-types", + "/scripts/generate-openclaw-config.mts", + ]; + case "hermes": + return [ + "/usr/local/bin/node", + "--experimental-strip-types", + "/opt/nemoclaw-hermes-config/generate-config.ts", + ]; + case "langchain-deepagents-code": + return [ + "/usr/local/bin/node", + "--experimental-strip-types", + "/opt/nemoclaw-deepagents-code/generate-config.ts", + ]; + } +} + +function messagingCommand( + agent: ManagedStartupMessagingAgent, + phase: "runtime-setup" | "post-agent-install", +): readonly string[] { + return [ + "/usr/local/bin/node", + "--experimental-strip-types", + "/src/lib/messaging/applier/build/messaging-build-applier.mts", + "--agent", + agent, + "--phase", + phase, + ...(phase === "post-agent-install" ? ["--managed-startup-runtime"] : []), + ]; +} + +function assertActionAgent( + inputAgent: ManagedStartupAgent, + actionAgent: ManagedStartupAgent, +): void { + if (inputAgent !== actionAgent) { + fail(`action for ${actionAgent} cannot be used by ${inputAgent}`); + } +} + +/** + * Convert the closed application-action vocabulary into immutable image + * commands. The vocabulary deliberately cannot express agent installation, + * package-manager access, command execution, or runtime activation. + */ +export function buildManagedStartupImageActionPlan( + input: ManagedStartupImageActionPlanInput, +): readonly ManagedStartupImageActionCommand[] { + const inputAgent = exactAgent(input.agent); + const commands: ManagedStartupImageActionCommand[] = []; + let dashboardActions = 0; + let generateActions = 0; + let runtimeMessagingActions = 0; + let postMessagingActions = 0; + + for (const action of input.actions) { + switch (action.kind) { + case "configure-dashboard": { + if (action.dashboard.agent !== input.agent) { + fail(`dashboard for ${action.dashboard.agent} cannot be used by ${input.agent}`); + } + dashboardActions += 1; + break; + } + case "generate-agent-config": { + assertActionAgent(inputAgent, exactAgent(action.agent)); + if (action.runAs !== "sandbox") { + fail("agent configuration generation must run as sandbox"); + } + generateActions += 1; + commands.push({ + action: "generate-agent-config", + runAs: action.runAs, + argv: generatorCommand(action.agent), + }); + break; + } + case "apply-messaging-plan": { + assertActionAgent(inputAgent, exactAgent(action.agent)); + if (action.mode !== "apply" && action.mode !== "clear") { + fail("messaging intent must be apply or clear"); + } + if (action.phase === "runtime-setup") { + if (action.runAs !== "root") { + fail("messaging runtime setup must run as root"); + } + runtimeMessagingActions += 1; + commands.push({ + action: "messaging-runtime-setup", + runAs: action.runAs, + argv: messagingCommand(action.agent, action.phase), + }); + } else if (action.phase === "post-agent-install") { + if (action.runAs !== "sandbox") { + fail("messaging post-agent configuration must run as sandbox"); + } + postMessagingActions += 1; + commands.push({ + action: "messaging-post-agent-install", + runAs: action.runAs, + argv: messagingCommand(action.agent, action.phase), + }); + } else { + fail("unsupported messaging construction phase"); + } + break; + } + default: + fail("unsupported managed startup construction action"); + } + } + + if (dashboardActions !== 1) fail("exactly one dashboard construction action is required"); + if (generateActions !== 1) fail("exactly one agent config construction action is required"); + const expectedMessagingActions = inputAgent === "langchain-deepagents-code" ? 0 : 1; + if ( + runtimeMessagingActions !== expectedMessagingActions || + postMessagingActions !== expectedMessagingActions + ) { + fail( + `${inputAgent} requires ${String(expectedMessagingActions)} action for each messaging phase`, + ); + } + const expectedOrder = + inputAgent === "langchain-deepagents-code" + ? ["generate-agent-config"] + : ["messaging-runtime-setup", "generate-agent-config", "messaging-post-agent-install"]; + if (commands.some((command, index) => command.action !== expectedOrder[index])) { + fail(`${inputAgent} image actions are not in the required construction order`); + } + + return Object.freeze( + commands.map((command) => + Object.freeze({ + ...command, + argv: Object.freeze([...command.argv]), + }), + ), + ); +} diff --git a/src/lib/onboard/managed-startup/onboard-profile.ts b/src/lib/onboard/managed-startup/onboard-profile.ts new file mode 100644 index 00000000000..b7efddbb387 --- /dev/null +++ b/src/lib/onboard/managed-startup/onboard-profile.ts @@ -0,0 +1,219 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxMessagingPlan } from "../../messaging/manifest"; +import type { ToolDisclosure } from "../../tool-disclosure"; +import type { ResolvedCorporateCa } from "../corporate-ca-types"; +import type { DcodeAutoApprovalMode } from "../dcode-auto-approval"; +import type { HermesDashboardOnboardState } from "../hermes-dashboard"; +import { hasCredentialBearingHostProxyEnvironment } from "../host-proxy-env"; +import { + MANAGED_STARTUP_AGENTS, + type ManagedStartupAgent, + type ManagedStartupDashboard, +} from "./profile"; +import { + type BuiltManagedStartupProfile, + buildManagedStartupProfile, + type ManagedStartupResolvedInferenceInput, +} from "./profile-builder"; + +const PROFILE_ENVIRONMENT_INPUTS = { + openclaw: [ + "NEMOCLAW_AGENT_HEARTBEAT_EVERY", + "NEMOCLAW_AGENT_TIMEOUT", + "NEMOCLAW_CONTEXT_WINDOW", + "NEMOCLAW_EXTRA_AGENTS_JSON", + "NEMOCLAW_EXTRA_AGENTS_JSON_B64", + "NEMOCLAW_INFERENCE_INPUTS", + "NEMOCLAW_MAX_TOKENS", + "NEMOCLAW_MINIMAL_BOOTSTRAP", + "NEMOCLAW_OPENCLAW_OTEL", + "NEMOCLAW_OPENCLAW_OTEL_ENDPOINT", + "NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE", + "NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME", + "NEMOCLAW_PROXY_HOST", + "NEMOCLAW_PROXY_PORT", + "NEMOCLAW_REASONING", + "NEMOCLAW_REASONING_EFFORT", + ], + hermes: ["NEMOCLAW_CONTEXT_WINDOW", "NEMOCLAW_PROXY_HOST", "NEMOCLAW_PROXY_PORT"], + "langchain-deepagents-code": ["NEMOCLAW_PROXY_HOST", "NEMOCLAW_PROXY_PORT"], +} as const satisfies Record; + +const HOST_PROXY_URL_INPUTS = ["HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"] as const; +const HOST_NO_PROXY_INPUTS = ["NO_PROXY", "no_proxy"] as const; + +export interface ManagedStartupOnboardProfileInput { + readonly agentName: string; + readonly inference: ManagedStartupResolvedInferenceInput; + readonly chatUiUrl: string; + readonly effectiveDashboardPort: number; + readonly manageDashboard: boolean; + readonly dashboardBindAddress: string | undefined; + readonly wslExposure: boolean; + readonly hermesDashboardState: HermesDashboardOnboardState; + readonly webSearch: { + readonly fetchEnabled: boolean; + readonly provider?: "brave" | "tavily"; + } | null; + readonly toolDisclosure: ToolDisclosure; + readonly hermesToolGateways: readonly string[]; + readonly messagingPlan: SandboxMessagingPlan | null; + readonly dcodeAutoApprovalMode: DcodeAutoApprovalMode; + readonly observabilityEnabled: boolean; + readonly environment: NodeJS.ProcessEnv; + /** CA material that the host resolved and validated before profile construction. */ + readonly corporateCa: ResolvedCorporateCa | null; +} + +export type BuiltManagedStartupOnboardProfile = BuiltManagedStartupProfile & { + /** + * Non-secret replay intent. Proxy credentials remain launch-only and never + * enter the canonical startup profile or durable receipt. + */ + readonly credentialProxyReplayRequired: boolean; +}; + +export class ManagedStartupOnboardProfileError extends Error { + constructor(message: string) { + super(`Cannot prepare managed onboarding profile: ${message}`); + this.name = "ManagedStartupOnboardProfileError"; + } +} + +function exactManagedAgent(agentName: string): ManagedStartupAgent { + if ((MANAGED_STARTUP_AGENTS as readonly string[]).includes(agentName)) { + return agentName as ManagedStartupAgent; + } + throw new ManagedStartupOnboardProfileError(`unsupported agent '${agentName}'`); +} + +function requireDashboardPort(port: number): number { + if (!Number.isInteger(port) || port < 1024 || port > 65_535) { + throw new ManagedStartupOnboardProfileError("dashboard port is missing or invalid"); + } + return port; +} + +function dashboardForInput( + agent: ManagedStartupAgent, + input: ManagedStartupOnboardProfileInput, +): ManagedStartupDashboard { + if (agent === "langchain-deepagents-code") { + if (input.manageDashboard) { + throw new ManagedStartupOnboardProfileError("DCode must not enable a dashboard"); + } + return { agent, mode: "disabled" }; + } + + if (!input.manageDashboard) { + throw new ManagedStartupOnboardProfileError(`${agent} requires managed dashboard state`); + } + + if (agent === "openclaw") { + const requestedBind = input.dashboardBindAddress?.trim(); + if (requestedBind && requestedBind !== "0.0.0.0") { + throw new ManagedStartupOnboardProfileError( + "dashboard bind address must be empty or 0.0.0.0", + ); + } + const bindAddress = requestedBind === "0.0.0.0" ? "0.0.0.0" : "127.0.0.1"; + const remote = + bindAddress === "0.0.0.0" || + input.wslExposure || + !["127.0.0.1", "localhost", "::1", "[::1]"].includes(new URL(input.chatUiUrl).hostname); + return { + agent, + mode: remote ? "remote" : "loopback", + url: input.chatUiUrl, + port: requireDashboardPort(input.effectiveDashboardPort), + bindAddress, + wslExposure: input.wslExposure, + }; + } + + const config = input.hermesDashboardState.config; + if (!input.hermesDashboardState.enabled) { + return { + agent, + mode: "disabled", + url: input.chatUiUrl, + publicPort: null, + internalPort: null, + tuiEnabled: false, + }; + } + if (!config?.enabled) { + throw new ManagedStartupOnboardProfileError( + "Hermes dashboard is enabled without resolved configuration", + ); + } + return { + agent, + mode: "loopback-forwarded", + url: input.chatUiUrl, + publicPort: requireDashboardPort(config.port), + internalPort: requireDashboardPort(config.internalPort), + tuiEnabled: config.tuiEnabled, + }; +} + +/** + * Copy only knobs whose values remain environment-owned at the managed-image + * boundary. Model, route, dashboard, tool, messaging, and agent-specific + * selections are passed explicitly so CLI precedence cannot be reversed by a + * stale ambient variable. + */ +function profileEnvironment( + agent: ManagedStartupAgent, + environment: NodeJS.ProcessEnv, +): NodeJS.ProcessEnv { + const selected: NodeJS.ProcessEnv = {}; + for (const name of PROFILE_ENVIRONMENT_INPUTS[agent]) { + const value = environment[name]; + if (value !== undefined) selected[name] = value; + } + const credentialProxy = hasCredentialBearingHostProxyEnvironment(environment); + if (!credentialProxy) { + for (const name of HOST_PROXY_URL_INPUTS) { + const value = environment[name]?.trim(); + if (!value) continue; + selected[name] = value; + } + const hasCredentialFreeProxy = HOST_PROXY_URL_INPUTS.some((name) => selected[name]); + if (hasCredentialFreeProxy) { + for (const name of HOST_NO_PROXY_INPUTS) { + const value = environment[name]; + if (value !== undefined) selected[name] = value; + } + } + } + return selected; +} + +export function buildManagedStartupOnboardProfile( + input: ManagedStartupOnboardProfileInput, +): BuiltManagedStartupOnboardProfile { + const agent = exactManagedAgent(input.agentName); + const dashboard = dashboardForInput(agent, input); + const environment = profileEnvironment(agent, input.environment); + const credentialProxyReplayRequired = + agent !== "langchain-deepagents-code" && + hasCredentialBearingHostProxyEnvironment(input.environment); + const built = buildManagedStartupProfile({ + agent, + inference: input.inference, + dashboard, + webSearch: agent === "langchain-deepagents-code" ? null : input.webSearch, + toolDisclosure: input.toolDisclosure, + hermesToolGateways: agent === "hermes" ? input.hermesToolGateways : [], + messagingPlan: agent === "langchain-deepagents-code" ? null : input.messagingPlan, + dcodeAutoApprovalMode: + agent === "langchain-deepagents-code" ? input.dcodeAutoApprovalMode : null, + observabilityEnabled: agent === "langchain-deepagents-code" ? input.observabilityEnabled : null, + environment, + corporateCa: input.corporateCa, + }); + return Object.freeze({ ...built, credentialProxyReplayRequired }); +} diff --git a/src/lib/onboard/managed-startup/profile-builder.ts b/src/lib/onboard/managed-startup/profile-builder.ts new file mode 100644 index 00000000000..4f906ba6bc8 --- /dev/null +++ b/src/lib/onboard/managed-startup/profile-builder.ts @@ -0,0 +1,951 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash, X509Certificate } from "node:crypto"; + +import { MAX_AUTODETECTED_OLLAMA_CONTEXT_WINDOW } from "../../inference/ollama-runtime-context"; +import { hydrateDerivedSandboxMessagingPlanFields } from "../../messaging/hydration"; +import { parseSandboxMessagingPlan } from "../../messaging/plan-validation"; +import { withLocalNoProxy } from "../../proxy/local-no-proxy"; +import { + MAX_CORPORATE_CA_BYTES, + MAX_CORPORATE_CA_CERTS, + PEM_CERTIFICATE_RE_GLOBAL, +} from "../corporate-ca-policy"; +import type { ResolvedCorporateCa } from "../corporate-ca-types"; +import { normalizeCertificateBlocks } from "../corporate-ca-validation"; +import { + encodeManagedStartupProfile, + MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY, + MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + MANAGED_STARTUP_REASONING_EFFORTS, + type ManagedStartupAgent, + type ManagedStartupDashboard, + type ManagedStartupDcodeAutoApprovalMode, + type ManagedStartupExtraAgents, + type ManagedStartupHermesToolGateway, + type ManagedStartupInputModality, + type ManagedStartupJsonObject, + type ManagedStartupProfile, + type ManagedStartupReasoningEffort, + type ManagedStartupToolDisclosure, + type ManagedStartupWebSearch, + validateManagedStartupProfile, +} from "./profile"; + +const DEFAULT_MANAGED_PROXY_HOST = "10.200.0.1"; +const DEFAULT_MANAGED_PROXY_PORT = 3128; +const DEFAULT_CONTEXT_WINDOW = 131_072; +const DEFAULT_OPENCLAW_MAX_TOKENS = 4096; +const DEFAULT_OPENCLAW_AGENT_TIMEOUT_SECONDS = 600; +const DEFAULT_OPENCLAW_OTEL_ENDPOINT = "http://host.openshell.internal:4318"; +const DEFAULT_OPENCLAW_OTEL_SERVICE_NAME = "openclaw-gateway"; +const MIN_HERMES_CONTEXT_WINDOW = 64_000; +const MAX_PROFILE_TUNING_INTEGER = 1_000_000_000; +const FALSE_VALUES = new Set(["0", "false", "no", "off"]); +const TRUE_VALUES = new Set(["1", "true", "yes", "on"]); +const STANDARD_BASE64_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u; + +/** + * These digests deliberately bind the builder to every classified stock + * Docker/runtime affordance, including its profile path and representation. + * Adding or reclassifying an affordance must update the builder in the same + * change; otherwise construction fails before a sandbox is launched. + */ +const EXPECTED_AFFORDANCE_INVENTORY_SHA256 = { + openclaw: "9b722441e33f0b0d7580f74cd185c0174979de9c1a784556ff56ff931b2c9904", + hermes: "26c2dc3750274e5c2a79bf382a4b18b3cf26c0ef64938e91b694427aa23756e8", + "langchain-deepagents-code": "780ec15dd97efaea5b75614d657a4baf93cba4c7933e50f78fb72814cb427c85", +} as const satisfies Record; + +const INVENTORY_INPUTS = new Set( + Object.values(MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY).flatMap((entries) => + entries.map((entry) => entry.input), + ), +); + +export interface ManagedStartupResolvedInferenceInput { + readonly routeProvider: string; + readonly upstreamProvider: string; + readonly model: string; + readonly routedBaseUrl: string; + readonly upstreamEndpointUrl: string | null; + readonly api: ManagedStartupProfile["inference"]["api"]; + readonly primaryModelRef: string | null; + readonly compatibility: Readonly> | null; +} + +export interface ManagedStartupProfileBuilderInput { + readonly agent: ManagedStartupAgent; + /** Fully resolved host semantics; the portable builder never selects providers. */ + readonly inference: ManagedStartupResolvedInferenceInput; + readonly dashboard: ManagedStartupDashboard; + readonly webSearch: { + readonly fetchEnabled: boolean; + readonly provider?: "brave" | "tavily"; + } | null; + readonly toolDisclosure: ManagedStartupToolDisclosure; + readonly hermesToolGateways: readonly string[]; + readonly messagingPlan: unknown | null; + readonly dcodeAutoApprovalMode: ManagedStartupDcodeAutoApprovalMode | null; + readonly observabilityEnabled: boolean | null; + /** + * Host onboarding knobs only. The builder reads an exact allowlist and never + * copies this object wholesale, so ambient credentials cannot enter a profile. + */ + readonly environment: NodeJS.ProcessEnv; + /** Validated host CA material returned by the existing corporate-CA resolver. */ + readonly corporateCa?: ResolvedCorporateCa | null; +} + +declare const VALIDATED_MANAGED_STARTUP_PROFILE_TRANSPORT: unique symbol; + +/** + * A profile transport whose bounded profile and nested messaging plan were + * validated by the host-side construction boundary before encoding. + * + * This brand does not replace validation after transport. The application + * boundary must decode the profile and re-run parseSandboxMessagingPlan before + * it changes configuration or shared state. + */ +export type ValidatedManagedStartupProfileTransport = string & { + readonly [VALIDATED_MANAGED_STARTUP_PROFILE_TRANSPORT]: true; +}; + +export interface BuiltManagedStartupProfile { + readonly profile: ManagedStartupProfile; + readonly encodedProfile: ValidatedManagedStartupProfileTransport; + /** Digest of the canonical, bounded, credential-screened profile transport. */ + readonly startupProfileSha256: string; + /** Exact normalized PEM bytes, encoded separately from the profile. */ + readonly corporateCaB64?: string; +} + +export class ManagedStartupProfileBuilderError extends Error { + constructor(message: string) { + super(`Cannot build managed startup profile: ${message}`); + this.name = "ManagedStartupProfileBuilderError"; + } +} + +function fail(message: string): never { + throw new ManagedStartupProfileBuilderError(message); +} + +function presentEnvironmentValue(environment: NodeJS.ProcessEnv, name: string): string | null { + const value = environment[name]; + if (typeof value !== "string") return null; + const trimmed = value.trim(); + return trimmed === "" ? null : trimmed; +} + +function parsePositiveInteger( + environment: NodeJS.ProcessEnv, + name: string, + fallback: number | null, + options: { readonly minimum?: number; readonly maximum?: number } = {}, +): number | null { + const raw = presentEnvironmentValue(environment, name); + if (raw === null) return fallback; + if (!/^[1-9][0-9]*$/u.test(raw)) { + fail(`${name} must be a positive integer`); + } + const parsed = Number(raw); + const minimum = options.minimum ?? 1; + const maximum = options.maximum ?? MAX_PROFILE_TUNING_INTEGER; + if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { + fail(`${name} must be between ${String(minimum)} and ${String(maximum)}`); + } + return parsed; +} + +function parsePort(environment: NodeJS.ProcessEnv, name: string, fallback: number): number { + const port = parsePositiveInteger(environment, name, fallback, { maximum: 65_535 }); + if (port === null) fail(`${name} must resolve to a TCP port`); + return port; +} + +function parseZeroOneFlag( + environment: NodeJS.ProcessEnv, + name: string, + fallback: boolean, +): boolean { + const raw = presentEnvironmentValue(environment, name); + if (raw === null) return fallback; + if (raw === "1") return true; + if (raw === "0") return false; + fail(`${name} must be "0" or "1"`); +} + +function parseHumanBoolean( + environment: NodeJS.ProcessEnv, + name: string, + fallback: boolean, +): boolean { + const raw = presentEnvironmentValue(environment, name); + if (raw === null) return fallback; + const normalized = raw.toLowerCase(); + if (TRUE_VALUES.has(normalized)) return true; + if (FALSE_VALUES.has(normalized)) return false; + fail(`${name} must be a boolean value`); +} + +function parseOpenClawOtelEnabled(environment: NodeJS.ProcessEnv): boolean { + const raw = presentEnvironmentValue(environment, "NEMOCLAW_OPENCLAW_OTEL"); + return raw !== null && !FALSE_VALUES.has(raw.toLowerCase()); +} + +function parseReasoning(environment: NodeJS.ProcessEnv): boolean { + const raw = presentEnvironmentValue(environment, "NEMOCLAW_REASONING"); + if (raw === null) return false; + if (raw === "true") return true; + if (raw === "false") return false; + fail('NEMOCLAW_REASONING must be "true" or "false"'); +} + +function parseReasoningEffort(environment: NodeJS.ProcessEnv): ManagedStartupReasoningEffort { + const raw = presentEnvironmentValue(environment, "NEMOCLAW_REASONING_EFFORT"); + const normalized = raw === null ? "default" : raw.toLowerCase(); + if ((MANAGED_STARTUP_REASONING_EFFORTS as readonly string[]).includes(normalized)) { + return normalized as ManagedStartupReasoningEffort; + } + fail(`NEMOCLAW_REASONING_EFFORT must be one of: ${MANAGED_STARTUP_REASONING_EFFORTS.join(", ")}`); +} + +function parseInputModalities( + environment: NodeJS.ProcessEnv, +): readonly ManagedStartupInputModality[] { + const raw = presentEnvironmentValue(environment, "NEMOCLAW_INFERENCE_INPUTS"); + if (raw === null) return ["text"]; + const values = raw.split(",").map((value) => value.trim()); + if ( + values.length === 0 || + values.some((value) => value !== "text" && value !== "image") || + new Set(values).size !== values.length + ) { + fail("NEMOCLAW_INFERENCE_INPUTS must be a unique comma-separated list of text and image"); + } + return values as ManagedStartupInputModality[]; +} + +function parseHeartbeat(environment: NodeJS.ProcessEnv): string | null { + const raw = presentEnvironmentValue(environment, "NEMOCLAW_AGENT_HEARTBEAT_EVERY"); + if (raw === null) return null; + if (!/^\d+(?:s|m|h)$/u.test(raw)) { + fail("NEMOCLAW_AGENT_HEARTBEAT_EVERY must be a duration ending in s, m, or h"); + } + return raw; +} + +function parseStrictBase64Json(raw: string, name: string): unknown { + if (!STANDARD_BASE64_RE.test(raw)) fail(`${name} must be canonical base64`); + const bytes = Buffer.from(raw, "base64"); + if (bytes.toString("base64") !== raw) fail(`${name} must be canonical base64`); + try { + return JSON.parse(bytes.toString("utf8")) as unknown; + } catch { + fail(`${name} must contain valid UTF-8 JSON`); + } +} + +function parseRawJson(raw: string, name: string): unknown { + try { + return JSON.parse(raw) as unknown; + } catch { + fail(`${name} must contain valid JSON`); + } +} + +function isPlainObject(value: unknown): value is Record { + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function normalizeExtraAgentsCandidate(value: unknown): ManagedStartupExtraAgents { + const emptyDefaults = { subagents: {} }; + if (value === null || value === undefined) { + return { agents: [], defaults: emptyDefaults, main: {} }; + } + if (Array.isArray(value)) { + return { + agents: value as ManagedStartupJsonObject[], + defaults: emptyDefaults, + main: {}, + }; + } + if (!isPlainObject(value)) { + fail( + "NEMOCLAW_EXTRA_AGENTS_JSON must be an array or an object with agents, defaults, and main", + ); + } + const unknownKeys = Object.keys(value).filter( + (key) => key !== "agents" && key !== "defaults" && key !== "main", + ); + if (unknownKeys.length > 0) { + fail("NEMOCLAW_EXTRA_AGENTS_JSON contains unsupported top-level fields"); + } + const agents = value.agents ?? []; + const defaults = value.defaults ?? emptyDefaults; + const main = value.main ?? {}; + if (!Array.isArray(agents) || !agents.every((agent) => isPlainObject(agent))) { + fail("NEMOCLAW_EXTRA_AGENTS_JSON.agents must be an object list"); + } + if (!isPlainObject(defaults) || !isPlainObject(main)) { + fail("NEMOCLAW_EXTRA_AGENTS_JSON defaults and main must be objects"); + } + return { + agents: agents as ManagedStartupJsonObject[], + defaults: defaults as ManagedStartupJsonObject, + main: main as ManagedStartupJsonObject, + }; +} + +function parseExtraAgents(environment: NodeJS.ProcessEnv): ManagedStartupExtraAgents { + const raw = presentEnvironmentValue(environment, "NEMOCLAW_EXTRA_AGENTS_JSON"); + const encoded = presentEnvironmentValue(environment, "NEMOCLAW_EXTRA_AGENTS_JSON_B64"); + if (raw !== null && encoded !== null) { + fail("NEMOCLAW_EXTRA_AGENTS_JSON and NEMOCLAW_EXTRA_AGENTS_JSON_B64 must not both be set"); + } + return normalizeExtraAgentsCandidate( + raw !== null + ? parseRawJson(raw, "NEMOCLAW_EXTRA_AGENTS_JSON") + : encoded !== null + ? parseStrictBase64Json(encoded, "NEMOCLAW_EXTRA_AGENTS_JSON_B64") + : null, + ); +} + +function normalizeWebSearch( + agent: ManagedStartupAgent, + config: ManagedStartupProfileBuilderInput["webSearch"], +): ManagedStartupWebSearch | null { + if (agent === "langchain-deepagents-code") { + if (config !== null) { + fail("langchain-deepagents-code does not support web-search profile intent"); + } + return null; + } + if (config !== null) { + if (!isPlainObject(config)) fail("webSearch must be null or a configuration object"); + const unknownKeys = Object.keys(config).filter( + (key) => key !== "fetchEnabled" && key !== "provider", + ); + if (unknownKeys.length > 0 || typeof config.fetchEnabled !== "boolean") { + fail("webSearch contains unsupported or malformed fields"); + } + if ( + config.provider !== undefined && + config.provider !== "brave" && + config.provider !== "tavily" + ) { + fail("webSearch.provider must be brave or tavily"); + } + } + const provider = + agent === "hermes" && config?.provider === undefined ? "tavily" : (config?.provider ?? "brave"); + if (agent === "hermes" && provider !== "tavily") { + fail("Hermes supports only the Tavily web-search provider"); + } + return { enabled: config?.fetchEnabled === true, provider }; +} + +function normalizeMessagingPlan( + agent: ManagedStartupAgent, + value: unknown | null, +): ManagedStartupJsonObject | null { + if (agent === "langchain-deepagents-code") { + if (value !== null) { + fail("langchain-deepagents-code messagingPlan must be null"); + } + return null; + } + if (value === null) return null; + const plan = parseSandboxMessagingPlan(value, { agent }); + if (!plan) fail(`messagingPlan must be a valid ${agent} SandboxMessagingPlan`); + const hydrated = hydrateDerivedSandboxMessagingPlanFields(plan); + const reparsed = parseSandboxMessagingPlan(hydrated, { agent }); + if (!reparsed) fail(`messagingPlan hydration produced an invalid ${agent} plan`); + return JSON.parse(JSON.stringify(reparsed)) as ManagedStartupJsonObject; +} + +function resolveAliasedEnvironmentValue( + environment: NodeJS.ProcessEnv, + upper: string, + lower: string, +): string | null { + const upperValue = presentEnvironmentValue(environment, upper); + const lowerValue = presentEnvironmentValue(environment, lower); + if (upperValue !== null && lowerValue !== null && upperValue !== lowerValue) { + fail(`${upper} and ${lower} must not express conflicting values`); + } + return upperValue ?? lowerValue; +} + +function normalizeNoProxyList(raw: string | null): string[] { + if (raw === null) return []; + const values = raw + .split(",") + .map((value) => value.trim()) + .filter(Boolean); + return [...new Set(values)]; +} + +function resolveHostProxy( + _agent: ManagedStartupAgent, + environment: NodeJS.ProcessEnv, +): Pick { + const hostHttpUrl = resolveAliasedEnvironmentValue(environment, "HTTP_PROXY", "http_proxy"); + const hostHttpsUrl = resolveAliasedEnvironmentValue(environment, "HTTPS_PROXY", "https_proxy"); + const noProxy = resolveAliasedEnvironmentValue(environment, "NO_PROXY", "no_proxy"); + if (hostHttpUrl === null && hostHttpsUrl === null) { + if (noProxy !== null) { + fail("NO_PROXY/no_proxy requires an HTTP_PROXY or HTTPS_PROXY intent"); + } + return { hostHttpUrl: null, hostHttpsUrl: null, hostNoProxy: [] }; + } + + const normalizedEnvironment: Record = {}; + if (hostHttpUrl !== null) { + normalizedEnvironment.HTTP_PROXY = hostHttpUrl; + normalizedEnvironment.http_proxy = hostHttpUrl; + } + if (hostHttpsUrl !== null) { + normalizedEnvironment.HTTPS_PROXY = hostHttpsUrl; + normalizedEnvironment.https_proxy = hostHttpsUrl; + } + if (noProxy !== null) { + normalizedEnvironment.NO_PROXY = noProxy; + normalizedEnvironment.no_proxy = noProxy; + } + withLocalNoProxy(normalizedEnvironment); + const upperNoProxy = normalizeNoProxyList(normalizedEnvironment.NO_PROXY ?? null); + const lowerNoProxy = normalizeNoProxyList(normalizedEnvironment.no_proxy ?? null); + return { + hostHttpUrl, + hostHttpsUrl, + hostNoProxy: [...new Set([...upperNoProxy, ...lowerNoProxy])], + }; +} + +function resolveCorporateCaMaterial(corporateCa: ResolvedCorporateCa | null | undefined): { + readonly bundleSha256: string | null; + readonly corporateCaB64?: string; +} { + if (!corporateCa) return { bundleSha256: null }; + const bytes = Buffer.byteLength(corporateCa.pem, "utf8"); + if (bytes === 0 || bytes > MAX_CORPORATE_CA_BYTES) { + fail(`corporate CA material must be between 1 and ${String(MAX_CORPORATE_CA_BYTES)} bytes`); + } + const blocks = corporateCa.pem.match(PEM_CERTIFICATE_RE_GLOBAL); + if (!blocks || blocks.length === 0 || blocks.length > MAX_CORPORATE_CA_CERTS) { + fail( + `corporate CA material must contain between 1 and ${String(MAX_CORPORATE_CA_CERTS)} certificates`, + ); + } + for (const block of blocks) { + let certificate: X509Certificate; + try { + certificate = new X509Certificate(block); + } catch { + fail("corporate CA material contains an invalid X.509 certificate"); + } + if (!certificate.ca) { + fail("corporate CA material contains a certificate without CA basic constraints"); + } + } + const normalizedPem = normalizeCertificateBlocks(blocks); + if (corporateCa.pem.trim() !== normalizedPem.trim()) { + fail("corporate CA material must contain only normalized PEM certificate blocks"); + } + return { + bundleSha256: createHash("sha256").update(normalizedPem, "utf8").digest("hex"), + corporateCaB64: Buffer.from(normalizedPem, "utf8").toString("base64"), + }; +} + +function assertAgentSpecificInput(input: ManagedStartupProfileBuilderInput): void { + if (input.dashboard.agent !== input.agent) { + fail("dashboard.agent must match agent"); + } + if (input.agent === "openclaw") { + if ( + input.inference.upstreamEndpointUrl !== null || + input.hermesToolGateways.length > 0 || + input.dcodeAutoApprovalMode !== null || + input.observabilityEnabled !== null + ) { + fail("OpenClaw input contains state owned by another agent"); + } + return; + } + if (input.agent === "hermes") { + if ( + input.inference.upstreamEndpointUrl !== null || + input.dcodeAutoApprovalMode !== null || + input.observabilityEnabled !== null + ) { + fail("Hermes input contains state owned by another agent"); + } + return; + } + if (input.webSearch !== null) { + fail("langchain-deepagents-code does not support web-search profile intent"); + } + if (input.hermesToolGateways.length > 0) { + fail("langchain-deepagents-code does not support Hermes tool gateways"); + } + if (input.messagingPlan !== null) { + fail("langchain-deepagents-code messagingPlan must be null"); + } + if ( + input.dcodeAutoApprovalMode !== "disabled" && + input.dcodeAutoApprovalMode !== "thread-opt-in" + ) { + fail("DCode approval state must be explicit"); + } + if (typeof input.observabilityEnabled !== "boolean") { + fail("DCode observability state must be explicit"); + } +} + +function assertNoWrongAgentEnvironment( + agent: ManagedStartupAgent, + environment: NodeJS.ProcessEnv, +): void { + const supported = new Set( + MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[agent].map((entry) => entry.input), + ); + for (const name of INVENTORY_INPUTS) { + if (presentEnvironmentValue(environment, name) === null || supported.has(name)) continue; + // NEMOCLAW_DASHBOARD_PORT is a host-side input used to resolve the explicit + // Hermes dashboard state even though the sandbox consumes its Hermes alias. + if (agent === "hermes" && name === "NEMOCLAW_DASHBOARD_PORT") continue; + fail(`${name} is not supported by ${agent}`); + } + const rawExtraAgents = presentEnvironmentValue(environment, "NEMOCLAW_EXTRA_AGENTS_JSON"); + if (agent !== "openclaw" && rawExtraAgents !== null) { + fail(`NEMOCLAW_EXTRA_AGENTS_JSON is not supported by ${agent}`); + } +} + +function inventoryDigest(agent: ManagedStartupAgent): string { + return createHash("sha256") + .update(JSON.stringify(MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[agent])) + .digest("hex"); +} + +/** Fail closed if the authoritative affordance inventory changes without this builder. */ +export function assertManagedStartupProfileBuilderInventoryCoverage(): void { + for (const agent of Object.keys(EXPECTED_AFFORDANCE_INVENTORY_SHA256) as ManagedStartupAgent[]) { + if (inventoryDigest(agent) !== EXPECTED_AFFORDANCE_INVENTORY_SHA256[agent]) { + fail(`${agent} affordance inventory changed without a builder mapping update`); + } + } +} + +function profilePathExists(profile: ManagedStartupProfile, profilePath: string): boolean { + let value: unknown = profile; + for (const segment of profilePath.split(".")) { + if (!isPlainObject(value) || !Object.hasOwn(value, segment)) return false; + value = value[segment]; + } + return true; +} + +function assertInventoryPathsResolved(profile: ManagedStartupProfile): void { + for (const affordance of MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[profile.agent]) { + if (!profilePathExists(profile, affordance.profilePath)) { + fail(`${affordance.input} has no resolved value at ${affordance.profilePath}`); + } + } +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map((item) => canonicalJson(item)).join(",")}]`; + } + if (isPlainObject(value)) { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +function assertEquivalent(name: string, actual: unknown, expected: unknown): void { + if (canonicalJson(actual) !== canonicalJson(expected)) { + fail(`${name} conflicts with the resolved semantic onboarding state`); + } +} + +function parseEnvironmentJson(environment: NodeJS.ProcessEnv, name: string): unknown | undefined { + const raw = presentEnvironmentValue(environment, name); + return raw === null ? undefined : parseStrictBase64Json(raw, name); +} + +/** + * Environment may carry legacy Docker inputs while callers migrate. Parse and + * compare each supplied value instead of silently preferring one source. + */ +function assertEnvironmentConsistency( + profile: ManagedStartupProfile, + environment: NodeJS.ProcessEnv, +): void { + const stringValues: Readonly> = { + NEMOCLAW_MODEL: profile.inference.model, + NEMOCLAW_INFERENCE_PROVIDER_ID: profile.inference.routeProvider, + NEMOCLAW_UPSTREAM_PROVIDER: profile.inference.upstreamProvider, + NEMOCLAW_PRIMARY_MODEL_REF: profile.inference.primaryModelRef, + NEMOCLAW_INFERENCE_BASE_URL: profile.inference.routedBaseUrl, + NEMOCLAW_INFERENCE_API: profile.inference.api, + NEMOCLAW_TOOL_DISCLOSURE: profile.tools.disclosure, + CHAT_UI_URL: + profile.dashboard.agent === "langchain-deepagents-code" ? null : profile.dashboard.url, + }; + for (const [name, expected] of Object.entries(stringValues)) { + const raw = presentEnvironmentValue(environment, name); + if (raw !== null) assertEquivalent(name, raw, expected); + } + + const numericValues: Readonly> = { + NEMOCLAW_CONTEXT_WINDOW: profile.tuning.contextWindow, + NEMOCLAW_MAX_TOKENS: profile.tuning.maxTokens, + NEMOCLAW_PROXY_PORT: profile.proxy.managedPort, + NEMOCLAW_AGENT_TIMEOUT: + profile.agentConfig.agent === "openclaw" ? profile.agentConfig.agentTimeoutSeconds : null, + NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE: + profile.agentConfig.agent === "openclaw" ? profile.agentConfig.otel.sampleRate : null, + }; + for (const [name, expected] of Object.entries(numericValues)) { + const raw = presentEnvironmentValue(environment, name); + if (raw !== null) assertEquivalent(name, Number(raw), expected); + } + + const managedHost = presentEnvironmentValue(environment, "NEMOCLAW_PROXY_HOST"); + if (managedHost !== null) + assertEquivalent("NEMOCLAW_PROXY_HOST", managedHost, profile.proxy.managedHost); + + const upstreamEndpoint = presentEnvironmentValue(environment, "NEMOCLAW_UPSTREAM_ENDPOINT_URL"); + if (upstreamEndpoint !== null) { + assertEquivalent( + "NEMOCLAW_UPSTREAM_ENDPOINT_URL", + upstreamEndpoint, + profile.inference.upstreamEndpointUrl, + ); + } + + if (profile.agentConfig.agent === "openclaw") { + const config = profile.agentConfig; + const dashboard = profile.dashboard; + if (dashboard.agent !== "openclaw") fail("OpenClaw dashboard state is inconsistent"); + const directValues: Readonly> = { + NEMOCLAW_REASONING: String(profile.tuning.reasoning), + NEMOCLAW_AGENT_HEARTBEAT_EVERY: config.heartbeatEvery, + NEMOCLAW_DISABLE_DEVICE_AUTH: config.deviceAuth.disabled ? "1" : "0", + NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE: config.deviceAuth.optOutSource, + NEMOCLAW_WEB_SEARCH_ENABLED: config.webSearch.enabled ? "1" : "0", + NEMOCLAW_WEB_SEARCH_PROVIDER: config.webSearch.provider, + NEMOCLAW_OPENCLAW_OTEL_ENDPOINT: config.otel.endpointUrl, + NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME: config.otel.serviceName, + NEMOCLAW_DASHBOARD_BIND: dashboard.bindAddress === "0.0.0.0" ? "0.0.0.0" : null, + NEMOCLAW_WSL_DASHBOARD_EXPOSURE: dashboard.wslExposure ? "1" : "0", + NEMOCLAW_DASHBOARD_PORT: dashboard.port, + }; + for (const [name, expected] of Object.entries(directValues)) { + const raw = presentEnvironmentValue(environment, name); + if (raw === null) continue; + assertEquivalent(name, typeof expected === "number" ? Number(raw) : raw, expected); + } + const reasoningEffort = presentEnvironmentValue(environment, "NEMOCLAW_REASONING_EFFORT"); + if (reasoningEffort !== null) { + assertEquivalent( + "NEMOCLAW_REASONING_EFFORT", + reasoningEffort.toLowerCase(), + profile.tuning.reasoningEffort, + ); + } + if (presentEnvironmentValue(environment, "NEMOCLAW_OPENCLAW_OTEL") !== null) { + assertEquivalent( + "NEMOCLAW_OPENCLAW_OTEL", + parseOpenClawOtelEnabled(environment), + config.otel.enabled, + ); + } + if (presentEnvironmentValue(environment, "NEMOCLAW_MINIMAL_BOOTSTRAP") !== null) { + assertEquivalent( + "NEMOCLAW_MINIMAL_BOOTSTRAP", + parseZeroOneFlag(environment, "NEMOCLAW_MINIMAL_BOOTSTRAP", false), + config.minimalBootstrap, + ); + } + if (presentEnvironmentValue(environment, "NEMOCLAW_INFERENCE_INPUTS") !== null) { + assertEquivalent( + "NEMOCLAW_INFERENCE_INPUTS", + [...parseInputModalities(environment)].sort(), + profile.inference.inputModalities, + ); + } + const compatibility = parseEnvironmentJson(environment, "NEMOCLAW_INFERENCE_COMPAT_B64"); + if (compatibility !== undefined) { + assertEquivalent( + "NEMOCLAW_INFERENCE_COMPAT_B64", + compatibility, + profile.inference.compatibility, + ); + } + const extraAgents = parseEnvironmentJson(environment, "NEMOCLAW_EXTRA_AGENTS_JSON_B64"); + if (extraAgents !== undefined) { + assertEquivalent( + "NEMOCLAW_EXTRA_AGENTS_JSON_B64", + normalizeExtraAgentsCandidate(extraAgents), + config.extraAgents, + ); + } + } else if (profile.agentConfig.agent === "hermes") { + const config = profile.agentConfig; + const dashboard = profile.dashboard; + if (dashboard.agent !== "hermes") fail("Hermes dashboard state is inconsistent"); + const directValues: Readonly> = { + NEMOCLAW_WEB_SEARCH_ENABLED: config.webSearch.enabled ? "1" : "0", + NEMOCLAW_WEB_SEARCH_PROVIDER: config.webSearch.provider, + NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER: profile.tools.enabledGateways.length > 0 ? "1" : "0", + NEMOCLAW_HERMES_DASHBOARD_PORT: dashboard.publicPort, + NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT: dashboard.internalPort, + }; + for (const [name, expected] of Object.entries(directValues)) { + const raw = presentEnvironmentValue(environment, name); + if (raw === null) continue; + assertEquivalent(name, typeof expected === "number" ? Number(raw) : raw, expected); + } + for (const [name, expected] of [ + ["NEMOCLAW_HERMES_DASHBOARD", dashboard.mode === "loopback-forwarded"], + ["NEMOCLAW_HERMES_DASHBOARD_TUI", dashboard.tuiEnabled], + ] as const) { + if (presentEnvironmentValue(environment, name) !== null) { + assertEquivalent(name, parseHumanBoolean(environment, name, false), expected); + } + } + const hostDashboardPort = presentEnvironmentValue(environment, "NEMOCLAW_DASHBOARD_PORT"); + if (hostDashboardPort !== null) { + assertEquivalent("NEMOCLAW_DASHBOARD_PORT", Number(hostDashboardPort), dashboard.publicPort); + } + const presets = parseEnvironmentJson(environment, "NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64"); + if (presets !== undefined) { + if (!Array.isArray(presets)) { + fail("NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64 must encode an array"); + } + assertEquivalent( + "NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64", + [...presets].sort(), + profile.tools.enabledGateways, + ); + } + } else { + const config = profile.agentConfig; + const approval = presentEnvironmentValue(environment, "NEMOCLAW_DCODE_AUTO_APPROVAL"); + if (approval !== null) { + assertEquivalent("NEMOCLAW_DCODE_AUTO_APPROVAL", approval, config.autoApprovalMode); + } + const observability = presentEnvironmentValue(environment, "NEMOCLAW_OBSERVABILITY"); + if (observability !== null) { + assertEquivalent( + "NEMOCLAW_OBSERVABILITY", + parseZeroOneFlag(environment, "NEMOCLAW_OBSERVABILITY", false), + config.observabilityEnabled, + ); + } + } + + const messaging = parseEnvironmentJson(environment, "NEMOCLAW_MESSAGING_PLAN_B64"); + if (messaging !== undefined) { + assertEquivalent( + "NEMOCLAW_MESSAGING_PLAN_B64", + normalizeMessagingPlan(profile.agent, messaging), + profile.messaging.plan, + ); + } + if (presentEnvironmentValue(environment, "NEMOCLAW_CORPORATE_CA_B64") !== null) { + fail("NEMOCLAW_CORPORATE_CA_B64 must use the separate corporateCa input"); + } +} + +function buildCandidate(input: ManagedStartupProfileBuilderInput): { + readonly profile: ManagedStartupProfile; + readonly corporateCaB64?: string; +} { + assertManagedStartupProfileBuilderInventoryCoverage(); + assertAgentSpecificInput(input); + assertNoWrongAgentEnvironment(input.agent, input.environment); + + const inference = input.inference; + const hostProxy = resolveHostProxy(input.agent, input.environment); + const managedHost = + presentEnvironmentValue(input.environment, "NEMOCLAW_PROXY_HOST") ?? DEFAULT_MANAGED_PROXY_HOST; + const managedPort = parsePort( + input.environment, + "NEMOCLAW_PROXY_PORT", + DEFAULT_MANAGED_PROXY_PORT, + ); + const messagingPlan = normalizeMessagingPlan(input.agent, input.messagingPlan); + const corporateCa = resolveCorporateCaMaterial(input.corporateCa); + const webSearch = normalizeWebSearch(input.agent, input.webSearch); + + let agentConfig: ManagedStartupProfile["agentConfig"]; + let tuning: ManagedStartupProfile["tuning"]; + if (input.agent === "openclaw") { + if (!webSearch) fail("OpenClaw web-search state is missing"); + const otelSampleRaw = + presentEnvironmentValue(input.environment, "NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE") ?? "1.0"; + const otelSampleRate = Number(otelSampleRaw); + if (!Number.isFinite(otelSampleRate) || otelSampleRate < 0 || otelSampleRate > 1) { + fail("NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE must be between 0 and 1"); + } + const otelEndpoint = + presentEnvironmentValue(input.environment, "NEMOCLAW_OPENCLAW_OTEL_ENDPOINT") ?? + DEFAULT_OPENCLAW_OTEL_ENDPOINT; + const otelServiceName = + presentEnvironmentValue(input.environment, "NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME") ?? + DEFAULT_OPENCLAW_OTEL_SERVICE_NAME; + agentConfig = { + agent: "openclaw", + webSearch, + otel: { + enabled: parseOpenClawOtelEnabled(input.environment), + endpointUrl: otelEndpoint, + serviceName: otelServiceName, + sampleRate: otelSampleRate, + }, + agentTimeoutSeconds: + parsePositiveInteger( + input.environment, + "NEMOCLAW_AGENT_TIMEOUT", + DEFAULT_OPENCLAW_AGENT_TIMEOUT_SECONDS, + ) ?? DEFAULT_OPENCLAW_AGENT_TIMEOUT_SECONDS, + heartbeatEvery: parseHeartbeat(input.environment), + extraAgents: parseExtraAgents(input.environment), + // Managed onboarding currently applies this compatibility opt-out to + // every stock OpenClaw image, independently of dashboard exposure. + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + minimalBootstrap: parseZeroOneFlag(input.environment, "NEMOCLAW_MINIMAL_BOOTSTRAP", false), + }; + tuning = { + contextWindow: + parsePositiveInteger(input.environment, "NEMOCLAW_CONTEXT_WINDOW", DEFAULT_CONTEXT_WINDOW, { + maximum: MAX_AUTODETECTED_OLLAMA_CONTEXT_WINDOW, + }) ?? DEFAULT_CONTEXT_WINDOW, + maxTokens: + parsePositiveInteger( + input.environment, + "NEMOCLAW_MAX_TOKENS", + DEFAULT_OPENCLAW_MAX_TOKENS, + ) ?? DEFAULT_OPENCLAW_MAX_TOKENS, + reasoning: parseReasoning(input.environment), + reasoningEffort: parseReasoningEffort(input.environment), + }; + } else if (input.agent === "hermes") { + if (!webSearch) fail("Hermes web-search state is missing"); + agentConfig = { agent: "hermes", webSearch }; + tuning = { + contextWindow: parsePositiveInteger(input.environment, "NEMOCLAW_CONTEXT_WINDOW", null, { + minimum: MIN_HERMES_CONTEXT_WINDOW, + maximum: MAX_AUTODETECTED_OLLAMA_CONTEXT_WINDOW, + }), + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }; + } else { + if (input.dcodeAutoApprovalMode === null || input.observabilityEnabled === null) { + fail("DCode approval and observability state must be explicit"); + } + agentConfig = { + agent: "langchain-deepagents-code", + autoApprovalMode: input.dcodeAutoApprovalMode, + observabilityEnabled: input.observabilityEnabled, + }; + tuning = { + contextWindow: null, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }; + } + + const candidate: ManagedStartupProfile = { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent: input.agent, + agentConfig, + inference: { + routeProvider: inference.routeProvider, + upstreamProvider: inference.upstreamProvider, + model: inference.model, + routedBaseUrl: inference.routedBaseUrl, + upstreamEndpointUrl: inference.upstreamEndpointUrl, + api: inference.api, + primaryModelRef: inference.primaryModelRef, + // Docker's legacy JSON encoder maps a null compatibility result to {}, + // and the OpenClaw generator consumes an object in all cases. + compatibility: + input.agent === "openclaw" + ? (JSON.parse(JSON.stringify(inference.compatibility ?? {})) as ManagedStartupJsonObject) + : null, + inputModalities: input.agent === "openclaw" ? parseInputModalities(input.environment) : null, + }, + proxy: { + managedHost, + managedPort, + ...hostProxy, + }, + dashboard: input.dashboard, + tools: { + disclosure: input.toolDisclosure, + enabledGateways: + input.agent === "hermes" + ? (input.hermesToolGateways as readonly ManagedStartupHermesToolGateway[]) + : [], + }, + messaging: { plan: messagingPlan }, + tuning, + corporateCa: { bundleSha256: corporateCa.bundleSha256 }, + }; + + const profile = validateManagedStartupProfile(candidate); + assertInventoryPathsResolved(profile); + assertEnvironmentConsistency(profile, input.environment); + return corporateCa.corporateCaB64 === undefined + ? { profile } + : { profile, corporateCaB64: corporateCa.corporateCaB64 }; +} + +/** Build a driver-neutral startup profile for later runtime-provider consumption. */ +export function buildManagedStartupProfile( + input: ManagedStartupProfileBuilderInput, +): BuiltManagedStartupProfile { + try { + const built = buildCandidate(input); + // buildCandidate performs the initial messaging parse, hydration, second + // parse, profile validation, and credential screening before this brand is + // applied. Consumption remains a separate trust boundary. + const encodedProfile = encodeManagedStartupProfile( + built.profile, + ) as ValidatedManagedStartupProfileTransport; + const startupProfileSha256 = createHash("sha256").update(encodedProfile, "utf8").digest("hex"); + return Object.freeze( + built.corporateCaB64 === undefined + ? { profile: built.profile, encodedProfile, startupProfileSha256 } + : { + profile: built.profile, + encodedProfile, + startupProfileSha256, + corporateCaB64: built.corporateCaB64, + }, + ); + } catch (error) { + if (error instanceof ManagedStartupProfileBuilderError) throw error; + const message = error instanceof Error ? error.message : "unknown validation failure"; + throw new ManagedStartupProfileBuilderError(message); + } +} From 25d32e0a554480745c1faccac4f77415053e5662 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 18:13:28 -0700 Subject: [PATCH 006/117] feat(onboard): map and coordinate startup profiles Signed-off-by: Aaron Erickson --- ...nerate-managed-startup-profile-fixture.mts | 229 +++++ .../managed-startup-agent-environment.test.ts | 780 +++++++++++++++++ .../managed-startup-application.test.ts | 408 +++++++++ .../managed-startup-coordinator.test.ts | 254 ++++++ .../managed-startup/agent-environment.ts | 468 ++++++++++ .../onboard/managed-startup/application.ts | 826 ++++++++++++++++++ .../onboard/managed-startup/coordinator.ts | 154 ++++ 7 files changed, 3119 insertions(+) create mode 100755 scripts/checks/generate-managed-startup-profile-fixture.mts create mode 100644 src/lib/onboard/managed-startup-agent-environment.test.ts create mode 100644 src/lib/onboard/managed-startup-application.test.ts create mode 100644 src/lib/onboard/managed-startup-coordinator.test.ts create mode 100644 src/lib/onboard/managed-startup/agent-environment.ts create mode 100644 src/lib/onboard/managed-startup/application.ts create mode 100644 src/lib/onboard/managed-startup/coordinator.ts diff --git a/scripts/checks/generate-managed-startup-profile-fixture.mts b/scripts/checks/generate-managed-startup-profile-fixture.mts new file mode 100755 index 00000000000..72499b1cfbd --- /dev/null +++ b/scripts/checks/generate-managed-startup-profile-fixture.mts @@ -0,0 +1,229 @@ +#!/usr/bin/env -S node --experimental-strip-types + +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { + encodeManagedStartupProfile, + MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + type ManagedStartupAgent, + type ManagedStartupProfile, +} from "../../src/lib/onboard/managed-startup/profile.ts"; + +const AGENTS = new Set(["openclaw", "hermes", "langchain-deepagents-code"]); + +export const MANAGED_STARTUP_E2E_HTTP_PROXY = "http://fixture-http-proxy.example.test:18080"; +export const MANAGED_STARTUP_E2E_HTTPS_PROXY = "http://fixture-https-proxy.example.test:18443"; +export const MANAGED_STARTUP_E2E_NO_PROXY = ["localhost", "127.0.0.1", ".example.test"] as const; + +// Real self-signed X.509 CA used by the no-network managed-image lifecycle +// gate. DCode additionally proves its hardened fetch transport selects the +// root-owned merged bundle containing these exact bytes. +export const MANAGED_STARTUP_E2E_CORPORATE_CA_PEM = `-----BEGIN CERTIFICATE----- +MIIDKzCCAhOgAwIBAgIUL3YNpyohvjOEzlwisLKfyiU3dRwwDQYJKoZIhvcNAQEL +BQAwJTEjMCEGA1UEAwwaTmVtb0NsYXcgVGVzdCBDb3Jwb3JhdGUgQ0EwHhcNMjYw +NzA2MDQwMjM2WhcNMzYwNzAzMDQwMjM2WjAlMSMwIQYDVQQDDBpOZW1vQ2xhdyBU +ZXN0IENvcnBvcmF0ZSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALVbV5tyMc65jEH39ejvQvBk7dvI8rz8rSZl+5BWSK2a4TzKm3jD3U+qCDZPicrA +ETCDcO09bN6YIAgpB6rYg5BIURJWxFuljBIBMCZEdO6AVlbURPaGsw6RKLA3cmhx +ZekT0qMcoOKm3N+Hb5MHXsWZ8EUf0co2LsWwJgDZrdwY26gF6w+9wr3iGLE92ZbO +LHhjHUYR1oWXmkXS3YW8MN2h5I+oyL71jBiwLHUi59wogxA/LTAD97/GqwJ6DC4C +UERbIpGYhZfrbiKmT+ASJuKRXaUp/0My3IzH90RqqY70d1E/pkAsd5M8SQ332qAZ +OgW4GgO3n7gAlaN/ILwunZ8CAwEAAaNTMFEwHQYDVR0OBBYEFMa5M8bvDm85eFQi +1D5fNATE/rawMB8GA1UdIwQYMBaAFMa5M8bvDm85eFQi1D5fNATE/rawMA8GA1Ud +EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAB8NR/0HBUH1WbbDOmGNDzge +o+4Pz0KWR5fPDSx9CrmvUk8ijKpJQcSjQcmrXuhCoRs6aExXLh+wImKkOyMIVXfd +YFWjCffSJzeBQfDlMVW+wiAjUh7xaIqpA6Z8EmpdfyoNWd30AuHjs9m8dAa8M/lP +0qhzCbjDiHNHfYSrAuBHlMJ5RsUrNVtSZGpg1dtaSBa+8XFWWNBeJrUANxb8i7Ax +MAhrfNQcxSkZH2lVY+TA2JO83v12nKXzaW1dC94SlsFf0tVSvM3QTeWVgijpr0q+ +J0N7VBg2CdK6jRjKLQOSOPq3ySCicHhVRI8hxIWotif7mK3jj6D8NRalwmlHgNM= +-----END CERTIFICATE----- +`; + +export function managedStartupE2eProfile( + agent: ManagedStartupAgent, + changed = false, + withCorporateCa = false, + withoutHostProxy = false, +): ManagedStartupProfile { + const model = changed ? "nvidia/nemotron-3-super-120b-a12b" : "nvidia/nemotron-3-ultra-550b-a55b"; + const common = { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + inference: { + routeProvider: "inference", + upstreamProvider: "nvidia", + model, + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions" as const, + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: withoutHostProxy ? null : MANAGED_STARTUP_E2E_HTTP_PROXY, + hostHttpsUrl: withoutHostProxy ? null : MANAGED_STARTUP_E2E_HTTPS_PROXY, + hostNoProxy: withoutHostProxy ? [] : MANAGED_STARTUP_E2E_NO_PROXY, + }, + tools: { + disclosure: "progressive" as const, + enabledGateways: [], + }, + messaging: { plan: null }, + corporateCa: { + bundleSha256: withCorporateCa + ? createHash("sha256").update(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM).digest("hex") + : null, + }, + }; + + switch (agent) { + case "openclaw": + return { + ...common, + agent, + agentConfig: { + agent, + webSearch: { enabled: false, provider: "brave" }, + otel: { + enabled: false, + endpointUrl: "http://host.openshell.internal:4318", + serviceName: "openclaw-gateway", + sampleRate: 1, + }, + agentTimeoutSeconds: 600, + heartbeatEvery: null, + extraAgents: { agents: [], defaults: {}, main: {} }, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + minimalBootstrap: true, + }, + inference: { + ...common.inference, + primaryModelRef: `inference/${model}`, + compatibility: {}, + inputModalities: ["text"], + }, + dashboard: { + agent, + mode: "loopback", + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }, + tuning: { + contextWindow: 131_072, + maxTokens: 8192, + reasoning: false, + reasoningEffort: "default", + }, + }; + case "hermes": + return { + ...common, + agent, + agentConfig: { + agent, + webSearch: { enabled: false, provider: "tavily" }, + }, + inference: { + ...common.inference, + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + dashboard: { + agent, + mode: "disabled", + url: "http://127.0.0.1:18789", + publicPort: null, + internalPort: null, + tuiEnabled: false, + }, + tuning: { + contextWindow: 131_072, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + }; + case "langchain-deepagents-code": + return { + ...common, + agent, + agentConfig: { + agent, + autoApprovalMode: "disabled", + observabilityEnabled: false, + }, + inference: { + ...common.inference, + upstreamEndpointUrl: "https://integrate.api.nvidia.com/v1", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + dashboard: { + agent, + mode: "disabled", + }, + tuning: { + contextWindow: null, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + }; + } +} + +function readAgent(value: string | undefined): ManagedStartupAgent { + if (value && AGENTS.has(value as ManagedStartupAgent)) { + return value as ManagedStartupAgent; + } + throw new Error("--agent must identify a shipped managed-image agent"); +} + +function main(argv: readonly string[]): void { + if (argv.length === 1 && argv[0] === "--corporate-ca-b64") { + process.stdout.write( + `${Buffer.from(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, "utf8").toString("base64")}\n`, + ); + return; + } + const agentIndex = argv.indexOf("--agent"); + if (agentIndex < 0) throw new Error("--agent is required"); + const unexpected = argv.filter( + (value, index) => + index !== agentIndex && + index !== agentIndex + 1 && + value !== "--changed" && + value !== "--corporate-ca" && + value !== "--without-host-proxy", + ); + if (unexpected.length > 0) { + throw new Error(`unsupported arguments: ${unexpected.join(" ")}`); + } + const agent = readAgent(argv[agentIndex + 1]); + process.stdout.write( + `${encodeManagedStartupProfile( + managedStartupE2eProfile( + agent, + argv.includes("--changed"), + argv.includes("--corporate-ca"), + argv.includes("--without-host-proxy"), + ), + )}\n`, + ); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { + main(process.argv.slice(2)); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/src/lib/onboard/managed-startup-agent-environment.test.ts b/src/lib/onboard/managed-startup-agent-environment.test.ts new file mode 100644 index 00000000000..3695c179ef3 --- /dev/null +++ b/src/lib/onboard/managed-startup-agent-environment.test.ts @@ -0,0 +1,780 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { Buffer } from "node:buffer"; + +import { describe, expect, it } from "vitest"; + +import { readHermesBuildSettings } from "../../../agents/hermes/config/build-env"; +import { buildConfig as buildOpenClawConfig } from "../../../scripts/generate-openclaw-config.mts"; +import { + type ManagedStartupAgentEnvironment, + mapManagedStartupProfileToAgentEnvironment, +} from "./managed-startup/agent-environment"; +import { + MANAGED_STARTUP_AGENTS, + MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY, + MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + type ManagedStartupAgent, + type ManagedStartupJsonObject, + type ManagedStartupProfile, +} from "./managed-startup/profile"; + +const CA_SHA256 = "a".repeat(64); + +function messagingPlan(agent: "openclaw" | "hermes"): ManagedStartupJsonObject { + return { + schemaVersion: 1, + sandboxName: `${agent}-sandbox`, + agent, + workflow: "onboard", + channels: [], + disabledChannels: [], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + runtimeSetup: { + nodePreloads: [], + envAliases: [], + secretScans: [], + }, + stateUpdates: [], + healthChecks: [], + }; +} + +function openClawProfile(): ManagedStartupProfile { + return { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent: "openclaw", + agentConfig: { + agent: "openclaw", + webSearch: { enabled: true, provider: "brave" }, + otel: { + enabled: true, + endpointUrl: "http://host.openshell.internal:4318", + serviceName: "openclaw-gateway", + sampleRate: 0.5, + }, + agentTimeoutSeconds: 900, + heartbeatEvery: "30m", + extraAgents: { + agents: [ + { + id: "reviewer", + workspace: "/sandbox/.openclaw/workspace-reviewer", + agentDir: "/sandbox/.openclaw/agents/reviewer", + tools: { profile: "minimal", allow: ["read"], deny: ["exec"] }, + }, + ], + defaults: { subagents: { maxSpawnDepth: 3 } }, + main: { tools: { profile: "minimal", allow: ["read"], deny: ["exec"] } }, + }, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + minimalBootstrap: true, + }, + inference: { + routeProvider: "inference", + upstreamProvider: "nvidia-prod", + model: "nvidia/nemotron-3-ultra-550b-a55b", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses", + primaryModelRef: "inference/nvidia/nemotron-3-ultra-550b-a55b", + compatibility: { maxRetries: 2, supportsDeveloperRole: true }, + inputModalities: ["text", "image"], + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: "http://proxy.example.test:8080", + hostHttpsUrl: "https://connect-proxy.example.test:8443", + hostNoProxy: ["localhost", "inference.local", "127.0.0.1"], + }, + dashboard: { + agent: "openclaw", + mode: "remote", + url: "https://dashboard.example.test:18789", + port: 18_789, + bindAddress: "0.0.0.0", + wslExposure: true, + }, + tools: { + disclosure: "progressive", + enabledGateways: [], + }, + messaging: { plan: messagingPlan("openclaw") }, + tuning: { + contextWindow: 131_072, + maxTokens: 8192, + reasoning: true, + reasoningEffort: "high", + }, + corporateCa: { bundleSha256: CA_SHA256 }, + }; +} + +function hermesProfile(): ManagedStartupProfile { + return { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent: "hermes", + agentConfig: { + agent: "hermes", + webSearch: { enabled: true, provider: "tavily" }, + }, + inference: { + routeProvider: "custom", + upstreamProvider: "anthropic-prod", + model: "claude-sonnet-4-5", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "anthropic-messages", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + proxy: { + managedHost: "proxy_name", + managedPort: 43128, + hostHttpUrl: "http://proxy.example.test:8080", + hostHttpsUrl: "http://proxy.example.test:3128", + hostNoProxy: ["localhost", "127.0.0.1"], + }, + dashboard: { + agent: "hermes", + mode: "loopback-forwarded", + url: "http://127.0.0.1:19189", + publicPort: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }, + tools: { + disclosure: "direct", + enabledGateways: ["nous-web", "nous-image", "nous-audio", "nous-browser", "nous-code"], + }, + messaging: { plan: messagingPlan("hermes") }, + tuning: { + contextWindow: 65_536, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + corporateCa: { bundleSha256: CA_SHA256 }, + }; +} + +function dcodeProfile(): ManagedStartupProfile { + return { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent: "langchain-deepagents-code", + agentConfig: { + agent: "langchain-deepagents-code", + autoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + }, + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + model: "openai/gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }, + dashboard: { + agent: "langchain-deepagents-code", + mode: "disabled", + }, + tools: { + disclosure: "progressive", + enabledGateways: [], + }, + messaging: { plan: null }, + tuning: { + contextWindow: null, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + corporateCa: { bundleSha256: CA_SHA256 }, + }; +} + +function decodeBase64Json(encoded: string): unknown { + return JSON.parse(Buffer.from(encoded, "base64").toString("utf8")) as unknown; +} + +function representedLegacyInputs(result: ManagedStartupAgentEnvironment): string[] { + return [ + ...new Set([ + ...Object.keys(result.configurationEnvironment), + ...Object.keys(result.runtimeEnvironment), + ...result.materials.map((material) => material.legacyInput), + ]), + ].sort(); +} + +const PROFILES: Readonly ManagedStartupProfile>> = { + openclaw: openClawProfile, + hermes: hermesProfile, + "langchain-deepagents-code": dcodeProfile, +}; + +describe("managed startup agent environment", () => { + it("maps every OpenClaw profile field to the existing generator and entrypoint contracts", () => { + const result = mapManagedStartupProfileToAgentEnvironment(openClawProfile()); + + expect(result.schemaVersion).toBe(1); + expect(result.agent).toBe("openclaw"); + expect(result.configurationEnvironment).toEqual({ + CHAT_UI_URL: "https://dashboard.example.test:18789", + NEMOCLAW_AGENT_HEARTBEAT_EVERY: "30m", + NEMOCLAW_AGENT_TIMEOUT: "900", + NEMOCLAW_CONTEXT_WINDOW: "131072", + NEMOCLAW_DASHBOARD_BIND: "0.0.0.0", + NEMOCLAW_DISABLE_DEVICE_AUTH: "1", + NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE: "managed-onboard", + NEMOCLAW_EXTRA_AGENTS_JSON_B64: expect.any(String), + NEMOCLAW_INFERENCE_API: "openai-responses", + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_INFERENCE_COMPAT_B64: expect.any(String), + NEMOCLAW_INFERENCE_INPUTS: "image,text", + NEMOCLAW_INFERENCE_PROVIDER_ID: "inference", + NEMOCLAW_MAX_TOKENS: "8192", + NEMOCLAW_MESSAGING_PLAN_B64: expect.any(String), + NEMOCLAW_MODEL: "nvidia/nemotron-3-ultra-550b-a55b", + NEMOCLAW_OPENCLAW_OTEL: "1", + NEMOCLAW_OPENCLAW_OTEL_ENDPOINT: "http://host.openshell.internal:4318", + NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE: "0.5", + NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME: "openclaw-gateway", + NEMOCLAW_PRIMARY_MODEL_REF: "inference/nvidia/nemotron-3-ultra-550b-a55b", + NEMOCLAW_PROXY_HOST: "10.200.0.1", + NEMOCLAW_PROXY_PORT: "3128", + NEMOCLAW_REASONING: "true", + NEMOCLAW_REASONING_EFFORT: "high", + NEMOCLAW_TOOL_DISCLOSURE: "progressive", + NEMOCLAW_UPSTREAM_PROVIDER: "nvidia-prod", + NEMOCLAW_WEB_SEARCH_ENABLED: "1", + NEMOCLAW_WEB_SEARCH_PROVIDER: "brave", + NEMOCLAW_WSL_DASHBOARD_EXPOSURE: "1", + }); + const expectedOpenClawRuntime = { ...result.configurationEnvironment }; + delete expectedOpenClawRuntime.NEMOCLAW_MESSAGING_PLAN_B64; + expect(result.runtimeEnvironment).toEqual({ + ...expectedOpenClawRuntime, + HTTP_PROXY: "http://proxy.example.test:8080", + HTTPS_PROXY: "https://connect-proxy.example.test:8443", + NO_PROXY: "127.0.0.1,inference.local,localhost", + NEMOCLAW_DASHBOARD_PORT: "18789", + NEMOCLAW_MINIMAL_BOOTSTRAP: "1", + http_proxy: "http://proxy.example.test:8080", + https_proxy: "https://connect-proxy.example.test:8443", + no_proxy: "127.0.0.1,inference.local,localhost", + }); + expect(Object.hasOwn(result.runtimeEnvironment, "NEMOCLAW_MESSAGING_PLAN_B64")).toBe(false); + + expect( + decodeBase64Json(result.configurationEnvironment.NEMOCLAW_INFERENCE_COMPAT_B64 ?? ""), + ).toEqual({ + maxRetries: 2, + supportsDeveloperRole: true, + }); + expect( + decodeBase64Json(result.configurationEnvironment.NEMOCLAW_EXTRA_AGENTS_JSON_B64 ?? ""), + ).toEqual({ + agents: [ + { + agentDir: "/sandbox/.openclaw/agents/reviewer", + id: "reviewer", + tools: { allow: ["read"], deny: ["exec"], profile: "minimal" }, + workspace: "/sandbox/.openclaw/workspace-reviewer", + }, + ], + defaults: { subagents: { maxSpawnDepth: 3 } }, + main: { tools: { allow: ["read"], deny: ["exec"], profile: "minimal" } }, + }); + const encodedPlan = result.configurationEnvironment.NEMOCLAW_MESSAGING_PLAN_B64 ?? ""; + expect(encodedPlan).toMatch(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/); + expect(decodeBase64Json(encodedPlan)).toMatchObject({ + schemaVersion: 1, + sandboxName: "openclaw-sandbox", + agent: "openclaw", + }); + expect(decodeBase64Json(encodedPlan)).not.toHaveProperty("workflow"); + + expect(result.materials).toEqual([ + { + kind: "corporate-ca-handoff", + legacyInput: "NEMOCLAW_CORPORATE_CA_B64", + expectedSha256: CA_SHA256, + }, + ]); + expect(result.actions).toEqual([ + { + kind: "apply-messaging-plan", + agent: "openclaw", + mode: "apply", + phase: "runtime-setup", + runAs: "root", + }, + { kind: "generate-agent-config", agent: "openclaw", runAs: "sandbox" }, + { + kind: "apply-messaging-plan", + agent: "openclaw", + mode: "apply", + phase: "post-agent-install", + runAs: "sandbox", + }, + { + kind: "configure-dashboard", + dashboard: openClawProfile().dashboard, + }, + ]); + }); + + it("maps every Hermes profile field, including gateway presets and dashboard forwarding", () => { + const result = mapManagedStartupProfileToAgentEnvironment(hermesProfile()); + + expect(result.configurationEnvironment).toEqual({ + CHAT_UI_URL: "http://127.0.0.1:19189", + NEMOCLAW_CONTEXT_WINDOW: "65536", + NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER: "1", + NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64: expect.any(String), + NEMOCLAW_INFERENCE_API: "anthropic-messages", + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_INFERENCE_PROVIDER_ID: "custom", + NEMOCLAW_MESSAGING_PLAN_B64: expect.any(String), + NEMOCLAW_MODEL: "claude-sonnet-4-5", + NEMOCLAW_TOOL_DISCLOSURE: "direct", + NEMOCLAW_UPSTREAM_PROVIDER: "anthropic-prod", + NEMOCLAW_WEB_SEARCH_ENABLED: "1", + NEMOCLAW_WEB_SEARCH_PROVIDER: "tavily", + }); + expect( + decodeBase64Json( + result.configurationEnvironment.NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64 ?? "", + ), + ).toEqual(["nous-audio", "nous-browser", "nous-code", "nous-image", "nous-web"]); + expect(result.runtimeEnvironment).toEqual({ + CHAT_UI_URL: "http://127.0.0.1:19189", + HTTP_PROXY: "http://proxy.example.test:8080", + HTTPS_PROXY: "http://proxy.example.test:3128", + NO_PROXY: "127.0.0.1,localhost", + NEMOCLAW_CONTEXT_WINDOW: "65536", + NEMOCLAW_DASHBOARD_PORT: "19189", + NEMOCLAW_HERMES_DASHBOARD: "1", + NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT: "29189", + NEMOCLAW_HERMES_DASHBOARD_PORT: "19189", + NEMOCLAW_HERMES_DASHBOARD_TUI: "1", + NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER: "1", + NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64: + result.configurationEnvironment.NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64, + NEMOCLAW_INFERENCE_API: "anthropic-messages", + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_INFERENCE_PROVIDER_ID: "custom", + NEMOCLAW_MODEL: "claude-sonnet-4-5", + NEMOCLAW_PROXY_HOST: "proxy_name", + NEMOCLAW_PROXY_PORT: "43128", + NEMOCLAW_TOOL_DISCLOSURE: "direct", + NEMOCLAW_UPSTREAM_PROVIDER: "anthropic-prod", + NEMOCLAW_WEB_SEARCH_ENABLED: "1", + NEMOCLAW_WEB_SEARCH_PROVIDER: "tavily", + http_proxy: "http://proxy.example.test:8080", + https_proxy: "http://proxy.example.test:3128", + no_proxy: "127.0.0.1,localhost", + }); + expect(result.actions).toContainEqual({ + kind: "apply-messaging-plan", + agent: "hermes", + mode: "apply", + phase: "runtime-setup", + runAs: "root", + }); + expect(result.actions).toContainEqual({ + kind: "apply-messaging-plan", + agent: "hermes", + mode: "apply", + phase: "post-agent-install", + runAs: "sandbox", + }); + expect(result.actions).toContainEqual({ + kind: "configure-dashboard", + dashboard: hermesProfile().dashboard, + }); + }); + + it("keeps DCode routing and auto-approval in root-owned files instead of ambient runtime env", () => { + const result = mapManagedStartupProfileToAgentEnvironment(dcodeProfile()); + + expect(result.configurationEnvironment).toEqual({ + HTTP_PROXY: "", + HTTPS_PROXY: "", + NEMOCLAW_INFERENCE_API: "openai-completions", + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_INFERENCE_PROVIDER_ID: "inference", + NEMOCLAW_MODEL: "openai/gpt-5.4", + NEMOCLAW_TOOL_DISCLOSURE: "progressive", + NEMOCLAW_UPSTREAM_ENDPOINT_URL: "https://openrouter.ai/api/v1", + NEMOCLAW_UPSTREAM_PROVIDER: "openrouter", + NO_PROXY: "", + http_proxy: "", + https_proxy: "", + no_proxy: "", + }); + const expectedDcodeRuntime = { ...result.configurationEnvironment }; + delete expectedDcodeRuntime.NEMOCLAW_INFERENCE_BASE_URL; + for (const name of [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + ]) { + delete expectedDcodeRuntime[name]; + } + expect(result.runtimeEnvironment).toEqual({ + ...expectedDcodeRuntime, + NEMOCLAW_OBSERVABILITY: "1", + }); + for (const environment of [result.configurationEnvironment, result.runtimeEnvironment]) { + expect(environment).not.toHaveProperty("NEMOCLAW_DCODE_AUTO_APPROVAL"); + expect(environment).not.toHaveProperty("NEMOCLAW_MESSAGING_PLAN_B64"); + expect(environment).not.toHaveProperty("NEMOCLAW_PROXY_HOST"); + expect(environment).not.toHaveProperty("NEMOCLAW_PROXY_PORT"); + } + expect(result.runtimeEnvironment).not.toHaveProperty("HTTP_PROXY"); + expect(result.runtimeEnvironment).not.toHaveProperty("HTTPS_PROXY"); + expect(result.runtimeEnvironment).not.toHaveProperty("NEMOCLAW_INFERENCE_BASE_URL"); + + expect(result.materials).toEqual([ + { + kind: "corporate-ca-handoff", + legacyInput: "NEMOCLAW_CORPORATE_CA_B64", + expectedSha256: CA_SHA256, + }, + { + kind: "root-owned-file", + legacyInput: "NEMOCLAW_DCODE_AUTO_APPROVAL", + path: "/usr/local/share/nemoclaw/dcode-auto-approval", + contents: "thread-opt-in\n", + owner: "root", + group: "root", + mode: 0o444, + }, + { + kind: "root-owned-file", + legacyInput: "NEMOCLAW_INFERENCE_BASE_URL", + path: "/usr/local/share/nemoclaw/dcode-inference-base-url", + contents: "https://inference.local/v1\n", + owner: "root", + group: "root", + mode: 0o444, + }, + { + kind: "root-owned-file", + legacyInput: "NEMOCLAW_PROXY_HOST", + path: "/usr/local/share/nemoclaw/dcode-proxy-host", + contents: "10.200.0.1\n", + owner: "root", + group: "root", + mode: 0o444, + }, + { + kind: "root-owned-file", + legacyInput: "NEMOCLAW_PROXY_PORT", + path: "/usr/local/share/nemoclaw/dcode-proxy-port", + contents: "3128\n", + owner: "root", + group: "root", + mode: 0o444, + }, + ]); + expect(result.actions).toEqual([ + { + kind: "generate-agent-config", + agent: "langchain-deepagents-code", + runAs: "sandbox", + }, + { + kind: "configure-dashboard", + dashboard: { agent: "langchain-deepagents-code", mode: "disabled" }, + }, + ]); + }); + + it("feeds the existing OpenClaw and Hermes config consumers without translation", () => { + const openclaw = mapManagedStartupProfileToAgentEnvironment(openClawProfile()); + const openclawConfig = buildOpenClawConfig({ + ...openclaw.configurationEnvironment, + ...openclaw.runtimeEnvironment, + }); + expect(openclawConfig).toMatchObject({ + agents: { + defaults: { + heartbeat: { every: "30m" }, + subagents: { maxSpawnDepth: 3 }, + timeoutSeconds: 900, + }, + list: [{ default: true, id: "main" }, { id: "reviewer" }], + }, + models: { + providers: { + inference: { + api: "openai-responses", + baseUrl: "https://inference.local/v1", + }, + }, + }, + }); + + const hermes = mapManagedStartupProfileToAgentEnvironment(hermesProfile()); + const hermesSettings = readHermesBuildSettings({ + ...hermes.configurationEnvironment, + ...hermes.runtimeEnvironment, + }); + expect(hermesSettings).toMatchObject({ + model: "claude-sonnet-4-5", + baseUrl: "https://inference.local/v1", + providerKey: "custom", + upstreamProvider: "anthropic-prod", + inferenceApi: "anthropic-messages", + contextWindow: 65_536, + toolDisclosure: "direct", + webSearchProvider: "tavily", + managedToolGateways: { + brokerEnabled: true, + presets: ["nous-audio", "nous-browser", "nous-code", "nous-image", "nous-web"], + }, + }); + }); + + it.each( + MANAGED_STARTUP_AGENTS, + )("represents the complete $0 Docker/start affordance inventory", (agent) => { + const result = mapManagedStartupProfileToAgentEnvironment(PROFILES[agent]()); + expect(representedLegacyInputs(result)).toEqual( + MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[agent] + .map((affordance) => affordance.input) + .sort(), + ); + const messagingActions = result.actions.filter( + (action) => action.kind === "apply-messaging-plan", + ); + expect(messagingActions.map(({ phase, runAs }) => [phase, runAs])).toEqual( + agent === "langchain-deepagents-code" + ? [] + : [ + ["runtime-setup", "root"], + ["post-agent-install", "sandbox"], + ], + ); + expect(messagingActions.map((action) => String(action.phase))).not.toContain("agent-install"); + }); + + it("uses explicit clear states without erasing launch-only ambient proxy credentials", () => { + const openclawBase = openClawProfile(); + assert(openclawBase.agentConfig.agent === "openclaw", "fixture mismatch"); + const openclaw: ManagedStartupProfile = { + ...openclawBase, + agentConfig: { + ...openclawBase.agentConfig, + heartbeatEvery: null, + minimalBootstrap: false, + }, + proxy: { + ...openclawBase.proxy, + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }, + dashboard: { + agent: "openclaw", + mode: "loopback", + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }, + messaging: { plan: null }, + corporateCa: { bundleSha256: null }, + }; + + const openclawResult = mapManagedStartupProfileToAgentEnvironment(openclaw); + expect(openclawResult.configurationEnvironment.NEMOCLAW_AGENT_HEARTBEAT_EVERY).toBe(""); + expect(openclawResult.configurationEnvironment.NEMOCLAW_DASHBOARD_BIND).toBe(""); + expect(openclawResult.runtimeEnvironment.NEMOCLAW_MINIMAL_BOOTSTRAP).toBe("0"); + for (const name of [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + ]) { + expect(openclawResult.runtimeEnvironment).not.toHaveProperty(name); + } + expect(openclawResult.configurationEnvironment).not.toHaveProperty( + "NEMOCLAW_MESSAGING_PLAN_B64", + ); + expect(openclawResult.actions).toContainEqual({ + kind: "apply-messaging-plan", + agent: "openclaw", + mode: "clear", + phase: "runtime-setup", + runAs: "root", + }); + expect(openclawResult.actions).toContainEqual({ + kind: "apply-messaging-plan", + agent: "openclaw", + mode: "clear", + phase: "post-agent-install", + runAs: "sandbox", + }); + expect(openclawResult.materials[0]).toMatchObject({ expectedSha256: null }); + + const hermes: ManagedStartupProfile = { + ...hermesProfile(), + proxy: { + ...hermesProfile().proxy, + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }, + dashboard: { + agent: "hermes", + mode: "disabled", + url: "http://127.0.0.1:18789", + publicPort: null, + internalPort: null, + tuiEnabled: false, + }, + tuning: { + contextWindow: null, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + }; + const hermesResult = mapManagedStartupProfileToAgentEnvironment(hermes); + expect(hermesResult.configurationEnvironment.NEMOCLAW_CONTEXT_WINDOW).toBe(""); + expect(hermesResult.runtimeEnvironment).toMatchObject({ + NEMOCLAW_DASHBOARD_PORT: "", + NEMOCLAW_HERMES_DASHBOARD: "0", + NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT: "", + NEMOCLAW_HERMES_DASHBOARD_PORT: "", + NEMOCLAW_HERMES_DASHBOARD_TUI: "0", + }); + for (const name of [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + ]) { + expect(hermesResult.runtimeEnvironment).not.toHaveProperty(name); + } + + const dcodeBase = dcodeProfile(); + const dcode: ManagedStartupProfile = { + ...dcodeBase, + inference: { ...dcodeBase.inference, upstreamEndpointUrl: null }, + }; + const dcodeResult = mapManagedStartupProfileToAgentEnvironment(dcode); + expect(dcodeResult.configurationEnvironment.NEMOCLAW_UPSTREAM_ENDPOINT_URL).toBe(""); + for (const name of [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + ]) { + expect(dcodeResult.configurationEnvironment).toHaveProperty(name, ""); + expect(dcodeResult.runtimeEnvironment).not.toHaveProperty(name); + } + }); + + it("is deterministic across profile key order and never emits certificate or credential bytes", () => { + const profile = openClawProfile(); + const cloned = JSON.parse(JSON.stringify(profile)) as ManagedStartupProfile; + const reordered: ManagedStartupProfile = { + ...cloned, + inference: { + api: profile.inference.api, + upstreamEndpointUrl: profile.inference.upstreamEndpointUrl, + compatibility: profile.inference.compatibility, + inputModalities: profile.inference.inputModalities, + routeProvider: profile.inference.routeProvider, + upstreamProvider: profile.inference.upstreamProvider, + primaryModelRef: profile.inference.primaryModelRef, + routedBaseUrl: profile.inference.routedBaseUrl, + model: profile.inference.model, + }, + }; + const first = mapManagedStartupProfileToAgentEnvironment(profile); + const second = mapManagedStartupProfileToAgentEnvironment(reordered); + + expect(JSON.stringify(first)).toBe(JSON.stringify(second)); + const serialized = JSON.stringify(first); + expect(serialized).not.toContain("BEGIN CERTIFICATE"); + expect(serialized).not.toContain("nvapi-"); + expect(serialized).not.toContain("NVIDIA_API_KEY"); + expect(serialized).toContain(CA_SHA256); + }); + + it("revalidates mismatched or unsupported messaging profiles", () => { + const profile: ManagedStartupProfile = { + ...openClawProfile(), + messaging: { plan: messagingPlan("hermes") }, + }; + expect(() => mapManagedStartupProfileToAgentEnvironment(profile)).toThrow( + /messaging.plan must be a version 1 plan for the selected agent/, + ); + + const dcode: ManagedStartupProfile = { + ...dcodeProfile(), + messaging: { plan: messagingPlan("openclaw") }, + }; + expect(() => mapManagedStartupProfileToAgentEnvironment(dcode)).toThrow( + /messaging.plan must be null for langchain-deepagents-code/, + ); + }); + + it("revalidates typed input while keeping DCode host proxy intent outside its pinned runtime", () => { + const dcodeBase = dcodeProfile(); + const profile: ManagedStartupProfile = { + ...dcodeBase, + proxy: { + ...dcodeBase.proxy, + hostHttpUrl: "http://proxy.example.test:8080", + }, + }; + const mappedDcode = mapManagedStartupProfileToAgentEnvironment(profile); + expect(mappedDcode.runtimeEnvironment.HTTP_PROXY).toBeUndefined(); + + const openclawBase = openClawProfile(); + const credentialBearing: ManagedStartupProfile = { + ...openclawBase, + inference: { + ...openclawBase.inference, + routedBaseUrl: "https://user:password@inference.local/v1", + }, + }; + expect(() => mapManagedStartupProfileToAgentEnvironment(credentialBearing)).toThrow( + /credential/, + ); + }); +}); diff --git a/src/lib/onboard/managed-startup-application.test.ts b/src/lib/onboard/managed-startup-application.test.ts new file mode 100644 index 00000000000..31d13e32039 --- /dev/null +++ b/src/lib/onboard/managed-startup-application.test.ts @@ -0,0 +1,408 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { LEAF_PEM, PEM } from "./__test-helpers__/corporate-ca-fixtures"; +import { + commitManagedStartupApplication, + type ManagedStartupApplicationTestRuntime, + prepareManagedStartupApplication, +} from "./managed-startup/application"; +import { + encodeManagedStartupProfile, + MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + type ManagedStartupAgent, + type ManagedStartupAgentConfig, + type ManagedStartupProfile, +} from "./managed-startup/profile"; + +function sha256(bytes: string | Buffer): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function agentConfigFor(agent: ManagedStartupAgent): ManagedStartupAgentConfig { + switch (agent) { + case "openclaw": + return { + agent, + webSearch: { enabled: false, provider: "brave" }, + otel: { + enabled: false, + endpointUrl: "http://host.openshell.internal:4318", + serviceName: "openclaw-gateway", + sampleRate: 1, + }, + agentTimeoutSeconds: 900, + heartbeatEvery: null, + extraAgents: { agents: [], defaults: {}, main: {} }, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + minimalBootstrap: true, + }; + case "hermes": + return { agent, webSearch: { enabled: false, provider: "tavily" } }; + case "langchain-deepagents-code": + return { agent, autoApprovalMode: "thread-opt-in", observabilityEnabled: true }; + } +} + +function profileFor( + agent: ManagedStartupAgent, + corporateCa: string | null = PEM, +): ManagedStartupProfile { + const inference = + agent === "openclaw" + ? { + routeProvider: "inference", + upstreamProvider: "nvidia", + model: "nvidia/nemotron-3-ultra-550b-a55b", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses" as const, + primaryModelRef: "inference/nvidia/nemotron-3-ultra-550b-a55b", + compatibility: null, + inputModalities: ["text"] as const, + } + : { + routeProvider: "inference", + upstreamProvider: agent === "hermes" ? "nvidia" : "openrouter", + model: "nvidia/nemotron-3-ultra-550b-a55b", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: + agent === "langchain-deepagents-code" ? "https://openrouter.ai/api/v1" : null, + api: "openai-completions" as const, + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }; + const dashboard = + agent === "openclaw" + ? { + agent, + mode: "loopback" as const, + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1" as const, + wslExposure: false, + } + : agent === "hermes" + ? { + agent, + mode: "disabled" as const, + url: "http://127.0.0.1:19189", + publicPort: null, + internalPort: null, + tuiEnabled: false as const, + } + : { + agent, + mode: "disabled" as const, + }; + return { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent, + agentConfig: agentConfigFor(agent), + inference, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }, + dashboard, + tools: { + disclosure: "progressive", + enabledGateways: [], + }, + messaging: { plan: null }, + tuning: { + contextWindow: agent === "langchain-deepagents-code" ? null : 65_536, + maxTokens: agent === "openclaw" ? 8192 : null, + reasoning: agent === "openclaw" ? true : null, + reasoningEffort: agent === "openclaw" ? "default" : null, + }, + corporateCa: { + bundleSha256: corporateCa === null ? null : sha256(corporateCa), + }, + }; +} + +describe("managed startup application", () => { + let fixtureRoot: string; + let stateDirectory: string; + let runtime: ManagedStartupApplicationTestRuntime; + + beforeEach(() => { + vi.spyOn(process, "geteuid").mockReturnValue(0); + fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-startup-")); + fs.chmodSync(fixtureRoot, 0o700); + stateDirectory = path.join(fixtureRoot, "state"); + runtime = { + rootUid: process.getuid?.() ?? 0, + rootGid: process.getgid?.() ?? 0, + }; + }); + + afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(fixtureRoot, { force: true, recursive: true }); + }); + + function prepare( + agent: ManagedStartupAgent, + corporateCa: string | null = PEM, + corporateCaB64: string | undefined = corporateCa === null + ? undefined + : Buffer.from(corporateCa, "utf8").toString("base64"), + ) { + return prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(profileFor(agent, corporateCa)), + expectedAgent: agent, + corporateCaB64, + stateDirectory, + }, + runtime, + ); + } + + it("requires effective uid 0 before touching state", () => { + vi.mocked(process.geteuid as () => number).mockReturnValue(1000); + expect(() => + prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(profileFor("openclaw")), + expectedAgent: "openclaw", + corporateCaB64: Buffer.from(PEM).toString("base64"), + stateDirectory, + }, + runtime, + ), + ).toThrow(/effective uid 0/u); + expect(fs.existsSync(stateDirectory)).toBe(false); + }); + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("prepares and commits a root-owned envelope for %s", (agent) => { + const prepared = prepare(agent); + + expect(prepared.status).toBe("prepared"); + expect(prepared.profile.agent).toBe(agent); + expect(fs.existsSync(path.join(stateDirectory, "committed.json"))).toBe(false); + expect(fs.existsSync(path.join(stateDirectory, "pending.json"))).toBe(true); + expect(fs.readFileSync(prepared.profilePath, "utf8")).toBe( + JSON.stringify(JSON.parse(fs.readFileSync(prepared.profilePath, "utf8"))), + ); + expect(fs.readFileSync(prepared.corporateCaPath as string)).toEqual(Buffer.from(PEM)); + + const stateStat = fs.statSync(stateDirectory); + const profileStat = fs.statSync(prepared.profilePath); + expect(stateStat.mode & 0o777).toBe(0o700); + expect(profileStat.mode & 0o777).toBe(0o600); + expect(profileStat.uid).toBe(runtime.rootUid); + expect(profileStat.gid).toBe(runtime.rootGid); + + const committed = commitManagedStartupApplication(prepared, runtime); + expect(committed.status).toBe("committed"); + expect(fs.existsSync(path.join(stateDirectory, "committed.json"))).toBe(true); + expect(fs.existsSync(path.join(stateDirectory, "pending.json"))).toBe(false); + }); + + it("rejects a canonical profile for the wrong image agent", () => { + expect(() => + prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(profileFor("hermes")), + expectedAgent: "openclaw", + corporateCaB64: Buffer.from(PEM).toString("base64"), + stateDirectory, + }, + runtime, + ), + ).toThrow(/targets hermes, expected openclaw/u); + }); + + it("requires the CA transport exactly when the profile records a digest", () => { + expect(() => + prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(profileFor("openclaw")), + expectedAgent: "openclaw", + stateDirectory, + }, + runtime, + ), + ).toThrow(/canonical standard base64/u); + + expect(() => + prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(profileFor("openclaw", null)), + expectedAgent: "openclaw", + corporateCaB64: Buffer.from(PEM).toString("base64"), + stateDirectory, + }, + runtime, + ), + ).toThrow(/must be absent/u); + + const prepared = prepare("openclaw", null); + expect(prepared.corporateCaPath).toBeNull(); + }); + + it.each([ + { + label: "non-canonical standard base64", + pem: PEM, + encoded: `${Buffer.from(PEM).toString("base64")}\n`, + message: /canonical standard base64/u, + }, + { + label: "wrong digest", + pem: PEM, + profilePem: LEAF_PEM, + encoded: Buffer.from(PEM).toString("base64"), + message: /SHA-256 digest/u, + }, + { + label: "invalid X.509", + pem: "-----BEGIN CERTIFICATE-----\nMIIBfake\n-----END CERTIFICATE-----\n", + message: /invalid X\.509/u, + }, + { + label: "non-CA certificate", + pem: LEAF_PEM, + message: /CA:TRUE/u, + }, + { + label: "trailing material", + pem: `${PEM}not-a-certificate`, + message: /trailing non-PEM material/u, + }, + { + label: "too many certificates", + pem: PEM.repeat(25), + message: /1-24 PEM CA certificates/u, + }, + ])("rejects a corporate CA with $label", ({ pem, encoded, message, ...testCase }) => { + const profilePem = + "profilePem" in testCase && typeof testCase.profilePem === "string" + ? testCase.profilePem + : encoded === undefined + ? pem + : PEM; + expect(() => + prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(profileFor("openclaw", profilePem)), + expectedAgent: "openclaw", + corporateCaB64: encoded ?? Buffer.from(pem).toString("base64"), + stateDirectory, + }, + runtime, + ), + ).toThrow(message); + }); + + it("rejects a symlinked state directory", () => { + const redirected = path.join(fixtureRoot, "redirected"); + fs.mkdirSync(redirected, { mode: 0o700 }); + fs.symlinkSync(redirected, stateDirectory); + + expect(() => prepare("openclaw")).toThrow(/real directory/u); + }); + + it("rejects permissive or non-root-owned state components", () => { + fs.mkdirSync(stateDirectory, { mode: 0o700 }); + fs.chmodSync(stateDirectory, 0o755); + expect(() => prepare("openclaw")).toThrow(/mode 0700/u); + + fs.chmodSync(stateDirectory, 0o700); + expect(() => + prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(profileFor("openclaw")), + expectedAgent: "openclaw", + corporateCaB64: Buffer.from(PEM).toString("base64"), + stateDirectory, + }, + { ...runtime, rootUid: runtime.rootUid + 1 }, + ), + ).toThrow(/root:root/u); + }); + + it("rejects hardlinked generation files before commit", () => { + const prepared = prepare("openclaw"); + const outside = path.join(fixtureRoot, "outside-profile"); + fs.writeFileSync(outside, fs.readFileSync(prepared.profilePath), { mode: 0o600 }); + fs.unlinkSync(prepared.profilePath); + fs.linkSync(outside, prepared.profilePath); + + expect(() => commitManagedStartupApplication(prepared, runtime)).toThrow(/hardlinked/u); + }); + + it("is idempotent for one committed fingerprint and rejects profile changes", () => { + const first = prepare("openclaw"); + commitManagedStartupApplication(first, runtime); + + const repeated = prepare("openclaw"); + expect(repeated.status).toBe("already-committed"); + expect(() => commitManagedStartupApplication(repeated, runtime)).not.toThrow(); + + const changed = { + ...profileFor("openclaw"), + inference: { + ...profileFor("openclaw").inference, + model: "nvidia/a-different-model", + primaryModelRef: "inference/nvidia/a-different-model", + }, + }; + expect(() => + prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(changed), + expectedAgent: "openclaw", + corporateCaB64: Buffer.from(PEM).toString("base64"), + stateDirectory, + }, + runtime, + ), + ).toThrow(/recreate the sandbox/u); + }); + + it("recovers a crash before commit without accepting the profile as applied", () => { + const first = prepare("hermes"); + const abandoned = path.join(stateDirectory, `.prepare-999-${"a".repeat(24)}`); + fs.mkdirSync(abandoned, { mode: 0o700 }); + fs.writeFileSync(path.join(abandoned, "profile.json"), "partial", { mode: 0o600 }); + + const recovered = prepare("hermes"); + expect(recovered.status).toBe("prepared"); + expect(recovered.fingerprint).toBe(first.fingerprint); + expect(fs.existsSync(abandoned)).toBe(false); + expect(fs.existsSync(path.join(stateDirectory, "committed.json"))).toBe(false); + + commitManagedStartupApplication(recovered, runtime); + expect(fs.existsSync(path.join(stateDirectory, "committed.json"))).toBe(true); + }); + + it("never accepts a partial committed generation", () => { + const prepared = prepare("langchain-deepagents-code"); + commitManagedStartupApplication(prepared, runtime); + fs.truncateSync(prepared.profilePath, 10); + + expect(() => prepare("langchain-deepagents-code")).toThrow( + /not valid JSON|canonical managed startup profile/u, + ); + }); +}); diff --git a/src/lib/onboard/managed-startup-coordinator.test.ts b/src/lib/onboard/managed-startup-coordinator.test.ts new file mode 100644 index 00000000000..dc99e569ca1 --- /dev/null +++ b/src/lib/onboard/managed-startup-coordinator.test.ts @@ -0,0 +1,254 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + type CommittedManagedStartupApplication, + type PreparedManagedStartupApplication, + type PrepareManagedStartupApplicationInput, +} from "./managed-startup/application"; +import { + coordinateManagedStartupApplication, + type ManagedStartupAgentAdapter, + type ManagedStartupCoordinatorDependencies, +} from "./managed-startup/coordinator"; +import { type ManagedStartupAgent, type ManagedStartupProfile } from "./managed-startup/profile"; + +function inputFor(agent: ManagedStartupAgent): PrepareManagedStartupApplicationInput { + return { + encodedProfile: `encoded-${agent}`, + expectedAgent: agent, + }; +} + +function preparedFor( + agent: ManagedStartupAgent, + status: PreparedManagedStartupApplication["status"] = "prepared", +): PreparedManagedStartupApplication { + return { + status, + stateDirectory: "/var/lib/nemoclaw/startup-profile", + generationDirectory: `/var/lib/nemoclaw/startup-profile/generation-${"a".repeat(64)}`, + profilePath: `/var/lib/nemoclaw/startup-profile/generation-${"a".repeat(64)}/profile.json`, + corporateCaPath: null, + fingerprint: "a".repeat(64), + expectedAgent: agent, + profile: { agent } as ManagedStartupProfile, + }; +} + +function committedFrom( + prepared: PreparedManagedStartupApplication, +): CommittedManagedStartupApplication { + const { status: _status, ...application } = prepared; + return { ...application, status: "committed" }; +} + +function dependenciesFor( + prepared: PreparedManagedStartupApplication, + order: string[] = [], +): ManagedStartupCoordinatorDependencies & { + prepareApplication: ReturnType; + commitApplication: ReturnType; +} { + return { + prepareApplication: vi.fn(async () => { + order.push("prepare"); + return prepared; + }), + commitApplication: vi.fn(async (application: PreparedManagedStartupApplication) => { + order.push("commit"); + return committedFrom(application); + }), + }; +} + +function adaptersFor(order: string[] = []): { + readonly adapters: ManagedStartupAgentAdapter[]; + readonly applyByAgent: Record>; +} { + const applyByAgent = { + openclaw: vi.fn(async () => { + order.push("apply:openclaw"); + }), + hermes: vi.fn(async () => { + order.push("apply:hermes"); + }), + "langchain-deepagents-code": vi.fn(async () => { + order.push("apply:langchain-deepagents-code"); + }), + }; + return { + adapters: [ + { agent: "openclaw", apply: applyByAgent.openclaw }, + { agent: "hermes", apply: applyByAgent.hermes }, + { + agent: "langchain-deepagents-code", + apply: applyByAgent["langchain-deepagents-code"], + }, + ], + applyByAgent, + }; +} + +describe("managed startup coordinator", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("dispatches exactly the %s adapter before commit", async (agent) => { + const order: string[] = []; + const prepared = preparedFor(agent); + const dependencies = dependenciesFor(prepared, order); + const { adapters, applyByAgent } = adaptersFor(order); + + const result = await coordinateManagedStartupApplication( + inputFor(agent), + adapters, + dependencies, + ); + + expect(result.adapterApplied).toBe(true); + expect(result.application.status).toBe("committed"); + expect(order).toEqual(["prepare", `apply:${agent}`, "commit"]); + expect(applyByAgent[agent]).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + agent, + profile: prepared.profile, + fingerprint: prepared.fingerprint, + }), + ); + for (const otherAgent of ["openclaw", "hermes", "langchain-deepagents-code"] as const) { + expect(applyByAgent[otherAgent]).toHaveBeenCalledTimes(otherAgent === agent ? 1 : 0); + } + expect(dependencies.commitApplication).toHaveBeenCalledWith(prepared); + }); + + it("does not reapply mutable config for an already committed profile", async () => { + const prepared = preparedFor("openclaw", "already-committed"); + const dependencies = dependenciesFor(prepared); + const { adapters, applyByAgent } = adaptersFor(); + + const result = await coordinateManagedStartupApplication( + inputFor("openclaw"), + adapters, + dependencies, + ); + + expect(result.adapterApplied).toBe(false); + expect(dependencies.commitApplication).toHaveBeenCalledExactlyOnceWith(prepared); + for (const apply of Object.values(applyByAgent)) { + expect(apply).not.toHaveBeenCalled(); + } + }); + + it("rejects a missing adapter before preparing state", async () => { + const prepared = preparedFor("openclaw"); + const dependencies = dependenciesFor(prepared); + const { adapters } = adaptersFor(); + + await expect( + coordinateManagedStartupApplication( + inputFor("openclaw"), + adapters.filter((adapter) => adapter.agent !== "hermes"), + dependencies, + ), + ).rejects.toThrow(/missing adapter for hermes/u); + expect(dependencies.prepareApplication).not.toHaveBeenCalled(); + }); + + it("rejects a duplicate adapter before preparing state", async () => { + const prepared = preparedFor("openclaw"); + const dependencies = dependenciesFor(prepared); + const { adapters } = adaptersFor(); + + await expect( + coordinateManagedStartupApplication( + inputFor("openclaw"), + [...adapters, adapters[0] as ManagedStartupAgentAdapter], + dependencies, + ), + ).rejects.toThrow(/duplicate adapter registered for openclaw/u); + expect(dependencies.prepareApplication).not.toHaveBeenCalled(); + }); + + it("rejects an adapter for an unshipped agent before preparing state", async () => { + const prepared = preparedFor("openclaw"); + const dependencies = dependenciesFor(prepared); + const { adapters } = adaptersFor(); + const wrong = { + agent: "not-a-shipped-agent", + apply: vi.fn(), + } as unknown as ManagedStartupAgentAdapter; + + await expect( + coordinateManagedStartupApplication(inputFor("openclaw"), [...adapters, wrong], dependencies), + ).rejects.toThrow(/one shipped agent/u); + expect(dependencies.prepareApplication).not.toHaveBeenCalled(); + }); + + it("fails closed instead of cross-dispatching a mismatched prepared profile", async () => { + const prepared = { + ...preparedFor("openclaw"), + profile: { agent: "hermes" } as ManagedStartupProfile, + }; + const dependencies = dependenciesFor(prepared); + const { adapters, applyByAgent } = adaptersFor(); + + await expect( + coordinateManagedStartupApplication(inputFor("openclaw"), adapters, dependencies), + ).rejects.toThrow(/targets hermes, expected openclaw/u); + expect(dependencies.commitApplication).not.toHaveBeenCalled(); + for (const apply of Object.values(applyByAgent)) { + expect(apply).not.toHaveBeenCalled(); + } + }); + + it("does not commit an adapter failure and can retry the pending profile", async () => { + const prepared = preparedFor("hermes"); + const dependencies = dependenciesFor(prepared); + const { adapters, applyByAgent } = adaptersFor(); + applyByAgent.hermes.mockRejectedValueOnce(new Error("adapter failed")); + + await expect( + coordinateManagedStartupApplication(inputFor("hermes"), adapters, dependencies), + ).rejects.toThrow("adapter failed"); + expect(dependencies.commitApplication).not.toHaveBeenCalled(); + + const retried = await coordinateManagedStartupApplication( + inputFor("hermes"), + adapters, + dependencies, + ); + expect(retried.application.status).toBe("committed"); + expect(applyByAgent.hermes).toHaveBeenCalledTimes(2); + expect(dependencies.commitApplication).toHaveBeenCalledTimes(1); + }); + + it("reapplies a pending adapter after a crash at the commit boundary", async () => { + const prepared = preparedFor("langchain-deepagents-code"); + const dependencies = dependenciesFor(prepared); + const { adapters, applyByAgent } = adaptersFor(); + dependencies.commitApplication.mockRejectedValueOnce( + new Error("simulated process interruption"), + ); + + await expect( + coordinateManagedStartupApplication( + inputFor("langchain-deepagents-code"), + adapters, + dependencies, + ), + ).rejects.toThrow("simulated process interruption"); + + const retried = await coordinateManagedStartupApplication( + inputFor("langchain-deepagents-code"), + adapters, + dependencies, + ); + expect(retried.application.status).toBe("committed"); + expect(applyByAgent["langchain-deepagents-code"]).toHaveBeenCalledTimes(2); + expect(dependencies.commitApplication).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/lib/onboard/managed-startup/agent-environment.ts b/src/lib/onboard/managed-startup/agent-environment.ts new file mode 100644 index 00000000000..a154667b24b --- /dev/null +++ b/src/lib/onboard/managed-startup/agent-environment.ts @@ -0,0 +1,468 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; + +import { parseSandboxMessagingPlan } from "../../messaging/plan-validation"; +import { + type ManagedStartupAgent, + type ManagedStartupDashboard, + type ManagedStartupProfile, + validateManagedStartupProfile, +} from "./profile"; + +export type ManagedStartupConfigAgent = ManagedStartupAgent; +export type ManagedStartupMessagingAgent = "openclaw" | "hermes"; + +export interface ManagedStartupCorporateCaMaterial { + readonly kind: "corporate-ca-handoff"; + readonly legacyInput: "NEMOCLAW_CORPORATE_CA_B64"; + /** + * The certificate bytes use a separate bounded transport. Keeping only its + * digest here prevents the driver-neutral profile mapper from becoming a + * secret or arbitrary-file transport. + */ + readonly expectedSha256: string | null; +} + +export interface ManagedStartupRootOwnedFileMaterial { + readonly kind: "root-owned-file"; + readonly legacyInput: + | "NEMOCLAW_DCODE_AUTO_APPROVAL" + | "NEMOCLAW_INFERENCE_BASE_URL" + | "NEMOCLAW_PROXY_HOST" + | "NEMOCLAW_PROXY_PORT"; + readonly path: + | "/usr/local/share/nemoclaw/dcode-auto-approval" + | "/usr/local/share/nemoclaw/dcode-inference-base-url" + | "/usr/local/share/nemoclaw/dcode-proxy-host" + | "/usr/local/share/nemoclaw/dcode-proxy-port"; + readonly contents: string; + readonly owner: "root"; + readonly group: "root"; + readonly mode: 0o444; +} + +export type ManagedStartupAgentMaterial = + | ManagedStartupCorporateCaMaterial + | ManagedStartupRootOwnedFileMaterial; + +export interface ManagedStartupGenerateConfigAction { + readonly kind: "generate-agent-config"; + readonly agent: ManagedStartupConfigAgent; + readonly runAs: "sandbox"; +} + +interface ManagedStartupApplyMessagingActionBase { + readonly kind: "apply-messaging-plan"; + readonly agent: ManagedStartupMessagingAgent; + readonly mode: "apply" | "clear"; + /** + * Complete managed images already contain the reviewed dependency union. + * The runtime action vocabulary intentionally cannot express the + * package-install phase. + */ + readonly phase: "runtime-setup" | "post-agent-install"; +} + +export interface ManagedStartupApplyMessagingRuntimeAction + extends ManagedStartupApplyMessagingActionBase { + readonly phase: "runtime-setup"; + /** Writes the reduced, root-owned messaging runtime-plan artifact. */ + readonly runAs: "root"; +} + +export interface ManagedStartupApplyMessagingConfigAction + extends ManagedStartupApplyMessagingActionBase { + readonly phase: "post-agent-install"; + /** Renders only sandbox-owned agent configuration from preinstalled assets. */ + readonly runAs: "sandbox"; +} + +export type ManagedStartupApplyMessagingAction = + | ManagedStartupApplyMessagingRuntimeAction + | ManagedStartupApplyMessagingConfigAction; + +export interface ManagedStartupConfigureDashboardAction { + readonly kind: "configure-dashboard"; + readonly dashboard: ManagedStartupDashboard; +} + +export type ManagedStartupAgentAction = + | ManagedStartupGenerateConfigAction + | ManagedStartupApplyMessagingAction + | ManagedStartupConfigureDashboardAction; + +export interface ManagedStartupAgentEnvironment { + readonly schemaVersion: ManagedStartupProfile["schemaVersion"]; + readonly agent: ManagedStartupAgent; + /** + * Inputs scoped to the trusted configuration-application phase. The + * application boundary must not blindly retain this whole map in the agent + * process environment. + */ + readonly configurationEnvironment: Readonly>; + /** + * Non-secret values intentionally retained for existing entrypoints and + * agent runtime adapters after generated configuration is committed. + */ + readonly runtimeEnvironment: Readonly>; + readonly materials: readonly ManagedStartupAgentMaterial[]; + readonly actions: readonly ManagedStartupAgentAction[]; +} + +export class ManagedStartupAgentEnvironmentError extends Error { + constructor(message: string) { + super(`Cannot map managed startup profile: ${message}`); + this.name = "ManagedStartupAgentEnvironmentError"; + } +} + +type MutableEnvironment = Record; + +function booleanFlag(value: boolean): "0" | "1" { + return value ? "1" : "0"; +} + +function canonicalizeJson(value: unknown): unknown { + if (Array.isArray(value)) return value.map((item) => canonicalizeJson(item)); + if (value === null || typeof value !== "object") return value; + const record = value as Record; + return Object.fromEntries( + Object.keys(record) + .sort() + .map((key) => [key, canonicalizeJson(record[key])]), + ); +} + +function encodeCanonicalJson(value: unknown): string { + return Buffer.from(JSON.stringify(canonicalizeJson(value)), "utf8").toString("base64"); +} + +function sortedEnvironment(environment: MutableEnvironment): Readonly> { + return Object.freeze( + Object.fromEntries( + Object.entries(environment).sort(([left], [right]) => + left < right ? -1 : left > right ? 1 : 0, + ), + ), + ); +} + +function commonConfigurationEnvironment(profile: ManagedStartupProfile): MutableEnvironment { + return { + NEMOCLAW_INFERENCE_API: profile.inference.api, + NEMOCLAW_INFERENCE_BASE_URL: profile.inference.routedBaseUrl, + NEMOCLAW_INFERENCE_PROVIDER_ID: profile.inference.routeProvider, + NEMOCLAW_MODEL: profile.inference.model, + NEMOCLAW_TOOL_DISCLOSURE: profile.tools.disclosure, + NEMOCLAW_UPSTREAM_PROVIDER: profile.inference.upstreamProvider, + }; +} + +function appendHostProxyEnvironment( + environment: MutableEnvironment, + profile: ManagedStartupProfile, + options: { readonly preserveAmbientWhenAbsent?: boolean } = {}, +): void { + if ( + options.preserveAmbientWhenAbsent === true && + profile.proxy.hostHttpUrl === null && + profile.proxy.hostHttpsUrl === null && + profile.proxy.hostNoProxy.length === 0 + ) { + return; + } + const httpProxy = profile.proxy.hostHttpUrl ?? ""; + const httpsProxy = profile.proxy.hostHttpsUrl ?? ""; + const noProxy = profile.proxy.hostNoProxy.join(","); + environment.HTTP_PROXY = httpProxy; + environment.HTTPS_PROXY = httpsProxy; + environment.NO_PROXY = noProxy; + environment.http_proxy = httpProxy; + environment.https_proxy = httpsProxy; + environment.no_proxy = noProxy; +} + +function messagingEnvironment( + profile: ManagedStartupProfile, + expectedAgent: ManagedStartupMessagingAgent, +): MutableEnvironment { + if (profile.messaging.plan === null) return {}; + const plan = parseSandboxMessagingPlan(profile.messaging.plan, { agent: expectedAgent }); + if (!plan) { + throw new ManagedStartupAgentEnvironmentError( + `messaging.plan must contain a validated ${expectedAgent} messaging plan`, + ); + } + const { workflow: _workflow, ...imageBuildPlan } = plan; + return { + NEMOCLAW_MESSAGING_PLAN_B64: encodeCanonicalJson(imageBuildPlan), + }; +} + +function corporateCaMaterial(profile: ManagedStartupProfile): ManagedStartupCorporateCaMaterial { + return Object.freeze({ + kind: "corporate-ca-handoff", + legacyInput: "NEMOCLAW_CORPORATE_CA_B64", + expectedSha256: profile.corporateCa.bundleSha256, + }); +} + +function rootOwnedFile( + legacyInput: ManagedStartupRootOwnedFileMaterial["legacyInput"], + path: ManagedStartupRootOwnedFileMaterial["path"], + value: string, +): ManagedStartupRootOwnedFileMaterial { + return Object.freeze({ + kind: "root-owned-file", + legacyInput, + path, + contents: `${value}\n`, + owner: "root", + group: "root", + mode: 0o444, + }); +} + +function dashboardAction( + dashboard: ManagedStartupDashboard, +): ManagedStartupConfigureDashboardAction { + return Object.freeze({ + kind: "configure-dashboard", + dashboard: Object.freeze(structuredClone(dashboard)), + }); +} + +function applicationActions( + profile: ManagedStartupProfile, + messagingAgent: ManagedStartupMessagingAgent | null, +): readonly ManagedStartupAgentAction[] { + const actions: ManagedStartupAgentAction[] = []; + if (messagingAgent !== null) { + actions.push( + Object.freeze({ + kind: "apply-messaging-plan", + agent: messagingAgent, + mode: profile.messaging.plan === null ? "clear" : "apply", + phase: "runtime-setup", + runAs: "root", + }), + ); + } + actions.push( + Object.freeze({ + kind: "generate-agent-config", + agent: profile.agent, + runAs: "sandbox", + }), + ); + if (messagingAgent !== null) { + actions.push( + Object.freeze({ + kind: "apply-messaging-plan", + agent: messagingAgent, + mode: profile.messaging.plan === null ? "clear" : "apply", + phase: "post-agent-install", + runAs: "sandbox", + }), + ); + } + actions.push(dashboardAction(profile.dashboard)); + return Object.freeze(actions); +} + +function mapOpenClawProfile(profile: ManagedStartupProfile): ManagedStartupAgentEnvironment { + if ( + profile.agent !== "openclaw" || + profile.agentConfig.agent !== "openclaw" || + profile.dashboard.agent !== "openclaw" || + profile.inference.primaryModelRef === null || + profile.inference.inputModalities === null || + profile.tuning.contextWindow === null || + profile.tuning.maxTokens === null || + profile.tuning.reasoning === null || + profile.tuning.reasoningEffort === null + ) { + throw new ManagedStartupAgentEnvironmentError("OpenClaw profile state is inconsistent"); + } + + const configurationEnvironment: MutableEnvironment = { + ...commonConfigurationEnvironment(profile), + ...messagingEnvironment(profile, "openclaw"), + CHAT_UI_URL: profile.dashboard.url, + NEMOCLAW_AGENT_HEARTBEAT_EVERY: profile.agentConfig.heartbeatEvery ?? "", + NEMOCLAW_AGENT_TIMEOUT: String(profile.agentConfig.agentTimeoutSeconds), + NEMOCLAW_CONTEXT_WINDOW: String(profile.tuning.contextWindow), + NEMOCLAW_DASHBOARD_BIND: + profile.dashboard.bindAddress === "0.0.0.0" ? profile.dashboard.bindAddress : "", + NEMOCLAW_DISABLE_DEVICE_AUTH: booleanFlag(profile.agentConfig.deviceAuth.disabled), + NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE: profile.agentConfig.deviceAuth.optOutSource, + NEMOCLAW_EXTRA_AGENTS_JSON_B64: encodeCanonicalJson(profile.agentConfig.extraAgents), + NEMOCLAW_INFERENCE_COMPAT_B64: encodeCanonicalJson(profile.inference.compatibility), + NEMOCLAW_INFERENCE_INPUTS: profile.inference.inputModalities.join(","), + NEMOCLAW_MAX_TOKENS: String(profile.tuning.maxTokens), + NEMOCLAW_OPENCLAW_OTEL: booleanFlag(profile.agentConfig.otel.enabled), + NEMOCLAW_OPENCLAW_OTEL_ENDPOINT: profile.agentConfig.otel.endpointUrl, + NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE: String(profile.agentConfig.otel.sampleRate), + NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME: profile.agentConfig.otel.serviceName, + NEMOCLAW_PRIMARY_MODEL_REF: profile.inference.primaryModelRef, + NEMOCLAW_PROXY_HOST: profile.proxy.managedHost, + NEMOCLAW_PROXY_PORT: String(profile.proxy.managedPort), + NEMOCLAW_REASONING: String(profile.tuning.reasoning), + NEMOCLAW_REASONING_EFFORT: profile.tuning.reasoningEffort, + NEMOCLAW_WEB_SEARCH_ENABLED: booleanFlag(profile.agentConfig.webSearch.enabled), + NEMOCLAW_WEB_SEARCH_PROVIDER: profile.agentConfig.webSearch.provider, + NEMOCLAW_WSL_DASHBOARD_EXPOSURE: booleanFlag(profile.dashboard.wslExposure), + }; + + const runtimeEnvironment: MutableEnvironment = { ...configurationEnvironment }; + delete runtimeEnvironment.NEMOCLAW_MESSAGING_PLAN_B64; + runtimeEnvironment.NEMOCLAW_DASHBOARD_PORT = String(profile.dashboard.port); + runtimeEnvironment.NEMOCLAW_MINIMAL_BOOTSTRAP = booleanFlag(profile.agentConfig.minimalBootstrap); + appendHostProxyEnvironment(runtimeEnvironment, profile, { preserveAmbientWhenAbsent: true }); + + return Object.freeze({ + schemaVersion: profile.schemaVersion, + agent: profile.agent, + configurationEnvironment: sortedEnvironment(configurationEnvironment), + runtimeEnvironment: sortedEnvironment(runtimeEnvironment), + materials: Object.freeze([corporateCaMaterial(profile)]), + actions: applicationActions(profile, "openclaw"), + }); +} + +function mapHermesProfile(profile: ManagedStartupProfile): ManagedStartupAgentEnvironment { + if ( + profile.agent !== "hermes" || + profile.agentConfig.agent !== "hermes" || + profile.dashboard.agent !== "hermes" + ) { + throw new ManagedStartupAgentEnvironmentError("Hermes profile state is inconsistent"); + } + + const configurationEnvironment: MutableEnvironment = { + ...commonConfigurationEnvironment(profile), + ...messagingEnvironment(profile, "hermes"), + CHAT_UI_URL: profile.dashboard.url, + NEMOCLAW_CONTEXT_WINDOW: + profile.tuning.contextWindow === null ? "" : String(profile.tuning.contextWindow), + NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER: booleanFlag(profile.tools.enabledGateways.length > 0), + NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64: encodeCanonicalJson(profile.tools.enabledGateways), + NEMOCLAW_WEB_SEARCH_ENABLED: booleanFlag(profile.agentConfig.webSearch.enabled), + NEMOCLAW_WEB_SEARCH_PROVIDER: profile.agentConfig.webSearch.provider, + }; + + const runtimeEnvironment: MutableEnvironment = { ...configurationEnvironment }; + delete runtimeEnvironment.NEMOCLAW_MESSAGING_PLAN_B64; + runtimeEnvironment.NEMOCLAW_DASHBOARD_PORT = + profile.dashboard.publicPort === null ? "" : String(profile.dashboard.publicPort); + runtimeEnvironment.NEMOCLAW_HERMES_DASHBOARD = + profile.dashboard.mode === "loopback-forwarded" ? "1" : "0"; + runtimeEnvironment.NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT = + profile.dashboard.internalPort === null ? "" : String(profile.dashboard.internalPort); + runtimeEnvironment.NEMOCLAW_HERMES_DASHBOARD_PORT = + profile.dashboard.publicPort === null ? "" : String(profile.dashboard.publicPort); + runtimeEnvironment.NEMOCLAW_HERMES_DASHBOARD_TUI = booleanFlag(profile.dashboard.tuiEnabled); + runtimeEnvironment.NEMOCLAW_PROXY_HOST = profile.proxy.managedHost; + runtimeEnvironment.NEMOCLAW_PROXY_PORT = String(profile.proxy.managedPort); + appendHostProxyEnvironment(runtimeEnvironment, profile, { preserveAmbientWhenAbsent: true }); + + return Object.freeze({ + schemaVersion: profile.schemaVersion, + agent: profile.agent, + configurationEnvironment: sortedEnvironment(configurationEnvironment), + runtimeEnvironment: sortedEnvironment(runtimeEnvironment), + materials: Object.freeze([corporateCaMaterial(profile)]), + actions: applicationActions(profile, "hermes"), + }); +} + +function mapDcodeProfile(profile: ManagedStartupProfile): ManagedStartupAgentEnvironment { + if ( + profile.agent !== "langchain-deepagents-code" || + profile.agentConfig.agent !== "langchain-deepagents-code" || + profile.dashboard.agent !== "langchain-deepagents-code" || + profile.messaging.plan !== null + ) { + throw new ManagedStartupAgentEnvironmentError( + "LangChain Deep Agents Code profile state is inconsistent", + ); + } + + const configurationEnvironment: MutableEnvironment = { + ...commonConfigurationEnvironment(profile), + NEMOCLAW_UPSTREAM_ENDPOINT_URL: profile.inference.upstreamEndpointUrl ?? "", + }; + appendHostProxyEnvironment(configurationEnvironment, profile); + const runtimeEnvironment: MutableEnvironment = { + ...configurationEnvironment, + NEMOCLAW_OBSERVABILITY: booleanFlag(profile.agentConfig.observabilityEnabled), + }; + // The config generator needs the routed base URL, but the long-running + // DCode process trusts only the root-owned file consumed by + // managed-dcode-runtime.py. + delete runtimeEnvironment.NEMOCLAW_INFERENCE_BASE_URL; + for (const name of [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + ]) { + delete runtimeEnvironment[name]; + } + const materials: readonly ManagedStartupAgentMaterial[] = Object.freeze([ + corporateCaMaterial(profile), + rootOwnedFile( + "NEMOCLAW_DCODE_AUTO_APPROVAL", + "/usr/local/share/nemoclaw/dcode-auto-approval", + profile.agentConfig.autoApprovalMode, + ), + rootOwnedFile( + "NEMOCLAW_INFERENCE_BASE_URL", + "/usr/local/share/nemoclaw/dcode-inference-base-url", + profile.inference.routedBaseUrl, + ), + rootOwnedFile( + "NEMOCLAW_PROXY_HOST", + "/usr/local/share/nemoclaw/dcode-proxy-host", + profile.proxy.managedHost, + ), + rootOwnedFile( + "NEMOCLAW_PROXY_PORT", + "/usr/local/share/nemoclaw/dcode-proxy-port", + String(profile.proxy.managedPort), + ), + ]); + + return Object.freeze({ + schemaVersion: profile.schemaVersion, + agent: profile.agent, + configurationEnvironment: sortedEnvironment(configurationEnvironment), + runtimeEnvironment: sortedEnvironment(runtimeEnvironment), + materials, + actions: applicationActions(profile, null), + }); +} + +/** + * Convert a secret-free validated profile into existing agent-generator and + * entrypoint inputs without depending on Docker, Podman, or another compute + * driver. Validation is repeated at this trust boundary so callers cannot use + * a TypeScript assertion to bypass agent capability checks. + */ +export function mapManagedStartupProfileToAgentEnvironment( + profile: ManagedStartupProfile, +): ManagedStartupAgentEnvironment { + const validated = validateManagedStartupProfile(profile); + switch (validated.agent) { + case "openclaw": + return mapOpenClawProfile(validated); + case "hermes": + return mapHermesProfile(validated); + case "langchain-deepagents-code": + return mapDcodeProfile(validated); + } +} diff --git a/src/lib/onboard/managed-startup/application.ts b/src/lib/onboard/managed-startup/application.ts new file mode 100644 index 00000000000..07e5d9adf19 --- /dev/null +++ b/src/lib/onboard/managed-startup/application.ts @@ -0,0 +1,826 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash, randomBytes, X509Certificate } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { TextDecoder } from "node:util"; + +import { + decodeManagedStartupProfile, + fingerprintManagedStartupProfile, + MANAGED_STARTUP_PROFILE_MAX_BYTES, + type ManagedStartupAgent, + type ManagedStartupProfile, + serializeManagedStartupProfile, + validateManagedStartupProfile, +} from "./profile"; + +export const MANAGED_STARTUP_APPLICATION_STATE_DIR = "/var/lib/nemoclaw/startup-profile"; +export const MANAGED_STARTUP_CA_MAX_BYTES = 128 * 1024; +export const MANAGED_STARTUP_CA_MAX_CERTIFICATES = 24; + +const STATE_SCHEMA_VERSION = 1 as const; +const STATE_DIRECTORY_MODE = 0o700; +const STATE_FILE_MODE = 0o600; +const MAX_CONTROL_FILE_BYTES = 512; +const MAX_STATE_ENTRIES = 32; +const SHA256_RE = /^[a-f0-9]{64}$/u; +const GENERATION_RE = /^generation-([a-f0-9]{64})$/u; +const PREPARE_TEMP_RE = /^\.prepare-[0-9]+-[a-f0-9]{24}$/u; +const CONTROL_TEMP_RE = /^\.(?:committed|pending)\.json-[a-f0-9]{24}\.tmp$/u; +const PEM_CERTIFICATE_RE = + /-----BEGIN CERTIFICATE-----\r?\n[A-Za-z0-9+/=\r\n]+?-----END CERTIFICATE-----/gu; +const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true }); + +interface ManagedStartupApplicationRuntime { + readonly rootUid: number; + readonly rootGid: number; +} + +const DEFAULT_RUNTIME: ManagedStartupApplicationRuntime = { + rootUid: 0, + rootGid: 0, +}; + +/** + * Explicit filesystem seam for unit tests that cannot create uid-0 files. + * Image entrypoints must omit this argument so uid/gid 0 remain mandatory. + */ +export interface ManagedStartupApplicationTestRuntime { + readonly rootUid: number; + readonly rootGid: number; +} + +export interface PrepareManagedStartupApplicationInput { + readonly encodedProfile: string; + readonly expectedAgent: ManagedStartupAgent; + readonly corporateCaB64?: string; + readonly stateDirectory?: string; +} + +export interface PreparedManagedStartupApplication { + readonly status: "prepared" | "already-committed"; + readonly stateDirectory: string; + readonly generationDirectory: string; + readonly profilePath: string; + readonly corporateCaPath: string | null; + readonly fingerprint: string; + readonly expectedAgent: ManagedStartupAgent; + readonly profile: ManagedStartupProfile; +} + +export interface CommittedManagedStartupApplication + extends Omit { + readonly status: "committed"; +} + +interface StateControl { + readonly schemaVersion: typeof STATE_SCHEMA_VERSION; + readonly fingerprint: string; + readonly generation: string; +} + +interface ValidatedGeneration { + readonly directory: string; + readonly profilePath: string; + readonly corporateCaPath: string | null; + readonly profile: ManagedStartupProfile; + readonly fingerprint: string; +} + +export class ManagedStartupApplicationError extends Error { + constructor(message: string) { + super(`Managed startup application failed: ${message}`); + this.name = "ManagedStartupApplicationError"; + } +} + +function fail(message: string): never { + throw new ManagedStartupApplicationError(message); +} + +function runtimeFor( + override: ManagedStartupApplicationTestRuntime | undefined, +): ManagedStartupApplicationRuntime { + return override ?? DEFAULT_RUNTIME; +} + +function requireContainerRoot(): void { + if (process.geteuid?.() !== 0) { + fail("the image-side applicator must run with effective uid 0"); + } +} + +function modeOf(stat: fs.Stats): number { + return stat.mode & 0o777; +} + +function requireOwner(stat: fs.Stats, target: string, runtime: ManagedStartupApplicationRuntime) { + if (stat.uid !== runtime.rootUid || stat.gid !== runtime.rootGid) { + fail(`${target} must be owned by root:root`); + } +} + +function requireSecureDirectory( + target: string, + runtime: ManagedStartupApplicationRuntime, + exactMode: boolean, +): void { + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch { + fail(`state directory component is missing or unreadable: ${target}`); + } + if (stat.isSymbolicLink() || !stat.isDirectory()) { + fail(`state directory component must be a real directory: ${target}`); + } + requireOwner(stat, target, runtime); + const mode = modeOf(stat); + if ((exactMode && mode !== STATE_DIRECTORY_MODE) || (!exactMode && (mode & 0o022) !== 0)) { + fail( + exactMode + ? `${target} must have mode 0700` + : `${target} must not be group- or world-writable`, + ); + } +} + +function ensureStateDirectory( + rawStateDirectory: string | undefined, + runtime: ManagedStartupApplicationRuntime, +): string { + const stateDirectory = rawStateDirectory ?? MANAGED_STARTUP_APPLICATION_STATE_DIR; + if (!path.isAbsolute(stateDirectory) || stateDirectory.includes("\0")) { + fail("stateDirectory must be an absolute path"); + } + const normalized = path.resolve(stateDirectory); + const parent = path.dirname(normalized); + requireSecureDirectory(parent, runtime, false); + try { + fs.mkdirSync(normalized, { mode: STATE_DIRECTORY_MODE }); + fs.chownSync(normalized, runtime.rootUid, runtime.rootGid); + fs.chmodSync(normalized, STATE_DIRECTORY_MODE); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") { + fail(`could not create the managed startup state directory: ${normalized}`); + } + } + requireSecureDirectory(normalized, runtime, true); + return normalized; +} + +function requireSecureRegularFileStat( + stat: fs.Stats, + target: string, + runtime: ManagedStartupApplicationRuntime, +): void { + if (!stat.isFile() || stat.isSymbolicLink()) { + fail(`${target} must be a regular file`); + } + if (stat.nlink !== 1) { + fail(`${target} must not be hardlinked`); + } + requireOwner(stat, target, runtime); + if (modeOf(stat) !== STATE_FILE_MODE) { + fail(`${target} must have mode 0600`); + } +} + +function readSecureFile( + target: string, + maxBytes: number, + runtime: ManagedStartupApplicationRuntime, +): Buffer { + let descriptor: number; + try { + descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + } catch { + fail(`state file is missing, unreadable, or a symlink: ${target}`); + } + try { + const stat = fs.fstatSync(descriptor); + requireSecureRegularFileStat(stat, target, runtime); + if (stat.size < 1 || stat.size > maxBytes) { + fail(`${target} is empty or exceeds its size limit`); + } + const content = fs.readFileSync(descriptor); + if (content.length !== stat.size) { + fail(`${target} changed while it was being read`); + } + return content; + } finally { + fs.closeSync(descriptor); + } +} + +function writeSecureNewFile( + target: string, + content: string | Buffer, + runtime: ManagedStartupApplicationRuntime, +): void { + let descriptor: number; + try { + descriptor = fs.openSync( + target, + fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW, + STATE_FILE_MODE, + ); + } catch { + fail(`refused to replace an existing state file: ${target}`); + } + try { + fs.fchownSync(descriptor, runtime.rootUid, runtime.rootGid); + fs.fchmodSync(descriptor, STATE_FILE_MODE); + fs.writeFileSync(descriptor, content); + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +function syncDirectory(target: string): void { + const descriptor = fs.openSync(target, fs.constants.O_RDONLY); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +function randomToken(): string { + return randomBytes(12).toString("hex"); +} + +function stateControl(fingerprint: string): StateControl { + return { + schemaVersion: STATE_SCHEMA_VERSION, + fingerprint, + generation: `generation-${fingerprint}`, + }; +} + +function serializeStateControl(control: StateControl): string { + return JSON.stringify({ + fingerprint: control.fingerprint, + generation: control.generation, + schemaVersion: control.schemaVersion, + }); +} + +function parseStateControl( + target: string, + runtime: ManagedStartupApplicationRuntime, +): StateControl { + const bytes = readSecureFile(target, MAX_CONTROL_FILE_BYTES, runtime); + let raw: string; + try { + raw = UTF8_DECODER.decode(bytes); + } catch { + fail(`${target} is not valid UTF-8`); + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + fail(`${target} is not valid JSON`); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + fail(`${target} does not contain a valid state control`); + } + const record = parsed as Record; + if ( + Object.keys(record).sort().join(",") !== "fingerprint,generation,schemaVersion" || + record.schemaVersion !== STATE_SCHEMA_VERSION || + typeof record.fingerprint !== "string" || + !SHA256_RE.test(record.fingerprint) || + record.generation !== `generation-${record.fingerprint}` + ) { + fail(`${target} does not contain a valid state control`); + } + const control = stateControl(record.fingerprint); + if (serializeStateControl(control) !== raw) { + fail(`${target} is not in canonical form`); + } + return control; +} + +function atomicWriteStateControl( + stateDirectory: string, + basename: "committed.json" | "pending.json", + control: StateControl, + runtime: ManagedStartupApplicationRuntime, +): void { + const target = path.join(stateDirectory, basename); + const temporary = path.join(stateDirectory, `.${basename}-${randomToken()}.tmp`); + writeSecureNewFile(temporary, serializeStateControl(control), runtime); + try { + fs.renameSync(temporary, target); + syncDirectory(stateDirectory); + } catch { + try { + fs.unlinkSync(temporary); + } catch { + // Preserve the primary atomic-write error. + } + fail(`could not atomically write ${basename}`); + } +} + +function validateCorporateCaBytes(bytes: Buffer): void { + if (bytes.length < 1 || bytes.length > MANAGED_STARTUP_CA_MAX_BYTES) { + fail(`corporate CA bundle must contain 1-${String(MANAGED_STARTUP_CA_MAX_BYTES)} bytes`); + } + let pem: string; + try { + pem = UTF8_DECODER.decode(bytes); + } catch { + fail("corporate CA bundle must be valid UTF-8 PEM"); + } + + const matches = [...pem.matchAll(PEM_CERTIFICATE_RE)]; + if ( + matches.length < 1 || + matches.length > MANAGED_STARTUP_CA_MAX_CERTIFICATES || + matches[0]?.index !== 0 + ) { + fail( + `corporate CA bundle must contain 1-${String( + MANAGED_STARTUP_CA_MAX_CERTIFICATES, + )} PEM CA certificates`, + ); + } + + let cursor = 0; + for (const match of matches) { + const index = match.index; + if (index === undefined || (!/^(?:\r?\n)+$/u.test(pem.slice(cursor, index)) && index !== 0)) { + fail("corporate CA bundle contains non-PEM material between certificates"); + } + const block = match[0]; + let certificate: X509Certificate; + try { + certificate = new X509Certificate(block); + } catch { + fail("corporate CA bundle contains an invalid X.509 certificate"); + } + if (!certificate.ca) { + fail("corporate CA bundle contains a certificate without basicConstraints CA:TRUE"); + } + cursor = index + block.length; + } + if (!/^(?:\r?\n)?$/u.test(pem.slice(cursor))) { + fail("corporate CA bundle contains trailing non-PEM material"); + } +} + +export function validateManagedStartupCorporateCaTransport( + encoded: string | undefined, + profile: ManagedStartupProfile, +): Buffer | null { + const expectedDigest = profile.corporateCa.bundleSha256; + if (expectedDigest === null) { + if (encoded !== undefined) { + fail("corporate CA transport must be absent when the profile has no CA digest"); + } + return null; + } + if ( + typeof encoded !== "string" || + encoded.length === 0 || + encoded.length > Math.ceil(MANAGED_STARTUP_CA_MAX_BYTES / 3) * 4 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(encoded) + ) { + fail("corporate CA transport must be canonical standard base64"); + } + const bytes = Buffer.from(encoded, "base64"); + if (bytes.toString("base64") !== encoded) { + fail("corporate CA transport must be canonical standard base64"); + } + validateCorporateCaBytes(bytes); + const actualDigest = createHash("sha256").update(bytes).digest("hex"); + if (actualDigest !== expectedDigest) { + fail("corporate CA bundle does not match the profile SHA-256 digest"); + } + return bytes; +} + +function readCanonicalProfile( + profilePath: string, + runtime: ManagedStartupApplicationRuntime, +): { + profile: ManagedStartupProfile; + fingerprint: string; +} { + const bytes = readSecureFile(profilePath, MANAGED_STARTUP_PROFILE_MAX_BYTES, runtime); + let raw: string; + try { + raw = UTF8_DECODER.decode(bytes); + } catch { + fail(`${profilePath} is not valid UTF-8`); + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + fail(`${profilePath} is not valid JSON`); + } + let profile: ManagedStartupProfile; + try { + profile = validateManagedStartupProfile(parsed); + } catch (error) { + fail(`${profilePath} is invalid: ${(error as Error).message}`); + } + if (serializeManagedStartupProfile(profile) !== raw) { + fail(`${profilePath} is not a canonical managed startup profile`); + } + return { + profile, + fingerprint: fingerprintManagedStartupProfile(profile), + }; +} + +function validateGeneration( + stateDirectory: string, + control: StateControl, + runtime: ManagedStartupApplicationRuntime, + expectedAgent?: ManagedStartupAgent, +): ValidatedGeneration { + if (!GENERATION_RE.test(control.generation)) { + fail("state control names an invalid generation"); + } + const directory = path.join(stateDirectory, control.generation); + requireSecureDirectory(directory, runtime, true); + const entries = fs.readdirSync(directory).sort(); + if ( + entries.some((entry) => entry !== "profile.json" && entry !== "corporate-ca.pem") || + !entries.includes("profile.json") + ) { + fail(`${directory} contains missing or unsupported state files`); + } + const profilePath = path.join(directory, "profile.json"); + const { profile, fingerprint } = readCanonicalProfile(profilePath, runtime); + if (fingerprint !== control.fingerprint) { + fail(`${directory} does not match its recorded profile fingerprint`); + } + if (expectedAgent !== undefined && profile.agent !== expectedAgent) { + fail(`managed startup profile targets ${profile.agent}, expected ${expectedAgent}`); + } + + const caPath = path.join(directory, "corporate-ca.pem"); + let corporateCaPath: string | null = null; + if (profile.corporateCa.bundleSha256 === null) { + if (entries.includes("corporate-ca.pem")) { + fail(`${directory} contains a CA bundle that is absent from the profile`); + } + } else { + if (!entries.includes("corporate-ca.pem")) { + fail(`${directory} is missing the CA bundle recorded by the profile`); + } + const caBytes = readSecureFile(caPath, MANAGED_STARTUP_CA_MAX_BYTES, runtime); + validateCorporateCaBytes(caBytes); + if (createHash("sha256").update(caBytes).digest("hex") !== profile.corporateCa.bundleSha256) { + fail(`${directory} contains a CA bundle with the wrong SHA-256 digest`); + } + corporateCaPath = caPath; + } + + return { + directory, + profilePath, + corporateCaPath, + profile, + fingerprint, + }; +} + +function validateDisposableDirectory( + target: string, + runtime: ManagedStartupApplicationRuntime, +): void { + requireSecureDirectory(target, runtime, true); + const entries = fs.readdirSync(target); + if ( + entries.length > 2 || + entries.some((entry) => entry !== "profile.json" && entry !== "corporate-ca.pem") + ) { + fail(`${target} is not a recognized disposable generation`); + } + for (const entry of entries) { + const file = path.join(target, entry); + const stat = fs.lstatSync(file); + requireSecureRegularFileStat(stat, file, runtime); + } +} + +function discardDirectory(target: string, runtime: ManagedStartupApplicationRuntime): void { + validateDisposableDirectory(target, runtime); + fs.rmSync(target, { recursive: true }); +} + +function unlinkSecureControlOrTemp( + target: string, + runtime: ManagedStartupApplicationRuntime, +): void { + const stat = fs.lstatSync(target); + requireSecureRegularFileStat(stat, target, runtime); + if (stat.size > MAX_CONTROL_FILE_BYTES) { + fail(`${target} exceeds the state-control size limit`); + } + fs.unlinkSync(target); +} + +function listStateEntries(stateDirectory: string): string[] { + const entries = fs.readdirSync(stateDirectory).sort(); + if (entries.length > MAX_STATE_ENTRIES) { + fail(`state directory exceeds ${String(MAX_STATE_ENTRIES)} entries`); + } + return entries; +} + +function cleanAtomicTemps( + stateDirectory: string, + entries: readonly string[], + runtime: ManagedStartupApplicationRuntime, +): void { + for (const entry of entries) { + const target = path.join(stateDirectory, entry); + if (PREPARE_TEMP_RE.test(entry)) { + discardDirectory(target, runtime); + } else if (CONTROL_TEMP_RE.test(entry)) { + unlinkSecureControlOrTemp(target, runtime); + } + } +} + +function requireKnownStateEntries(stateDirectory: string, entries: readonly string[]): void { + for (const entry of entries) { + if ( + entry === "committed.json" || + entry === "pending.json" || + GENERATION_RE.test(entry) || + PREPARE_TEMP_RE.test(entry) || + CONTROL_TEMP_RE.test(entry) + ) { + continue; + } + fail(`${stateDirectory} contains unsupported state component ${entry}`); + } +} + +function discardGenerationsExcept( + stateDirectory: string, + keepGeneration: string | null, + runtime: ManagedStartupApplicationRuntime, +): void { + for (const entry of listStateEntries(stateDirectory)) { + if (GENERATION_RE.test(entry) && entry !== keepGeneration) { + discardDirectory(path.join(stateDirectory, entry), runtime); + } + } +} + +function optionalStateControl( + stateDirectory: string, + basename: "committed.json" | "pending.json", + runtime: ManagedStartupApplicationRuntime, +): StateControl | null { + const target = path.join(stateDirectory, basename); + try { + fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + fail(`could not inspect ${target}`); + } + return parseStateControl(target, runtime); +} + +function removePendingControl( + stateDirectory: string, + runtime: ManagedStartupApplicationRuntime, +): void { + unlinkSecureControlOrTemp(path.join(stateDirectory, "pending.json"), runtime); + syncDirectory(stateDirectory); +} + +function recoverState( + stateDirectory: string, + requested: StateControl, + expectedAgent: ManagedStartupAgent, + runtime: ManagedStartupApplicationRuntime, +): { + committed: ValidatedGeneration | null; + pending: ValidatedGeneration | null; +} { + const initialEntries = listStateEntries(stateDirectory); + requireKnownStateEntries(stateDirectory, initialEntries); + cleanAtomicTemps(stateDirectory, initialEntries, runtime); + + const committedControl = optionalStateControl(stateDirectory, "committed.json", runtime); + const pendingControl = optionalStateControl(stateDirectory, "pending.json", runtime); + if (committedControl) { + const committed = validateGeneration(stateDirectory, committedControl, runtime, expectedAgent); + if ( + committedControl.fingerprint !== requested.fingerprint || + committedControl.generation !== requested.generation + ) { + fail("a different startup profile is already committed; recreate the sandbox to change it"); + } + if (pendingControl) { + if ( + pendingControl.fingerprint !== committedControl.fingerprint || + pendingControl.generation !== committedControl.generation + ) { + fail("committed and pending startup state disagree"); + } + removePendingControl(stateDirectory, runtime); + } + discardGenerationsExcept(stateDirectory, committedControl.generation, runtime); + return { committed, pending: null }; + } + + if (pendingControl) { + if ( + pendingControl.fingerprint === requested.fingerprint && + pendingControl.generation === requested.generation + ) { + const pending = validateGeneration(stateDirectory, pendingControl, runtime, expectedAgent); + discardGenerationsExcept(stateDirectory, pendingControl.generation, runtime); + return { committed: null, pending }; + } + discardGenerationsExcept(stateDirectory, null, runtime); + removePendingControl(stateDirectory, runtime); + return { committed: null, pending: null }; + } + + discardGenerationsExcept(stateDirectory, null, runtime); + return { committed: null, pending: null }; +} + +function createGeneration( + stateDirectory: string, + control: StateControl, + profileJson: string, + corporateCa: Buffer | null, + runtime: ManagedStartupApplicationRuntime, +): ValidatedGeneration { + const temporaryName = `.prepare-${String(process.pid)}-${randomToken()}`; + const temporary = path.join(stateDirectory, temporaryName); + const generation = path.join(stateDirectory, control.generation); + try { + fs.mkdirSync(temporary, { mode: STATE_DIRECTORY_MODE }); + fs.chownSync(temporary, runtime.rootUid, runtime.rootGid); + fs.chmodSync(temporary, STATE_DIRECTORY_MODE); + writeSecureNewFile(path.join(temporary, "profile.json"), profileJson, runtime); + if (corporateCa) { + writeSecureNewFile(path.join(temporary, "corporate-ca.pem"), corporateCa, runtime); + } + syncDirectory(temporary); + fs.renameSync(temporary, generation); + syncDirectory(stateDirectory); + } catch (error) { + try { + fs.lstatSync(temporary); + discardDirectory(temporary, runtime); + } catch { + // Preserve the generation error. + } + if (error instanceof ManagedStartupApplicationError) throw error; + fail(`could not atomically prepare generation ${control.generation}`); + } + return validateGeneration(stateDirectory, control, runtime); +} + +function toPrepared( + status: PreparedManagedStartupApplication["status"], + stateDirectory: string, + generation: ValidatedGeneration, + expectedAgent: ManagedStartupAgent, +): PreparedManagedStartupApplication { + return { + status, + stateDirectory, + generationDirectory: generation.directory, + profilePath: generation.profilePath, + corporateCaPath: generation.corporateCaPath, + fingerprint: generation.fingerprint, + expectedAgent, + profile: generation.profile, + }; +} + +/** + * Validate the secret-free envelope and atomically prepare immutable state. + * + * Agent-specific adapters may read the returned generation, make their own + * configuration changes, and then call commitManagedStartupApplication. A + * prepared generation is deliberately not treated as applied. + */ +export function prepareManagedStartupApplication( + input: PrepareManagedStartupApplicationInput, + testRuntime?: ManagedStartupApplicationTestRuntime, +): PreparedManagedStartupApplication { + const runtime = runtimeFor(testRuntime); + requireContainerRoot(); + + let profile: ManagedStartupProfile; + try { + profile = decodeManagedStartupProfile(input.encodedProfile); + } catch (error) { + fail((error as Error).message); + } + if (profile.agent !== input.expectedAgent) { + fail(`managed startup profile targets ${profile.agent}, expected ${input.expectedAgent}`); + } + const corporateCa = validateManagedStartupCorporateCaTransport(input.corporateCaB64, profile); + const profileJson = serializeManagedStartupProfile(profile); + const control = stateControl(fingerprintManagedStartupProfile(profile)); + const stateDirectory = ensureStateDirectory(input.stateDirectory, runtime); + const recovered = recoverState(stateDirectory, control, input.expectedAgent, runtime); + if (recovered.committed) { + return toPrepared( + "already-committed", + stateDirectory, + recovered.committed, + input.expectedAgent, + ); + } + if (recovered.pending) { + return toPrepared("prepared", stateDirectory, recovered.pending, input.expectedAgent); + } + + const generation = createGeneration(stateDirectory, control, profileJson, corporateCa, runtime); + atomicWriteStateControl(stateDirectory, "pending.json", control, runtime); + return toPrepared("prepared", stateDirectory, generation, input.expectedAgent); +} + +function validatePreparedHandle(handle: PreparedManagedStartupApplication): StateControl { + if ( + !path.isAbsolute(handle.stateDirectory) || + !SHA256_RE.test(handle.fingerprint) || + handle.generationDirectory !== + path.join(handle.stateDirectory, `generation-${handle.fingerprint}`) || + handle.profilePath !== path.join(handle.generationDirectory, "profile.json") || + (handle.corporateCaPath !== null && + handle.corporateCaPath !== path.join(handle.generationDirectory, "corporate-ca.pem")) + ) { + fail("prepared startup handle is malformed"); + } + return stateControl(handle.fingerprint); +} + +/** + * Mark a prepared profile applied only after every agent-specific adapter has + * completed. The marker rename is the sole commit point. + */ +export function commitManagedStartupApplication( + prepared: PreparedManagedStartupApplication, + testRuntime?: ManagedStartupApplicationTestRuntime, +): CommittedManagedStartupApplication { + const runtime = runtimeFor(testRuntime); + requireContainerRoot(); + const requested = validatePreparedHandle(prepared); + const stateDirectory = ensureStateDirectory(prepared.stateDirectory, runtime); + const committedControl = optionalStateControl(stateDirectory, "committed.json", runtime); + if (committedControl) { + if ( + committedControl.fingerprint !== requested.fingerprint || + committedControl.generation !== requested.generation + ) { + fail("a different startup profile is already committed"); + } + const generation = validateGeneration( + stateDirectory, + committedControl, + runtime, + prepared.expectedAgent, + ); + return { + ...toPrepared("already-committed", stateDirectory, generation, prepared.expectedAgent), + status: "committed", + }; + } + + const pendingControl = optionalStateControl(stateDirectory, "pending.json", runtime); + if ( + !pendingControl || + pendingControl.fingerprint !== requested.fingerprint || + pendingControl.generation !== requested.generation + ) { + fail("the prepared startup generation is not the active pending generation"); + } + const generation = validateGeneration( + stateDirectory, + pendingControl, + runtime, + prepared.expectedAgent, + ); + atomicWriteStateControl(stateDirectory, "committed.json", pendingControl, runtime); + removePendingControl(stateDirectory, runtime); + return { + ...toPrepared("already-committed", stateDirectory, generation, prepared.expectedAgent), + status: "committed", + }; +} diff --git a/src/lib/onboard/managed-startup/coordinator.ts b/src/lib/onboard/managed-startup/coordinator.ts new file mode 100644 index 00000000000..a2b99281c24 --- /dev/null +++ b/src/lib/onboard/managed-startup/coordinator.ts @@ -0,0 +1,154 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + type CommittedManagedStartupApplication, + commitManagedStartupApplication, + type PreparedManagedStartupApplication, + type PrepareManagedStartupApplicationInput, + prepareManagedStartupApplication, +} from "./application"; +import { + MANAGED_STARTUP_AGENTS, + type ManagedStartupAgent, + type ManagedStartupProfile, +} from "./profile"; + +export interface ManagedStartupAdapterContext { + readonly agent: ManagedStartupAgent; + readonly profile: ManagedStartupProfile; + readonly fingerprint: string; + readonly generationDirectory: string; + readonly profilePath: string; + readonly corporateCaPath: string | null; +} + +export interface ManagedStartupAgentAdapter { + readonly agent: ManagedStartupAgent; + readonly apply: (context: ManagedStartupAdapterContext) => void | Promise; +} + +export interface ManagedStartupCoordinatorDependencies { + readonly prepareApplication: ( + input: PrepareManagedStartupApplicationInput, + ) => PreparedManagedStartupApplication | Promise; + readonly commitApplication: ( + prepared: PreparedManagedStartupApplication, + ) => CommittedManagedStartupApplication | Promise; +} + +export interface ManagedStartupCoordinationResult { + readonly adapterApplied: boolean; + readonly application: CommittedManagedStartupApplication; +} + +type AdapterRegistry = Readonly>; + +const SHIPPED_AGENT_SET = new Set(MANAGED_STARTUP_AGENTS); + +const DEFAULT_DEPENDENCIES: ManagedStartupCoordinatorDependencies = { + prepareApplication: (input) => prepareManagedStartupApplication(input), + commitApplication: (prepared) => commitManagedStartupApplication(prepared), +}; + +export class ManagedStartupCoordinatorError extends Error { + constructor(message: string) { + super(`Managed startup coordination failed: ${message}`); + this.name = "ManagedStartupCoordinatorError"; + } +} + +function fail(message: string): never { + throw new ManagedStartupCoordinatorError(message); +} + +function createAdapterRegistry(adapters: readonly ManagedStartupAgentAdapter[]): AdapterRegistry { + const byAgent = new Map(); + for (const adapter of adapters) { + if ( + typeof adapter !== "object" || + adapter === null || + !SHIPPED_AGENT_SET.has(adapter.agent) || + typeof adapter.apply !== "function" + ) { + fail("every adapter must identify one shipped agent and provide an apply function"); + } + if (byAgent.has(adapter.agent)) { + fail(`duplicate adapter registered for ${adapter.agent}`); + } + byAgent.set(adapter.agent, adapter); + } + + const missing = MANAGED_STARTUP_AGENTS.filter((agent) => !byAgent.has(agent)); + if (missing.length > 0) { + fail(`missing adapter for ${missing.join(", ")}`); + } + if (byAgent.size !== MANAGED_STARTUP_AGENTS.length) { + fail("adapter registry must contain exactly the shipped agents"); + } + + return Object.freeze( + Object.fromEntries( + MANAGED_STARTUP_AGENTS.map((agent) => { + const adapter = byAgent.get(agent); + if (!adapter) fail(`missing adapter for ${agent}`); + return [agent, adapter]; + }), + ), + ) as AdapterRegistry; +} + +function requirePreparedIdentity( + prepared: PreparedManagedStartupApplication, + requestedAgent: ManagedStartupAgent, +): void { + if (prepared.expectedAgent !== requestedAgent || prepared.profile.agent !== requestedAgent) { + fail(`prepared profile targets ${prepared.profile.agent}, expected ${requestedAgent}`); + } +} + +function adapterContext(prepared: PreparedManagedStartupApplication): ManagedStartupAdapterContext { + return Object.freeze({ + agent: prepared.profile.agent, + profile: prepared.profile, + fingerprint: prepared.fingerprint, + generationDirectory: prepared.generationDirectory, + profilePath: prepared.profilePath, + corporateCaPath: prepared.corporateCaPath, + }); +} + +/** + * Coordinate one managed startup without depending on a host container driver. + * + * A complete, duplicate-free adapter registry is required before application + * state is prepared. New pending profiles dispatch exactly their matching + * adapter and commit only after it succeeds. An already committed profile is + * revalidated through commit without reapplying mutable agent configuration. + */ +export async function coordinateManagedStartupApplication( + input: PrepareManagedStartupApplicationInput, + adapters: readonly ManagedStartupAgentAdapter[], + dependencies: ManagedStartupCoordinatorDependencies = DEFAULT_DEPENDENCIES, +): Promise { + const registry = createAdapterRegistry(adapters); + const prepared = await dependencies.prepareApplication(input); + requirePreparedIdentity(prepared, input.expectedAgent); + + if (prepared.status === "already-committed") { + return { + adapterApplied: false, + application: await dependencies.commitApplication(prepared), + }; + } + + const adapter = registry[prepared.profile.agent]; + if (adapter.agent !== prepared.profile.agent) { + fail(`adapter registry cross-dispatch detected for ${prepared.profile.agent}`); + } + await adapter.apply(adapterContext(prepared)); + return { + adapterApplied: true, + application: await dependencies.commitApplication(prepared), + }; +} From 094d177744f3295fd21f51a85aea38690c5f3c46 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 18:17:58 -0700 Subject: [PATCH 007/117] feat(onboard): apply startup profiles in managed images Signed-off-by: Aaron Erickson --- ci/env-var-doc-allowlist.json | 4 + scripts/lib/entrypoint-env-wrapper.sh | 140 +++ .../applier/build/messaging-build-applier.mts | 62 +- .../messaging/post-agent-install-selection.ts | 77 ++ .../managed-startup-agent-environment.test.ts | 112 +- .../managed-startup-image-runtime.test.ts | 520 +++++++- .../onboard/managed-startup-profile.test.ts | 6 +- .../managed-startup/agent-environment.ts | 97 +- .../onboard/managed-startup/image-runtime.ts | 1089 ++++++++++++++++- src/lib/onboard/managed-startup/profile.ts | 6 +- src/lib/onboard/sandbox-create-launch.test.ts | 14 +- src/lib/onboard/sandbox-create-launch.ts | 2 + test/entrypoint-env-wrapper.test.ts | 218 ++++ test/messaging-build-applier.test.ts | 27 +- 14 files changed, 2308 insertions(+), 66 deletions(-) create mode 100755 scripts/lib/entrypoint-env-wrapper.sh create mode 100644 src/lib/messaging/post-agent-install-selection.ts create mode 100644 test/entrypoint-env-wrapper.test.ts diff --git a/ci/env-var-doc-allowlist.json b/ci/env-var-doc-allowlist.json index aed046f4cbb..eaac8c08b27 100644 --- a/ci/env-var-doc-allowlist.json +++ b/ci/env-var-doc-allowlist.json @@ -90,5 +90,9 @@ { "name": "NEMOCLAW_DOCKER_GPU_PATCH_NETWORK", "reason": "Internal one-process handoff from Docker GPU patch preparation into sandbox creation. Rebuild scopes and restores it; users must not set it." + }, + { + "name": "NEMOCLAW_MANAGED_HERMES_HASH_B64", + "reason": "Internal one-process transport from the root image applicator to its sandbox-owned Hermes compatibility-hash writer. The value is a bounded base64-encoded hash receipt that only the private internal writer consumes; users must not set it." } ] diff --git a/scripts/lib/entrypoint-env-wrapper.sh b/scripts/lib/entrypoint-env-wrapper.sh new file mode 100755 index 00000000000..0e9e90368ed --- /dev/null +++ b/scripts/lib/entrypoint-env-wrapper.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Normalize OpenShell's sandbox-create command when an OCI runtime invokes the +# image ENTRYPOINT with the literal argv: +# +# env NAME=value ... nemoclaw-start [agent command...] +# +# This runs before any managed-startup gate. Only environment names emitted by +# NemoClaw's launch renderer are promoted into the root entrypoint process; +# interpreter/loader variables such as NODE_OPTIONS, BASH_ENV, PATH, and +# LD_PRELOAD therefore cannot be smuggled into the trusted profile applicator. +# +# Result: NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV contains the command tail. +nemoclaw_normalize_entrypoint_env_wrapper() { + NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV=("$@") + NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC="$#" + [ "$#" -gt 0 ] || return 0 + + case "$1" in + nemoclaw-start | /usr/local/bin/nemoclaw-start) + shift + NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV=("$@") + NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC="$#" + return 0 + ;; + env) ;; + *) return 0 ;; + esac + + local -a _nemoclaw_original_argv=("$@") + local -a _nemoclaw_assignments=() + local _nemoclaw_self_index=-1 + local _nemoclaw_index + local _nemoclaw_token + local _nemoclaw_name + local _nemoclaw_seen_names="|" + + # Locate only the exact self-wrapper grammar. A normal explicit command such + # as `env FOO=bar printenv` remains a user command and is not interpreted by + # this root entrypoint normalization. + for ((_nemoclaw_index = 1; _nemoclaw_index < ${#_nemoclaw_original_argv[@]}; _nemoclaw_index += 1)); do + _nemoclaw_token="${_nemoclaw_original_argv[$_nemoclaw_index]}" + case "$_nemoclaw_token" in + nemoclaw-start | /usr/local/bin/nemoclaw-start) + _nemoclaw_self_index="$_nemoclaw_index" + break + ;; + *=*) ;; + *) break ;; + esac + done + + if [ "$_nemoclaw_self_index" -lt 0 ]; then + # A managed handoff must never silently degrade into an unmanaged command + # because the self-wrapper was absent or malformed. + for _nemoclaw_token in "${_nemoclaw_original_argv[@]:1}"; do + case "$_nemoclaw_token" in + NEMOCLAW_STARTUP_PROFILE_B64=* | NEMOCLAW_CORPORATE_CA_B64=*) + printf '%s\n' \ + '[SECURITY] Malformed managed startup env wrapper; expected nemoclaw-start after assignments.' >&2 + return 1 + ;; + esac + done + return 0 + fi + + if [ "$_nemoclaw_self_index" -gt 65 ]; then + printf '%s\n' '[SECURITY] Managed startup env wrapper has too many assignments.' >&2 + return 1 + fi + + for ((_nemoclaw_index = 1; _nemoclaw_index < _nemoclaw_self_index; _nemoclaw_index += 1)); do + _nemoclaw_token="${_nemoclaw_original_argv[$_nemoclaw_index]}" + _nemoclaw_name="${_nemoclaw_token%%=*}" + if [ "${#_nemoclaw_token}" -gt 122880 ] \ + || [[ ! "$_nemoclaw_name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] \ + || [[ "$_nemoclaw_token" == *$'\n'* ]] \ + || [[ "$_nemoclaw_token" == *$'\r'* ]]; then + printf '%s\n' '[SECURITY] Managed startup env wrapper contains a malformed assignment.' >&2 + return 1 + fi + case "$_nemoclaw_name" in + AWS_EC2_METADATA_DISABLED | \ + CHAT_UI_URL | \ + HTTP_PROXY | HTTPS_PROXY | NO_PROXY | \ + http_proxy | https_proxy | no_proxy | \ + OPENCLAW_HOME | OPENCLAW_STATE_DIR | OPENCLAW_WORKSPACE_DIR | \ + NEMOCLAW_AUTO_PAIR_DEADLINE_SECS | \ + NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS | \ + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS | \ + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS | \ + NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS | \ + NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS | \ + NEMOCLAW_CORPORATE_CA_B64 | \ + NEMOCLAW_DASHBOARD_BIND | NEMOCLAW_DASHBOARD_PORT | \ + NEMOCLAW_EXTRA_PLACEHOLDER_KEYS | \ + NEMOCLAW_HERMES_DASHBOARD | \ + NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT | \ + NEMOCLAW_HERMES_DASHBOARD_PORT | \ + NEMOCLAW_HERMES_DASHBOARD_TUI | \ + NEMOCLAW_MINIMAL_BOOTSTRAP | \ + NEMOCLAW_OBSERVABILITY | \ + NEMOCLAW_PROXY_HOST | NEMOCLAW_PROXY_PORT | \ + NEMOCLAW_SANDBOX_NAME | \ + NEMOCLAW_STARTUP_PROFILE_B64) ;; + *) + printf '%s\n' \ + "[SECURITY] Managed startup env wrapper contains unsupported variable '${_nemoclaw_name}'." >&2 + return 1 + ;; + esac + case "$_nemoclaw_seen_names" in + *"|${_nemoclaw_name}|"*) + printf '%s\n' \ + "[SECURITY] Managed startup env wrapper repeats variable '${_nemoclaw_name}'." >&2 + return 1 + ;; + esac + _nemoclaw_assignments+=("$_nemoclaw_token") + _nemoclaw_seen_names="${_nemoclaw_seen_names}${_nemoclaw_name}|" + done + + # Export only after the complete vector has passed validation so malformed + # input cannot leave a partially mutated root process. + if [ "$_nemoclaw_self_index" -gt 1 ]; then + for _nemoclaw_token in "${_nemoclaw_assignments[@]}"; do + export "${_nemoclaw_token?}" + done + fi + # shellcheck disable=SC2034 # output array is consumed by the sourcing entrypoint + NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV=( + "${_nemoclaw_original_argv[@]:$((_nemoclaw_self_index + 1))}" + ) + # shellcheck disable=SC2034 # output count is consumed by the sourcing entrypoint + NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC=$((\ + ${#_nemoclaw_original_argv[@]} - _nemoclaw_self_index - 1)) +} diff --git a/src/lib/messaging/applier/build/messaging-build-applier.mts b/src/lib/messaging/applier/build/messaging-build-applier.mts index e6cfdab7835..ea03cea82f5 100755 --- a/src/lib/messaging/applier/build/messaging-build-applier.mts +++ b/src/lib/messaging/applier/build/messaging-build-applier.mts @@ -28,6 +28,11 @@ import { telegramManifest } from "../../channels/telegram/manifest.ts"; import { wechatManifest } from "../../channels/wechat/manifest.ts"; import { whatsappManifest } from "../../channels/whatsapp/manifest.ts"; import type { ChannelAgentPackageRuntimeLockSpec, ChannelManifest } from "../../manifest/types.ts"; +import { + selectActiveMessagingChannelIds, + selectEnabledMessagingAgentRender, + selectEnabledPostAgentInstallBuildFiles, +} from "../../post-agent-install-selection.ts"; type Env = Record; type JsonObject = Record; @@ -341,19 +346,7 @@ export function applyMessagingAgentRenderToLocalFiles( export function activeChannels(plan: MessagingBuildPlan | null): string[] { if (!plan) return []; - const seen = new Set(); - const channels: string[] = []; - for (const item of plan.channels) { - const channel = String(item.channelId || "") - .trim() - .toLowerCase(); - if (!channel || seen.has(channel)) continue; - if (item.active === true && item.disabled !== true) { - seen.add(channel); - channels.push(channel); - } - } - return channels; + return selectActiveMessagingChannelIds(plan); } export function messagingRuntimePlanPath(env: Env = process.env): string { @@ -974,10 +967,7 @@ function resolveAgentRenderTarget( } function enabledAgentRender(plan: MessagingBuildPlan): MessagingRenderEntry[] { - const active = new Set(activeChannels(plan)); - return plan.agentRender.filter( - (render) => render.agent === plan.agent && active.has(render.channelId), - ); + return selectEnabledMessagingAgentRender(plan); } function enabledBuildStepsForPhase( @@ -985,6 +975,9 @@ function enabledBuildStepsForPhase( phase: MessagingHookPhase, ): MessagingBuildStep[] { if (!plan) return []; + if (phase === "post-agent-install") { + return selectEnabledPostAgentInstallBuildFiles(plan); + } return enabledBuildSteps(plan).filter((step) => buildStepMatchesPhase(plan, step, phase)); } @@ -1714,11 +1707,25 @@ function formatError(error: unknown): string { export type MessagingBuildPhase = "runtime-setup" | "agent-install" | "post-agent-install"; +export interface MessagingBuildPhaseOptions { + /** + * A managed image already contains the reviewed capability union. Apply only + * the explicit render and build-file plan to its durable home directory. + */ + readonly managedStartupRuntime?: boolean; +} + export function applyMessagingBuildPhase( plan: MessagingBuildPlan | null, phase: MessagingBuildPhase, env: Env = process.env, + options: MessagingBuildPhaseOptions = {}, ): readonly string[] { + if (options.managedStartupRuntime && phase !== "post-agent-install") { + throw new MessagingBuildApplierError( + "Managed startup runtime mode is only valid for post-agent-install", + ); + } if (phase === "runtime-setup") { const target = writeMessagingRuntimePlanArtifact(plan, messagingRuntimePlanPath(env)); return target ? [target] : []; @@ -1732,7 +1739,7 @@ export function applyMessagingBuildPhase( ...applyPostAgentInstallBuildFilesToLocalFiles(plan), ]; const appliedTargets = applyPostAgentInstallOutputs(); - if (plan?.agent === "openclaw") { + if (plan?.agent === "openclaw" && !options.managedStartupRuntime) { runOpenClawMessagingDoctor(plan, env); return uniqueStrings([...appliedTargets, ...applyPostAgentInstallOutputs()]); } @@ -1802,23 +1809,25 @@ export function describeMessagingBuildPhase( } export function main(argv: readonly string[] = process.argv.slice(2)): void { - const { agent, phase, dryRun } = parseMessagingBuildArgs(argv); + const { agent, phase, dryRun, managedStartupRuntime } = parseMessagingBuildArgs(argv); const plan = readMessagingBuildPlanFromEnv(process.env, agent); if (dryRun) { console.log(JSON.stringify(describeMessagingBuildPhase(plan, phase, process.env), null, 2)); return; } - applyMessagingBuildPhase(plan, phase, process.env); + applyMessagingBuildPhase(plan, phase, process.env, { managedStartupRuntime }); } function parseMessagingBuildArgs(argv: readonly string[]): { readonly agent: MessagingAgentId; readonly phase: MessagingBuildPhase; readonly dryRun: boolean; + readonly managedStartupRuntime: boolean; } { let agent: MessagingAgentId | undefined; let phase: MessagingBuildPhase | undefined; let dryRun = false; + let managedStartupRuntime = false; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; @@ -1826,6 +1835,10 @@ function parseMessagingBuildArgs(argv: readonly string[]): { dryRun = true; continue; } + if (arg === "--managed-startup-runtime") { + managedStartupRuntime = true; + continue; + } if (arg === "--agent") { agent = readAgentArg(argv[index + 1]); index += 1; @@ -1851,10 +1864,17 @@ function parseMessagingBuildArgs(argv: readonly string[]): { throw new MessagingBuildApplierError(`Unknown messaging build applier argument: ${arg}`); } + const resolvedPhase = phase ?? "post-agent-install"; + if (managedStartupRuntime && resolvedPhase !== "post-agent-install") { + throw new MessagingBuildApplierError( + "--managed-startup-runtime requires --phase post-agent-install", + ); + } return { agent: agent ?? "openclaw", - phase: phase ?? "post-agent-install", + phase: resolvedPhase, dryRun, + managedStartupRuntime, }; } diff --git a/src/lib/messaging/post-agent-install-selection.ts b/src/lib/messaging/post-agent-install-selection.ts new file mode 100644 index 00000000000..63c9afb380c --- /dev/null +++ b/src/lib/messaging/post-agent-install-selection.ts @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +interface SelectionHook { + readonly id: string; + readonly phase: string; +} + +interface SelectionChannel { + readonly channelId: string; + readonly active?: boolean; + readonly disabled?: boolean; + readonly hooks?: readonly SelectionHook[]; +} + +interface SelectionPlanBase { + readonly channels: readonly SelectionChannel[]; +} + +/** + * Canonical active-channel selection for the image applier. Each selection + * consumer must resolve the same active channels and mutable outputs. + */ +export function selectActiveMessagingChannelIds(plan: SelectionPlanBase): string[] { + const seen = new Set(); + const channels: string[] = []; + for (const item of plan.channels) { + const channel = String(item.channelId || "") + .trim() + .toLowerCase(); + if (!channel || seen.has(channel)) continue; + if (item.active === true && item.disabled !== true) { + seen.add(channel); + channels.push(channel); + } + } + return channels; +} + +export function selectEnabledMessagingAgentRender< + Render extends { + readonly agent: string; + readonly channelId: string; + }, +>( + plan: SelectionPlanBase & { + readonly agent: string; + readonly agentRender: readonly Render[]; + }, +): Render[] { + const active = new Set(selectActiveMessagingChannelIds(plan)); + return plan.agentRender.filter( + (render) => render.agent === plan.agent && active.has(render.channelId), + ); +} + +export function selectEnabledPostAgentInstallBuildFiles< + Step extends { + readonly channelId: string; + readonly kind: string; + readonly hookId?: string; + }, +>( + plan: SelectionPlanBase & { + readonly buildSteps: readonly Step[]; + }, +): Step[] { + const active = new Set(selectActiveMessagingChannelIds(plan)); + return plan.buildSteps.filter((step) => { + if (!active.has(step.channelId) || step.kind !== "build-file") return false; + if (!step.hookId) return true; + const hookPhase = plan.channels + .find((channel) => channel.channelId === step.channelId) + ?.hooks?.find((hook) => hook.id === step.hookId)?.phase; + return hookPhase === undefined || hookPhase === "post-agent-install"; + }); +} diff --git a/src/lib/onboard/managed-startup-agent-environment.test.ts b/src/lib/onboard/managed-startup-agent-environment.test.ts index 3695c179ef3..88be8dc6b6d 100644 --- a/src/lib/onboard/managed-startup-agent-environment.test.ts +++ b/src/lib/onboard/managed-startup-agent-environment.test.ts @@ -16,12 +16,26 @@ import { MANAGED_STARTUP_AGENTS, MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY, MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + MANAGED_STARTUP_RUNTIME_CLEANUP_OBLIGATIONS, type ManagedStartupAgent, type ManagedStartupJsonObject, type ManagedStartupProfile, } from "./managed-startup/profile"; const CA_SHA256 = "a".repeat(64); +const OPENCLAW_APPLICATION_RUNTIME_NAMES = [ + "NEMOCLAW_AUTO_PAIR_DEADLINE_SECS", + "NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS", + "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS", + "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS", + "NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS", + "NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS", +] as const; +const UNSUPPORTED_AGENT_RUNTIME_UNSETS = [ + ...OPENCLAW_APPLICATION_RUNTIME_NAMES, + "NEMOCLAW_DASHBOARD_BIND", + "NEMOCLAW_MINIMAL_BOOTSTRAP", +] as const; function messagingPlan(agent: "openclaw" | "hermes"): ManagedStartupJsonObject { return { @@ -233,7 +247,14 @@ const PROFILES: Readonly ManagedStartupProfile describe("managed startup agent environment", () => { it("maps every OpenClaw profile field to the existing generator and entrypoint contracts", () => { - const result = mapManagedStartupProfileToAgentEnvironment(openClawProfile()); + const result = mapManagedStartupProfileToAgentEnvironment(openClawProfile(), { + NEMOCLAW_AUTO_PAIR_DEADLINE_SECS: " 30 ", + NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS: "3e0", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: "0.25", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "03", + NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS: "10.0", + NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS: "6e2", + }); expect(result.schemaVersion).toBe(1); expect(result.agent).toBe("openclaw"); @@ -283,6 +304,20 @@ describe("managed startup agent environment", () => { no_proxy: "127.0.0.1,inference.local,localhost", }); expect(Object.hasOwn(result.runtimeEnvironment, "NEMOCLAW_MESSAGING_PLAN_B64")).toBe(false); + expect(result.applicationRuntime).toEqual({ + exportEnvironment: { + NEMOCLAW_AUTO_PAIR_DEADLINE_SECS: "30", + NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS: "3", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: "0.25", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "3", + NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS: "10", + NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS: "600", + }, + unsetEnvironment: [], + }); + expect(Object.isFrozen(result.applicationRuntime)).toBe(true); + expect(Object.isFrozen(result.applicationRuntime.exportEnvironment)).toBe(true); + expect(Object.isFrozen(result.applicationRuntime.unsetEnvironment)).toBe(true); expect( decodeBase64Json(result.configurationEnvironment.NEMOCLAW_INFERENCE_COMPAT_B64 ?? ""), @@ -343,8 +378,69 @@ describe("managed startup agent environment", () => { ]); }); + it.each([ + ["NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS", "0", /positive safe integer/u], + ["NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS", "1.5", /positive safe integer/u], + [ + "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS", + String(Number.MAX_SAFE_INTEGER + 1), + /positive safe integer/u, + ], + ["NEMOCLAW_AUTO_PAIR_DEADLINE_SECS", "Infinity", /finite positive seconds/u], + ["NEMOCLAW_AUTO_PAIR_DEADLINE_SECS", "NaN", /finite positive seconds/u], + ["NEMOCLAW_AUTO_PAIR_DEADLINE_SECS", "not-a-number", /finite positive seconds/u], + ["NEMOCLAW_AUTO_PAIR_DEADLINE_SECS", "1\n", /single-line text/u], + ["NEMOCLAW_AUTO_PAIR_DEADLINE_SECS", "\r1", /single-line text/u], + ["NEMOCLAW_AUTO_PAIR_DEADLINE_SECS", "1\0", /single-line text/u], + ["NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS", "-0.1", /finite positive seconds/u], + ["NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS", " ", /finite positive seconds/u], + ] as const)("rejects invalid application runtime input %s=%s", (name, value, message) => { + expect(() => + mapManagedStartupProfileToAgentEnvironment(openClawProfile(), { [name]: value }), + ).toThrow(message); + }); + + it.each( + MANAGED_STARTUP_AGENTS, + )("derives every unsupported $0 runtime unset from the closed contract", (agent) => { + const result = mapManagedStartupProfileToAgentEnvironment(PROFILES[agent](), { + NEMOCLAW_AUTO_PAIR_DEADLINE_SECS: "30", + NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS: "3", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: "0.25", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "3", + NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS: "10", + NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS: "600", + }); + const unsets = new Set(result.applicationRuntime.unsetEnvironment); + for (const obligation of MANAGED_STARTUP_RUNTIME_CLEANUP_OBLIGATIONS) { + expect(unsets.has(obligation.input)).toBe(!obligation.supportedFor.includes(agent)); + } + for (const name of OPENCLAW_APPLICATION_RUNTIME_NAMES) { + expect(unsets.has(name)).toBe(agent !== "openclaw"); + } + }); + + it("keeps the profile mapper independent from mutable process-global runtime input", () => { + const name = "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS"; + const previous = process.env[name]; + process.env[name] = "not-a-number"; + try { + expect( + mapManagedStartupProfileToAgentEnvironment(openClawProfile()).applicationRuntime, + ).toEqual({ + exportEnvironment: {}, + unsetEnvironment: [], + }); + } finally { + if (previous === undefined) delete process.env[name]; + else process.env[name] = previous; + } + }); + it("maps every Hermes profile field, including gateway presets and dashboard forwarding", () => { - const result = mapManagedStartupProfileToAgentEnvironment(hermesProfile()); + const result = mapManagedStartupProfileToAgentEnvironment(hermesProfile(), { + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "not-a-number", + }); expect(result.configurationEnvironment).toEqual({ CHAT_UI_URL: "http://127.0.0.1:19189", @@ -394,6 +490,10 @@ describe("managed startup agent environment", () => { https_proxy: "http://proxy.example.test:3128", no_proxy: "127.0.0.1,localhost", }); + expect(result.applicationRuntime).toEqual({ + exportEnvironment: {}, + unsetEnvironment: UNSUPPORTED_AGENT_RUNTIME_UNSETS, + }); expect(result.actions).toContainEqual({ kind: "apply-messaging-plan", agent: "hermes", @@ -415,7 +515,9 @@ describe("managed startup agent environment", () => { }); it("keeps DCode routing and auto-approval in root-owned files instead of ambient runtime env", () => { - const result = mapManagedStartupProfileToAgentEnvironment(dcodeProfile()); + const result = mapManagedStartupProfileToAgentEnvironment(dcodeProfile(), { + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: "not-a-number", + }); expect(result.configurationEnvironment).toEqual({ HTTP_PROXY: "", @@ -448,6 +550,10 @@ describe("managed startup agent environment", () => { ...expectedDcodeRuntime, NEMOCLAW_OBSERVABILITY: "1", }); + expect(result.applicationRuntime).toEqual({ + exportEnvironment: {}, + unsetEnvironment: UNSUPPORTED_AGENT_RUNTIME_UNSETS, + }); for (const environment of [result.configurationEnvironment, result.runtimeEnvironment]) { expect(environment).not.toHaveProperty("NEMOCLAW_DCODE_AUTO_APPROVAL"); expect(environment).not.toHaveProperty("NEMOCLAW_MESSAGING_PLAN_B64"); diff --git a/src/lib/onboard/managed-startup-image-runtime.test.ts b/src/lib/onboard/managed-startup-image-runtime.test.ts index f643e15bf80..d0ec6588e61 100644 --- a/src/lib/onboard/managed-startup-image-runtime.test.ts +++ b/src/lib/onboard/managed-startup-image-runtime.test.ts @@ -1,13 +1,42 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { createHash, X509Certificate } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +const coordinatorMock = vi.hoisted(() => ({ + coordinateManagedStartupApplication: vi.fn(), +})); +vi.mock("./managed-startup/coordinator", () => coordinatorMock); + +import { + MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, + managedStartupE2eProfile, +} from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { mapManagedStartupProfileToAgentEnvironment } from "./managed-startup/agent-environment"; import { + applyManagedStartupCommandEnvironmentPlan, + applyManagedStartupImageProfile, buildManagedStartupImageActionPlan, + MANAGED_STARTUP_MERGED_CA_FILE, + MANAGED_STARTUP_PROFILE_ENV, + MANAGED_STARTUP_RUNTIME_ENV_FILE, type ManagedStartupImageActionPlanInput, + normalizeHermesManagedConfigDescriptor, + readStableRegularFile, + serializeManagedStartupRuntimeEnvironment, } from "./managed-startup/image-runtime"; -import type { ManagedStartupAgent, ManagedStartupDashboard } from "./managed-startup/profile"; +import { + encodeManagedStartupProfile, + fingerprintManagedStartupProfile, + MANAGED_STARTUP_AGENTS, + type ManagedStartupAgent, + type ManagedStartupDashboard, + validateManagedStartupProfile, +} from "./managed-startup/profile"; function dashboard(agent: ManagedStartupAgent): ManagedStartupDashboard { switch (agent) { @@ -229,3 +258,490 @@ describe("buildManagedStartupImageActionPlan", () => { ).toThrow(message); }); }); + +const PROXY_ENV_NAMES = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", +] as const; +const OPENCLAW_APPLICATION_RUNTIME_NAMES = [ + "NEMOCLAW_AUTO_PAIR_DEADLINE_SECS", + "NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS", + "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS", + "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS", + "NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS", + "NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS", +] as const; + +describe("managed startup image runtime", () => { + let temporaryDirectoryPath = ""; + + beforeEach(() => { + coordinatorMock.coordinateManagedStartupApplication.mockReset(); + temporaryDirectoryPath = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-startup-")); + }); + afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(temporaryDirectoryPath, { force: true, recursive: true }); + }); + + function temporaryDirectory(): string { + return temporaryDirectoryPath; + } + + function mockDescriptorOwnership(uid: bigint, gid: bigint): void { + const realFstatSync = fs.fstatSync.bind(fs); + const realLstatSync = fs.lstatSync.bind(fs); + const ownership = new Map([ + ["uid", uid], + ["gid", gid], + ]); + const owned = (stat: fs.BigIntStats): fs.BigIntStats => + new Proxy(stat, { + get(inner, property) { + const value = ownership.has(property) + ? ownership.get(property) + : (Reflect.get(inner, property, inner) as unknown); + return typeof value === "function" ? value.bind(inner) : value; + }, + }); + vi.spyOn(fs, "fstatSync").mockImplementation(((descriptor: number, options: { bigint: true }) => + owned(realFstatSync(descriptor, options))) as typeof fs.fstatSync); + vi.spyOn(fs, "lstatSync").mockImplementation(((file: fs.PathLike, options: { bigint: true }) => + owned(realLstatSync(file, options))) as typeof fs.lstatSync); + } + + function mockRootReplayFilesystem(runtimeWrites: string[]): void { + const directories = new Set([ + "/", + "/run", + "/run/nemoclaw", + "/var", + "/var/lib", + "/var/lib/nemoclaw", + ]); + let runtimeFileWritten = false; + const stat = (kind: "directory" | "file", mode: number) => + ({ + gid: 0, + isDirectory: () => kind === "directory", + isFile: () => kind === "file", + isSymbolicLink: () => false, + mode, + nlink: 1, + uid: 0, + }) as fs.Stats; + const missing = () => Object.assign(new Error("missing"), { code: "ENOENT" }); + + vi.spyOn(process, "geteuid").mockReturnValue(0); + vi.spyOn(fs, "lstatSync").mockImplementation(((target: fs.PathLike) => { + const resolved = String(target); + if (directories.has(resolved)) return stat("directory", 0o755); + if (resolved === MANAGED_STARTUP_RUNTIME_ENV_FILE && runtimeFileWritten) { + return stat("file", 0o400); + } + throw missing(); + }) as typeof fs.lstatSync); + vi.spyOn(fs, "mkdirSync").mockImplementation(() => undefined); + vi.spyOn(fs, "chownSync").mockImplementation(() => undefined); + vi.spyOn(fs, "chmodSync").mockImplementation(() => undefined); + vi.spyOn(fs, "existsSync").mockReturnValue(false); + vi.spyOn(fs, "openSync").mockReturnValue(91); + vi.spyOn(fs, "fchownSync").mockImplementation(() => undefined); + vi.spyOn(fs, "writeFileSync").mockImplementation(((target: fs.PathOrFileDescriptor, value) => { + if (target === 91) runtimeWrites.push(String(value)); + }) as typeof fs.writeFileSync); + vi.spyOn(fs, "fchmodSync").mockImplementation(() => undefined); + vi.spyOn(fs, "fsyncSync").mockImplementation(() => undefined); + vi.spyOn(fs, "closeSync").mockImplementation(() => undefined); + vi.spyOn(fs, "renameSync").mockImplementation((_source, target) => { + if (String(target) === MANAGED_STARTUP_RUNTIME_ENV_FILE) runtimeFileWritten = true; + }); + vi.spyOn(fs, "unlinkSync").mockImplementation(() => { + throw missing(); + }); + } + + it("rejects invalid OpenClaw launch controls before filesystem or coordinator mutation", async () => { + const profile = managedStartupE2eProfile("openclaw"); + const lstat = vi.spyOn(fs, "lstatSync"); + vi.spyOn(process, "geteuid").mockReturnValue(0); + + await expect( + applyManagedStartupImageProfile("openclaw", { + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "1", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: "NaN", + [MANAGED_STARTUP_PROFILE_ENV]: encodeManagedStartupProfile(profile), + }), + ).rejects.toThrow(/finite positive seconds/u); + expect(lstat).not.toHaveBeenCalled(); + expect(coordinatorMock.coordinateManagedStartupApplication).not.toHaveBeenCalled(); + }); + + it("refreshes admitted launch controls on committed replay without changing the profile", async () => { + const profile = managedStartupE2eProfile("openclaw"); + const encodedProfile = encodeManagedStartupProfile(profile); + const fingerprint = fingerprintManagedStartupProfile(profile); + const runtimeWrites: string[] = []; + mockRootReplayFilesystem(runtimeWrites); + coordinatorMock.coordinateManagedStartupApplication.mockResolvedValue({ + adapterApplied: false, + application: { + status: "committed", + stateDirectory: "/var/lib/nemoclaw/managed-startup", + generationDirectory: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}`, + profilePath: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}/profile.json`, + corporateCaPath: null, + fingerprint, + expectedAgent: "openclaw", + profile, + }, + }); + + const first = await applyManagedStartupImageProfile("openclaw", { + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "1", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "3", + [MANAGED_STARTUP_PROFILE_ENV]: encodedProfile, + }); + const second = await applyManagedStartupImageProfile("openclaw", { + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "1", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "5", + [MANAGED_STARTUP_PROFILE_ENV]: encodedProfile, + }); + + expect(first).toMatchObject({ adapterApplied: false, fingerprint }); + expect(second).toMatchObject({ adapterApplied: false, fingerprint }); + expect(runtimeWrites).toHaveLength(2); + expect(runtimeWrites[0]).toContain("export NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS='3'"); + expect(runtimeWrites[1]).toContain("export NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS='5'"); + expect(coordinatorMock.coordinateManagedStartupApplication).toHaveBeenCalledTimes(2); + }); + + it.each( + MANAGED_STARTUP_AGENTS, + )("maps the complete %s profile into the reviewed image command contract", (agent) => { + const mapped = mapManagedStartupProfileToAgentEnvironment(managedStartupE2eProfile(agent)); + const plan = buildManagedStartupImageActionPlan({ + agent: mapped.agent, + actions: mapped.actions, + }); + + expect(plan.map(({ action }) => action)).toEqual( + agent === "langchain-deepagents-code" + ? ["generate-agent-config"] + : ["messaging-runtime-setup", "generate-agent-config", "messaging-post-agent-install"], + ); + expect(plan.some((command) => command.argv.includes("agent-install"))).toBe(false); + }); + + it.each( + MANAGED_STARTUP_AGENTS, + )("provides valid same-profile and changed-profile fixtures for %s recreation checks", (agent) => { + const initial = validateManagedStartupProfile(managedStartupE2eProfile(agent)); + const same = validateManagedStartupProfile(managedStartupE2eProfile(agent)); + const changed = validateManagedStartupProfile(managedStartupE2eProfile(agent, true)); + + expect(fingerprintManagedStartupProfile(same)).toBe(fingerprintManagedStartupProfile(initial)); + expect(fingerprintManagedStartupProfile(changed)).not.toBe( + fingerprintManagedStartupProfile(initial), + ); + }); + + it("binds the real corporate-CA fixture into every agent profile by exact digest", () => { + expect(() => new X509Certificate(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM)).not.toThrow(); + const digest = createHash("sha256").update(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM).digest("hex"); + + for (const agent of MANAGED_STARTUP_AGENTS) { + expect(managedStartupE2eProfile(agent, false, true).corporateCa.bundleSha256).toBe(digest); + } + }); + + it("writes a deterministic root-sourced runtime environment without profile transport", () => { + const applicationRuntime = { + exportEnvironment: { + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: "0.25", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "3", + }, + unsetEnvironment: ["NEMOCLAW_MINIMAL_BOOTSTRAP"], + }; + const script = serializeManagedStartupRuntimeEnvironment( + { + NEMOCLAW_MODEL: "model-with-'quote", + NEMOCLAW_OBSERVABILITY: "0", + }, + true, + { + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_MODEL: "model-with-'quote", + }, + applicationRuntime, + ); + + expect(script).toContain("unset NEMOCLAW_INFERENCE_BASE_URL"); + expect(script).toContain("unset NEMOCLAW_MINIMAL_BOOTSTRAP"); + expect(script).toContain("export NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS='0.25'"); + expect(script).toContain("export NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS='3'"); + expect(script).toContain("export NEMOCLAW_MANAGED_STARTUP_APPLIED='1'"); + expect(script).toContain("export NEMOCLAW_MODEL='model-with-'\"'\"'quote'"); + expect(script).toContain(`export SSL_CERT_FILE='${MANAGED_STARTUP_MERGED_CA_FILE}'`); + expect(script).toContain("export _NEMOCLAW_CORPORATE_CA_MERGED='1'"); + expect(script).not.toContain("NEMOCLAW_STARTUP_PROFILE_B64"); + expect(script).not.toContain("NEMOCLAW_CORPORATE_CA_B64"); + expect(script.endsWith("\n")).toBe(true); + expect( + serializeManagedStartupRuntimeEnvironment( + { + NEMOCLAW_MODEL: "model-with-'quote", + NEMOCLAW_OBSERVABILITY: "0", + }, + true, + { + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_MODEL: "model-with-'quote", + }, + applicationRuntime, + ), + ).toBe(script); + }); + + it("validates runtime plans while removing launch-only exports and unsets from child commands", () => { + const ambient = { + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "stale", + NEMOCLAW_MINIMAL_BOOTSTRAP: "1", + PRESERVED: "yes", + }; + const applied = applyManagedStartupCommandEnvironmentPlan(ambient, { + exportEnvironment: { NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "3" }, + unsetEnvironment: ["NEMOCLAW_MINIMAL_BOOTSTRAP"], + }); + + expect(applied).toEqual({ + PRESERVED: "yes", + }); + expect(ambient).toHaveProperty("NEMOCLAW_MINIMAL_BOOTSTRAP", "1"); + expect(() => + applyManagedStartupCommandEnvironmentPlan(ambient, { + exportEnvironment: { NEMOCLAW_MINIMAL_BOOTSTRAP: "1" }, + unsetEnvironment: ["NEMOCLAW_MINIMAL_BOOTSTRAP"], + }), + ).toThrow(/both export and unset NEMOCLAW_MINIMAL_BOOTSTRAP/u); + expect(ambient).toHaveProperty("NEMOCLAW_MINIMAL_BOOTSTRAP", "1"); + }); + + it.each([ + "hermes", + "langchain-deepagents-code", + ] as const)("removes OpenClaw launch controls and cleanup obligations from %s children and runtime", (agent) => { + const mapped = mapManagedStartupProfileToAgentEnvironment(managedStartupE2eProfile(agent), { + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "invalid-for-this-agent", + }); + const ambient = { + ...Object.fromEntries(OPENCLAW_APPLICATION_RUNTIME_NAMES.map((name) => [name, "ambient"])), + NEMOCLAW_DASHBOARD_BIND: "0.0.0.0", + NEMOCLAW_MINIMAL_BOOTSTRAP: "1", + PRESERVED: "yes", + }; + const child = applyManagedStartupCommandEnvironmentPlan(ambient, mapped.applicationRuntime); + const script = serializeManagedStartupRuntimeEnvironment( + mapped.runtimeEnvironment, + false, + mapped.configurationEnvironment, + mapped.applicationRuntime, + ); + + expect(child).toEqual({ PRESERVED: "yes" }); + for (const name of [ + ...OPENCLAW_APPLICATION_RUNTIME_NAMES, + "NEMOCLAW_DASHBOARD_BIND", + "NEMOCLAW_MINIMAL_BOOTSTRAP", + ]) { + expect(script).toContain(`unset ${name}`); + expect(script).not.toContain(`export ${name}=`); + } + }); + + it("rejects a serialized runtime export that conflicts with an explicit unset", () => { + expect(() => + serializeManagedStartupRuntimeEnvironment( + { NEMOCLAW_MINIMAL_BOOTSTRAP: "1" }, + false, + {}, + { exportEnvironment: {}, unsetEnvironment: ["NEMOCLAW_MINIMAL_BOOTSTRAP"] }, + ), + ).toThrow(/runtime environment cannot both export and unset NEMOCLAW_MINIMAL_BOOTSTRAP/u); + }); + + it.each([ + [ + { exportEnvironment: { "BAD-NAME": "value" }, unsetEnvironment: [] }, + /invalid application runtime environment key/u, + ], + [ + { exportEnvironment: { VALID_NAME: "line 1\nline 2" }, unsetEnvironment: [] }, + /must be single-line text/u, + ], + [ + { exportEnvironment: {}, unsetEnvironment: ["DUPLICATE", "DUPLICATE"] }, + /duplicate application runtime unset/u, + ], + ])("rejects a malformed application runtime plan before command mutation", (plan, message) => { + const ambient = { PRESERVED: "yes" }; + expect(() => applyManagedStartupCommandEnvironmentPlan(ambient, plan)).toThrow(message); + expect(ambient).toEqual({ PRESERVED: "yes" }); + }); + + it.each(["openclaw", "hermes"] as const)("preserves launch-only proxy env for %s", (agent) => { + const mapped = mapManagedStartupProfileToAgentEnvironment( + managedStartupE2eProfile(agent, false, false, true), + ); + const script = serializeManagedStartupRuntimeEnvironment( + mapped.runtimeEnvironment, + false, + mapped.configurationEnvironment, + ); + for (const name of PROXY_ENV_NAMES) { + expect(script).not.toMatch(new RegExp(`(?:export|unset) ${name}(?:=|$)`, "mu")); + } + }); + + it("clears launch-only proxy env when DCode pins managed routing", () => { + const mapped = mapManagedStartupProfileToAgentEnvironment( + managedStartupE2eProfile("langchain-deepagents-code", false, false, true), + ); + const script = serializeManagedStartupRuntimeEnvironment( + mapped.runtimeEnvironment, + false, + mapped.configurationEnvironment, + ); + for (const name of PROXY_ENV_NAMES) { + expect(script).toContain(`unset ${name}`); + } + }); + + it("rejects multiline runtime values before producing a sourceable file", () => { + expect(() => + serializeManagedStartupRuntimeEnvironment({ NEMOCLAW_MODEL: "bad\nvalue" }, false), + ).toThrow(/single-line/u); + }); + + it("refuses a symlink instead of opening its target", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, "target"); + const link = path.join(directory, "link"); + fs.writeFileSync(target, "trusted\n"); + fs.symlinkSync(target, link); + + expect(() => readStableRegularFile(link, 1024)).toThrow(/unsafe or unreadable/u); + }); + + it("rejects descriptor metadata drift after a bounded read", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, "material"); + fs.writeFileSync(target, "trusted\n", { mode: 0o600 }); + const realReadSync = fs.readSync.bind(fs); + vi.spyOn(fs, "readSync") + .mockImplementationOnce((( + descriptor: number, + buffer: NodeJS.ArrayBufferView, + offset: number, + length: number, + position: number | null, + ) => { + const bytesRead = realReadSync(descriptor, buffer, offset, length, position); + fs.chmodSync(target, 0o644); + return bytesRead; + }) as typeof fs.readSync) + .mockImplementation(realReadSync as typeof fs.readSync); + + expect(() => readStableRegularFile(target, 1024)).toThrow(/changed while it was read/u); + }); + + it("normalizes mutable sandbox-owned Hermes config descriptors to mode 0640", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, "config.yaml"); + fs.writeFileSync(target, "model: managed\n", { mode: 0o600 }); + mockDescriptorOwnership(501n, 20n); + + normalizeHermesManagedConfigDescriptor(target, { + uid: 501, + gid: 20, + }); + + expect(fs.readFileSync(target, "utf8")).toBe("model: managed\n"); + expect(fs.statSync(target).mode & 0o777).toBe(0o640); + }); + + it("preserves a root-owned shields-up Hermes descriptor without chmod", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, ".env"); + fs.writeFileSync(target, "OPENAI_API_KEY=managed\n", { mode: 0o444 }); + mockDescriptorOwnership(0n, 0n); + const chmod = vi.spyOn(fs, "fchmodSync"); + + normalizeHermesManagedConfigDescriptor(target, { + uid: 501, + gid: 20, + }); + + expect(chmod).not.toHaveBeenCalled(); + expect(fs.readFileSync(target, "utf8")).toBe("OPENAI_API_KEY=managed\n"); + }); + + it.each([0o440, 0o644, 0o660])("fails closed on unexpected mutable Hermes mode %s", (mode) => { + const directory = temporaryDirectory(); + const target = path.join(directory, "config.yaml"); + fs.writeFileSync(target, "model: managed\n", { mode }); + fs.chmodSync(target, mode); + mockDescriptorOwnership(501n, 20n); + + expect(() => + normalizeHermesManagedConfigDescriptor(target, { + uid: 501, + gid: 20, + }), + ).toThrow(/unexpected Hermes managed config descriptor/u); + expect(fs.statSync(target).mode & 0o777).toBe(mode); + }); + + it("fails closed on an unexpected Hermes descriptor owner", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, "config.yaml"); + fs.writeFileSync(target, "model: managed\n", { mode: 0o600 }); + mockDescriptorOwnership(502n, 21n); + + expect(() => + normalizeHermesManagedConfigDescriptor(target, { + uid: 501, + gid: 20, + }), + ).toThrow(/unexpected Hermes managed config descriptor/u); + expect(fs.statSync(target).mode & 0o777).toBe(0o600); + }); + + it("detects a path replacement while normalizing through the trusted descriptor", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, "config.yaml"); + const displaced = path.join(directory, "displaced.yaml"); + const replacement = path.join(directory, "replacement.yaml"); + fs.writeFileSync(target, "model: managed\n", { mode: 0o600 }); + fs.writeFileSync(replacement, "model: replaced\n", { mode: 0o640 }); + mockDescriptorOwnership(501n, 20n); + const realFchmodSync = fs.fchmodSync.bind(fs); + vi.spyOn(fs, "fchmodSync").mockImplementation((descriptor, mode) => { + realFchmodSync(descriptor, mode); + fs.renameSync(target, displaced); + fs.renameSync(replacement, target); + }); + + expect(() => + normalizeHermesManagedConfigDescriptor(target, { + uid: 501, + gid: 20, + }), + ).toThrow(/changed during normalization/u); + expect(fs.readFileSync(target, "utf8")).toBe("model: replaced\n"); + }); +}); diff --git a/src/lib/onboard/managed-startup-profile.test.ts b/src/lib/onboard/managed-startup-profile.test.ts index 256c77bb0b2..21f60a6a419 100644 --- a/src/lib/onboard/managed-startup-profile.test.ts +++ b/src/lib/onboard/managed-startup-profile.test.ts @@ -277,6 +277,8 @@ const STOCK_RUNTIME_INPUT_AGENTS = { HTTP_PROXY: MANAGED_STARTUP_AGENTS, NEMOCLAW_AUTO_PAIR_DEADLINE_SECS: ["openclaw"], NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS: ["openclaw"], + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: ["openclaw"], + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: ["openclaw"], NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS: ["openclaw"], NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS: ["openclaw"], NEMOCLAW_DASHBOARD_BIND: ["openclaw"], @@ -504,8 +506,8 @@ describe("managed startup profile", () => { ).toEqual({ NEMOCLAW_AUTO_PAIR_DEADLINE_SECS: "managed-launch-forwarded", NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS: "managed-launch-forwarded", - NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: "image-consumed-not-forwarded", - NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "image-consumed-not-forwarded", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: "managed-launch-forwarded", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "managed-launch-forwarded", NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS: "managed-launch-forwarded", NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS: "managed-launch-forwarded", }); diff --git a/src/lib/onboard/managed-startup/agent-environment.ts b/src/lib/onboard/managed-startup/agent-environment.ts index a154667b24b..fe34d8dae73 100644 --- a/src/lib/onboard/managed-startup/agent-environment.ts +++ b/src/lib/onboard/managed-startup/agent-environment.ts @@ -5,6 +5,7 @@ import { Buffer } from "node:buffer"; import { parseSandboxMessagingPlan } from "../../messaging/plan-validation"; import { + MANAGED_STARTUP_RUNTIME_CLEANUP_OBLIGATIONS, type ManagedStartupAgent, type ManagedStartupDashboard, type ManagedStartupProfile, @@ -47,6 +48,13 @@ export type ManagedStartupAgentMaterial = | ManagedStartupCorporateCaMaterial | ManagedStartupRootOwnedFileMaterial; +export interface ManagedStartupApplicationRuntimePlan { + /** Validated launch-only values that remain available to image setup and the agent runtime. */ + readonly exportEnvironment: Readonly>; + /** Ambient launch values that are unsupported for the selected agent and must be removed. */ + readonly unsetEnvironment: readonly string[]; +} + export interface ManagedStartupGenerateConfigAction { readonly kind: "generate-agent-config"; readonly agent: ManagedStartupConfigAgent; @@ -107,6 +115,7 @@ export interface ManagedStartupAgentEnvironment { * agent runtime adapters after generated configuration is committed. */ readonly runtimeEnvironment: Readonly>; + readonly applicationRuntime: ManagedStartupApplicationRuntimePlan; readonly materials: readonly ManagedStartupAgentMaterial[]; readonly actions: readonly ManagedStartupAgentAction[]; } @@ -119,6 +128,17 @@ export class ManagedStartupAgentEnvironmentError extends Error { } type MutableEnvironment = Record; +type ApplicationEnvironment = Readonly>; +const EMPTY_APPLICATION_ENVIRONMENT: ApplicationEnvironment = Object.freeze({}); + +const OPENCLAW_APPLICATION_RUNTIME_INPUTS = Object.freeze([ + ["NEMOCLAW_AUTO_PAIR_DEADLINE_SECS", "positive-finite-seconds"], + ["NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS", "positive-finite-seconds"], + ["NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS", "positive-finite-seconds"], + ["NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS", "positive-safe-integer"], + ["NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS", "positive-finite-seconds"], + ["NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS", "positive-finite-seconds"], +] as const); function booleanFlag(value: boolean): "0" | "1" { return value ? "1" : "0"; @@ -149,6 +169,58 @@ function sortedEnvironment(environment: MutableEnvironment): Readonly 0 + : Number.isFinite(value) && value > 0; + if (!valid) { + throw new ManagedStartupAgentEnvironmentError( + `${name} must be ${ + kind === "positive-safe-integer" ? "a positive safe integer" : "finite positive seconds" + }`, + ); + } + return String(value); +} + +function applicationRuntimePlan( + profile: ManagedStartupProfile, + environment: ApplicationEnvironment, +): ManagedStartupApplicationRuntimePlan { + const exportEnvironment: MutableEnvironment = {}; + if (profile.agent === "openclaw") { + for (const [name, kind] of OPENCLAW_APPLICATION_RUNTIME_INPUTS) { + const raw = environment[name]; + if (raw !== undefined) { + exportEnvironment[name] = canonicalApplicationRuntimeValue(name, raw, kind); + } + } + } + const unsetEnvironment = new Set( + MANAGED_STARTUP_RUNTIME_CLEANUP_OBLIGATIONS.filter( + ({ supportedFor }) => !supportedFor.includes(profile.agent), + ).map(({ input }) => input), + ); + if (profile.agent !== "openclaw") { + for (const [name] of OPENCLAW_APPLICATION_RUNTIME_INPUTS) { + unsetEnvironment.add(name); + } + } + return Object.freeze({ + exportEnvironment: sortedEnvironment(exportEnvironment), + unsetEnvironment: Object.freeze([...unsetEnvironment].sort()), + }); +} + function commonConfigurationEnvironment(profile: ManagedStartupProfile): MutableEnvironment { return { NEMOCLAW_INFERENCE_API: profile.inference.api, @@ -272,7 +344,10 @@ function applicationActions( return Object.freeze(actions); } -function mapOpenClawProfile(profile: ManagedStartupProfile): ManagedStartupAgentEnvironment { +function mapOpenClawProfile( + profile: ManagedStartupProfile, + environment: ApplicationEnvironment, +): ManagedStartupAgentEnvironment { if ( profile.agent !== "openclaw" || profile.agentConfig.agent !== "openclaw" || @@ -327,12 +402,16 @@ function mapOpenClawProfile(profile: ManagedStartupProfile): ManagedStartupAgent agent: profile.agent, configurationEnvironment: sortedEnvironment(configurationEnvironment), runtimeEnvironment: sortedEnvironment(runtimeEnvironment), + applicationRuntime: applicationRuntimePlan(profile, environment), materials: Object.freeze([corporateCaMaterial(profile)]), actions: applicationActions(profile, "openclaw"), }); } -function mapHermesProfile(profile: ManagedStartupProfile): ManagedStartupAgentEnvironment { +function mapHermesProfile( + profile: ManagedStartupProfile, + environment: ApplicationEnvironment, +): ManagedStartupAgentEnvironment { if ( profile.agent !== "hermes" || profile.agentConfig.agent !== "hermes" || @@ -373,12 +452,16 @@ function mapHermesProfile(profile: ManagedStartupProfile): ManagedStartupAgentEn agent: profile.agent, configurationEnvironment: sortedEnvironment(configurationEnvironment), runtimeEnvironment: sortedEnvironment(runtimeEnvironment), + applicationRuntime: applicationRuntimePlan(profile, environment), materials: Object.freeze([corporateCaMaterial(profile)]), actions: applicationActions(profile, "hermes"), }); } -function mapDcodeProfile(profile: ManagedStartupProfile): ManagedStartupAgentEnvironment { +function mapDcodeProfile( + profile: ManagedStartupProfile, + environment: ApplicationEnvironment, +): ManagedStartupAgentEnvironment { if ( profile.agent !== "langchain-deepagents-code" || profile.agentConfig.agent !== "langchain-deepagents-code" || @@ -442,6 +525,7 @@ function mapDcodeProfile(profile: ManagedStartupProfile): ManagedStartupAgentEnv agent: profile.agent, configurationEnvironment: sortedEnvironment(configurationEnvironment), runtimeEnvironment: sortedEnvironment(runtimeEnvironment), + applicationRuntime: applicationRuntimePlan(profile, environment), materials, actions: applicationActions(profile, null), }); @@ -455,14 +539,15 @@ function mapDcodeProfile(profile: ManagedStartupProfile): ManagedStartupAgentEnv */ export function mapManagedStartupProfileToAgentEnvironment( profile: ManagedStartupProfile, + environment: ApplicationEnvironment = EMPTY_APPLICATION_ENVIRONMENT, ): ManagedStartupAgentEnvironment { const validated = validateManagedStartupProfile(profile); switch (validated.agent) { case "openclaw": - return mapOpenClawProfile(validated); + return mapOpenClawProfile(validated, environment); case "hermes": - return mapHermesProfile(validated); + return mapHermesProfile(validated, environment); case "langchain-deepagents-code": - return mapDcodeProfile(validated); + return mapDcodeProfile(validated, environment); } } diff --git a/src/lib/onboard/managed-startup/image-runtime.ts b/src/lib/onboard/managed-startup/image-runtime.ts index 1c2f0c8af10..37642a97493 100644 --- a/src/lib/onboard/managed-startup/image-runtime.ts +++ b/src/lib/onboard/managed-startup/image-runtime.ts @@ -1,16 +1,49 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +import { + type ManagedStartupAgentEnvironment, + type ManagedStartupAgentMaterial, + type ManagedStartupApplicationRuntimePlan, + mapManagedStartupProfileToAgentEnvironment, +} from "./agent-environment"; +import { + coordinateManagedStartupApplication, + type ManagedStartupAdapterContext, + type ManagedStartupAgentAdapter, +} from "./coordinator"; import { + decodeManagedStartupProfile, MANAGED_STARTUP_AGENTS, type ManagedStartupAgent, type ManagedStartupDashboard, } from "./profile"; +import { MANAGED_STARTUP_CA_ENV, MANAGED_STARTUP_PROFILE_ENV } from "./transport"; + +export { MANAGED_STARTUP_CA_ENV, MANAGED_STARTUP_PROFILE_ENV } from "./transport"; +export const MANAGED_STARTUP_RUNTIME_ENV_FILE = "/run/nemoclaw/managed-startup-runtime.env"; +export const MANAGED_STARTUP_RUNTIME_EXECUTABLE = + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs"; +export const MANAGED_STARTUP_MERGED_CA_FILE = "/run/nemoclaw/managed-startup-ca-bundle.pem"; + +const MANAGED_STARTUP_CORPORATE_CA_FILE = "/usr/local/share/nemoclaw/corporate-ca.pem"; +const MESSAGING_RUNTIME_PLAN_FILE = "/usr/local/share/nemoclaw/messaging-runtime-plan.json"; +const ROOT_STATE_PARENT = "/var/lib/nemoclaw"; +const ROOT_RUNTIME_DIRECTORY = "/run/nemoclaw"; +const ROOT_OWNED_DIRECTORY_MODE = 0o755; +const MAX_TRUST_BUNDLE_BYTES = 4 * 1024 * 1024; +const HERMES_MANAGED_CONFIG_FILES = [ + "/sandbox/.hermes/config.yaml", + "/sandbox/.hermes/.env", +] as const; +const FIXED_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; +const SHA256_RE = /^[a-f0-9]{64}$/u; -/** - * This module owns only pure managed-image command construction. It does not - * execute commands, mutate sandbox state, or activate a compute driver. - */ export type ManagedStartupImageIdentity = "root" | "sandbox"; export type ManagedStartupMessagingAgent = "openclaw" | "hermes"; @@ -70,6 +103,13 @@ export interface ManagedStartupImageActionCommand { readonly argv: readonly string[]; } +export interface ManagedStartupImageApplyResult { + readonly agent: ManagedStartupAgent; + readonly adapterApplied: boolean; + readonly fingerprint: string; + readonly runtimeEnvironmentFile: string; +} + export class ManagedStartupImageActionPlanError extends Error { constructor(message: string) { super(`Cannot build managed startup image action plan: ${message}`); @@ -77,10 +117,97 @@ export class ManagedStartupImageActionPlanError extends Error { } } -function fail(message: string): never { +export class ManagedStartupImageRuntimeError extends Error { + constructor(message: string) { + super(`Managed startup image application failed: ${message}`); + this.name = "ManagedStartupImageRuntimeError"; + } +} + +type Environment = Record; + +interface CommandResult { + readonly status: number; + readonly stdout: string; + readonly stderr: string; +} + +function failActionPlan(message: string): never { throw new ManagedStartupImageActionPlanError(message); } +function exactActionPlanAgent(value: string): ManagedStartupAgent { + if ((MANAGED_STARTUP_AGENTS as readonly string[]).includes(value)) { + return value as ManagedStartupAgent; + } + return failActionPlan(`unsupported agent ${JSON.stringify(value)}`); +} + +function fail(message: string): never { + throw new ManagedStartupImageRuntimeError(message); +} + +export function validateManagedStartupApplicationRuntimePlan( + plan: ManagedStartupApplicationRuntimePlan, +): ManagedStartupApplicationRuntimePlan { + if (typeof plan !== "object" || plan === null) { + return fail("application runtime plan must be an object"); + } + const exportEnvironment = plan.exportEnvironment; + const unsetEnvironment = plan.unsetEnvironment; + if ( + typeof exportEnvironment !== "object" || + exportEnvironment === null || + Array.isArray(exportEnvironment) || + !Array.isArray(unsetEnvironment) + ) { + return fail("application runtime plan must contain exports and unsets"); + } + const exports: Record = {}; + for (const [name, value] of Object.entries(exportEnvironment)) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(name)) { + return fail(`invalid application runtime environment key ${JSON.stringify(name)}`); + } + if (typeof value !== "string" || value.includes("\0") || /[\r\n]/u.test(value)) { + return fail(`application runtime environment value for ${name} must be single-line text`); + } + exports[name] = value; + } + const unsets = new Set(); + for (const name of unsetEnvironment) { + if (typeof name !== "string" || !/^[A-Za-z_][A-Za-z0-9_]*$/u.test(name)) { + return fail(`invalid application runtime unset ${JSON.stringify(name)}`); + } + if (unsets.has(name)) { + return fail(`duplicate application runtime unset ${name}`); + } + if (Object.hasOwn(exports, name)) { + return fail(`application runtime cannot both export and unset ${name}`); + } + unsets.add(name); + } + return Object.freeze({ + exportEnvironment: Object.freeze( + Object.fromEntries( + Object.entries(exports).sort(([left], [right]) => left.localeCompare(right)), + ), + ), + unsetEnvironment: Object.freeze([...unsets].sort()), + }); +} + +export function applyManagedStartupCommandEnvironmentPlan( + environment: Readonly, + plan: ManagedStartupApplicationRuntimePlan, +): NodeJS.ProcessEnv { + const validated = validateManagedStartupApplicationRuntimePlan(plan); + const applied: NodeJS.ProcessEnv = { ...environment }; + for (const name of [...Object.keys(validated.exportEnvironment), ...validated.unsetEnvironment]) { + delete applied[name]; + } + return applied; +} + function exactAgent(value: string): ManagedStartupAgent { if ((MANAGED_STARTUP_AGENTS as readonly string[]).includes(value)) { return value as ManagedStartupAgent; @@ -88,6 +215,240 @@ function exactAgent(value: string): ManagedStartupAgent { return fail(`unsupported agent ${JSON.stringify(value)}`); } +function requireRoot(): void { + if (process.geteuid?.() !== 0) { + fail("managed startup requires container effective uid 0"); + } +} + +function modeOf(stat: fs.Stats): number { + return stat.mode & 0o777; +} + +function requireRootOwnedDirectory(target: string, mode: number): void { + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch { + fail(`required root-owned directory is missing: ${target}`); + } + if ( + stat.isSymbolicLink() || + !stat.isDirectory() || + stat.uid !== 0 || + stat.gid !== 0 || + modeOf(stat) !== mode + ) { + fail(`${target} must be a root:root directory with mode ${mode.toString(8)}`); + } +} + +function ensureRootOwnedDirectory(target: string, mode = ROOT_OWNED_DIRECTORY_MODE): void { + const parent = path.dirname(target); + const parentStat = fs.lstatSync(parent); + if ( + parentStat.isSymbolicLink() || + !parentStat.isDirectory() || + parentStat.uid !== 0 || + parentStat.gid !== 0 || + (modeOf(parentStat) & 0o022) !== 0 + ) { + fail(`refusing unsafe parent directory for ${target}`); + } + try { + fs.mkdirSync(target, { mode }); + fs.chownSync(target, 0, 0); + fs.chmodSync(target, mode); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") { + fail(`could not create ${target}`); + } + } + requireRootOwnedDirectory(target, mode); +} + +function requireSafeExistingRootTarget(target: string): void { + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + fail(`could not inspect ${target}`); + } + if ( + stat.isSymbolicLink() || + !stat.isFile() || + stat.nlink !== 1 || + stat.uid !== 0 || + stat.gid !== 0 + ) { + fail(`refusing to replace unsafe root-owned file ${target}`); + } +} + +function atomicWriteRootFile(target: string, contents: string | Buffer, mode: number): void { + const parent = path.dirname(target); + const parentStat = fs.lstatSync(parent); + if ( + parentStat.isSymbolicLink() || + !parentStat.isDirectory() || + parentStat.uid !== 0 || + parentStat.gid !== 0 || + (modeOf(parentStat) & 0o022) !== 0 + ) { + fail(`refusing unsafe root-owned file parent ${parent}`); + } + requireSafeExistingRootTarget(target); + const temporary = path.join( + parent, + `.${path.basename(target)}.${randomBytes(12).toString("hex")}`, + ); + let descriptor: number | undefined; + try { + descriptor = fs.openSync( + temporary, + fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW, + 0o600, + ); + fs.fchownSync(descriptor, 0, 0); + fs.writeFileSync(descriptor, contents); + fs.fchmodSync(descriptor, mode); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = undefined; + fs.renameSync(temporary, target); + } catch (error) { + if (descriptor !== undefined) fs.closeSync(descriptor); + try { + fs.unlinkSync(temporary); + } catch { + // Preserve the primary write failure. + } + fail(`could not atomically write ${target}: ${(error as Error).message}`); + } + const stat = fs.lstatSync(target); + if ( + stat.isSymbolicLink() || + !stat.isFile() || + stat.nlink !== 1 || + stat.uid !== 0 || + stat.gid !== 0 || + modeOf(stat) !== mode + ) { + fail(`root-owned output failed metadata verification: ${target}`); + } +} + +function removeSafeRootFile(target: string): void { + requireSafeExistingRootTarget(target); + try { + fs.unlinkSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + fail(`could not remove ${target}`); + } + } +} + +function trustedExecutable(target: string): boolean { + try { + const stat = fs.lstatSync(target); + return ( + !stat.isSymbolicLink() && + stat.isFile() && + stat.uid === 0 && + stat.gid === 0 && + (modeOf(stat) & 0o022) === 0 && + (modeOf(stat) & 0o111) !== 0 + ); + } catch { + return false; + } +} + +function readSandboxIdentity(): { readonly uid: string; readonly gid: string } { + const readId = (flag: "-u" | "-g"): string => { + const result = spawnSync("/usr/bin/id", [flag, "sandbox"], { + encoding: "utf8", + env: { PATH: FIXED_PATH }, + }); + const value = result.stdout.trim(); + if (result.status !== 0 || !/^[1-9][0-9]*$/u.test(value)) { + fail("could not resolve the sandbox account"); + } + return value; + }; + return { uid: readId("-u"), gid: readId("-g") }; +} + +function sandboxPrefix(): readonly string[] { + if (trustedExecutable("/usr/local/bin/gosu")) { + return ["/usr/local/bin/gosu", "sandbox"]; + } + if (trustedExecutable("/usr/bin/setpriv")) { + const identity = readSandboxIdentity(); + return [ + "/usr/bin/setpriv", + `--reuid=${identity.uid}`, + `--regid=${identity.gid}`, + "--init-groups", + "--", + ]; + } + return fail("a trusted gosu or setpriv executable is required"); +} + +function commandEnvironment( + configurationEnvironment: Readonly>, + applicationRuntime: ManagedStartupApplicationRuntimePlan, +): NodeJS.ProcessEnv { + const env = applyManagedStartupCommandEnvironmentPlan( + { + ...process.env, + ...configurationEnvironment, + HOME: "/sandbox", + PATH: FIXED_PATH, + NPM_CONFIG_OFFLINE: "true", + npm_config_offline: "true", + PIP_DISABLE_PIP_VERSION_CHECK: "1", + PIP_NO_INDEX: "1", + UV_OFFLINE: "1", + }, + applicationRuntime, + ); + delete env[MANAGED_STARTUP_PROFILE_ENV]; + delete env[MANAGED_STARTUP_CA_ENV]; + return env; +} + +function execute( + argv: readonly string[], + runAs: ManagedStartupImageIdentity, + configurationEnvironment: Readonly>, + applicationRuntime: ManagedStartupApplicationRuntimePlan, + capture = false, +): CommandResult { + if (argv.length === 0) fail("refusing an empty managed startup command"); + const command = runAs === "sandbox" ? [...sandboxPrefix(), ...argv] : [...argv]; + const result = spawnSync(command[0] as string, command.slice(1), { + encoding: "utf8", + env: commandEnvironment(configurationEnvironment, applicationRuntime), + stdio: capture ? "pipe" : "inherit", + }); + if (result.error) { + fail(`could not execute ${argv[0]}: ${result.error.message}`); + } + if (result.status !== 0) { + const detail = capture ? `: ${(result.stderr || result.stdout).trim()}` : ""; + fail(`${argv[0]} exited with status ${String(result.status ?? "unknown")}${detail}`); + } + return { + status: result.status, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + }; +} + function generatorCommand(agent: ManagedStartupAgent): readonly string[] { switch (agent) { case "openclaw": @@ -132,19 +493,19 @@ function assertActionAgent( actionAgent: ManagedStartupAgent, ): void { if (inputAgent !== actionAgent) { - fail(`action for ${actionAgent} cannot be used by ${inputAgent}`); + failActionPlan(`action for ${actionAgent} cannot be used by ${inputAgent}`); } } /** * Convert the closed application-action vocabulary into immutable image - * commands. The vocabulary deliberately cannot express agent installation, - * package-manager access, command execution, or runtime activation. + * commands. The vocabulary cannot express agent installation, package-manager + * access, arbitrary command execution, or runtime activation. */ export function buildManagedStartupImageActionPlan( input: ManagedStartupImageActionPlanInput, ): readonly ManagedStartupImageActionCommand[] { - const inputAgent = exactAgent(input.agent); + const inputAgent = exactActionPlanAgent(input.agent); const commands: ManagedStartupImageActionCommand[] = []; let dashboardActions = 0; let generateActions = 0; @@ -155,15 +516,17 @@ export function buildManagedStartupImageActionPlan( switch (action.kind) { case "configure-dashboard": { if (action.dashboard.agent !== input.agent) { - fail(`dashboard for ${action.dashboard.agent} cannot be used by ${input.agent}`); + failActionPlan( + `dashboard for ${action.dashboard.agent} cannot be used by ${input.agent}`, + ); } dashboardActions += 1; break; } case "generate-agent-config": { - assertActionAgent(inputAgent, exactAgent(action.agent)); + assertActionAgent(inputAgent, exactActionPlanAgent(action.agent)); if (action.runAs !== "sandbox") { - fail("agent configuration generation must run as sandbox"); + failActionPlan("agent configuration generation must run as sandbox"); } generateActions += 1; commands.push({ @@ -174,13 +537,13 @@ export function buildManagedStartupImageActionPlan( break; } case "apply-messaging-plan": { - assertActionAgent(inputAgent, exactAgent(action.agent)); + assertActionAgent(inputAgent, exactActionPlanAgent(action.agent)); if (action.mode !== "apply" && action.mode !== "clear") { - fail("messaging intent must be apply or clear"); + failActionPlan("messaging intent must be apply or clear"); } if (action.phase === "runtime-setup") { if (action.runAs !== "root") { - fail("messaging runtime setup must run as root"); + failActionPlan("messaging runtime setup must run as root"); } runtimeMessagingActions += 1; commands.push({ @@ -190,7 +553,7 @@ export function buildManagedStartupImageActionPlan( }); } else if (action.phase === "post-agent-install") { if (action.runAs !== "sandbox") { - fail("messaging post-agent configuration must run as sandbox"); + failActionPlan("messaging post-agent configuration must run as sandbox"); } postMessagingActions += 1; commands.push({ @@ -199,23 +562,27 @@ export function buildManagedStartupImageActionPlan( argv: messagingCommand(action.agent, action.phase), }); } else { - fail("unsupported messaging construction phase"); + failActionPlan("unsupported messaging construction phase"); } break; } default: - fail("unsupported managed startup construction action"); + failActionPlan("unsupported managed startup construction action"); } } - if (dashboardActions !== 1) fail("exactly one dashboard construction action is required"); - if (generateActions !== 1) fail("exactly one agent config construction action is required"); + if (dashboardActions !== 1) { + failActionPlan("exactly one dashboard construction action is required"); + } + if (generateActions !== 1) { + failActionPlan("exactly one agent config construction action is required"); + } const expectedMessagingActions = inputAgent === "langchain-deepagents-code" ? 0 : 1; if ( runtimeMessagingActions !== expectedMessagingActions || postMessagingActions !== expectedMessagingActions ) { - fail( + failActionPlan( `${inputAgent} requires ${String(expectedMessagingActions)} action for each messaging phase`, ); } @@ -224,7 +591,7 @@ export function buildManagedStartupImageActionPlan( ? ["generate-agent-config"] : ["messaging-runtime-setup", "generate-agent-config", "messaging-post-agent-install"]; if (commands.some((command, index) => command.action !== expectedOrder[index])) { - fail(`${inputAgent} image actions are not in the required construction order`); + failActionPlan(`${inputAgent} image actions are not in the required construction order`); } return Object.freeze( @@ -236,3 +603,681 @@ export function buildManagedStartupImageActionPlan( ), ); } + +function prepareMessagingRuntimeTarget(mode: "apply" | "clear"): void { + if (mode === "clear") { + removeSafeRootFile(MESSAGING_RUNTIME_PLAN_FILE); + return; + } + requireSafeExistingRootTarget(MESSAGING_RUNTIME_PLAN_FILE); + try { + fs.unlinkSync(MESSAGING_RUNTIME_PLAN_FILE); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + fail("could not prepare the messaging runtime-plan target"); + } + } +} + +function verifyMessagingRuntimeTarget(mode: "apply" | "clear"): void { + if (mode === "clear") { + if (fs.existsSync(MESSAGING_RUNTIME_PLAN_FILE)) { + fail("clear messaging profile left a runtime-plan artifact"); + } + return; + } + const stat = fs.lstatSync(MESSAGING_RUNTIME_PLAN_FILE); + if ( + stat.isSymbolicLink() || + !stat.isFile() || + stat.nlink !== 1 || + stat.uid !== 0 || + stat.gid !== 0 || + modeOf(stat) !== 0o644 + ) { + fail("messaging runtime-plan artifact failed root ownership validation"); + } +} + +function runInternalSandboxAction( + action: "write-openclaw-hash" | "write-hermes-compat-hash", + configurationEnvironment: Readonly>, + applicationRuntime: ManagedStartupApplicationRuntimePlan, + extraEnvironment: Readonly> = {}, +): void { + execute( + ["/usr/local/bin/node", MANAGED_STARTUP_RUNTIME_EXECUTABLE, `--internal-${action}`], + "sandbox", + { ...configurationEnvironment, ...extraEnvironment }, + applicationRuntime, + ); +} + +function sealOpenClawConfiguration( + configurationEnvironment: Readonly>, + applicationRuntime: ManagedStartupApplicationRuntimePlan, +): void { + const validation = execute( + ["/usr/local/bin/openclaw", "config", "validate", "--json"], + "sandbox", + { + ...configurationEnvironment, + OPENCLAW_CONFIG_PATH: "/sandbox/.openclaw/openclaw.json", + }, + applicationRuntime, + true, + ); + let parsed: unknown; + try { + parsed = JSON.parse(validation.stdout); + } catch { + fail("OpenClaw config validation did not emit JSON"); + } + if ( + typeof parsed !== "object" || + parsed === null || + (parsed as Record).valid !== true + ) { + fail("OpenClaw rejected the generated managed startup config"); + } + runInternalSandboxAction("write-openclaw-hash", configurationEnvironment, applicationRuntime); +} + +interface StableRegularFile { + readonly bytes: Buffer; + readonly stat: fs.BigIntStats; +} + +interface NumericIdentity { + readonly uid: number; + readonly gid: number; +} + +function sameStableFileMetadata(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.mode === right.mode && + left.nlink === right.nlink && + left.uid === right.uid && + left.gid === right.gid && + left.size === right.size && + left.mtimeNs === right.mtimeNs && + left.ctimeNs === right.ctimeNs + ); +} + +function readStableRegularFileSnapshot(target: string, maxBytes: number): StableRegularFile { + if (typeof fs.constants.O_NOFOLLOW !== "number") { + fail("O_NOFOLLOW is unavailable for managed startup file reads"); + } + const nonblock = typeof fs.constants.O_NONBLOCK === "number" ? fs.constants.O_NONBLOCK : 0; + let descriptor: number; + try { + descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | nonblock); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") throw error; + fail(`refusing unsafe or unreadable file ${target}`); + } + + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + before.size < 1n || + before.size > BigInt(maxBytes) + ) { + fail(`refusing unsafe or oversized file ${target}`); + } + + const bytes = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < bytes.length) { + const bytesRead = fs.readSync(descriptor, bytes, offset, bytes.length - offset, offset); + if (bytesRead === 0) break; + offset += bytesRead; + } + const overflow = Buffer.alloc(1); + const overflowBytes = fs.readSync(descriptor, overflow, 0, 1, offset); + const after = fs.fstatSync(descriptor, { bigint: true }); + if (offset !== bytes.length || overflowBytes !== 0 || !sameStableFileMetadata(before, after)) { + fail(`${target} changed while it was read`); + } + return { bytes, stat: before }; + } finally { + try { + fs.closeSync(descriptor); + } catch { + fail(`could not close safely opened file ${target}`); + } + } +} + +export function readStableRegularFile(target: string, maxBytes: number): Buffer { + return readStableRegularFileSnapshot(target, maxBytes).bytes; +} + +/** + * Restore the mutable Hermes image contract after its sandbox-side generator + * atomically replaces config.yaml or .env with mode 0600. The mode transition + * is performed through the already-authenticated descriptor, never by path. + * + * Shields-up turns these files into root:root 0444 trust anchors. That state is + * valid on an already-committed replay and must not be made mutable again. + */ +export function normalizeHermesManagedConfigDescriptor( + target: string, + sandboxIdentity: NumericIdentity, +): void { + if ( + !Number.isSafeInteger(sandboxIdentity.uid) || + sandboxIdentity.uid <= 0 || + !Number.isSafeInteger(sandboxIdentity.gid) || + sandboxIdentity.gid <= 0 + ) { + fail("invalid sandbox identity for Hermes descriptor normalization"); + } + if (typeof fs.constants.O_NOFOLLOW !== "number") { + fail("O_NOFOLLOW is unavailable for Hermes descriptor normalization"); + } + const nonblock = typeof fs.constants.O_NONBLOCK === "number" ? fs.constants.O_NONBLOCK : 0; + let descriptor: number; + try { + descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | nonblock); + } catch { + fail(`refusing unsafe Hermes managed config descriptor ${target}`); + } + + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + const beforeMode = Number(before.mode & 0o777n); + const mutable = + before.uid === BigInt(sandboxIdentity.uid) && + before.gid === BigInt(sandboxIdentity.gid) && + (beforeMode === 0o600 || beforeMode === 0o640); + const shielded = before.uid === 0n && before.gid === 0n && beforeMode === 0o444; + if (!before.isFile() || before.nlink !== 1n || (!mutable && !shielded)) { + fail(`refusing unexpected Hermes managed config descriptor ${target}`); + } + + const expectedMode = mutable ? 0o640 : 0o444; + if (mutable && beforeMode === 0o600) { + try { + fs.fchmodSync(descriptor, expectedMode); + } catch { + fail(`could not normalize Hermes managed config descriptor ${target}`); + } + } + + const after = fs.fstatSync(descriptor, { bigint: true }); + let pathAfter: fs.BigIntStats; + try { + pathAfter = fs.lstatSync(target, { bigint: true }); + } catch { + fail(`Hermes managed config descriptor disappeared during normalization: ${target}`); + } + const expectedUid = mutable ? BigInt(sandboxIdentity.uid) : 0n; + const expectedGid = mutable ? BigInt(sandboxIdentity.gid) : 0n; + if ( + !after.isFile() || + after.nlink !== 1n || + after.dev !== before.dev || + after.ino !== before.ino || + after.uid !== expectedUid || + after.gid !== expectedGid || + Number(after.mode & 0o777n) !== expectedMode || + after.size !== before.size || + after.mtimeNs !== before.mtimeNs || + pathAfter.isSymbolicLink() || + !pathAfter.isFile() || + !sameStableFileMetadata(after, pathAfter) + ) { + fail(`Hermes managed config descriptor changed during normalization: ${target}`); + } + } finally { + try { + fs.closeSync(descriptor); + } catch { + fail(`could not close Hermes managed config descriptor ${target}`); + } + } +} + +function normalizeHermesManagedConfiguration(): void { + const identity = readSandboxIdentity(); + const sandboxIdentity = { + uid: Number(identity.uid), + gid: Number(identity.gid), + }; + for (const target of HERMES_MANAGED_CONFIG_FILES) { + normalizeHermesManagedConfigDescriptor(target, sandboxIdentity); + } +} + +function sealHermesConfiguration( + configurationEnvironment: Readonly>, + applicationRuntime: ManagedStartupApplicationRuntimePlan, +): void { + const configPath = "/sandbox/.hermes/config.yaml"; + const envPath = "/sandbox/.hermes/.env"; + const config = readStableRegularFile(configPath, 4 * 1024 * 1024); + const env = readStableRegularFile(envPath, 512 * 1024); + const digest = execute( + [ + "/opt/hermes/.venv/bin/python3", + "-I", + "/usr/local/lib/nemoclaw/build-hermes-mcp-digest.py", + "--guard", + "/usr/local/lib/nemoclaw/hermes-runtime-config-guard.py", + "--config", + configPath, + ], + "root", + configurationEnvironment, + applicationRuntime, + true, + ).stdout.trim(); + if (!SHA256_RE.test(digest)) { + fail("Hermes MCP digest helper returned an invalid digest"); + } + const hashText = [ + `${createHash("sha256").update(config).digest("hex")} ${configPath}`, + `${createHash("sha256").update(env).digest("hex")} ${envPath}`, + `# nemoclaw-hermes-mcp-state-v1 intended=${digest} applied=${digest}`, + "", + ].join("\n"); + atomicWriteRootFile("/etc/nemoclaw/hermes.config-hash", hashText, 0o444); + runInternalSandboxAction( + "write-hermes-compat-hash", + configurationEnvironment, + applicationRuntime, + { + NEMOCLAW_MANAGED_HERMES_HASH_B64: Buffer.from(hashText, "utf8").toString("base64"), + }, + ); +} + +function installRootOwnedMaterials(materials: readonly ManagedStartupAgentMaterial[]): void { + for (const material of materials) { + if (material.kind !== "root-owned-file") continue; + if (material.owner !== "root" || material.group !== "root" || material.mode !== 0o444) { + fail(`unsupported root-owned material contract for ${material.path}`); + } + atomicWriteRootFile(material.path, material.contents, material.mode); + } +} + +function verifyRootOwnedMaterials(materials: readonly ManagedStartupAgentMaterial[]): void { + for (const material of materials) { + if (material.kind !== "root-owned-file") continue; + const expected = Buffer.from(material.contents, "utf8"); + const { bytes, stat } = readStableRegularFileSnapshot(material.path, expected.length); + if ( + stat.nlink !== 1n || + stat.uid !== 0n || + stat.gid !== 0n || + Number(stat.mode & 0o777n) !== material.mode || + !bytes.equals(expected) + ) { + fail(`committed root-owned material drifted: ${material.path}`); + } + } +} + +function installCorporateCa(corporateCaPath: string | null): void { + if (corporateCaPath === null) { + removeSafeRootFile(MANAGED_STARTUP_CORPORATE_CA_FILE); + return; + } + const bytes = readStableRegularFile(corporateCaPath, 128 * 1024); + atomicWriteRootFile(MANAGED_STARTUP_CORPORATE_CA_FILE, bytes, 0o444); +} + +function safeTrustBundle(target: string): Buffer | null { + try { + const { bytes, stat } = readStableRegularFileSnapshot(target, MAX_TRUST_BUNDLE_BYTES); + if (Number(stat.mode & 0o022n) !== 0) { + fail(`refusing unsafe trust bundle ${target}`); + } + return bytes; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } +} + +function mergeCorporateCa(corporateCaPath: string | null): boolean { + if (corporateCaPath === null) { + removeSafeRootFile(MANAGED_STARTUP_MERGED_CA_FILE); + return false; + } + const corporate = readStableRegularFile(corporateCaPath, 128 * 1024); + const candidates = [ + "/etc/openshell-tls/ca-bundle.pem", + process.env.SSL_CERT_FILE ?? "", + "/etc/ssl/certs/ca-certificates.crt", + ].filter( + (candidate, index, values) => + candidate && + candidate !== MANAGED_STARTUP_MERGED_CA_FILE && + values.indexOf(candidate) === index, + ); + let base: Buffer | null = null; + for (const candidate of candidates) { + base = safeTrustBundle(candidate); + if (base) break; + } + const merged = Buffer.concat([ + ...(base ? [base, Buffer.from("\n", "utf8")] : []), + corporate, + ...(corporate.at(-1) === 0x0a ? [] : [Buffer.from("\n", "utf8")]), + ]); + atomicWriteRootFile(MANAGED_STARTUP_MERGED_CA_FILE, merged, 0o444); + return true; +} + +function shellSingleQuote(value: string): string { + if (value.includes("\0") || /[\r\n]/u.test(value)) { + fail("runtime environment values must be single-line text"); + } + return `'${value.replaceAll("'", `'\"'\"'`)}'`; +} + +export function serializeManagedStartupRuntimeEnvironment( + environment: Readonly>, + corporateCaMerged: boolean, + configurationEnvironment: Readonly> = {}, + applicationRuntime: ManagedStartupApplicationRuntimePlan = { + exportEnvironment: {}, + unsetEnvironment: [], + }, +): string { + const validatedApplicationRuntime = + validateManagedStartupApplicationRuntimePlan(applicationRuntime); + const output: Record = { + ...environment, + ...validatedApplicationRuntime.exportEnvironment, + NEMOCLAW_MANAGED_STARTUP_APPLIED: "1", + }; + if (corporateCaMerged) { + for (const name of [ + "CURL_CA_BUNDLE", + "GIT_SSL_CAINFO", + "NODE_EXTRA_CA_CERTS", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_FILE", + ]) { + output[name] = MANAGED_STARTUP_MERGED_CA_FILE; + } + output._NEMOCLAW_CORPORATE_CA_MERGED = "1"; + } + const unsetNames = new Set([ + ...Object.keys(configurationEnvironment).filter((name) => !Object.hasOwn(output, name)), + ...validatedApplicationRuntime.unsetEnvironment, + ]); + for (const name of validatedApplicationRuntime.unsetEnvironment) { + if (Object.hasOwn(output, name)) { + fail(`runtime environment cannot both export and unset ${name}`); + } + } + const unsetLines = [...unsetNames].sort().map((name) => { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(name)) { + fail(`invalid runtime environment key ${JSON.stringify(name)}`); + } + return `unset ${name}`; + }); + const exportLines = Object.entries(output) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([name, value]) => { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(name)) { + fail(`invalid runtime environment key ${JSON.stringify(name)}`); + } + return `export ${name}=${shellSingleQuote(value)}`; + }); + return `${[...unsetLines, ...exportLines].join("\n")}\n`; +} + +function applyAdapter( + context: ManagedStartupAdapterContext, + mapped: ManagedStartupAgentEnvironment, +): void { + if (mapped.agent !== context.agent) { + fail(`mapped ${mapped.agent} environment for ${context.agent}`); + } + const commandPlan = buildManagedStartupImageActionPlan({ + agent: mapped.agent, + actions: mapped.actions, + }); + let commandIndex = 0; + for (const action of mapped.actions) { + if (action.kind === "configure-dashboard") continue; + const command = commandPlan[commandIndex]; + if (!command) fail(`missing image command for ${action.kind}`); + commandIndex += 1; + if (action.kind === "apply-messaging-plan") { + if (action.phase === "runtime-setup") { + prepareMessagingRuntimeTarget(action.mode); + } + execute( + command.argv, + command.runAs, + mapped.configurationEnvironment, + mapped.applicationRuntime, + ); + if (action.phase === "runtime-setup") { + verifyMessagingRuntimeTarget(action.mode); + } + continue; + } + execute( + command.argv, + command.runAs, + mapped.configurationEnvironment, + mapped.applicationRuntime, + ); + } + if (commandIndex !== commandPlan.length) { + fail("image action plan contains an unmatched command"); + } + + switch (context.agent) { + case "openclaw": + sealOpenClawConfiguration(mapped.configurationEnvironment, mapped.applicationRuntime); + break; + case "hermes": + sealHermesConfiguration(mapped.configurationEnvironment, mapped.applicationRuntime); + // Normalize before the coordinator commits a newly applied profile so + // the durable transaction never records generator-created 0600 files as + // a completed mutable image contract. + normalizeHermesManagedConfiguration(); + break; + case "langchain-deepagents-code": + break; + } + installRootOwnedMaterials(mapped.materials); + installCorporateCa(context.corporateCaPath); + mergeCorporateCa(context.corporateCaPath); +} + +function adapters(mapped: ManagedStartupAgentEnvironment): readonly ManagedStartupAgentAdapter[] { + return MANAGED_STARTUP_AGENTS.map((agent) => ({ + agent, + apply: (context: ManagedStartupAdapterContext) => applyAdapter(context, mapped), + })); +} + +export async function applyManagedStartupImageProfile( + expectedAgentInput: string, + env: Environment = process.env, +): Promise { + requireRoot(); + const expectedAgent = exactAgent(expectedAgentInput); + if (env.NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION !== "1") { + fail("startup profiles require a complete managed image"); + } + const encodedProfile = env[MANAGED_STARTUP_PROFILE_ENV]; + if (!encodedProfile) fail(`${MANAGED_STARTUP_PROFILE_ENV} is required`); + let profile; + try { + profile = decodeManagedStartupProfile(encodedProfile); + } catch (error) { + fail((error as Error).message); + } + if (profile.agent !== expectedAgent) { + fail(`managed startup profile targets ${profile.agent}, expected ${expectedAgent}`); + } + const mapped = mapManagedStartupProfileToAgentEnvironment(profile, env); + validateManagedStartupApplicationRuntimePlan(mapped.applicationRuntime); + + ensureRootOwnedDirectory(ROOT_STATE_PARENT); + ensureRootOwnedDirectory(ROOT_RUNTIME_DIRECTORY); + const result = await coordinateManagedStartupApplication( + { + encodedProfile, + expectedAgent, + ...(env[MANAGED_STARTUP_CA_ENV] === undefined + ? {} + : { corporateCaB64: env[MANAGED_STARTUP_CA_ENV] }), + }, + adapters(mapped), + ); + if (mapped.agent !== result.application.profile.agent) { + fail(`mapped ${mapped.agent} environment for ${result.application.profile.agent}`); + } + if (expectedAgent === "hermes" && !result.adapterApplied) { + // Committed startup replays still repair generator-created 0600 files, + // while the descriptor guard preserves root-owned shields-up files. + normalizeHermesManagedConfiguration(); + } + let corporateCaMerged: boolean; + if (result.adapterApplied) { + corporateCaMerged = result.application.corporateCaPath !== null; + } else { + verifyRootOwnedMaterials(mapped.materials); + if (result.application.corporateCaPath === null) { + if (fs.existsSync(MANAGED_STARTUP_CORPORATE_CA_FILE)) { + fail("committed profile without a corporate CA has a stale CA material"); + } + } else { + const expected = readStableRegularFile(result.application.corporateCaPath, 128 * 1024); + const installed = readStableRegularFile(MANAGED_STARTUP_CORPORATE_CA_FILE, 128 * 1024); + if (!expected.equals(installed)) { + fail("committed corporate CA material drifted"); + } + } + corporateCaMerged = mergeCorporateCa(result.application.corporateCaPath); + } + atomicWriteRootFile( + MANAGED_STARTUP_RUNTIME_ENV_FILE, + serializeManagedStartupRuntimeEnvironment( + mapped.runtimeEnvironment, + corporateCaMerged, + mapped.configurationEnvironment, + mapped.applicationRuntime, + ), + 0o400, + ); + return { + agent: expectedAgent, + adapterApplied: result.adapterApplied, + fingerprint: result.application.fingerprint, + runtimeEnvironmentFile: MANAGED_STARTUP_RUNTIME_ENV_FILE, + }; +} + +function writeSandboxFileAtomically(target: string, contents: string, mode: number): void { + const parent = path.dirname(target); + const parentStat = fs.lstatSync(parent); + if ( + parentStat.isSymbolicLink() || + !parentStat.isDirectory() || + parentStat.uid !== process.geteuid?.() || + parentStat.gid !== process.getegid?.() + ) { + fail(`refusing unsafe sandbox-owned directory ${parent}`); + } + const temporary = path.join( + parent, + `.${path.basename(target)}.${randomBytes(12).toString("hex")}`, + ); + let descriptor: number | undefined; + try { + descriptor = fs.openSync( + temporary, + fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW, + 0o600, + ); + fs.writeFileSync(descriptor, contents); + fs.fchmodSync(descriptor, mode); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = undefined; + fs.renameSync(temporary, target); + } catch (error) { + if (descriptor !== undefined) fs.closeSync(descriptor); + try { + fs.unlinkSync(temporary); + } catch { + // Preserve the primary write failure. + } + fail(`could not write sandbox-owned file ${target}: ${(error as Error).message}`); + } +} + +function internalWriteOpenClawHash(): void { + if (process.geteuid?.() === 0) fail("sandbox hash writer must not run as root"); + const configPath = "/sandbox/.openclaw/openclaw.json"; + const config = readStableRegularFile(configPath, 16 * 1024 * 1024); + const text = `${createHash("sha256").update(config).digest("hex")} openclaw.json\n`; + writeSandboxFileAtomically("/sandbox/.openclaw/.config-hash", text, 0o660); +} + +function internalWriteHermesCompatHash(): void { + if (process.geteuid?.() === 0) fail("sandbox hash writer must not run as root"); + const encoded = process.env.NEMOCLAW_MANAGED_HERMES_HASH_B64 ?? ""; + if ( + encoded.length === 0 || + encoded.length > 4096 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(encoded) + ) { + fail("Hermes compatibility hash transport is invalid"); + } + const decoded = Buffer.from(encoded, "base64"); + if (decoded.toString("base64") !== encoded) { + fail("Hermes compatibility hash transport is non-canonical"); + } + writeSandboxFileAtomically("/sandbox/.hermes/.config-hash", decoded.toString("utf8"), 0o640); +} + +function readCliAgent(argv: readonly string[]): string { + const index = argv.indexOf("--agent"); + if (index < 0 || index + 1 >= argv.length || argv.length !== 2) { + fail("usage: managed-startup-image-runtime --agent "); + } + return argv[index + 1] as string; +} + +export async function main(argv: readonly string[] = process.argv.slice(2)): Promise { + if (argv.length === 1 && argv[0] === "--internal-write-openclaw-hash") { + internalWriteOpenClawHash(); + return; + } + if (argv.length === 1 && argv[0] === "--internal-write-hermes-compat-hash") { + internalWriteHermesCompatHash(); + return; + } + const result = await applyManagedStartupImageProfile(readCliAgent(argv)); + console.log( + result.adapterApplied + ? `[managed-startup] applied ${result.agent} profile ${result.fingerprint}` + : `[managed-startup] ${result.agent} profile ${result.fingerprint} is already committed`, + ); +} + +if (require.main === module) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/src/lib/onboard/managed-startup/profile.ts b/src/lib/onboard/managed-startup/profile.ts index bacf4be6f3a..d57722afd4f 100644 --- a/src/lib/onboard/managed-startup/profile.ts +++ b/src/lib/onboard/managed-startup/profile.ts @@ -597,14 +597,12 @@ export const MANAGED_STARTUP_PROFILE_DEFERRED_RUNTIME_INPUTS = Object.freeze({ deferredRuntimeInput( "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS", "application-environment", - "the image consumes this documented scheduler control but managed launch must admit it in the application environment transaction", - "image-consumed-not-forwarded", + "operator scheduler tuning is applied by the application environment transaction", ), deferredRuntimeInput( "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS", "application-environment", - "the image consumes this documented scheduler control but managed launch must admit it in the application environment transaction", - "image-consumed-not-forwarded", + "operator scheduler tuning is applied by the application environment transaction", ), deferredRuntimeInput( "NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS", diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index e687681e3f4..d39c4611479 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -129,9 +129,10 @@ describe("prepareSandboxCreateLaunch", () => { env: { NEMOCLAW_AUTO_PAIR_DEADLINE_SECS: " 30 ", NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS: "3", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: " 0.25 ", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: " 99 ", NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS: "10", NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS: "600", - NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "99", NEMOCLAW_PROVIDER_KEY: "must-not-enter-the-sandbox", }, extraPlaceholderKeys: [], @@ -150,12 +151,11 @@ describe("prepareSandboxCreateLaunch", () => { "OPENCLAW_WORKSPACE_DIR=/sandbox/.openclaw/workspace", "NEMOCLAW_AUTO_PAIR_DEADLINE_SECS=30", "NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS=3", + "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS=0.25", + "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS=99", "NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS=10", "NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS=600", ]); - expect(result.sandboxStartupCommand.join(" ")).not.toContain( - "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS", - ); expect(result.sandboxStartupCommand.join(" ")).not.toContain("NEMOCLAW_PROVIDER_KEY"); }); @@ -164,7 +164,11 @@ describe("prepareSandboxCreateLaunch", () => { agent: loadAgent("hermes"), chatUiUrl: "http://127.0.0.1:18789/", createArgs: [], - env: { NEMOCLAW_AUTO_PAIR_DEADLINE_SECS: "30" }, + env: { + NEMOCLAW_AUTO_PAIR_DEADLINE_SECS: "30", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: "1", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "3", + }, extraPlaceholderKeys: [], getDashboardForwardPort: () => "18789", hermesDashboardState: { diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index 1e5431bb479..a0232e221bd 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -25,6 +25,8 @@ type OpenshellArgv = (args: string[]) => string[]; const OPENCLAW_AUTO_PAIR_RUNTIME_ENV_KEYS = [ "NEMOCLAW_AUTO_PAIR_DEADLINE_SECS", "NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS", + "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS", + "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS", "NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS", "NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS", ] as const; diff --git a/test/entrypoint-env-wrapper.test.ts b/test/entrypoint-env-wrapper.test.ts new file mode 100644 index 00000000000..8bd74015681 --- /dev/null +++ b/test/entrypoint-env-wrapper.test.ts @@ -0,0 +1,218 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import { sliceBlock } from "./helpers/corporate-ca-support"; + +const HELPER = path.join(import.meta.dirname, "..", "scripts", "lib", "entrypoint-env-wrapper.sh"); +const OPENCLAW_START = path.join(import.meta.dirname, "..", "scripts", "nemoclaw-start.sh"); + +function runNormalizer(argv: readonly string[]) { + const harness = [ + "set -euo pipefail", + 'helper="$1"', + "shift", + 'source "$helper"', + 'nemoclaw_normalize_entrypoint_env_wrapper "$@"', + 'if [ "$NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC" -eq 0 ]; then', + " set --", + "else", + ' set -- "${NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV[@]}"', + "fi", + `printf 'UID=%s\\n' "$(id -u)"`, + "printf 'PROFILE=%s\\n' \"${NEMOCLAW_STARTUP_PROFILE_B64-__UNSET__}\"", + "printf 'CA=%s\\n' \"${NEMOCLAW_CORPORATE_CA_B64-__UNSET__}\"", + "printf 'HTTP_PROXY=%s\\n' \"${HTTP_PROXY-__UNSET__}\"", + "printf 'NO_PROXY=%s\\n' \"${NO_PROXY-__UNSET__}\"", + "printf 'FAST_REENTRY_INTERVAL=%s\\n' \"${NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS-__UNSET__}\"", + "printf 'FAST_REENTRY_POLLS=%s\\n' \"${NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS-__UNSET__}\"", + `printf 'ARG=%s\\n' "$@"`, + ].join("\n"); + return spawnSync("/bin/bash", ["-c", harness, "entrypoint-env-wrapper-test", HELPER, ...argv], { + encoding: "utf8", + env: { + PATH: "/usr/local/bin:/usr/bin:/bin", + }, + }); +} + +describe("OCI entrypoint env-wrapper normalization", () => { + it("promotes the exact managed handoff before preserving the command tail", () => { + const uid = String(process.getuid?.() ?? ""); + const result = runNormalizer([ + "env", + "NEMOCLAW_STARTUP_PROFILE_B64=eyJzY2hlbWFWZXJzaW9uIjoxfQ", + "NEMOCLAW_CORPORATE_CA_B64=Y2E=", + "HTTP_PROXY=http://user:pass@proxy.example.test:18080", + "NO_PROXY=localhost,127.0.0.1", + "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS=0.25", + "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS=3", + "nemoclaw-start", + "/bin/sh", + "-c", + "printf managed command", + ]); + + expect(result.status).toBe(0); + expect(result.stdout).toContain(`UID=${uid}`); + expect(result.stdout).toContain("PROFILE=eyJzY2hlbWFWZXJzaW9uIjoxfQ"); + expect(result.stdout).toContain("CA=Y2E="); + expect(result.stdout).toContain("HTTP_PROXY=http://user:pass@proxy.example.test:18080"); + expect(result.stdout).toContain("NO_PROXY=localhost,127.0.0.1"); + expect(result.stdout).toContain("FAST_REENTRY_INTERVAL=0.25"); + expect(result.stdout).toContain("FAST_REENTRY_POLLS=3"); + expect(result.stdout).toContain("ARG=/bin/sh\nARG=-c\nARG=printf managed command\n"); + }); + + it("strips a direct self invocation without interpreting its command arguments", () => { + const result = runNormalizer(["/usr/local/bin/nemoclaw-start", "env", "FOO=bar", "printenv"]); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("PROFILE=__UNSET__"); + expect(result.stdout).toContain("ARG=env\nARG=FOO=bar\nARG=printenv\n"); + }); + + it("leaves an unrelated explicit env command untouched", () => { + const result = runNormalizer(["env", "FOO=bar", "printenv", "FOO"]); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("ARG=env\nARG=FOO=bar\nARG=printenv\nARG=FOO\n"); + }); + + it.each([ + { + argv: ["env", "NODE_OPTIONS=--require=/sandbox/untrusted.cjs", "nemoclaw-start"], + message: "unsupported variable 'NODE_OPTIONS'", + }, + { + argv: [ + "env", + "NEMOCLAW_STARTUP_PROFILE_B64=first", + "NEMOCLAW_STARTUP_PROFILE_B64=second", + "nemoclaw-start", + ], + message: "repeats variable 'NEMOCLAW_STARTUP_PROFILE_B64'", + }, + { + argv: ["env", "NEMOCLAW_STARTUP_PROFILE_B64=profile", "/usr/bin/true"], + message: "Malformed managed startup env wrapper", + }, + { + argv: ["env", "NEMOCLAW_CORPORATE_CA_B64=Y2E=", "not-an-assignment", "nemoclaw-start"], + message: "Malformed managed startup env wrapper", + }, + ])("fails closed for malformed or unsafe root handoff: $message", ({ argv, message }) => { + const result = runNormalizer(argv); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(message); + expect(result.stdout).toBe(""); + }); + + it("unwraps the sandbox-create env self-wrapper and applies dashboard port defaults", () => { + const normalizer = fs.readFileSync( + path.join(import.meta.dirname, "..", "scripts", "lib", "entrypoint-env-wrapper.sh"), + "utf-8", + ); + const openClawPortBlock = sliceBlock( + OPENCLAW_START, + 'NEMOCLAW_CMD=("$@")', + "# ── Config integrity check", + ); + const snippet = [ + normalizer, + 'nemoclaw_normalize_entrypoint_env_wrapper "$@"', + 'if [ "$NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC" -eq 0 ]; then set --; ' + + 'else set -- "${NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV[@]}"; fi', + openClawPortBlock, + ].join("\n"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-env-wrapper-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "run.sh"); + + function runScenario(setArgs: string, extraEnv: Record = {}) { + const script = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + setArgs, + snippet, + 'printf "CHAT_UI_URL=%s\\n" "$CHAT_UI_URL"', + 'printf "PUBLIC_PORT=%s\\n" "$PUBLIC_PORT"', + 'printf "OPENCLAW_GATEWAY_PORT=%s\\n" "$OPENCLAW_GATEWAY_PORT"', + 'printf "OPENCLAW_GATEWAY_URL=%s\\n" "$OPENCLAW_GATEWAY_URL"', + 'printf "SANDBOX_HOME=%s\\n" "$_SANDBOX_HOME"', + 'printf "OPENCLAW_HOME=%s\\n" "$OPENCLAW_HOME"', + 'printf "OPENCLAW_STATE_DIR=%s\\n" "$OPENCLAW_STATE_DIR"', + 'printf "OPENCLAW_CONFIG_PATH=%s\\n" "$OPENCLAW_CONFIG_PATH"', + 'printf "OPENCLAW_OAUTH_DIR=%s\\n" "$OPENCLAW_OAUTH_DIR"', + 'printf "CMD=%s\\n" "${NEMOCLAW_CMD[*]}"', + ].join("\n"); + fs.writeFileSync(scriptPath, script, { mode: 0o700 }); + return spawnSync("bash", [scriptPath], { + encoding: "utf-8", + timeout: 5000, + env: { ...process.env, PATH: `${fakeBin}:${process.env.PATH || ""}`, ...extraEnv }, + }); + } + + try { + fs.mkdirSync(fakeBin); + fs.writeFileSync(path.join(fakeBin, "openclaw"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + const injected = runScenario( + "set -- env CHAT_UI_URL=https://chat.example.test NEMOCLAW_DASHBOARD_PORT=19000 nemoclaw-start openclaw agent --agent main", + ); + expect(injected.status).toBe(0); + expect(injected.stdout).toContain("CHAT_UI_URL=http://127.0.0.1:19000"); + expect(injected.stdout).toContain("PUBLIC_PORT=19000"); + expect(injected.stdout).toContain("OPENCLAW_GATEWAY_PORT=19000"); + expect(injected.stdout).toContain("OPENCLAW_GATEWAY_URL=ws://127.0.0.1:19000"); + expect(injected.stdout).toContain("SANDBOX_HOME=/sandbox"); + expect(injected.stdout).toContain("OPENCLAW_HOME=/sandbox"); + expect(injected.stdout).toContain("OPENCLAW_STATE_DIR=/sandbox/.openclaw"); + expect(injected.stdout).toContain("OPENCLAW_CONFIG_PATH=/sandbox/.openclaw/openclaw.json"); + expect(injected.stdout).toContain("OPENCLAW_OAUTH_DIR=/sandbox/.openclaw/credentials"); + expect(injected.stdout).toContain("CMD=openclaw agent --agent main"); + + const bakedCustomPort = runScenario("set -- nemoclaw-start openclaw agent", { + CHAT_UI_URL: "http://127.0.0.1:18790", + }); + expect(bakedCustomPort.status).toBe(0); + expect(bakedCustomPort.stdout).toContain("CHAT_UI_URL=http://127.0.0.1:18790"); + expect(bakedCustomPort.stdout).toContain("PUBLIC_PORT=18790"); + expect(bakedCustomPort.stdout).toContain("OPENCLAW_GATEWAY_PORT=18790"); + expect(bakedCustomPort.stdout).toContain("OPENCLAW_GATEWAY_URL=ws://127.0.0.1:18790"); + expect(bakedCustomPort.stdout).toContain("OPENCLAW_STATE_DIR=/sandbox/.openclaw"); + expect(bakedCustomPort.stdout).toContain("OPENCLAW_OAUTH_DIR=/sandbox/.openclaw/credentials"); + expect(bakedCustomPort.stdout).toContain("CMD=openclaw agent"); + + const baked = runScenario("set -- nemoclaw-start openclaw agent", { + CHAT_UI_URL: "https://baked.example.test/ui", + }); + expect(baked.status).toBe(0); + expect(baked.stdout).toContain("CHAT_UI_URL=https://baked.example.test/ui"); + expect(baked.stdout).toContain("PUBLIC_PORT=18789"); + expect(baked.stdout).toContain("OPENCLAW_GATEWAY_PORT=18789"); + expect(baked.stdout).toContain("OPENCLAW_GATEWAY_URL=ws://127.0.0.1:18789"); + expect(baked.stdout).toContain("SANDBOX_HOME=/sandbox"); + expect(baked.stdout).toContain("OPENCLAW_STATE_DIR=/sandbox/.openclaw"); + expect(baked.stdout).toContain("CMD=openclaw agent"); + + const invalidHighPort = runScenario("set -- nemoclaw-start openclaw agent", { + NEMOCLAW_DASHBOARD_PORT: "70000", + }); + expect(invalidHighPort.status).toBe(1); + expect(invalidHighPort.stderr).toContain("Invalid NEMOCLAW_DASHBOARD_PORT='70000'"); + expect(invalidHighPort.stderr).toContain("must be an integer between 1024 and 65535"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/messaging-build-applier.test.ts b/test/messaging-build-applier.test.ts index 52e33cfec35..30e3fdb5775 100644 --- a/test/messaging-build-applier.test.ts +++ b/test/messaging-build-applier.test.ts @@ -158,6 +158,7 @@ function runApplierProcess( agent: "hermes" | "openclaw", phase: MessagingBuildPhase, dryRun = false, + managedStartupRuntime = false, ) { return spawnSync( "node", @@ -169,6 +170,7 @@ function runApplierProcess( "--phase", phase, ...(dryRun ? ["--dry-run"] : []), + ...(managedStartupRuntime ? ["--managed-startup-runtime"] : []), ], { encoding: "utf-8", @@ -1209,7 +1211,7 @@ describe("messaging-build-applier.mts: agent-install", () => { } }); - it("reapplies OpenClaw messaging render after doctor rewrites config", async () => { + it("keeps doctor rerendering while managed startup skips the broad doctor", async () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-doctor-rewrite-")); const tracePath = path.join(tmp, "openclaw.trace"); const fakeOpenclaw = path.join(tmp, "openclaw"); @@ -1271,6 +1273,29 @@ describe("messaging-build-applier.mts: agent-install", () => { expect(config.plugins?.entries?.slack).toEqual({ enabled: true }); expect(config.channels?.["openclaw-weixin"]?.accounts?.primary).toEqual({ enabled: true }); expect(config.channels?.wechat).toBeUndefined(); + + fs.writeFileSync( + path.join(tmp, ".openclaw", "openclaw.json"), + `${JSON.stringify({ channels: {}, plugins: { entries: {} } }, null, 2)}\n`, + ); + fs.writeFileSync(tracePath, ""); + const managedResult = runApplierProcess(env, "openclaw", "post-agent-install", false, true); + expect(managedResult.status, managedResult.stderr).toBe(0); + expect(fs.readFileSync(tracePath, "utf-8")).toBe(""); + const managedConfig = JSON.parse( + fs.readFileSync(path.join(tmp, ".openclaw", "openclaw.json"), "utf-8"), + ); + expect(managedConfig.channels?.telegram?.accounts?.default).toMatchObject({ + botToken: "openshell:resolve:env:TELEGRAM_BOT_TOKEN", + enabled: true, + }); + expect(managedConfig.channels?.discord?.enabled).toBe(true); + expect(managedConfig.plugins?.entries?.discord).toEqual({ enabled: true }); + expect(managedConfig.channels?.slack?.enabled).toBe(true); + expect(managedConfig.plugins?.entries?.slack).toEqual({ enabled: true }); + expect(managedConfig.channels?.["openclaw-weixin"]?.accounts?.primary).toEqual({ + enabled: true, + }); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } From 06f67e5dac1221c506690da9aef9adb19b710610 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 18:22:03 -0700 Subject: [PATCH 008/117] feat(onboard): add managed shared-state transactions Signed-off-by: Aaron Erickson --- .../managed-startup-image-runtime.test.ts | 219 +++- .../managed-startup-root-apply.test.ts | 96 ++ ...d-startup-shared-state-transaction.test.ts | 328 ++++++ .../managed-startup/docker-root-apply.test.ts | 282 +++++ .../managed-startup/docker-root-apply.ts | 210 ++++ .../docker-shared-state.test.ts | 287 +++++ .../managed-startup/docker-shared-state.ts | 271 +++++ .../onboard/managed-startup/image-runtime.ts | 392 ++++++- src/lib/onboard/managed-startup/root-apply.ts | 182 +++ .../shared-state-transaction.ts | 1025 +++++++++++++++++ 10 files changed, 3254 insertions(+), 38 deletions(-) create mode 100644 src/lib/onboard/managed-startup-root-apply.test.ts create mode 100644 src/lib/onboard/managed-startup-shared-state-transaction.test.ts create mode 100644 src/lib/onboard/managed-startup/docker-root-apply.test.ts create mode 100644 src/lib/onboard/managed-startup/docker-root-apply.ts create mode 100644 src/lib/onboard/managed-startup/docker-shared-state.test.ts create mode 100644 src/lib/onboard/managed-startup/docker-shared-state.ts create mode 100644 src/lib/onboard/managed-startup/root-apply.ts create mode 100644 src/lib/onboard/managed-startup/shared-state-transaction.ts diff --git a/src/lib/onboard/managed-startup-image-runtime.test.ts b/src/lib/onboard/managed-startup-image-runtime.test.ts index d0ec6588e61..f78e08e3e84 100644 --- a/src/lib/onboard/managed-startup-image-runtime.test.ts +++ b/src/lib/onboard/managed-startup-image-runtime.test.ts @@ -20,14 +20,18 @@ import { mapManagedStartupProfileToAgentEnvironment } from "./managed-startup/ag import { applyManagedStartupCommandEnvironmentPlan, applyManagedStartupImageProfile, + applyManagedStartupRootRequest, buildManagedStartupImageActionPlan, + MANAGED_STARTUP_COMPLETION_FILE, MANAGED_STARTUP_MERGED_CA_FILE, MANAGED_STARTUP_PROFILE_ENV, MANAGED_STARTUP_RUNTIME_ENV_FILE, type ManagedStartupImageActionPlanInput, normalizeHermesManagedConfigDescriptor, readStableRegularFile, + serializeManagedStartupCompletionMarker, serializeManagedStartupRuntimeEnvironment, + verifyManagedStartupImageCompletion, } from "./managed-startup/image-runtime"; import { encodeManagedStartupProfile, @@ -35,8 +39,10 @@ import { MANAGED_STARTUP_AGENTS, type ManagedStartupAgent, type ManagedStartupDashboard, + type ManagedStartupProfile, validateManagedStartupProfile, } from "./managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; function dashboard(agent: ManagedStartupAgent): ManagedStartupDashboard { switch (agent) { @@ -323,7 +329,10 @@ describe("managed startup image runtime", () => { "/var/lib", "/var/lib/nemoclaw", ]); - let runtimeFileWritten = false; + const files = new Map(); + const descriptorTargets = new Map(); + const pendingFiles = new Map(); + let nextDescriptor = 91; const stat = (kind: "directory" | "file", mode: number) => ({ gid: 0, @@ -334,31 +343,88 @@ describe("managed startup image runtime", () => { nlink: 1, uid: 0, }) as fs.Stats; + const bigFileStat = (bytes: Buffer) => + ({ + ctimeNs: 1n, + dev: 1n, + gid: 0n, + ino: 2n, + isFile: () => true, + mode: 0o100444n, + mtimeNs: 1n, + nlink: 1n, + size: BigInt(bytes.length), + uid: 0n, + }) as fs.BigIntStats; const missing = () => Object.assign(new Error("missing"), { code: "ENOENT" }); vi.spyOn(process, "geteuid").mockReturnValue(0); vi.spyOn(fs, "lstatSync").mockImplementation(((target: fs.PathLike) => { const resolved = String(target); if (directories.has(resolved)) return stat("directory", 0o755); - if (resolved === MANAGED_STARTUP_RUNTIME_ENV_FILE && runtimeFileWritten) { - return stat("file", 0o400); - } + if (files.has(resolved)) return stat("file", 0o444); throw missing(); }) as typeof fs.lstatSync); vi.spyOn(fs, "mkdirSync").mockImplementation(() => undefined); vi.spyOn(fs, "chownSync").mockImplementation(() => undefined); vi.spyOn(fs, "chmodSync").mockImplementation(() => undefined); vi.spyOn(fs, "existsSync").mockReturnValue(false); - vi.spyOn(fs, "openSync").mockReturnValue(91); + vi.spyOn(fs, "openSync").mockImplementation(((target: fs.PathLike) => { + const resolved = String(target); + if ( + (resolved === MANAGED_STARTUP_RUNTIME_ENV_FILE || + resolved === MANAGED_STARTUP_COMPLETION_FILE) && + !files.has(resolved) + ) { + throw missing(); + } + const descriptor = nextDescriptor; + nextDescriptor += 1; + descriptorTargets.set(descriptor, resolved); + return descriptor; + }) as typeof fs.openSync); + vi.spyOn(fs, "fstatSync").mockImplementation(((descriptor: number) => { + const target = descriptorTargets.get(descriptor); + const bytes = target === undefined ? undefined : files.get(target); + if (bytes === undefined) throw missing(); + return bigFileStat(bytes); + }) as typeof fs.fstatSync); + vi.spyOn(fs, "readSync").mockImplementation((( + descriptor: number, + buffer: NodeJS.ArrayBufferView, + offset: number, + length: number, + position: number | null, + ) => { + const target = descriptorTargets.get(descriptor); + const bytes = target === undefined ? undefined : files.get(target); + if (bytes === undefined) throw missing(); + const start = position ?? 0; + const count = Math.min(length, Math.max(0, bytes.length - start)); + bytes.copy(buffer as Buffer, offset, start, start + count); + return count; + }) as typeof fs.readSync); vi.spyOn(fs, "fchownSync").mockImplementation(() => undefined); vi.spyOn(fs, "writeFileSync").mockImplementation(((target: fs.PathOrFileDescriptor, value) => { - if (target === 91) runtimeWrites.push(String(value)); + if (typeof target !== "number") return; + const resolved = descriptorTargets.get(target); + if (resolved === undefined) throw missing(); + pendingFiles.set( + resolved, + Buffer.isBuffer(value) ? Buffer.from(value) : Buffer.from(String(value), "utf8"), + ); }) as typeof fs.writeFileSync); vi.spyOn(fs, "fchmodSync").mockImplementation(() => undefined); vi.spyOn(fs, "fsyncSync").mockImplementation(() => undefined); vi.spyOn(fs, "closeSync").mockImplementation(() => undefined); - vi.spyOn(fs, "renameSync").mockImplementation((_source, target) => { - if (String(target) === MANAGED_STARTUP_RUNTIME_ENV_FILE) runtimeFileWritten = true; + vi.spyOn(fs, "renameSync").mockImplementation((source, target) => { + const pending = pendingFiles.get(String(source)); + if (pending === undefined) throw missing(); + files.set(String(target), pending); + pendingFiles.delete(String(source)); + if (String(target) === MANAGED_STARTUP_RUNTIME_ENV_FILE) { + runtimeWrites.push(pending.toString("utf8")); + } }); vi.spyOn(fs, "unlinkSync").mockImplementation(() => { throw missing(); @@ -367,14 +433,16 @@ describe("managed startup image runtime", () => { it("rejects invalid OpenClaw launch controls before filesystem or coordinator mutation", async () => { const profile = managedStartupE2eProfile("openclaw"); + const request = createManagedStartupRootApplyRequest({ + agent: profile.agent, + encodedProfile: encodeManagedStartupProfile(profile), + }); const lstat = vi.spyOn(fs, "lstatSync"); vi.spyOn(process, "geteuid").mockReturnValue(0); await expect( - applyManagedStartupImageProfile("openclaw", { - NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "1", + applyManagedStartupRootRequest(request, { NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: "NaN", - [MANAGED_STARTUP_PROFILE_ENV]: encodeManagedStartupProfile(profile), }), ).rejects.toThrow(/finite positive seconds/u); expect(lstat).not.toHaveBeenCalled(); @@ -406,20 +474,69 @@ describe("managed startup image runtime", () => { NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "3", [MANAGED_STARTUP_PROFILE_ENV]: encodedProfile, }); - const second = await applyManagedStartupImageProfile("openclaw", { - NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "1", - NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "5", - [MANAGED_STARTUP_PROFILE_ENV]: encodedProfile, - }); + const second = await applyManagedStartupRootRequest( + createManagedStartupRootApplyRequest({ + agent: profile.agent, + encodedProfile, + }), + { + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "5", + }, + ); expect(first).toMatchObject({ adapterApplied: false, fingerprint }); - expect(second).toMatchObject({ adapterApplied: false, fingerprint }); + expect(second).toMatchObject({ + adapterApplied: false, + fingerprint, + transactionPending: false, + }); expect(runtimeWrites).toHaveLength(2); expect(runtimeWrites[0]).toContain("export NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS='3'"); expect(runtimeWrites[1]).toContain("export NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS='5'"); expect(coordinatorMock.coordinateManagedStartupApplication).toHaveBeenCalledTimes(2); }); + function writeCompletionFixture( + profile: ManagedStartupProfile, + corporateCaMerged = false, + ): { + readonly agent: ManagedStartupAgent; + readonly completionFile: string; + readonly fingerprint: string; + readonly runtimeEnvironmentFile: string; + } { + const mapped = mapManagedStartupProfileToAgentEnvironment(profile); + const runtimeEnvironment = serializeManagedStartupRuntimeEnvironment( + mapped.runtimeEnvironment, + corporateCaMerged, + mapped.configurationEnvironment, + ); + const fingerprint = fingerprintManagedStartupProfile(profile); + const completionFile = path.join(temporaryDirectory(), "managed-startup-complete.json"); + const runtimeEnvironmentFile = path.join(temporaryDirectory(), "managed-startup-runtime.env"); + fs.writeFileSync(runtimeEnvironmentFile, runtimeEnvironment, { mode: 0o444 }); + fs.chmodSync(runtimeEnvironmentFile, 0o444); + fs.writeFileSync( + completionFile, + serializeManagedStartupCompletionMarker({ + schemaVersion: 1, + agent: profile.agent, + profileFingerprint: fingerprint, + runtimeEnvironmentSha256: createHash("sha256") + .update(runtimeEnvironment, "utf8") + .digest("hex"), + corporateCaMerged, + }), + { mode: 0o444 }, + ); + fs.chmodSync(completionFile, 0o444); + return { + agent: profile.agent, + completionFile, + fingerprint, + runtimeEnvironmentFile, + }; + } it.each( MANAGED_STARTUP_AGENTS, )("maps the complete %s profile into the reviewed image command contract", (agent) => { @@ -450,6 +567,74 @@ describe("managed startup image runtime", () => { ); }); + it.each( + MANAGED_STARTUP_AGENTS, + )("accepts the root completion marker and exact runtime handoff for %s", (agent) => { + const fixture = writeCompletionFixture(managedStartupE2eProfile(agent)); + mockDescriptorOwnership(0n, 0n); + expect( + verifyManagedStartupImageCompletion( + agent, + fixture.fingerprint, + fixture.completionFile, + fixture.runtimeEnvironmentFile, + ), + ).toEqual({ agent, fingerprint: fixture.fingerprint }); + }); + + it("rejects a changed profile against the root completion fingerprint", () => { + const initial = writeCompletionFixture(managedStartupE2eProfile("openclaw")); + const changedProfile = managedStartupE2eProfile("openclaw", true); + mockDescriptorOwnership(0n, 0n); + expect(() => + verifyManagedStartupImageCompletion( + "openclaw", + fingerprintManagedStartupProfile(changedProfile), + initial.completionFile, + initial.runtimeEnvironmentFile, + ), + ).toThrow(/completion marker does not match the requested profile/u); + }); + + it("rejects runtime handoff drift after a matching completion", () => { + const fixture = writeCompletionFixture(managedStartupE2eProfile("hermes")); + mockDescriptorOwnership(0n, 0n); + fs.chmodSync(fixture.runtimeEnvironmentFile, 0o644); + fs.appendFileSync(fixture.runtimeEnvironmentFile, "export NEMOCLAW_MODEL='tampered/model'\n"); + fs.chmodSync(fixture.runtimeEnvironmentFile, 0o444); + + expect(() => + verifyManagedStartupImageCompletion( + "hermes", + fixture.fingerprint, + fixture.completionFile, + fixture.runtimeEnvironmentFile, + ), + ).toThrow(/runtime environment digest mismatch/u); + }); + + it("accepts merged CA paths without putting the CA payload in the readable handoff", () => { + const fixture = writeCompletionFixture( + managedStartupE2eProfile("langchain-deepagents-code", false, true), + true, + ); + mockDescriptorOwnership(0n, 0n); + expect( + verifyManagedStartupImageCompletion( + "langchain-deepagents-code", + fixture.fingerprint, + fixture.completionFile, + fixture.runtimeEnvironmentFile, + ), + ).toEqual({ + agent: "langchain-deepagents-code", + fingerprint: fixture.fingerprint, + }); + expect(fs.readFileSync(fixture.runtimeEnvironmentFile, "utf8")).not.toContain( + "NEMOCLAW_CORPORATE_CA_B64", + ); + }); + it("binds the real corporate-CA fixture into every agent profile by exact digest", () => { expect(() => new X509Certificate(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM)).not.toThrow(); const digest = createHash("sha256").update(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM).digest("hex"); diff --git a/src/lib/onboard/managed-startup-root-apply.test.ts b/src/lib/onboard/managed-startup-root-apply.test.ts new file mode 100644 index 00000000000..5ee4e59e6de --- /dev/null +++ b/src/lib/onboard/managed-startup-root-apply.test.ts @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, + managedStartupE2eProfile, +} from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { encodeManagedStartupProfile } from "./managed-startup/profile"; +import { + createManagedStartupRootApplyRequest, + MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES, + parseManagedStartupRootApplyRequest, + serializeManagedStartupRootApplyRequest, +} from "./managed-startup/root-apply"; + +function requestFor( + agent: "openclaw" | "hermes" | "langchain-deepagents-code", + withCorporateCa = false, +) { + return createManagedStartupRootApplyRequest({ + agent, + encodedProfile: encodeManagedStartupProfile( + managedStartupE2eProfile(agent, false, withCorporateCa), + ), + ...(withCorporateCa + ? { corporateCaB64: Buffer.from(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM).toString("base64") } + : {}), + }); +} + +describe("managed startup root-application envelope", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("round-trips one canonical bounded %s request", (agent) => { + const request = requestFor(agent, true); + const serialized = serializeManagedStartupRootApplyRequest(request); + + expect(Buffer.byteLength(serialized, "utf8")).toBeLessThan( + MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES, + ); + expect(parseManagedStartupRootApplyRequest(serialized)).toEqual(request); + }); + + it("rejects non-canonical JSON and unknown fields", () => { + const request = requestFor("openclaw"); + const reordered = `${JSON.stringify({ + schemaVersion: request.schemaVersion, + profileFingerprint: request.profileFingerprint, + encodedProfile: request.encodedProfile, + corporateCaB64: request.corporateCaB64, + agent: request.agent, + })}\n`; + const withUnknownField = `${JSON.stringify({ + ...JSON.parse(serializeManagedStartupRootApplyRequest(request)), + surprise: true, + })}\n`; + + expect(() => parseManagedStartupRootApplyRequest(reordered)).toThrow(/not canonical/u); + expect(() => parseManagedStartupRootApplyRequest(withUnknownField)).toThrow(/invalid schema/u); + }); + + it("rejects a tampered profile fingerprint", () => { + const request = requestFor("hermes"); + const parsed = JSON.parse(serializeManagedStartupRootApplyRequest(request)) as Record< + string, + unknown + >; + parsed.profileFingerprint = "f".repeat(64); + + expect(() => parseManagedStartupRootApplyRequest(`${JSON.stringify(parsed)}\n`)).toThrow( + /fingerprint does not match/u, + ); + }); + + it("rejects a canonical CA payload that does not match the profile digest", () => { + const profile = managedStartupE2eProfile("langchain-deepagents-code", false, true); + + expect(() => + createManagedStartupRootApplyRequest({ + agent: profile.agent, + encodedProfile: encodeManagedStartupProfile(profile), + corporateCaB64: Buffer.from("different-ca").toString("base64"), + }), + ).toThrow(/does not match the profile digest/u); + }); + + it("rejects an oversized serialized transport before parsing JSON", () => { + expect(() => + parseManagedStartupRootApplyRequest("x".repeat(MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES + 1)), + ).toThrow(/empty or too large/u); + }); +}); diff --git a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts new file mode 100644 index 00000000000..0d28be2cbe2 --- /dev/null +++ b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts @@ -0,0 +1,328 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import type { SandboxMessagingPlan } from "../messaging/manifest"; +import type { ManagedStartupAgent, ManagedStartupProfile } from "./managed-startup/profile"; +import { + beginManagedStartupSharedStateTransaction, + commitManagedStartupSharedStateTransaction, + type ManagedStartupSharedTransactionOptions, + rollbackManagedStartupSharedStateTransaction, +} from "./managed-startup/shared-state-transaction"; + +function mode(target: string): number { + return fs.lstatSync(target).mode & 0o7777; +} + +function unavailableIdentity(name: string): never { + throw new Error(`effective ${name} is unavailable`); +} + +function effectiveUid(): number { + return process.geteuid?.() ?? unavailableIdentity("uid"); +} + +function effectiveGid(): number { + return process.getegid?.() ?? unavailableIdentity("gid"); +} + +describe("managed startup shared-state transaction", () => { + let temporaryRoot = ""; + let sandboxRoot = ""; + let transactionDirectory = ""; + let options: ManagedStartupSharedTransactionOptions; + + beforeEach(() => { + temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-shared-transaction-")); + sandboxRoot = path.join(temporaryRoot, "sandbox"); + const transactionParent = path.join(temporaryRoot, "root-state"); + transactionDirectory = path.join(transactionParent, "transaction"); + fs.mkdirSync(sandboxRoot, { mode: 0o755 }); + fs.mkdirSync(transactionParent, { mode: 0o755 }); + fs.chmodSync(transactionParent, 0o755); + options = { + sandboxRoot, + transactionDirectory, + trustedUid: effectiveUid(), + trustedGid: effectiveGid(), + }; + }); + + afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(temporaryRoot, { force: true, recursive: true }); + }); + + function agentRoot(agent: ManagedStartupAgent): string { + return path.join( + sandboxRoot, + agent === "openclaw" ? ".openclaw" : agent === "hermes" ? ".hermes" : ".deepagents", + ); + } + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("restores exact %s bytes, ownership, modes, and absence receipts", (agent) => { + const root = agentRoot(agent); + fs.mkdirSync(root, { mode: 0o750 }); + fs.chmodSync(root, 0o750); + const originalFiles = + agent === "openclaw" + ? [["openclaw.json", "openclaw-original\n", 0o640] as const] + : agent === "hermes" + ? [ + ["config.yaml", "hermes-original\n", 0o640] as const, + [".env", "TOKEN=original\n", 0o600] as const, + ] + : [["config.toml", "dcode-original\n", 0o660] as const]; + for (const [name, contents, fileMode] of originalFiles) { + const target = path.join(root, name); + fs.writeFileSync(target, contents); + fs.chmodSync(target, fileMode); + } + + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile(agent), options); + for (const [name] of originalFiles) { + const target = path.join(root, name); + fs.writeFileSync(target, "changed\n"); + fs.chmodSync(target, 0o600); + } + const createManagedDrift: Record void> = { + openclaw: () => fs.writeFileSync(path.join(root, ".config-hash"), "new\n"), + hermes: () => fs.writeFileSync(path.join(root, ".config-hash"), "new\n"), + "langchain-deepagents-code": () => { + fs.mkdirSync(path.join(root, ".state")); + fs.mkdirSync(path.join(root, "skills")); + }, + }; + createManagedDrift[agent](); + + expect(rollbackManagedStartupSharedStateTransaction(agent, options)).toBe(true); + for (const [name, contents, fileMode] of originalFiles) { + const target = path.join(root, name); + expect(fs.readFileSync(target, "utf8")).toBe(contents); + expect(mode(target)).toBe(fileMode); + expect(fs.lstatSync(target).uid).toBe(effectiveUid()); + expect(fs.lstatSync(target).gid).toBe(effectiveGid()); + } + expect(mode(root)).toBe(0o750); + const absentManagedPaths: Record = { + openclaw: [".config-hash"], + hermes: [".config-hash"], + "langchain-deepagents-code": [".state", "skills"], + }; + for (const relativePath of absentManagedPaths[agent]) { + expect(fs.existsSync(path.join(root, relativePath))).toBe(false); + } + expect(fs.existsSync(transactionDirectory)).toBe(false); + }); + + it("tracks only active post-install messaging outputs and leaves disabled targets alone", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "{}\n"); + const plan: SandboxMessagingPlan = { + schemaVersion: 1, + sandboxName: "managed", + agent: "openclaw", + workflow: "onboard", + channels: [ + { + channelId: "wechat", + displayName: "WeChat", + authMode: "host-qr", + active: true, + selected: true, + configured: true, + disabled: false, + inputs: [], + hooks: [ + { + channelId: "wechat", + id: "seed", + phase: "post-agent-install", + handler: "wechat.seedOpenClawAccount", + }, + ], + }, + { + channelId: "telegram", + displayName: "Telegram", + authMode: "token-paste", + active: false, + selected: true, + configured: true, + disabled: true, + inputs: [], + hooks: [], + }, + ], + disabledChannels: ["telegram"], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [ + { + channelId: "wechat", + agent: "openclaw", + target: "~/.openclaw/active-render.json", + kind: "json-fragment", + path: "channels.wechat", + value: { enabled: true }, + templateRefs: [], + }, + { + channelId: "telegram", + agent: "openclaw", + target: "~/.openclaw/disabled-render.json", + kind: "json-fragment", + path: "channels.telegram", + value: { enabled: true }, + templateRefs: [], + }, + ], + buildSteps: [ + { + channelId: "wechat", + kind: "build-file", + hookId: "seed", + outputId: "accounts", + required: true, + value: { + path: "openclaw-weixin/accounts.json", + content: ["primary"], + }, + }, + { + channelId: "telegram", + kind: "build-file", + outputId: "disabled", + required: true, + value: { path: "disabled/new.json", content: {} }, + }, + ], + stateUpdates: [], + healthChecks: [], + }; + const profile = { + ...managedStartupE2eProfile("openclaw"), + messaging: { plan: plan as unknown as ManagedStartupProfile["messaging"]["plan"] }, + }; + beginManagedStartupSharedStateTransaction(profile, options); + + fs.writeFileSync(path.join(root, "active-render.json"), "active\n"); + fs.mkdirSync(path.join(root, "openclaw-weixin")); + fs.writeFileSync(path.join(root, "openclaw-weixin", "accounts.json"), "active\n"); + fs.writeFileSync(path.join(root, "disabled-render.json"), "keep\n"); + fs.mkdirSync(path.join(root, "disabled")); + fs.writeFileSync(path.join(root, "disabled", "new.json"), "keep\n"); + + expect(rollbackManagedStartupSharedStateTransaction("openclaw", options)).toBe(true); + expect(fs.existsSync(path.join(root, "active-render.json"))).toBe(false); + expect(fs.existsSync(path.join(root, "openclaw-weixin"))).toBe(false); + expect(fs.readFileSync(path.join(root, "disabled-render.json"), "utf8")).toBe("keep\n"); + expect(fs.readFileSync(path.join(root, "disabled", "new.json"), "utf8")).toBe("keep\n"); + }); + + it("commits applied output while removing its private backups", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + const config = path.join(root, "openclaw.json"); + fs.writeFileSync(config, "before\n"); + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options); + fs.writeFileSync(config, "after\n"); + + expect(commitManagedStartupSharedStateTransaction("openclaw", options)).toBe(true); + expect(fs.readFileSync(config, "utf8")).toBe("after\n"); + expect(fs.existsSync(transactionDirectory)).toBe(false); + expect(commitManagedStartupSharedStateTransaction("openclaw", options)).toBe(false); + }); + + it("resumes the same pending profile idempotently and rejects profile drift", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "{}\n"); + const profile = managedStartupE2eProfile("openclaw"); + expect(beginManagedStartupSharedStateTransaction(profile, options)).toBe(true); + expect(beginManagedStartupSharedStateTransaction(profile, options)).toBe(false); + expect(() => + beginManagedStartupSharedStateTransaction( + managedStartupE2eProfile("openclaw", true), + options, + ), + ).toThrow(/belongs to a different profile/u); + expect(rollbackManagedStartupSharedStateTransaction("openclaw", options)).toBe(true); + }); + + it("rejects planted target and ancestor symlinks before creating a receipt", () => { + const outside = path.join(temporaryRoot, "outside"); + fs.mkdirSync(outside); + const root = agentRoot("openclaw"); + fs.symlinkSync(outside, root); + expect(() => + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options), + ).toThrow(/ancestor is unsafe/u); + expect(fs.existsSync(transactionDirectory)).toBe(false); + + fs.unlinkSync(root); + fs.mkdirSync(root); + const outsideConfig = path.join(outside, "openclaw.json"); + fs.writeFileSync(outsideConfig, "outside\n"); + fs.symlinkSync(outsideConfig, path.join(root, "openclaw.json")); + expect(() => + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options), + ).toThrow(/not a safe regular file/u); + expect(fs.readFileSync(outsideConfig, "utf8")).toBe("outside\n"); + expect(fs.existsSync(transactionDirectory)).toBe(false); + }); + + it("does not rewrite unchanged shield-like files or directory metadata", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root, { mode: 0o755 }); + const config = path.join(root, "openclaw.json"); + fs.writeFileSync(config, "shielded\n"); + fs.chmodSync(config, 0o444); + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options); + const rename = vi.spyOn(fs, "renameSync"); + const chown = vi.spyOn(fs, "chownSync"); + + expect(rollbackManagedStartupSharedStateTransaction("openclaw", options)).toBe(true); + expect(rename).not.toHaveBeenCalled(); + expect(chown).not.toHaveBeenCalled(); + expect(fs.readFileSync(config, "utf8")).toBe("shielded\n"); + expect(mode(config)).toBe(0o444); + }); + + it("refuses to operate on a pending transaction for a different agent", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "{}\n"); + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options); + expect(() => rollbackManagedStartupSharedStateTransaction("hermes", options)).toThrow( + /targets openclaw, expected hermes/u, + ); + expect(fs.existsSync(transactionDirectory)).toBe(true); + expect(rollbackManagedStartupSharedStateTransaction("openclaw", options)).toBe(true); + }); + + it("rejects an oversized managed output before creating a receipt", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + const config = path.join(root, "openclaw.json"); + fs.writeFileSync(config, ""); + fs.truncateSync(config, 8 * 1024 * 1024 + 1); + + expect(() => + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options), + ).toThrow(/unsafe or oversized/u); + expect(fs.existsSync(transactionDirectory)).toBe(false); + }); +}); diff --git a/src/lib/onboard/managed-startup/docker-root-apply.test.ts b/src/lib/onboard/managed-startup/docker-root-apply.test.ts new file mode 100644 index 00000000000..b2140817bc6 --- /dev/null +++ b/src/lib/onboard/managed-startup/docker-root-apply.test.ts @@ -0,0 +1,282 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + applyDockerManagedStartupRootRequest, + getDockerManagedStartupFailureTransaction, +} from "./docker-root-apply"; +import { encodeManagedStartupProfile } from "./profile"; +import { + createManagedStartupRootApplyRequest, + parseManagedStartupRootApplyRequest, +} from "./root-apply"; +import { MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY } from "./shared-state-transaction"; + +const CONTAINER_ID = "b".repeat(64); +const IMAGE_ID = `sha256:${"c".repeat(64)}`; + +function requestFor(agent: "openclaw" | "hermes" | "langchain-deepagents-code") { + return createManagedStartupRootApplyRequest({ + agent, + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agent)), + }); +} + +function stableInspect(overrides: Record = {}): string { + return JSON.stringify([ + { + Id: CONTAINER_ID, + Image: IMAGE_ID, + State: { + Running: true, + Paused: false, + Restarting: false, + Dead: false, + }, + ...overrides, + }, + ]); +} + +function successfulSpawnResult() { + return { + status: 0, + signal: null, + stdout: "", + stderr: "", + output: [null, "", ""], + pid: 1, + error: undefined, + }; +} + +describe("Docker managed-startup root applicator", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("pins exact container/image identity and uses fixed root stdin for %s", (agent) => { + const request = requestFor(agent); + const dockerCapture = vi.fn(() => stableInspect()); + const dockerSpawnSync = vi.fn(() => successfulSpawnResult()); + + expect( + applyDockerManagedStartupRootRequest( + { containerId: CONTAINER_ID, request }, + { dockerCapture, dockerSpawnSync, environment: {} }, + ), + ).toEqual({ agent, containerId: CONTAINER_ID, image: IMAGE_ID }); + + expect(dockerCapture).toHaveBeenCalledWith(["inspect", "--type", "container", CONTAINER_ID], { + ignoreError: false, + timeout: 30_000, + }); + expect(dockerSpawnSync).toHaveBeenCalledTimes(2); + const [argv, options] = dockerSpawnSync.mock.calls[0] as unknown as [ + string[], + { input: string; timeout: number; encoding: string }, + ]; + expect(argv).toEqual([ + "exec", + "--interactive", + "--user", + "0:0", + "--workdir", + "/", + CONTAINER_ID, + "/usr/bin/env", + "-i", + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", + "--apply-root-stdin", + "--agent", + agent, + ]); + expect(argv.join(" ")).not.toContain(request.encodedProfile); + expect(parseManagedStartupRootApplyRequest(options.input)).toEqual(request); + expect(options).toMatchObject({ encoding: "utf8", timeout: 300_000 }); + expect(dockerSpawnSync.mock.calls[1]).toEqual([ + [ + "exec", + "--user", + "0:0", + "--workdir", + "/", + CONTAINER_ID, + "/usr/bin/env", + "-i", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/bin/sh", + "-c", + 'if [ -d "$1" ] && [ ! -L "$1" ]; then exit 0; fi; if [ ! -e "$1" ] && [ ! -L "$1" ]; then exit 1; fi; exit 2', + "nemoclaw-transaction-probe", + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + ], + { encoding: "utf8", timeout: 30_000 }, + ]); + }); + + it("forwards only allowlisted application-runtime controls through the clean root exec", () => { + const dockerSpawnSync = vi.fn(() => successfulSpawnResult()); + + applyDockerManagedStartupRootRequest( + { containerId: CONTAINER_ID, request: requestFor("openclaw") }, + { + dockerCapture: vi.fn(() => stableInspect()), + dockerSpawnSync, + environment: { + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: " 0.25 ", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "03", + NEMOCLAW_NOT_AN_APPLICATION_RUNTIME_INPUT: "must-not-cross", + }, + }, + ); + + const [argv] = dockerSpawnSync.mock.calls[0] as unknown as [string[]]; + expect(argv).toContain("NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS= 0.25 "); + expect(argv).toContain("NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS=03"); + expect(argv.join("\n")).not.toContain("NEMOCLAW_NOT_AN_APPLICATION_RUNTIME_INPUT"); + }); + + it("retries one lost acknowledgement with the identical pinned command and payload", () => { + const request = requestFor("openclaw"); + const dockerSpawnSync = vi + .fn() + .mockReturnValueOnce({ ...successfulSpawnResult(), status: 1, stderr: "lost ack" }) + .mockReturnValueOnce(successfulSpawnResult()) + .mockReturnValueOnce(successfulSpawnResult()); + + applyDockerManagedStartupRootRequest( + { containerId: CONTAINER_ID, request }, + { dockerCapture: vi.fn(() => stableInspect()), dockerSpawnSync }, + ); + + expect(dockerSpawnSync).toHaveBeenCalledTimes(3); + expect(dockerSpawnSync.mock.calls[1]).toEqual(dockerSpawnSync.mock.calls[0]); + }); + + it("returns no pending transaction when the same profile was already finalized", () => { + const request = requestFor("openclaw"); + const dockerSpawnSync = vi + .fn() + .mockReturnValueOnce(successfulSpawnResult()) + .mockReturnValueOnce({ ...successfulSpawnResult(), status: 1 }); + + expect( + applyDockerManagedStartupRootRequest( + { containerId: CONTAINER_ID, request }, + { dockerCapture: vi.fn(() => stableInspect()), dockerSpawnSync }, + ), + ).toBeNull(); + expect(dockerSpawnSync).toHaveBeenCalledTimes(2); + }); + + it("fails with rollback context when transaction state cannot be verified", () => { + const request = requestFor("hermes"); + const dockerSpawnSync = vi + .fn() + .mockReturnValueOnce(successfulSpawnResult()) + .mockReturnValueOnce({ + ...successfulSpawnResult(), + status: null, + error: new Error("probe unavailable"), + }); + + let failure: unknown; + try { + applyDockerManagedStartupRootRequest( + { containerId: CONTAINER_ID, request }, + { dockerCapture: vi.fn(() => stableInspect()), dockerSpawnSync }, + ); + } catch (error) { + failure = error; + } + + expect(failure).toEqual( + expect.objectContaining({ + message: expect.stringContaining("transaction state could not be verified"), + }), + ); + expect(getDockerManagedStartupFailureTransaction(failure)).toEqual({ + agent: "hermes", + containerId: CONTAINER_ID, + image: IMAGE_ID, + }); + }); + + it("attaches the pinned transaction when both idempotent attempts fail", () => { + const request = requestFor("hermes"); + const dockerSpawnSync = vi.fn(() => ({ + ...successfulSpawnResult(), + status: 1, + stderr: "exec failed", + })); + + let failure: unknown; + try { + applyDockerManagedStartupRootRequest( + { containerId: CONTAINER_ID, request }, + { dockerCapture: vi.fn(() => stableInspect()), dockerSpawnSync }, + ); + } catch (error) { + failure = error; + } + + expect(failure).toEqual( + expect.objectContaining({ message: expect.stringContaining("exec failed") }), + ); + expect(getDockerManagedStartupFailureTransaction(failure)).toEqual({ + agent: "hermes", + containerId: CONTAINER_ID, + image: IMAGE_ID, + }); + expect(dockerSpawnSync).toHaveBeenCalledTimes(2); + }); + + it.each([ + { + label: "changed exact identity", + containerId: CONTAINER_ID, + inspect: stableInspect({ Id: "d".repeat(64) }), + error: /identity changed/u, + }, + { + label: "mutable image identity", + containerId: CONTAINER_ID, + inspect: stableInspect({ Image: "registry.example/image:latest" }), + error: /immutable image identity/u, + }, + { + label: "unstable running state", + containerId: CONTAINER_ID, + inspect: stableInspect({ + State: { Running: true, Paused: false, Restarting: true, Dead: false }, + }), + error: /not stably running/u, + }, + { + label: "short caller identity", + containerId: "b".repeat(12), + inspect: stableInspect(), + error: /full lowercase Docker container ID/u, + }, + ])("rejects $label before root exec", ({ containerId, inspect, error }) => { + const dockerSpawnSync = vi.fn(); + + expect(() => + applyDockerManagedStartupRootRequest( + { containerId, request: requestFor("openclaw") }, + { dockerCapture: vi.fn(() => inspect), dockerSpawnSync }, + ), + ).toThrow(error); + expect(dockerSpawnSync).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/managed-startup/docker-root-apply.ts b/src/lib/onboard/managed-startup/docker-root-apply.ts new file mode 100644 index 00000000000..d6fa187aa08 --- /dev/null +++ b/src/lib/onboard/managed-startup/docker-root-apply.ts @@ -0,0 +1,210 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { dockerSpawnSync } from "../../adapters/docker/exec"; +import { dockerCapture } from "../../adapters/docker/run"; +import { isImmutableDockerImageId } from "../openshell-docker-sandbox-containers"; +import { MANAGED_STARTUP_RUNTIME_EXECUTABLE } from "./image-runtime"; +import { + type ManagedStartupRootApplyRequest, + selectManagedStartupApplicationRuntimeEnvironment, + serializeManagedStartupRootApplyRequest, +} from "./root-apply"; +import { MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY } from "./shared-state-transaction"; + +const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; +const ROOT_APPLY_TIMEOUT_MS = 300_000; +const FIXED_ROOT_ENV = [ + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +] as const; + +interface DockerManagedStartupInspect { + readonly Id?: string; + readonly Image?: string; + readonly State?: { + readonly Running?: boolean; + readonly Paused?: boolean; + readonly Restarting?: boolean; + readonly Dead?: boolean; + } | null; +} + +export interface DockerManagedStartupTransaction { + readonly agent: ManagedStartupRootApplyRequest["agent"]; + readonly containerId: string; + readonly image: string; +} + +export interface DockerManagedStartupRootApplyDeps { + readonly dockerCapture?: typeof dockerCapture; + readonly dockerSpawnSync?: typeof dockerSpawnSync; + readonly environment?: Readonly; +} + +export function getDockerManagedStartupFailureTransaction( + error: unknown, +): DockerManagedStartupTransaction | null { + if (typeof error === "object" && error !== null && "managedStartupTransaction" in error) { + return ( + (error as { managedStartupTransaction?: DockerManagedStartupTransaction }) + .managedStartupTransaction ?? null + ); + } + return null; +} + +function commandDetail(result: { + readonly status?: number | null; + readonly stdout?: string | Buffer | null; + readonly stderr?: string | Buffer | null; + readonly error?: Error | null; +}): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}` + .trim() + .slice(-1200); +} + +function inspectExactContainer( + containerId: string, + capture: typeof dockerCapture, +): { readonly containerId: string; readonly image: string } { + if (!FULL_CONTAINER_ID_RE.test(containerId)) { + throw new Error("Managed startup requires one full lowercase Docker container ID."); + } + const output = capture(["inspect", "--type", "container", containerId], { + ignoreError: false, + timeout: 30_000, + }); + let parsed: unknown; + try { + parsed = JSON.parse(output); + } catch { + throw new Error("Docker returned malformed inspect output for the managed-startup container."); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("Docker inspect did not resolve exactly one managed-startup container."); + } + const inspect = parsed[0] as DockerManagedStartupInspect; + const exactId = String(inspect.Id ?? "").toLowerCase(); + const image = String(inspect.Image ?? "").toLowerCase(); + if (exactId !== containerId) { + throw new Error("Managed-startup container identity changed before root application."); + } + if (!isImmutableDockerImageId(image)) { + throw new Error("Managed-startup container does not have an immutable image identity."); + } + if ( + inspect.State?.Running !== true || + inspect.State.Paused === true || + inspect.State.Restarting === true || + inspect.State.Dead === true + ) { + throw new Error("Managed-startup container is not stably running for root application."); + } + return { containerId: exactId, image }; +} + +export function applyDockerManagedStartupRootRequest( + input: { + readonly containerId: string; + readonly request: ManagedStartupRootApplyRequest; + }, + deps: DockerManagedStartupRootApplyDeps = {}, +): DockerManagedStartupTransaction | null { + const capture = deps.dockerCapture ?? dockerCapture; + const spawn = deps.dockerSpawnSync ?? dockerSpawnSync; + const pinned = inspectExactContainer(input.containerId, capture); + const transaction = { + agent: input.request.agent, + containerId: pinned.containerId, + image: pinned.image, + } satisfies DockerManagedStartupTransaction; + const payload = serializeManagedStartupRootApplyRequest(input.request); + const applicationRuntimeEnvironment = Object.entries( + selectManagedStartupApplicationRuntimeEnvironment(deps.environment ?? process.env), + ).map(([name, value]) => `${name}=${value}`); + const argv = [ + "exec", + "--interactive", + "--user", + "0:0", + "--workdir", + "/", + pinned.containerId, + "/usr/bin/env", + "-i", + ...FIXED_ROOT_ENV, + ...applicationRuntimeEnvironment, + "/usr/local/bin/node", + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + "--apply-root-stdin", + "--agent", + input.request.agent, + ]; + const receiptProbeArgv = [ + "exec", + "--user", + "0:0", + "--workdir", + "/", + pinned.containerId, + "/usr/bin/env", + "-i", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/bin/sh", + "-c", + 'if [ -d "$1" ] && [ ! -L "$1" ]; then exit 0; fi; if [ ! -e "$1" ] && [ ! -L "$1" ]; then exit 1; fi; exit 2', + "nemoclaw-transaction-probe", + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + ]; + + let lastFailure = ""; + // The image-side coordinator and transaction are idempotent. One retry + // reconciles the only ambiguous case: Docker lost the first exec + // acknowledgement after the completion marker was already published. + for (let attempt = 0; attempt < 2; attempt += 1) { + const result = spawn(argv, { + encoding: "utf8", + input: payload, + timeout: ROOT_APPLY_TIMEOUT_MS, + }); + if (result.status === 0) { + const receiptProbe = spawn(receiptProbeArgv, { + encoding: "utf8", + timeout: 30_000, + }); + if (receiptProbe.status === 0) return transaction; + if (receiptProbe.status === 1) return null; + const receiptProbeDetail = commandDetail(receiptProbe); + const error = new Error( + `Managed startup root application completed, but transaction state could not be verified in exact container ${pinned.containerId.slice(0, 12)}${ + receiptProbeDetail ? `: ${receiptProbeDetail}` : "" + }`, + ); + ( + error as Error & { + managedStartupTransaction?: DockerManagedStartupTransaction; + } + ).managedStartupTransaction = transaction; + throw error; + } + lastFailure = commandDetail(result); + } + const error = new Error( + `Managed startup root application failed in exact container ${pinned.containerId.slice(0, 12)}${ + lastFailure ? `: ${lastFailure}` : "" + }`, + ); + ( + error as Error & { + managedStartupTransaction?: DockerManagedStartupTransaction; + } + ).managedStartupTransaction = transaction; + throw error; +} diff --git a/src/lib/onboard/managed-startup/docker-shared-state.test.ts b/src/lib/onboard/managed-startup/docker-shared-state.test.ts new file mode 100644 index 00000000000..a604a19656d --- /dev/null +++ b/src/lib/onboard/managed-startup/docker-shared-state.test.ts @@ -0,0 +1,287 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { DockerGpuPatchResult } from "../docker-gpu-patch-types"; +import type { DockerManagedStartupTransaction } from "./docker-root-apply"; +import { finalizeDockerManagedStartupSharedState } from "./docker-shared-state"; +import { MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY } from "./shared-state-transaction"; + +const IMMUTABLE_IMAGE = `sha256:${"a".repeat(64)}`; + +function result(): DockerGpuPatchResult { + return { + applied: true, + oldContainerId: "old", + newContainerId: "new", + originalName: "openshell-alpha", + backupContainerName: "openshell-alpha-backup", + mode: { + kind: "startup-command", + label: "restart-safe startup command", + device: "", + args: [], + }, + backupRemoved: false, + }; +} + +function transaction(): DockerManagedStartupTransaction { + return { + agent: "openclaw", + containerId: "new", + image: IMMUTABLE_IMAGE, + }; +} + +function removeReceiptParents(...receiptPaths: readonly string[]): void { + for (const receiptPath of receiptPaths.filter((candidate) => candidate.length > 0)) { + fs.rmSync(path.dirname(receiptPath), { force: true, recursive: true }); + } +} + +describe("Docker managed-startup shared-state finalization", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("copies the bounded receipt before commit and removes the host copy on success", () => { + const calls: string[] = []; + let receiptPath = ""; + const dockerRun = vi + .fn() + .mockImplementationOnce((args: readonly string[]) => { + calls.push("copy"); + receiptPath = String(args[2]); + expect(args[1]).toBe(`new:${MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY}`); + return { status: 0 }; + }) + .mockImplementationOnce((args: readonly string[]) => { + calls.push("commit"); + expect(args).toEqual([ + "exec", + "--user", + "0:0", + "--env", + "NODE_OPTIONS=", + "--env", + "NODE_PATH=", + "--env", + "BASH_ENV=", + "--env", + "ENV=", + "new", + "/usr/bin/env", + "-i", + "HOME=/root", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", + "--commit-shared-state-transaction", + "--agent", + "openclaw", + ]); + return { status: 0 }; + }); + const dockerStop = vi.fn(); + + expect( + finalizeDockerManagedStartupSharedState( + { transaction: transaction(), patchResult: result(), supervisorReady: true }, + { dockerRun, dockerStop }, + ), + ).toEqual({ supervisorReady: true, failure: null }); + expect(calls).toEqual(["copy", "commit"]); + expect(dockerStop).not.toHaveBeenCalled(); + expect(fs.existsSync(path.dirname(receiptPath))).toBe(false); + }); + + it("uses the preserved pre-commit receipt after a lost commit acknowledgement", () => { + const calls: string[] = []; + let receiptPath = ""; + const dockerRun = vi + .fn() + .mockImplementationOnce((args: readonly string[]) => { + calls.push("copy"); + receiptPath = String(args[2]); + return { status: 0 }; + }) + .mockImplementationOnce(() => { + calls.push("commit-lost-ack"); + return { status: 1, stderr: "daemon acknowledgement lost" }; + }) + .mockImplementationOnce((args: readonly string[]) => { + calls.push("rollback-helper"); + expect(args).toEqual([ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--cap-add", + "CHOWN", + "--cap-add", + "DAC_OVERRIDE", + "--cap-add", + "FOWNER", + "--env", + "NODE_OPTIONS=", + "--env", + "NODE_PATH=", + "--env", + "BASH_ENV=", + "--env", + "ENV=", + "--volumes-from", + "new", + "--mount", + expect.stringMatching( + /^type=bind,src=.+,dst=\/run\/nemoclaw\/managed-startup-shared-rollback-receipt-v1,readonly$/u, + ), + "--entrypoint", + "/usr/local/bin/node", + IMMUTABLE_IMAGE, + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", + "--rollback-shared-state-transaction", + "--agent", + "openclaw", + "--read-only-receipt", + ]); + return { status: 0 }; + }); + const dockerStop = vi.fn(() => { + calls.push("stop"); + return { status: 0 }; + }); + + const outcome = finalizeDockerManagedStartupSharedState( + { transaction: transaction(), patchResult: result(), supervisorReady: true }, + { dockerRun, dockerStop }, + ); + expect(outcome.supervisorReady).toBe(false); + expect(outcome.failure?.message).toContain("commit failed"); + expect(calls).toEqual(["copy", "commit-lost-ack", "stop", "rollback-helper"]); + expect(fs.existsSync(path.dirname(receiptPath))).toBe(false); + }); + + it("uses unique receipt paths and treats already-completed cleanup idempotently", () => { + const receiptPaths: string[] = []; + const copyReceipt = (args: readonly string[]) => { + expect(args[0]).toBe("cp"); + const receiptPath = String(args[2]); + receiptPaths.push(receiptPath); + return { status: 0 }; + }; + const completeCommit = (args: readonly string[]) => { + expect(args[0]).toBe("exec"); + removeReceiptParents(receiptPaths.at(-1)!); + return { status: 0 }; + }; + const dockerRun = vi + .fn() + .mockImplementationOnce(copyReceipt) + .mockImplementationOnce(completeCommit) + .mockImplementationOnce(copyReceipt) + .mockImplementationOnce(completeCommit); + + for (let attempt = 0; attempt < 2; attempt += 1) { + expect( + finalizeDockerManagedStartupSharedState( + { transaction: transaction(), patchResult: result(), supervisorReady: true }, + { dockerRun }, + ), + ).toEqual({ supervisorReady: true, failure: null }); + } + expect(new Set(receiptPaths).size).toBe(2); + expect(receiptPaths.every((receiptPath) => !fs.existsSync(path.dirname(receiptPath)))).toBe( + true, + ); + }); + + it("quiesces a failed supervisor before copying and replaying the receipt", () => { + const calls: string[] = []; + const dockerStop = vi.fn(() => { + calls.push("stop"); + return { status: 0 }; + }); + const dockerRun = vi.fn((args: readonly string[]) => { + calls.push(args[0] === "cp" ? "copy" : "rollback-helper"); + return { status: 0 }; + }); + + expect( + finalizeDockerManagedStartupSharedState( + { transaction: transaction(), patchResult: result(), supervisorReady: false }, + { dockerRun, dockerStop }, + ), + ).toEqual({ supervisorReady: false, failure: null }); + expect(calls).toEqual(["stop", "copy", "rollback-helper"]); + }); + + it("stops a live workload when pre-commit receipt preservation fails", () => { + const dockerRun = vi.fn(() => ({ status: 1, stderr: "copy failed" })); + const dockerStop = vi.fn(() => ({ status: 0 })); + + expect(() => + finalizeDockerManagedStartupSharedState( + { transaction: transaction(), patchResult: result(), supervisorReady: true }, + { dockerRun, dockerStop }, + ), + ).toThrow(/Could not copy/u); + expect(dockerStop).toHaveBeenCalledOnce(); + expect(dockerRun).toHaveBeenCalledOnce(); + }); + + it("fails before container rollback when the immutable helper cannot verify restoration", () => { + const dockerStop = vi.fn(() => ({ status: 0 })); + let receiptPath = ""; + const dockerRun = vi + .fn() + .mockImplementationOnce((args: readonly string[]) => { + receiptPath = String(args[2]); + return { status: 0 }; + }) + .mockImplementationOnce(() => ({ + status: 1, + stderr: "receipt verification failed", + })); + + try { + expect(() => + finalizeDockerManagedStartupSharedState( + { transaction: transaction(), patchResult: result(), supervisorReady: false }, + { dockerRun, dockerStop }, + ), + ).toThrow(/could not restore and verify/u); + expect(dockerStop).toHaveBeenCalledOnce(); + expect(fs.existsSync(path.dirname(receiptPath))).toBe(true); + } finally { + removeReceiptParents(receiptPath); + } + }); + + it("is a no-op for non-managed container patches", () => { + const dockerRun = vi.fn(); + const dockerStop = vi.fn(); + expect( + finalizeDockerManagedStartupSharedState( + { transaction: null, patchResult: result(), supervisorReady: true }, + { dockerRun, dockerStop }, + ), + ).toEqual({ supervisorReady: true, failure: null }); + expect(dockerRun).not.toHaveBeenCalled(); + expect(dockerStop).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/managed-startup/docker-shared-state.ts b/src/lib/onboard/managed-startup/docker-shared-state.ts new file mode 100644 index 00000000000..41c36aea851 --- /dev/null +++ b/src/lib/onboard/managed-startup/docker-shared-state.ts @@ -0,0 +1,271 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + dockerRm as defaultDockerRm, + dockerStop as defaultDockerStop, +} from "../../adapters/docker/container"; +import { dockerRun as defaultDockerRun } from "../../adapters/docker/run"; +import { hasZeroDockerExitStatus } from "../docker-command-result"; +import { + DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + DOCKER_GPU_PATCH_TIMEOUT_MS, +} from "../docker-gpu-patch-constants"; +import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "../docker-gpu-patch-types"; +import { cleanupTempDir, secureTempFile } from "../temp-files"; +import type { DockerManagedStartupTransaction } from "./docker-root-apply"; +import { MANAGED_STARTUP_RUNTIME_EXECUTABLE } from "./image-runtime"; +import { + MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, +} from "./shared-state-transaction"; + +const RECEIPT_TEMP_PREFIX = "nemoclaw-managed-startup-receipt"; +const NEUTRALIZED_PROCESS_INJECTION_ENV = [ + "--env", + "NODE_OPTIONS=", + "--env", + "NODE_PATH=", + "--env", + "BASH_ENV=", + "--env", + "ENV=", +] as const; + +export interface DockerManagedStartupSharedStateOutcome { + /** + * True only when the new supervisor is still eligible for successful + * container cutover. A commit failure forces shared-state rollback first. + */ + readonly supervisorReady: boolean; + /** Original commit failure after a successful shared-state rollback. */ + readonly failure: Error | null; +} + +function commandDetail(result: { + readonly stderr?: string | Buffer | null; + readonly stdout?: string | Buffer | null; + readonly error?: Error | null; +}): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}` + .trim() + .slice(-800); +} + +function cleanupReceiptBestEffort(receiptPath: string): void { + try { + cleanupTempDir(receiptPath, RECEIPT_TEMP_PREFIX); + } catch (error) { + console.warn( + ` ⚠ Managed-startup shared state is finalized, but its protected host receipt could not be removed (${receiptPath}): ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +function transactionCommand(action: "commit" | "rollback", agent: string): string[] { + return [ + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + `--${action}-shared-state-transaction`, + "--agent", + agent, + ]; +} + +const DOCKER_MUTATION_OPTIONS = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, +} as const; + +function quiesceManagedStartupContainer( + transaction: DockerManagedStartupTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerStop = deps.dockerStop ?? defaultDockerStop; + const stopped = dockerStop(transaction.containerId, { + ...DOCKER_MUTATION_OPTIONS, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopped)) { + throw new Error( + `Could not quiesce the failed managed-startup container before shared-state rollback: ${commandDetail(stopped)}`, + ); + } +} + +function copyManagedStartupReceipt( + transaction: DockerManagedStartupTransaction, + deps: DockerGpuPatchDeps, +): string { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + const receiptPath = secureTempFile(RECEIPT_TEMP_PREFIX); + try { + const copy = dockerRun( + [ + "cp", + `${transaction.containerId}:${MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY}`, + receiptPath, + ], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(copy)) { + throw new Error( + `Could not copy the managed-startup rollback receipt from the failed container: ${commandDetail(copy)}`, + ); + } + if (receiptPath.includes(",") || /[\r\n\0]/u.test(receiptPath)) { + throw new Error("Managed-startup rollback receipt path is unsafe for a Docker bind mount"); + } + return receiptPath; + } catch (error) { + cleanupReceiptBestEffort(receiptPath); + throw error; + } +} + +function rollbackManagedStartupSharedState( + transaction: DockerManagedStartupTransaction, + receiptPath: string, + deps: DockerGpuPatchDeps, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + let restored = false; + try { + const helper = dockerRun( + [ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--cap-add", + "CHOWN", + "--cap-add", + "DAC_OVERRIDE", + "--cap-add", + "FOWNER", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + "--volumes-from", + transaction.containerId, + "--mount", + `type=bind,src=${receiptPath},dst=${MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY},readonly`, + "--entrypoint", + "/usr/local/bin/node", + transaction.image, + ...transactionCommand("rollback", transaction.agent), + "--read-only-receipt", + ], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(helper)) { + throw new Error( + `Immutable managed-startup helper could not restore and verify shared state: ${commandDetail(helper)}. ` + + `Protected receipt retained at ${receiptPath}`, + ); + } + restored = true; + } finally { + if (restored) { + cleanupReceiptBestEffort(receiptPath); + } + } +} + +function removeFailedUnbackedContainer( + transaction: DockerManagedStartupTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerRm = deps.dockerRm ?? defaultDockerRm; + const removed = dockerRm(transaction.containerId, DOCKER_MUTATION_OPTIONS); + if (!hasZeroDockerExitStatus(removed)) { + throw new Error( + `Could not remove the failed managed-startup container after shared-state rollback: ${commandDetail(removed)}`, + ); + } +} + +/** + * Finalize the shared-state half of managed container cutover before generic + * backup removal or rollback. A shared-state rollback failure deliberately + * throws so callers cannot remove the new container or restart the old one + * while `/sandbox` remains partially applied. + */ +export function finalizeDockerManagedStartupSharedState( + input: { + readonly transaction: DockerManagedStartupTransaction | null; + readonly patchResult?: DockerGpuPatchResult | null; + readonly supervisorReady: boolean; + }, + deps: DockerGpuPatchDeps = {}, +): DockerManagedStartupSharedStateOutcome { + const transaction = input.transaction; + if (!transaction) { + return { supervisorReady: input.supervisorReady, failure: null }; + } + const dockerRun = deps.dockerRun ?? defaultDockerRun; + if (input.supervisorReady) { + // Preserve a verified rollback source before commit deletes the + // container-local receipt. If Docker loses the exec acknowledgement after + // deletion, this copy still makes the cutover reversible. + let receiptPath: string; + try { + receiptPath = copyManagedStartupReceipt(transaction, deps); + } catch (error) { + try { + quiesceManagedStartupContainer(transaction, deps); + } catch (stopError) { + throw new Error( + `Managed-startup receipt preservation failed and the new workload could not be quiesced: ${ + error instanceof Error ? error.message : String(error) + }; ${stopError instanceof Error ? stopError.message : String(stopError)}`, + ); + } + throw error; + } + const commit = dockerRun( + [ + "exec", + "--user", + "0:0", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + transaction.containerId, + "/usr/bin/env", + "-i", + "HOME=/root", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + ...transactionCommand("commit", transaction.agent), + ], + DOCKER_MUTATION_OPTIONS, + ); + if (hasZeroDockerExitStatus(commit)) { + cleanupReceiptBestEffort(receiptPath); + return { supervisorReady: true, failure: null }; + } + const failure = new Error( + `OpenShell supervisor reconnected, but managed shared-state commit failed: ${commandDetail(commit)}`, + ); + quiesceManagedStartupContainer(transaction, deps); + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!input.patchResult) removeFailedUnbackedContainer(transaction, deps); + return { supervisorReady: false, failure }; + } + + quiesceManagedStartupContainer(transaction, deps); + const receiptPath = copyManagedStartupReceipt(transaction, deps); + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!input.patchResult) removeFailedUnbackedContainer(transaction, deps); + return { supervisorReady: false, failure: null }; +} diff --git a/src/lib/onboard/managed-startup/image-runtime.ts b/src/lib/onboard/managed-startup/image-runtime.ts index 37642a97493..c6252ad97c2 100644 --- a/src/lib/onboard/managed-startup/image-runtime.ts +++ b/src/lib/onboard/managed-startup/image-runtime.ts @@ -19,10 +19,24 @@ import { } from "./coordinator"; import { decodeManagedStartupProfile, + fingerprintManagedStartupProfile, MANAGED_STARTUP_AGENTS, type ManagedStartupAgent, type ManagedStartupDashboard, + type ManagedStartupProfile, } from "./profile"; +import { + MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES, + type ManagedStartupRootApplyRequest, + parseManagedStartupRootApplyRequest, + selectManagedStartupApplicationRuntimeEnvironment, +} from "./root-apply"; +import { + beginManagedStartupSharedStateTransaction, + commitManagedStartupSharedStateTransaction, + MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, + rollbackManagedStartupSharedStateTransaction, +} from "./shared-state-transaction"; import { MANAGED_STARTUP_CA_ENV, MANAGED_STARTUP_PROFILE_ENV } from "./transport"; export { MANAGED_STARTUP_CA_ENV, MANAGED_STARTUP_PROFILE_ENV } from "./transport"; @@ -30,6 +44,7 @@ export const MANAGED_STARTUP_RUNTIME_ENV_FILE = "/run/nemoclaw/managed-startup-r export const MANAGED_STARTUP_RUNTIME_EXECUTABLE = "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs"; export const MANAGED_STARTUP_MERGED_CA_FILE = "/run/nemoclaw/managed-startup-ca-bundle.pem"; +export const MANAGED_STARTUP_COMPLETION_FILE = "/run/nemoclaw/managed-startup-complete.json"; const MANAGED_STARTUP_CORPORATE_CA_FILE = "/usr/local/share/nemoclaw/corporate-ca.pem"; const MESSAGING_RUNTIME_PLAN_FILE = "/usr/local/share/nemoclaw/messaging-runtime-plan.json"; @@ -43,6 +58,9 @@ const HERMES_MANAGED_CONFIG_FILES = [ ] as const; const FIXED_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; const SHA256_RE = /^[a-f0-9]{64}$/u; +const MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION = 1; +const MAX_MANAGED_STARTUP_COMPLETION_BYTES = 4096; +const MAX_MANAGED_STARTUP_RUNTIME_ENVIRONMENT_BYTES = 512 * 1024; export type ManagedStartupImageIdentity = "root" | "sandbox"; export type ManagedStartupMessagingAgent = "openclaw" | "hermes"; @@ -110,6 +128,18 @@ export interface ManagedStartupImageApplyResult { readonly runtimeEnvironmentFile: string; } +export interface ManagedStartupRootApplyResult extends ManagedStartupImageApplyResult { + readonly transactionPending: boolean; +} + +export interface ManagedStartupCompletionMarker { + readonly schemaVersion: typeof MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION; + readonly agent: ManagedStartupAgent; + readonly profileFingerprint: string; + readonly runtimeEnvironmentSha256: string; + readonly corporateCaMerged: boolean; +} + export class ManagedStartupImageActionPlanError extends Error { constructor(message: string) { super(`Cannot build managed startup image action plan: ${message}`); @@ -215,6 +245,24 @@ function exactAgent(value: string): ManagedStartupAgent { return fail(`unsupported agent ${JSON.stringify(value)}`); } +function managedTransactionProfile( + expectedAgentInput: string, + env: Environment = process.env, +): ManagedStartupProfile { + requireRoot(); + const expectedAgent = exactAgent(expectedAgentInput); + if (env.NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION !== "1") { + fail("shared-state transactions require a complete managed image"); + } + const encodedProfile = env[MANAGED_STARTUP_PROFILE_ENV]; + if (!encodedProfile) fail(`${MANAGED_STARTUP_PROFILE_ENV} is required`); + const profile = decodeManagedStartupProfile(encodedProfile); + if (profile.agent !== expectedAgent) { + fail(`shared-state transaction profile targets ${profile.agent}, expected ${expectedAgent}`); + } + return profile; +} + function requireRoot(): void { if (process.geteuid?.() !== 0) { fail("managed startup requires container effective uid 0"); @@ -993,6 +1041,28 @@ export function serializeManagedStartupRuntimeEnvironment( unsetEnvironment: [], }, ): string { + const { output, unsetNames } = materializeManagedStartupRuntimeEnvironment( + environment, + corporateCaMerged, + configurationEnvironment, + applicationRuntime, + ); + const unsetLines = unsetNames.map((name) => `unset ${name}`); + const exportLines = Object.entries(output) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([name, value]) => `export ${name}=${shellSingleQuote(value)}`); + return `${[...unsetLines, ...exportLines].join("\n")}\n`; +} + +function materializeManagedStartupRuntimeEnvironment( + environment: Readonly>, + corporateCaMerged: boolean, + configurationEnvironment: Readonly> = {}, + applicationRuntime: ManagedStartupApplicationRuntimePlan = { + exportEnvironment: {}, + unsetEnvironment: [], + }, +): { output: Record; unsetNames: string[] } { const validatedApplicationRuntime = validateManagedStartupApplicationRuntimePlan(applicationRuntime); const output: Record = { @@ -1012,6 +1082,11 @@ export function serializeManagedStartupRuntimeEnvironment( } output._NEMOCLAW_CORPORATE_CA_MERGED = "1"; } + for (const name of [...Object.keys(configurationEnvironment), ...Object.keys(output)]) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(name)) { + fail(`invalid runtime environment key ${JSON.stringify(name)}`); + } + } const unsetNames = new Set([ ...Object.keys(configurationEnvironment).filter((name) => !Object.hasOwn(output, name)), ...validatedApplicationRuntime.unsetEnvironment, @@ -1021,21 +1096,146 @@ export function serializeManagedStartupRuntimeEnvironment( fail(`runtime environment cannot both export and unset ${name}`); } } - const unsetLines = [...unsetNames].sort().map((name) => { + for (const name of unsetNames) { if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(name)) { fail(`invalid runtime environment key ${JSON.stringify(name)}`); } - return `unset ${name}`; - }); - const exportLines = Object.entries(output) - .sort(([left], [right]) => left.localeCompare(right)) - .map(([name, value]) => { - if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(name)) { - fail(`invalid runtime environment key ${JSON.stringify(name)}`); + } + return { output, unsetNames: [...unsetNames].sort() }; +} + +export function serializeManagedStartupCompletionMarker( + marker: ManagedStartupCompletionMarker, +): string { + if ( + marker.schemaVersion !== MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION || + !(MANAGED_STARTUP_AGENTS as readonly string[]).includes(marker.agent) || + !SHA256_RE.test(marker.profileFingerprint) || + !SHA256_RE.test(marker.runtimeEnvironmentSha256) || + typeof marker.corporateCaMerged !== "boolean" + ) { + fail("managed startup completion marker is invalid"); + } + return `${JSON.stringify({ + agent: marker.agent, + corporateCaMerged: marker.corporateCaMerged, + profileFingerprint: marker.profileFingerprint, + runtimeEnvironmentSha256: marker.runtimeEnvironmentSha256, + schemaVersion: marker.schemaVersion, + })}\n`; +} + +function parseManagedStartupCompletionMarker(text: string): ManagedStartupCompletionMarker { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("managed startup completion marker is not valid JSON"); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + fail("managed startup completion marker must be an object"); + } + const record = parsed as Record; + const expectedKeys = [ + "agent", + "corporateCaMerged", + "profileFingerprint", + "runtimeEnvironmentSha256", + "schemaVersion", + ]; + if ( + Object.keys(record).sort().join(",") !== expectedKeys.sort().join(",") || + record.schemaVersion !== MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION || + typeof record.agent !== "string" || + !(MANAGED_STARTUP_AGENTS as readonly string[]).includes(record.agent) || + typeof record.profileFingerprint !== "string" || + !SHA256_RE.test(record.profileFingerprint) || + typeof record.runtimeEnvironmentSha256 !== "string" || + !SHA256_RE.test(record.runtimeEnvironmentSha256) || + typeof record.corporateCaMerged !== "boolean" + ) { + fail("managed startup completion marker has an invalid schema"); + } + const marker = { + schemaVersion: MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION, + agent: record.agent as ManagedStartupAgent, + profileFingerprint: record.profileFingerprint, + runtimeEnvironmentSha256: record.runtimeEnvironmentSha256, + corporateCaMerged: record.corporateCaMerged, + } as const; + if (serializeManagedStartupCompletionMarker(marker) !== text) { + fail("managed startup completion marker is not canonical"); + } + return marker; +} + +export function verifyManagedStartupImageCompletion( + expectedAgentInput: string, + expectedFingerprint: string, + completionFile: string = MANAGED_STARTUP_COMPLETION_FILE, + runtimeEnvironmentFile: string = MANAGED_STARTUP_RUNTIME_ENV_FILE, +): { readonly agent: ManagedStartupAgent; readonly fingerprint: string } { + const expectedAgent = exactAgent(expectedAgentInput); + if (!SHA256_RE.test(expectedFingerprint)) { + fail("startup completion expected profile fingerprint is invalid"); + } + const { bytes, stat } = readStableRegularFileSnapshot( + completionFile, + MAX_MANAGED_STARTUP_COMPLETION_BYTES, + ); + if ( + stat.nlink !== 1n || + stat.uid !== 0n || + stat.gid !== 0n || + Number(stat.mode & 0o777n) !== 0o444 + ) { + fail("managed startup completion marker must be root:root mode 0444"); + } + const marker = parseManagedStartupCompletionMarker(bytes.toString("utf8")); + if (marker.agent !== expectedAgent || marker.profileFingerprint !== expectedFingerprint) { + fail("managed startup completion marker does not match the requested profile"); + } + const runtimeEnvironment = readStableRegularFileSnapshot( + runtimeEnvironmentFile, + MAX_MANAGED_STARTUP_RUNTIME_ENVIRONMENT_BYTES, + ); + if ( + runtimeEnvironment.stat.nlink !== 1n || + runtimeEnvironment.stat.uid !== 0n || + runtimeEnvironment.stat.gid !== 0n || + Number(runtimeEnvironment.stat.mode & 0o777n) !== 0o444 + ) { + fail("managed startup runtime environment must be root:root mode 0444"); + } + const runtimeEnvironmentSha256 = createHash("sha256") + .update(runtimeEnvironment.bytes) + .digest("hex"); + if (runtimeEnvironmentSha256 !== marker.runtimeEnvironmentSha256) { + fail("managed startup completion marker runtime environment digest mismatch"); + } + return { agent: expectedAgent, fingerprint: expectedFingerprint }; +} + +export function waitForManagedStartupImageCompletion( + expectedAgentInput: string, + expectedFingerprint: string, + timeoutSeconds = 600, +): { readonly agent: ManagedStartupAgent; readonly fingerprint: string } { + if (!Number.isSafeInteger(timeoutSeconds) || timeoutSeconds < 1 || timeoutSeconds > 3600) { + fail("startup completion wait timeout must be an integer from 1 to 3600 seconds"); + } + const deadline = Date.now() + timeoutSeconds * 1000; + while (true) { + try { + return verifyManagedStartupImageCompletion(expectedAgentInput, expectedFingerprint); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if (Date.now() >= deadline) { + fail(`startup completion was not published within ${String(timeoutSeconds)} seconds`); } - return `export ${name}=${shellSingleQuote(value)}`; - }); - return `${[...unsetLines, ...exportLines].join("\n")}\n`; + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 250); + } + } } function applyAdapter( @@ -1168,15 +1368,29 @@ export async function applyManagedStartupImageProfile( } corporateCaMerged = mergeCorporateCa(result.application.corporateCaPath); } + const runtimeEnvironment = serializeManagedStartupRuntimeEnvironment( + mapped.runtimeEnvironment, + corporateCaMerged, + mapped.configurationEnvironment, + mapped.applicationRuntime, + ); + // This handoff is intentionally readable by the sandbox account only after + // the root-owned completion marker authenticates its exact digest. It + // contains the secret-free mapped profile environment, never provider + // credentials or the raw corporate-CA transport. + atomicWriteRootFile(MANAGED_STARTUP_RUNTIME_ENV_FILE, runtimeEnvironment, 0o444); atomicWriteRootFile( - MANAGED_STARTUP_RUNTIME_ENV_FILE, - serializeManagedStartupRuntimeEnvironment( - mapped.runtimeEnvironment, + MANAGED_STARTUP_COMPLETION_FILE, + serializeManagedStartupCompletionMarker({ + schemaVersion: MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION, + agent: expectedAgent, + profileFingerprint: result.application.fingerprint, + runtimeEnvironmentSha256: createHash("sha256") + .update(runtimeEnvironment, "utf8") + .digest("hex"), corporateCaMerged, - mapped.configurationEnvironment, - mapped.applicationRuntime, - ), - 0o400, + }), + 0o444, ); return { agent: expectedAgent, @@ -1186,6 +1400,72 @@ export async function applyManagedStartupImageProfile( }; } +function completionAlreadyPublished(request: ManagedStartupRootApplyRequest): boolean { + try { + verifyManagedStartupImageCompletion(request.agent, request.profileFingerprint); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + throw error; + } +} + +export async function applyManagedStartupRootRequest( + request: ManagedStartupRootApplyRequest, + env: Environment = process.env, +): Promise { + requireRoot(); + const profile = decodeManagedStartupProfile(request.encodedProfile); + if ( + profile.agent !== request.agent || + fingerprintManagedStartupProfile(profile) !== request.profileFingerprint + ) { + fail("root application request identity does not match its profile"); + } + const imageEnvironment = { + HOME: "/root", + PATH: FIXED_PATH, + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "1", + ...selectManagedStartupApplicationRuntimeEnvironment(env), + [MANAGED_STARTUP_PROFILE_ENV]: request.encodedProfile, + ...(request.corporateCaB64 === null + ? {} + : { [MANAGED_STARTUP_CA_ENV]: request.corporateCaB64 }), + }; + // Validate launch controls before completion inspection or transaction + // mutation. A completed same-profile replay must still be allowed to refresh + // these non-fingerprinted application-runtime values. + mapManagedStartupProfileToAgentEnvironment(profile, imageEnvironment); + const alreadyPublished = completionAlreadyPublished(request); + if (!alreadyPublished) { + ensureRootOwnedDirectory(ROOT_STATE_PARENT); + beginManagedStartupSharedStateTransaction(profile); + } + const result = await applyManagedStartupImageProfile(request.agent, imageEnvironment); + return { ...result, transactionPending: !alreadyPublished }; +} + +function readBoundedRootApplyStdin(): string { + const chunks: Buffer[] = []; + let total = 0; + while (true) { + const chunk = Buffer.alloc(16 * 1024); + const read = fs.readSync(0, chunk, 0, chunk.length, null); + if (read === 0) break; + total += read; + if (total > MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES) { + fail("root application stdin exceeds its bounded transport"); + } + chunks.push(chunk.subarray(0, read)); + } + const bytes = Buffer.concat(chunks, total); + const text = bytes.toString("utf8"); + if (text.includes("\0") || !Buffer.from(text, "utf8").equals(bytes)) { + fail("root application stdin must be valid UTF-8 without NUL bytes"); + } + return text; +} + function writeSandboxFileAtomically(target: string, contents: string, mode: number): void { const parent = path.dirname(target); const parentStat = fs.lstatSync(parent); @@ -1250,10 +1530,20 @@ function internalWriteHermesCompatHash(): void { writeSandboxFileAtomically("/sandbox/.hermes/.config-hash", decoded.toString("utf8"), 0o640); } -function readCliAgent(argv: readonly string[]): string { +function readCliAgent(argv: readonly string[], expectedLength = 2): string { const index = argv.indexOf("--agent"); - if (index < 0 || index + 1 >= argv.length || argv.length !== 2) { - fail("usage: managed-startup-image-runtime --agent "); + if (index < 0 || index + 1 >= argv.length || argv.length !== expectedLength) { + fail( + "usage: managed-startup-image-runtime [--apply-root-stdin|--wait-for-completion|--verify-completion|--begin-shared-state-transaction|--commit-shared-state-transaction] --agent ", + ); + } + return argv[index + 1] as string; +} + +function readCliFingerprint(argv: readonly string[]): string { + const index = argv.indexOf("--profile-fingerprint"); + if (index < 0 || index + 1 >= argv.length) { + fail("managed startup profile fingerprint argument is missing"); } return argv[index + 1] as string; } @@ -1267,6 +1557,66 @@ export async function main(argv: readonly string[] = process.argv.slice(2)): Pro internalWriteHermesCompatHash(); return; } + if (argv.length === 3 && argv[0] === "--apply-root-stdin") { + const expectedAgent = exactAgent(readCliAgent(argv, 3)); + const request = parseManagedStartupRootApplyRequest(readBoundedRootApplyStdin()); + if (request.agent !== expectedAgent) { + fail(`root application request targets ${request.agent}, expected ${expectedAgent}`); + } + const result = await applyManagedStartupRootRequest(request); + console.log( + result.transactionPending + ? `[managed-startup] applied ${result.agent} profile ${result.fingerprint}; transaction pending` + : `[managed-startup] ${result.agent} profile ${result.fingerprint} was already complete`, + ); + return; + } + if ( + argv.length === 5 && + (argv[0] === "--verify-completion" || argv[0] === "--wait-for-completion") + ) { + const agent = readCliAgent(argv, 5); + const fingerprint = readCliFingerprint(argv); + const result = + argv[0] === "--wait-for-completion" + ? waitForManagedStartupImageCompletion(agent, fingerprint) + : verifyManagedStartupImageCompletion(agent, fingerprint); + console.log( + `[managed-startup] verified ${result.agent} profile ${result.fingerprint} completion`, + ); + return; + } + if (argv.length === 3 && argv[0] === "--begin-shared-state-transaction") { + const profile = managedTransactionProfile(readCliAgent(argv, 3)); + ensureRootOwnedDirectory(ROOT_STATE_PARENT); + const created = beginManagedStartupSharedStateTransaction(profile); + process.stdout.write(created ? "created\n" : "pending\n"); + return; + } + if ( + argv.length === 4 && + argv[0] === "--rollback-shared-state-transaction" && + argv[3] === "--read-only-receipt" + ) { + requireRoot(); + const agent = exactAgent(readCliAgent(argv, 4)); + const rolledBack = rollbackManagedStartupSharedStateTransaction(agent, { + transactionDirectory: MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, + readOnlyReceipt: true, + }); + if (!rolledBack) fail("read-only shared-state rollback receipt is missing"); + console.log(`[managed-startup] verified and restored ${agent} shared state`); + return; + } + if (argv.length === 3 && argv[0] === "--commit-shared-state-transaction") { + requireRoot(); + const agent = exactAgent(readCliAgent(argv, 3)); + if (!commitManagedStartupSharedStateTransaction(agent)) { + fail("managed startup transaction is missing at commit"); + } + console.log(`[managed-startup] committed ${agent} shared state`); + return; + } const result = await applyManagedStartupImageProfile(readCliAgent(argv)); console.log( result.adapterApplied diff --git a/src/lib/onboard/managed-startup/root-apply.ts b/src/lib/onboard/managed-startup/root-apply.ts new file mode 100644 index 00000000000..6f5e43b502f --- /dev/null +++ b/src/lib/onboard/managed-startup/root-apply.ts @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { + decodeManagedStartupProfile, + fingerprintManagedStartupProfile, + MANAGED_STARTUP_AGENTS, + MANAGED_STARTUP_PROFILE_DEFERRED_RUNTIME_INPUTS, + MANAGED_STARTUP_PROFILE_MAX_ENCODED_BYTES, + type ManagedStartupAgent, +} from "./profile"; + +export const MANAGED_STARTUP_ROOT_APPLY_SCHEMA_VERSION = 1 as const; + +// One canonical profile (64 KiB decoded), one bounded corporate CA bundle +// (128 KiB decoded), and a small fixed JSON envelope. +export const MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES = 320 * 1024; + +const MAX_CORPORATE_CA_ENCODED_BYTES = 4 * Math.ceil((128 * 1024) / 3); +const SHA256_RE = /^[a-f0-9]{64}$/u; +const STANDARD_BASE64_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u; + +export const MANAGED_STARTUP_APPLICATION_RUNTIME_ENV_KEYS = Object.freeze( + MANAGED_STARTUP_PROFILE_DEFERRED_RUNTIME_INPUTS.openclaw + .filter( + ({ admission, owner }) => + admission === "managed-launch-forwarded" && owner === "application-environment", + ) + .map(({ input }) => input), +); + +export function selectManagedStartupApplicationRuntimeEnvironment( + environment: Readonly>, +): Readonly> { + const selected: Record = {}; + for (const name of MANAGED_STARTUP_APPLICATION_RUNTIME_ENV_KEYS) { + const value = environment[name]; + if (value !== undefined) selected[name] = value; + } + return Object.freeze(selected); +} + +export interface ManagedStartupRootApplyRequest { + readonly schemaVersion: typeof MANAGED_STARTUP_ROOT_APPLY_SCHEMA_VERSION; + readonly agent: ManagedStartupAgent; + readonly encodedProfile: string; + readonly profileFingerprint: string; + readonly corporateCaB64: string | null; +} + +function fail(message: string): never { + throw new Error(`Managed startup root application request is invalid: ${message}`); +} + +function exactAgent(value: unknown): ManagedStartupAgent { + if (typeof value === "string" && (MANAGED_STARTUP_AGENTS as readonly string[]).includes(value)) { + return value as ManagedStartupAgent; + } + return fail("agent is unsupported"); +} + +export function createManagedStartupRootApplyRequest(input: { + readonly agent: ManagedStartupAgent; + readonly encodedProfile: string; + readonly corporateCaB64?: string; +}): ManagedStartupRootApplyRequest { + const agent = exactAgent(input.agent); + if ( + input.encodedProfile.length === 0 || + input.encodedProfile.length > MANAGED_STARTUP_PROFILE_MAX_ENCODED_BYTES + ) { + fail("encoded profile exceeds its bounded transport"); + } + const profile = decodeManagedStartupProfile(input.encodedProfile); + if (profile.agent !== agent) { + fail(`profile targets ${profile.agent}, expected ${agent}`); + } + const corporateCaB64 = input.corporateCaB64 ?? null; + if ( + corporateCaB64 !== null && + (corporateCaB64.length === 0 || + corporateCaB64.length > MAX_CORPORATE_CA_ENCODED_BYTES || + !STANDARD_BASE64_RE.test(corporateCaB64) || + Buffer.from(corporateCaB64, "base64").toString("base64") !== corporateCaB64) + ) { + fail("corporate CA is not canonical bounded base64"); + } + if ((profile.corporateCa.bundleSha256 !== null) !== (corporateCaB64 !== null)) { + fail("corporate CA transport does not match the profile"); + } + if ( + corporateCaB64 !== null && + createHash("sha256").update(Buffer.from(corporateCaB64, "base64")).digest("hex") !== + profile.corporateCa.bundleSha256 + ) { + fail("corporate CA does not match the profile digest"); + } + return Object.freeze({ + schemaVersion: MANAGED_STARTUP_ROOT_APPLY_SCHEMA_VERSION, + agent, + encodedProfile: input.encodedProfile, + profileFingerprint: fingerprintManagedStartupProfile(profile), + corporateCaB64, + }); +} + +export function serializeManagedStartupRootApplyRequest( + request: ManagedStartupRootApplyRequest, +): string { + const normalized = createManagedStartupRootApplyRequest({ + agent: request.agent, + encodedProfile: request.encodedProfile, + ...(request.corporateCaB64 === null ? {} : { corporateCaB64: request.corporateCaB64 }), + }); + if ( + request.schemaVersion !== MANAGED_STARTUP_ROOT_APPLY_SCHEMA_VERSION || + request.profileFingerprint !== normalized.profileFingerprint || + !SHA256_RE.test(request.profileFingerprint) + ) { + fail("schema version or profile fingerprint is invalid"); + } + const serialized = `${JSON.stringify({ + agent: normalized.agent, + corporateCaB64: normalized.corporateCaB64, + encodedProfile: normalized.encodedProfile, + profileFingerprint: normalized.profileFingerprint, + schemaVersion: normalized.schemaVersion, + })}\n`; + if (Buffer.byteLength(serialized, "utf8") > MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES) { + fail("serialized request exceeds its bounded transport"); + } + return serialized; +} + +export function parseManagedStartupRootApplyRequest(text: string): ManagedStartupRootApplyRequest { + if (text.length === 0 || Buffer.byteLength(text, "utf8") > MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES) { + fail("serialized request is empty or too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("serialized request is not valid JSON"); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + fail("serialized request must be an object"); + } + const record = parsed as Record; + const expectedKeys = [ + "agent", + "corporateCaB64", + "encodedProfile", + "profileFingerprint", + "schemaVersion", + ]; + if ( + Object.keys(record).sort().join(",") !== expectedKeys.sort().join(",") || + record.schemaVersion !== MANAGED_STARTUP_ROOT_APPLY_SCHEMA_VERSION || + typeof record.encodedProfile !== "string" || + typeof record.profileFingerprint !== "string" || + (record.corporateCaB64 !== null && typeof record.corporateCaB64 !== "string") + ) { + fail("serialized request has an invalid schema"); + } + const request = createManagedStartupRootApplyRequest({ + agent: exactAgent(record.agent), + encodedProfile: record.encodedProfile, + ...(record.corporateCaB64 === null ? {} : { corporateCaB64: record.corporateCaB64 as string }), + }); + if ( + record.profileFingerprint !== request.profileFingerprint || + !SHA256_RE.test(record.profileFingerprint) + ) { + fail("profile fingerprint does not match the encoded profile"); + } + if (serializeManagedStartupRootApplyRequest(request) !== text) { + fail("serialized request is not canonical"); + } + return request; +} diff --git a/src/lib/onboard/managed-startup/shared-state-transaction.ts b/src/lib/onboard/managed-startup/shared-state-transaction.ts new file mode 100644 index 00000000000..d921ec36d89 --- /dev/null +++ b/src/lib/onboard/managed-startup/shared-state-transaction.ts @@ -0,0 +1,1025 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, randomBytes } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { parseSandboxMessagingPlan } from "../../messaging/plan-validation"; +import { + selectEnabledMessagingAgentRender, + selectEnabledPostAgentInstallBuildFiles, +} from "../../messaging/post-agent-install-selection"; +import { + fingerprintManagedStartupProfile, + type ManagedStartupAgent, + type ManagedStartupProfile, +} from "./profile"; + +const TRANSACTION_SCHEMA_VERSION = 1; +const MAX_TRANSACTION_FILES = 128; +const MAX_TRANSACTION_FILE_BYTES = 8 * 1024 * 1024; +const MAX_TRANSACTION_TOTAL_BYTES = 32 * 1024 * 1024; +const MAX_MANIFEST_BYTES = 256 * 1024; +const TRANSACTION_PARENT_DIRECTORY_MODE = 0o755; +const TRANSACTION_DIRECTORY_MODE = 0o700; +const TRANSACTION_FILE_MODE = 0o400; + +export const MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY = + "/var/lib/nemoclaw/managed-startup-shared-state-transaction-v1"; +export const MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY = + "/run/nemoclaw/managed-startup-shared-rollback-receipt-v1"; + +interface FilePresentReceipt { + readonly path: string; + readonly state: "file"; + readonly backup: string; + readonly sha256: string; + readonly size: number; + readonly uid: number; + readonly gid: number; + readonly mode: number; +} + +interface FileAbsentReceipt { + readonly path: string; + readonly state: "absent"; +} + +type FileReceipt = FilePresentReceipt | FileAbsentReceipt; + +interface DirectoryPresentReceipt { + readonly path: string; + readonly state: "directory"; + readonly uid: number; + readonly gid: number; + readonly mode: number; +} + +interface DirectoryAbsentReceipt { + readonly path: string; + readonly state: "absent"; +} + +type DirectoryReceipt = DirectoryPresentReceipt | DirectoryAbsentReceipt; + +interface TransactionManifest { + readonly schemaVersion: typeof TRANSACTION_SCHEMA_VERSION; + readonly agent: ManagedStartupAgent; + readonly profileFingerprint: string; + readonly files: readonly FileReceipt[]; + readonly directories: readonly DirectoryReceipt[]; +} + +export interface ManagedStartupSharedTransactionOptions { + readonly sandboxRoot?: string; + readonly transactionDirectory?: string; + /** Test seam. Production always retains the root:root defaults. */ + readonly trustedUid?: number; + /** Test seam. Production always retains the root:root defaults. */ + readonly trustedGid?: number; + /** + * Rollback-helper seam. The host copy is mounted read-only at a fixed path, + * so ownership may reflect the Docker CLI user instead of container root. + */ + readonly readOnlyReceipt?: boolean; +} + +interface ResolvedOptions { + readonly sandboxRoot: string; + readonly transactionParentDirectory: string; + readonly transactionDirectory: string; + readonly backupDirectory: string; + readonly manifestFile: string; + readonly trustedUid: number; + readonly trustedGid: number; + readonly readOnlyReceipt: boolean; +} + +interface StableFile { + readonly bytes: Buffer; + readonly stat: fs.BigIntStats; +} + +function fail(message: string): never { + throw new Error(`Managed startup shared-state transaction failed: ${message}`); +} + +function resolveOptions(options: ManagedStartupSharedTransactionOptions = {}): ResolvedOptions { + const sandboxRoot = path.resolve(options.sandboxRoot ?? "/sandbox"); + const transactionDirectory = path.resolve( + options.transactionDirectory ?? MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + ); + if ( + transactionDirectory === sandboxRoot || + transactionDirectory.startsWith(`${sandboxRoot}${path.sep}`) + ) { + fail("transaction receipts must not be stored in sandbox-shared state"); + } + return { + sandboxRoot, + transactionParentDirectory: path.dirname(transactionDirectory), + transactionDirectory, + backupDirectory: path.join(transactionDirectory, "backups"), + manifestFile: path.join(transactionDirectory, "manifest.json"), + trustedUid: options.trustedUid ?? 0, + trustedGid: options.trustedGid ?? 0, + readOnlyReceipt: options.readOnlyReceipt ?? false, + }; +} + +function modeOf(stat: fs.Stats | fs.BigIntStats): number { + if (typeof stat.mode === "bigint") { + return Number(stat.mode & 0o7777n); + } + return stat.mode & 0o7777; +} + +function requireTransactionIdentity(options: ResolvedOptions): void { + const expectedUid = options.readOnlyReceipt ? 0 : options.trustedUid; + const expectedGid = options.readOnlyReceipt ? 0 : options.trustedGid; + if (process.geteuid?.() !== expectedUid || process.getegid?.() !== expectedGid) { + fail("transaction control requires the trusted effective identity"); + } +} + +function pathExistsNoFollow(target: string): boolean { + try { + fs.lstatSync(target); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + fail(`could not inspect ${target}`); + } +} + +function requireDirectory( + target: string, + options: ResolvedOptions, + expectedMode: number | null = null, +): fs.Stats { + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch { + fail(`required directory is missing: ${target}`); + } + if (stat.isSymbolicLink() || !stat.isDirectory()) { + fail(`required directory is unsafe: ${target}`); + } + if ( + expectedMode !== null && + (stat.uid !== options.trustedUid || + stat.gid !== options.trustedGid || + modeOf(stat) !== expectedMode) + ) { + fail( + `${target} must be ${options.trustedUid}:${options.trustedGid} mode ${expectedMode.toString(8)}`, + ); + } + return stat; +} + +function requireTransactionBoundaries(options: ResolvedOptions): void { + requireDirectory(options.sandboxRoot, options); + requireDirectory(options.transactionParentDirectory, options, TRANSACTION_PARENT_DIRECTORY_MODE); +} + +function sameStableMetadata(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.mode === right.mode && + left.nlink === right.nlink && + left.uid === right.uid && + left.gid === right.gid && + left.size === right.size && + left.mtimeNs === right.mtimeNs && + left.ctimeNs === right.ctimeNs + ); +} + +function readStableFile(target: string, maxBytes: number): StableFile { + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") fail("O_NOFOLLOW is unavailable"); + let descriptor: number; + try { + descriptor = fs.openSync(target, fs.constants.O_RDONLY | noFollow); + } catch { + fail(`could not safely open ${target}`); + } + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + before.size < 0n || + before.size > BigInt(maxBytes) + ) { + fail(`refusing unsafe or oversized transaction file ${target}`); + } + const bytes = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < bytes.length) { + const count = fs.readSync(descriptor, bytes, offset, bytes.length - offset, offset); + if (count === 0) break; + offset += count; + } + const overflow = Buffer.alloc(1); + const overflowCount = fs.readSync(descriptor, overflow, 0, 1, offset); + const after = fs.fstatSync(descriptor, { bigint: true }); + if (offset !== bytes.length || overflowCount !== 0 || !sameStableMetadata(before, after)) { + fail(`${target} changed while it was captured`); + } + return { bytes, stat: before }; + } finally { + fs.closeSync(descriptor); + } +} + +function safeRelativePath(value: string): string { + if ( + value.length === 0 || + value.startsWith("/") || + value.includes("\\") || + /[\0-\x1f\x7f]/u.test(value) + ) { + fail(`unsafe transaction path ${JSON.stringify(value)}`); + } + const segments = value.split("/"); + if (segments.some((segment) => !segment || segment === "." || segment === "..")) { + fail(`unsafe transaction path ${JSON.stringify(value)}`); + } + return segments.join("/"); +} + +function absoluteTarget(relativePath: string, options: ResolvedOptions): string { + const safe = safeRelativePath(relativePath); + const target = path.resolve(options.sandboxRoot, safe); + if (!target.startsWith(`${options.sandboxRoot}${path.sep}`)) { + fail(`transaction target escapes the sandbox root: ${relativePath}`); + } + return target; +} + +function relativeTarget(target: string, options: ResolvedOptions): string { + return safeRelativePath(path.relative(options.sandboxRoot, target)); +} + +function validateExistingAncestors(target: string, options: ResolvedOptions): void { + const relative = relativeTarget(target, options); + const sandboxStat = requireDirectory(options.sandboxRoot, options); + let current = options.sandboxRoot; + const segments = relative.split("/").slice(0, -1); + for (const segment of segments) { + current = path.join(current, segment); + let stat: fs.Stats; + try { + stat = fs.lstatSync(current); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + fail(`could not inspect transaction path ancestor ${current}`); + } + if (stat.isSymbolicLink() || !stat.isDirectory()) { + fail(`transaction path ancestor is unsafe: ${current}`); + } + if (stat.dev !== sandboxStat.dev) { + fail(`transaction path crosses a nested filesystem mount: ${current}`); + } + } +} + +function agentRoot(agent: ManagedStartupAgent, sandboxRoot: string): string { + switch (agent) { + case "openclaw": + return path.join(sandboxRoot, ".openclaw"); + case "hermes": + return path.join(sandboxRoot, ".hermes"); + case "langchain-deepagents-code": + return path.join(sandboxRoot, ".deepagents"); + } +} + +function resolveUnderAgentRoot(root: string, relativePath: string): string { + const safe = safeRelativePath(relativePath); + const target = path.resolve(root, safe); + if (!target.startsWith(`${root}${path.sep}`)) { + fail(`managed output escapes the agent root: ${relativePath}`); + } + return target; +} + +function renderTarget(root: string, agent: ManagedStartupAgent, target: string): string { + if (agent === "openclaw" && target === "openclaw.json") { + return path.join(root, "openclaw.json"); + } + const prefix = agent === "openclaw" ? "~/.openclaw/" : agent === "hermes" ? "~/.hermes/" : null; + if (!prefix || !target.startsWith(prefix)) { + fail(`unsupported managed messaging render target ${JSON.stringify(target)}`); + } + return resolveUnderAgentRoot(root, target.slice(prefix.length)); +} + +function managedOutputTargets( + profile: ManagedStartupProfile, + options: ResolvedOptions, +): { readonly files: string[]; readonly directories: string[] } { + const root = agentRoot(profile.agent, options.sandboxRoot); + const files = new Set(); + const directories = new Set([root]); + switch (profile.agent) { + case "openclaw": + files.add(path.join(root, "openclaw.json")); + files.add(path.join(root, ".config-hash")); + break; + case "hermes": + files.add(path.join(root, "config.yaml")); + files.add(path.join(root, ".env")); + files.add(path.join(root, ".config-hash")); + break; + case "langchain-deepagents-code": + files.add(path.join(root, "config.toml")); + directories.add(path.join(root, ".state")); + directories.add(path.join(root, "skills")); + break; + } + + if (profile.messaging.plan !== null) { + const plan = parseSandboxMessagingPlan(profile.messaging.plan, { agent: profile.agent }); + if (!plan) fail("managed messaging plan is invalid"); + for (const render of selectEnabledMessagingAgentRender(plan)) { + if (typeof render.target !== "string") continue; + files.add(renderTarget(root, profile.agent, render.target)); + } + for (const step of selectEnabledPostAgentInstallBuildFiles(plan)) { + if (typeof step.value !== "object" || step.value === null) { + continue; + } + const outputPath = (step.value as Record).path; + if (typeof outputPath === "string") { + files.add(resolveUnderAgentRoot(root, outputPath)); + } + } + } + + for (const file of files) { + let parent = path.dirname(file); + while (parent !== options.sandboxRoot && parent.startsWith(`${root}${path.sep}`)) { + directories.add(parent); + if (parent === root) break; + parent = path.dirname(parent); + } + } + return { + files: [...files].sort(), + directories: [...directories].sort( + (left, right) => left.split(path.sep).length - right.split(path.sep).length, + ), + }; +} + +function snapshotFile( + target: string, + index: number, + options: ResolvedOptions, +): { readonly receipt: FileReceipt; readonly bytes: Buffer | null } { + validateExistingAncestors(target, options); + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return { + receipt: { path: relativeTarget(target, options), state: "absent" }, + bytes: null, + }; + } + fail(`could not inspect managed output ${target}`); + } + if (stat.isSymbolicLink() || !stat.isFile() || stat.nlink !== 1) { + fail(`managed output is not a safe regular file: ${target}`); + } + if (stat.dev !== requireDirectory(options.sandboxRoot, options).dev) { + fail(`managed output crosses a nested filesystem mount: ${target}`); + } + const stable = readStableFile(target, MAX_TRANSACTION_FILE_BYTES); + const size = Number(stable.stat.size); + const backup = `${String(index).padStart(3, "0")}.bin`; + return { + receipt: { + path: relativeTarget(target, options), + state: "file", + backup, + sha256: createHash("sha256").update(stable.bytes).digest("hex"), + size, + uid: Number(stable.stat.uid), + gid: Number(stable.stat.gid), + mode: Number(stable.stat.mode & 0o7777n), + }, + bytes: stable.bytes, + }; +} + +function snapshotDirectory(target: string, options: ResolvedOptions): DirectoryReceipt { + validateExistingAncestors(path.join(target, ".receipt"), options); + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return { path: relativeTarget(target, options), state: "absent" }; + } + fail(`could not inspect managed output directory ${target}`); + } + if (stat.isSymbolicLink() || !stat.isDirectory()) { + fail(`managed output directory is unsafe: ${target}`); + } + if (stat.dev !== requireDirectory(options.sandboxRoot, options).dev) { + fail(`managed output directory crosses a nested filesystem mount: ${target}`); + } + return { + path: relativeTarget(target, options), + state: "directory", + uid: stat.uid, + gid: stat.gid, + mode: modeOf(stat), + }; +} + +function atomicWriteTrustedFile( + target: string, + contents: string | Buffer, + mode: number, + uid: number, + gid: number, +): void { + const parent = path.dirname(target); + const temporary = path.join( + parent, + `.${path.basename(target)}.${randomBytes(12).toString("hex")}`, + ); + let descriptor: number | undefined; + try { + descriptor = fs.openSync( + temporary, + fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW, + 0o600, + ); + fs.writeFileSync(descriptor, contents); + fs.fchownSync(descriptor, uid, gid); + fs.fchmodSync(descriptor, mode); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = undefined; + fs.renameSync(temporary, target); + } catch (error) { + if (descriptor !== undefined) fs.closeSync(descriptor); + try { + fs.unlinkSync(temporary); + } catch { + // Preserve the primary failure. + } + fail(`could not atomically write ${target}: ${(error as Error).message}`); + } +} + +function canonicalManifest(manifest: TransactionManifest): string { + return `${JSON.stringify(manifest, null, 2)}\n`; +} + +function requireExactKeys(record: Record, keys: readonly string[]): void { + if (Object.keys(record).sort().join(",") !== [...keys].sort().join(",")) { + fail("transaction manifest contains unexpected fields"); + } +} + +function safeMetadata(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0; +} + +function parseManifest(text: string): TransactionManifest { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("transaction manifest is not valid JSON"); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + fail("transaction manifest must be an object"); + } + const record = parsed as Record; + requireExactKeys(record, [ + "agent", + "directories", + "files", + "profileFingerprint", + "schemaVersion", + ]); + if ( + record.schemaVersion !== TRANSACTION_SCHEMA_VERSION || + !["openclaw", "hermes", "langchain-deepagents-code"].includes(String(record.agent)) || + typeof record.profileFingerprint !== "string" || + !/^[a-f0-9]{64}$/u.test(record.profileFingerprint) || + !Array.isArray(record.files) || + !Array.isArray(record.directories) || + record.files.length > MAX_TRANSACTION_FILES || + record.directories.length > MAX_TRANSACTION_FILES * 4 + ) { + fail("transaction manifest has an invalid envelope"); + } + const files = record.files.map((value): FileReceipt => { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return fail("transaction file receipt must be an object"); + } + const receipt = value as Record; + if (typeof receipt.path !== "string") { + return fail("transaction file receipt path must be a string"); + } + const receiptPath = safeRelativePath(receipt.path); + if (receipt.state === "absent") { + requireExactKeys(receipt, ["path", "state"]); + return { path: receiptPath, state: "absent" }; + } + requireExactKeys(receipt, ["backup", "gid", "mode", "path", "sha256", "size", "state", "uid"]); + if ( + receipt.state !== "file" || + typeof receipt.backup !== "string" || + !/^[0-9]{3}\.bin$/u.test(receipt.backup) || + typeof receipt.sha256 !== "string" || + !/^[a-f0-9]{64}$/u.test(receipt.sha256) || + !safeMetadata(receipt.size) || + (receipt.size as number) > MAX_TRANSACTION_FILE_BYTES || + !safeMetadata(receipt.uid) || + !safeMetadata(receipt.gid) || + !safeMetadata(receipt.mode) || + (receipt.mode as number) > 0o7777 + ) { + return fail("transaction file receipt is invalid"); + } + return { + path: receiptPath, + state: "file", + backup: receipt.backup, + sha256: receipt.sha256, + size: receipt.size as number, + uid: receipt.uid as number, + gid: receipt.gid as number, + mode: receipt.mode as number, + }; + }); + const directories = record.directories.map((value): DirectoryReceipt => { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return fail("transaction directory receipt must be an object"); + } + const receipt = value as Record; + if (typeof receipt.path !== "string") { + return fail("transaction directory receipt path must be a string"); + } + const receiptPath = safeRelativePath(receipt.path); + if (receipt.state === "absent") { + requireExactKeys(receipt, ["path", "state"]); + return { path: receiptPath, state: "absent" }; + } + requireExactKeys(receipt, ["gid", "mode", "path", "state", "uid"]); + if ( + receipt.state !== "directory" || + !safeMetadata(receipt.uid) || + !safeMetadata(receipt.gid) || + !safeMetadata(receipt.mode) || + (receipt.mode as number) > 0o7777 + ) { + return fail("transaction directory receipt is invalid"); + } + return { + path: receiptPath, + state: "directory", + uid: receipt.uid as number, + gid: receipt.gid as number, + mode: receipt.mode as number, + }; + }); + const filePaths = files.map((receipt) => receipt.path); + const directoryPaths = directories.map((receipt) => receipt.path); + const backupNames = files + .filter((receipt): receipt is FilePresentReceipt => receipt.state === "file") + .map((receipt) => receipt.backup); + if ( + new Set(filePaths).size !== filePaths.length || + new Set(directoryPaths).size !== directoryPaths.length || + new Set(backupNames).size !== backupNames.length + ) { + fail("transaction manifest contains duplicate receipts"); + } + const manifest: TransactionManifest = { + schemaVersion: TRANSACTION_SCHEMA_VERSION, + agent: record.agent as ManagedStartupAgent, + profileFingerprint: record.profileFingerprint, + files, + directories, + }; + if (canonicalManifest(manifest) !== text) { + fail("transaction manifest is not canonical"); + } + return manifest; +} + +function requireTrustedTransactionPath( + target: string, + mode: number, + options: ResolvedOptions, +): void { + const stat = fs.lstatSync(target); + if ( + stat.isSymbolicLink() || + (mode === TRANSACTION_DIRECTORY_MODE ? !stat.isDirectory() : !stat.isFile()) || + (!options.readOnlyReceipt && + (stat.uid !== options.trustedUid || stat.gid !== options.trustedGid)) || + modeOf(stat) !== mode + ) { + fail(`transaction artifact has unsafe metadata: ${target}`); + } +} + +function requireReadOnlyReceiptMount(options: ResolvedOptions): void { + if (!options.readOnlyReceipt) return; + const probe = path.join(options.transactionDirectory, ".nemoclaw-write-probe"); + let descriptor: number | undefined; + try { + descriptor = fs.openSync( + probe, + fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW, + 0o600, + ); + fs.closeSync(descriptor); + descriptor = undefined; + fs.unlinkSync(probe); + } catch (error) { + if (descriptor !== undefined) fs.closeSync(descriptor); + if ((error as NodeJS.ErrnoException).code === "EROFS") return; + fail("rollback receipt must be mounted on a read-only filesystem"); + } + fail("rollback receipt mount is writable"); +} + +function loadManifest(options: ResolvedOptions): TransactionManifest | null { + requireTransactionBoundaries(options); + if (!pathExistsNoFollow(options.transactionDirectory)) return null; + requireTrustedTransactionPath(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE, options); + requireReadOnlyReceiptMount(options); + requireTrustedTransactionPath(options.backupDirectory, TRANSACTION_DIRECTORY_MODE, options); + requireTrustedTransactionPath(options.manifestFile, TRANSACTION_FILE_MODE, options); + const stable = readStableFile(options.manifestFile, MAX_MANIFEST_BYTES); + if ( + (!options.readOnlyReceipt && + (Number(stable.stat.uid) !== options.trustedUid || + Number(stable.stat.gid) !== options.trustedGid)) || + Number(stable.stat.mode & 0o7777n) !== TRANSACTION_FILE_MODE + ) { + fail("transaction manifest ownership changed while it was read"); + } + return parseManifest(stable.bytes.toString("utf8")); +} + +function verifyBackup(receipt: FilePresentReceipt, options: ResolvedOptions): Buffer { + const backupPath = path.join(options.backupDirectory, receipt.backup); + requireTrustedTransactionPath(backupPath, TRANSACTION_FILE_MODE, options); + const stable = readStableFile(backupPath, MAX_TRANSACTION_FILE_BYTES); + const digest = createHash("sha256").update(stable.bytes).digest("hex"); + if (stable.bytes.length !== receipt.size || digest !== receipt.sha256) { + fail(`transaction backup does not match its receipt: ${receipt.path}`); + } + return stable.bytes; +} + +function verifyAllBackups( + receipts: readonly FileReceipt[], + options: ResolvedOptions, +): ReadonlyMap { + const backups = new Map(); + for (const receipt of receipts) { + if (receipt.state === "file") { + backups.set(receipt.path, verifyBackup(receipt, options)); + } + } + return backups; +} + +function fileMatchesReceipt(target: string, receipt: FilePresentReceipt): boolean { + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + fail(`could not inspect managed output ${target}`); + } + if (stat.isSymbolicLink() || !stat.isFile() || stat.nlink !== 1) return false; + const stable = readStableFile(target, MAX_TRANSACTION_FILE_BYTES); + return ( + stable.bytes.length === receipt.size && + createHash("sha256").update(stable.bytes).digest("hex") === receipt.sha256 && + Number(stable.stat.uid) === receipt.uid && + Number(stable.stat.gid) === receipt.gid && + Number(stable.stat.mode & 0o7777n) === receipt.mode + ); +} + +function directoryMatchesReceipt(target: string, receipt: DirectoryPresentReceipt): boolean { + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + fail(`could not inspect managed output directory ${target}`); + } + return ( + !stat.isSymbolicLink() && + stat.isDirectory() && + stat.uid === receipt.uid && + stat.gid === receipt.gid && + modeOf(stat) === receipt.mode + ); +} + +function removeTransactionDirectory(options: ResolvedOptions): void { + requireTrustedTransactionPath(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE, options); + fs.rmSync(options.transactionDirectory, { force: false, recursive: true }); + if (pathExistsNoFollow(options.transactionDirectory)) { + fail("transaction directory remained after cleanup"); + } +} + +export function beginManagedStartupSharedStateTransaction( + profile: ManagedStartupProfile, + inputOptions: ManagedStartupSharedTransactionOptions = {}, +): boolean { + const options = resolveOptions(inputOptions); + requireTransactionIdentity(options); + if (options.readOnlyReceipt) { + fail("cannot begin a transaction from a read-only rollback receipt"); + } + requireTransactionBoundaries(options); + const profileFingerprint = fingerprintManagedStartupProfile(profile); + const pending = loadManifest(options); + if (pending) { + if (pending.agent !== profile.agent || pending.profileFingerprint !== profileFingerprint) { + fail("a pending managed startup transaction belongs to a different profile"); + } + verifyAllBackups(pending.files, options); + return false; + } + const targets = managedOutputTargets(profile, options); + if (targets.files.length > MAX_TRANSACTION_FILES) { + fail("managed startup transaction has too many file targets"); + } + const snapshots = targets.files.map((target, index) => snapshotFile(target, index, options)); + const totalBytes = snapshots.reduce((sum, snapshot) => sum + (snapshot.bytes?.length ?? 0), 0); + if (totalBytes > MAX_TRANSACTION_TOTAL_BYTES) { + fail("managed startup transaction backup exceeds the total size limit"); + } + const directories = targets.directories.map((target) => snapshotDirectory(target, options)); + const manifest: TransactionManifest = { + schemaVersion: TRANSACTION_SCHEMA_VERSION, + agent: profile.agent, + profileFingerprint, + files: snapshots.map(({ receipt }) => receipt), + directories, + }; + + let createdTransactionIdentity: + | { readonly dev: bigint; readonly ino: bigint; readonly uid: bigint; readonly gid: bigint } + | undefined; + try { + fs.mkdirSync(options.transactionDirectory, { mode: TRANSACTION_DIRECTORY_MODE }); + const created = fs.lstatSync(options.transactionDirectory, { bigint: true }); + if (!created.isDirectory() || created.isSymbolicLink()) { + fail("new transaction path is not a directory"); + } + createdTransactionIdentity = { + dev: created.dev, + ino: created.ino, + uid: created.uid, + gid: created.gid, + }; + fs.chownSync(options.transactionDirectory, options.trustedUid, options.trustedGid); + fs.chmodSync(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE); + fs.mkdirSync(options.backupDirectory, { mode: TRANSACTION_DIRECTORY_MODE }); + fs.chownSync(options.backupDirectory, options.trustedUid, options.trustedGid); + fs.chmodSync(options.backupDirectory, TRANSACTION_DIRECTORY_MODE); + for (const snapshot of snapshots) { + if (snapshot.receipt.state !== "file" || snapshot.bytes === null) continue; + atomicWriteTrustedFile( + path.join(options.backupDirectory, snapshot.receipt.backup), + snapshot.bytes, + TRANSACTION_FILE_MODE, + options.trustedUid, + options.trustedGid, + ); + } + atomicWriteTrustedFile( + options.manifestFile, + canonicalManifest(manifest), + TRANSACTION_FILE_MODE, + options.trustedUid, + options.trustedGid, + ); + loadManifest(options); + } catch (error) { + try { + if (createdTransactionIdentity && pathExistsNoFollow(options.transactionDirectory)) { + const current = fs.lstatSync(options.transactionDirectory, { bigint: true }); + if ( + !current.isSymbolicLink() && + current.isDirectory() && + current.dev === createdTransactionIdentity.dev && + current.ino === createdTransactionIdentity.ino && + current.uid === createdTransactionIdentity.uid && + current.gid === createdTransactionIdentity.gid + ) { + fs.chmodSync(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE); + fs.chownSync(options.transactionDirectory, options.trustedUid, options.trustedGid); + } + requireTrustedTransactionPath( + options.transactionDirectory, + TRANSACTION_DIRECTORY_MODE, + options, + ); + fs.rmSync(options.transactionDirectory, { force: true, recursive: true }); + } + } catch { + // Preserve the primary transaction preparation failure. + } + throw error; + } + return true; +} + +function ensureOriginalDirectories( + receipts: readonly DirectoryReceipt[], + options: ResolvedOptions, +): void { + for (const receipt of receipts) { + if (receipt.state !== "directory") continue; + const target = absoluteTarget(receipt.path, options); + validateExistingAncestors(path.join(target, ".restore"), options); + let stat: fs.Stats | null = null; + try { + stat = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + fail(`could not inspect restore directory ${target}`); + } + } + if (stat && (stat.isSymbolicLink() || !stat.isDirectory())) { + fail(`restore directory is unsafe: ${target}`); + } + if (stat && directoryMatchesReceipt(target, receipt)) continue; + if (!stat) fs.mkdirSync(target, { mode: receipt.mode }); + fs.chownSync(target, receipt.uid, receipt.gid); + fs.chmodSync(target, receipt.mode); + } +} + +function restoreFiles( + receipts: readonly FileReceipt[], + backups: ReadonlyMap, + options: ResolvedOptions, +): void { + for (const receipt of receipts) { + const target = absoluteTarget(receipt.path, options); + validateExistingAncestors(target, options); + if (receipt.state === "absent") { + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + fail(`could not inspect new managed output ${target}`); + } + if (stat.isDirectory()) { + fail(`new managed output unexpectedly became a directory: ${target}`); + } + fs.unlinkSync(target); + continue; + } + if (fileMatchesReceipt(target, receipt)) continue; + const bytes = backups.get(receipt.path); + if (!bytes) fail(`verified transaction backup is missing: ${receipt.path}`); + let current: fs.Stats | null = null; + try { + current = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + fail(`could not inspect managed output before restore: ${target}`); + } + } + if (current?.isDirectory()) { + fail(`managed output unexpectedly became a directory: ${target}`); + } + atomicWriteTrustedFile(target, bytes, receipt.mode, receipt.uid, receipt.gid); + } +} + +function restoreDirectoryMetadata( + receipts: readonly DirectoryReceipt[], + options: ResolvedOptions, +): void { + for (const receipt of [...receipts].reverse()) { + const target = absoluteTarget(receipt.path, options); + if (receipt.state === "absent") { + try { + fs.rmdirSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + fail(`could not remove newly created managed directory ${target}`); + } + continue; + } + if (directoryMatchesReceipt(target, receipt)) continue; + const stat = fs.lstatSync(target); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + fail(`managed directory changed type during restore: ${target}`); + } + fs.chownSync(target, receipt.uid, receipt.gid); + fs.chmodSync(target, receipt.mode); + } +} + +function verifyRestoration(manifest: TransactionManifest, options: ResolvedOptions): void { + for (const receipt of manifest.files) { + const target = absoluteTarget(receipt.path, options); + if (receipt.state === "absent") { + if (pathExistsNoFollow(target)) { + fail(`new managed output remained after rollback: ${target}`); + } + continue; + } + const stable = readStableFile(target, MAX_TRANSACTION_FILE_BYTES); + if ( + stable.bytes.length !== receipt.size || + createHash("sha256").update(stable.bytes).digest("hex") !== receipt.sha256 || + Number(stable.stat.uid) !== receipt.uid || + Number(stable.stat.gid) !== receipt.gid || + Number(stable.stat.mode & 0o7777n) !== receipt.mode + ) { + fail(`managed output was not restored exactly: ${target}`); + } + } + for (const receipt of manifest.directories) { + const target = absoluteTarget(receipt.path, options); + if (receipt.state === "absent") { + if (pathExistsNoFollow(target)) { + fail(`new managed directory remained after rollback: ${target}`); + } + continue; + } + const stat = fs.lstatSync(target); + if ( + stat.isSymbolicLink() || + !stat.isDirectory() || + stat.uid !== receipt.uid || + stat.gid !== receipt.gid || + modeOf(stat) !== receipt.mode + ) { + fail(`managed directory metadata was not restored exactly: ${target}`); + } + } +} + +export function rollbackManagedStartupSharedStateTransaction( + expectedAgent: ManagedStartupAgent, + inputOptions: ManagedStartupSharedTransactionOptions = {}, +): boolean { + const options = resolveOptions(inputOptions); + requireTransactionIdentity(options); + const manifest = loadManifest(options); + if (!manifest) return false; + if (manifest.agent !== expectedAgent) { + fail(`pending transaction targets ${manifest.agent}, expected ${expectedAgent}`); + } + const backups = verifyAllBackups(manifest.files, options); + ensureOriginalDirectories(manifest.directories, options); + restoreFiles(manifest.files, backups, options); + restoreDirectoryMetadata(manifest.directories, options); + verifyRestoration(manifest, options); + if (!options.readOnlyReceipt) { + removeTransactionDirectory(options); + } + return true; +} + +export function commitManagedStartupSharedStateTransaction( + expectedAgent: ManagedStartupAgent, + inputOptions: ManagedStartupSharedTransactionOptions = {}, +): boolean { + const options = resolveOptions(inputOptions); + requireTransactionIdentity(options); + if (options.readOnlyReceipt) { + fail("cannot commit a read-only rollback receipt"); + } + const manifest = loadManifest(options); + if (!manifest) return false; + if (manifest.agent !== expectedAgent) { + fail(`pending transaction targets ${manifest.agent}, expected ${expectedAgent}`); + } + removeTransactionDirectory(options); + return true; +} From 6da5eb278cdabcd380358a2f09cb8b8ce2bcac53 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 18:58:16 -0700 Subject: [PATCH 009/117] test(onboard): keep managed startup tests branchless Signed-off-by: Aaron Erickson --- .../managed-startup-agent-environment.test.ts | 4 ++-- .../managed-startup-image-runtime.test.ts | 22 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/lib/onboard/managed-startup-agent-environment.test.ts b/src/lib/onboard/managed-startup-agent-environment.test.ts index 88be8dc6b6d..98bd6c0b5a7 100644 --- a/src/lib/onboard/managed-startup-agent-environment.test.ts +++ b/src/lib/onboard/managed-startup-agent-environment.test.ts @@ -432,8 +432,8 @@ describe("managed startup agent environment", () => { unsetEnvironment: [], }); } finally { - if (previous === undefined) delete process.env[name]; - else process.env[name] = previous; + delete process.env[name]; + Object.assign(process.env, previous === undefined ? {} : { [name]: previous }); } }); diff --git a/src/lib/onboard/managed-startup-image-runtime.test.ts b/src/lib/onboard/managed-startup-image-runtime.test.ts index d0ec6588e61..51e842310ae 100644 --- a/src/lib/onboard/managed-startup-image-runtime.test.ts +++ b/src/lib/onboard/managed-startup-image-runtime.test.ts @@ -334,16 +334,18 @@ describe("managed startup image runtime", () => { nlink: 1, uid: 0, }) as fs.Stats; - const missing = () => Object.assign(new Error("missing"), { code: "ENOENT" }); + const missing = (): never => { + throw Object.assign(new Error("missing"), { code: "ENOENT" }); + }; vi.spyOn(process, "geteuid").mockReturnValue(0); vi.spyOn(fs, "lstatSync").mockImplementation(((target: fs.PathLike) => { const resolved = String(target); - if (directories.has(resolved)) return stat("directory", 0o755); - if (resolved === MANAGED_STARTUP_RUNTIME_ENV_FILE && runtimeFileWritten) { - return stat("file", 0o400); - } - throw missing(); + return directories.has(resolved) + ? stat("directory", 0o755) + : resolved === MANAGED_STARTUP_RUNTIME_ENV_FILE && runtimeFileWritten + ? stat("file", 0o400) + : missing(); }) as typeof fs.lstatSync); vi.spyOn(fs, "mkdirSync").mockImplementation(() => undefined); vi.spyOn(fs, "chownSync").mockImplementation(() => undefined); @@ -352,17 +354,15 @@ describe("managed startup image runtime", () => { vi.spyOn(fs, "openSync").mockReturnValue(91); vi.spyOn(fs, "fchownSync").mockImplementation(() => undefined); vi.spyOn(fs, "writeFileSync").mockImplementation(((target: fs.PathOrFileDescriptor, value) => { - if (target === 91) runtimeWrites.push(String(value)); + runtimeWrites.push(...(target === 91 ? [String(value)] : [])); }) as typeof fs.writeFileSync); vi.spyOn(fs, "fchmodSync").mockImplementation(() => undefined); vi.spyOn(fs, "fsyncSync").mockImplementation(() => undefined); vi.spyOn(fs, "closeSync").mockImplementation(() => undefined); vi.spyOn(fs, "renameSync").mockImplementation((_source, target) => { - if (String(target) === MANAGED_STARTUP_RUNTIME_ENV_FILE) runtimeFileWritten = true; - }); - vi.spyOn(fs, "unlinkSync").mockImplementation(() => { - throw missing(); + runtimeFileWritten ||= String(target) === MANAGED_STARTUP_RUNTIME_ENV_FILE; }); + vi.spyOn(fs, "unlinkSync").mockImplementation(missing); } it("rejects invalid OpenClaw launch controls before filesystem or coordinator mutation", async () => { From b14013f64516f252ee3374c0954f5347b99af3f2 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 20:21:22 -0700 Subject: [PATCH 010/117] test(onboard): cover managed startup fixture CLI Signed-off-by: Aaron Erickson --- ...nerate-managed-startup-profile-fixture.mts | 3 +- ...te-managed-startup-profile-fixture.test.ts | 113 ++++++++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 test/generate-managed-startup-profile-fixture.test.ts diff --git a/scripts/checks/generate-managed-startup-profile-fixture.mts b/scripts/checks/generate-managed-startup-profile-fixture.mts index 72499b1cfbd..bef51c8d347 100755 --- a/scripts/checks/generate-managed-startup-profile-fixture.mts +++ b/scripts/checks/generate-managed-startup-profile-fixture.mts @@ -8,12 +8,13 @@ import { resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { encodeManagedStartupProfile, + MANAGED_STARTUP_AGENTS, MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, type ManagedStartupAgent, type ManagedStartupProfile, } from "../../src/lib/onboard/managed-startup/profile.ts"; -const AGENTS = new Set(["openclaw", "hermes", "langchain-deepagents-code"]); +const AGENTS = new Set(MANAGED_STARTUP_AGENTS); export const MANAGED_STARTUP_E2E_HTTP_PROXY = "http://fixture-http-proxy.example.test:18080"; export const MANAGED_STARTUP_E2E_HTTPS_PROXY = "http://fixture-https-proxy.example.test:18443"; diff --git a/test/generate-managed-startup-profile-fixture.test.ts b/test/generate-managed-startup-profile-fixture.test.ts new file mode 100644 index 00000000000..9e6c7a644bb --- /dev/null +++ b/test/generate-managed-startup-profile-fixture.test.ts @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, + MANAGED_STARTUP_E2E_HTTP_PROXY, + MANAGED_STARTUP_E2E_HTTPS_PROXY, + MANAGED_STARTUP_E2E_NO_PROXY, +} from "../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + decodeManagedStartupProfile, + MANAGED_STARTUP_AGENTS, +} from "../src/lib/onboard/managed-startup/profile"; + +const SCRIPT_PATH = path.join( + import.meta.dirname, + "..", + "scripts", + "checks", + "generate-managed-startup-profile-fixture.mts", +); +const DEFAULT_MODEL = "nvidia/nemotron-3-ultra-550b-a55b"; +const CHANGED_MODEL = "nvidia/nemotron-3-super-120b-a12b"; +const CORPORATE_CA_SHA256 = createHash("sha256") + .update(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM) + .digest("hex"); + +function runFixture(args: readonly string[]) { + return spawnSync( + process.execPath, + ["--experimental-strip-types", "--no-warnings", SCRIPT_PATH, ...args], + { + encoding: "utf8", + timeout: 10_000, + }, + ); +} + +describe("generate-managed-startup-profile-fixture.mts CLI", () => { + it("emits the exact corporate CA as base64", () => { + const result = runFixture(["--corporate-ca-b64"]); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(Buffer.from(result.stdout.trim(), "base64").toString("utf8")).toBe( + MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, + ); + }); + + it.each([ + { + name: "missing --agent", + args: [] as const, + error: "--agent is required", + }, + { + name: "invalid agent", + args: ["--agent", "not-a-shipped-agent"] as const, + error: "--agent must identify a shipped managed-image agent", + }, + { + name: "unsupported argument", + args: ["--agent", "openclaw", "--unsupported"] as const, + error: "unsupported arguments: --unsupported", + }, + ])("rejects $name", ({ args, error }) => { + const result = runFixture(args); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr.trim()).toBe(error); + }); + + it.each(MANAGED_STARTUP_AGENTS)("emits a valid default profile for %s", (agent) => { + const result = runFixture(["--agent", agent]); + const profile = decodeManagedStartupProfile(result.stdout.trim()); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(profile.agent).toBe(agent); + expect(profile.inference.model).toBe(DEFAULT_MODEL); + expect(profile.proxy.hostHttpUrl).toBe(MANAGED_STARTUP_E2E_HTTP_PROXY); + expect(profile.proxy.hostHttpsUrl).toBe(MANAGED_STARTUP_E2E_HTTPS_PROXY); + expect(profile.proxy.hostNoProxy).toEqual([...MANAGED_STARTUP_E2E_NO_PROXY].sort()); + expect(profile.corporateCa.bundleSha256).toBeNull(); + }); + + it.each( + MANAGED_STARTUP_AGENTS, + )("honors every supported optional flag together for %s", (agent) => { + const result = runFixture([ + "--agent", + agent, + "--changed", + "--corporate-ca", + "--without-host-proxy", + ]); + const profile = decodeManagedStartupProfile(result.stdout.trim()); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(profile.agent).toBe(agent); + expect(profile.inference.model).toBe(CHANGED_MODEL); + expect(profile.proxy.hostHttpUrl).toBeNull(); + expect(profile.proxy.hostHttpsUrl).toBeNull(); + expect(profile.proxy.hostNoProxy).toEqual([]); + expect(profile.corporateCa.bundleSha256).toBe(CORPORATE_CA_SHA256); + }); +}); From c95bdf721510eb3aa79f3e1646a46c6dc8b4b3cd Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 21:01:00 -0700 Subject: [PATCH 011/117] fix(onboard): serialize managed startup transactions Signed-off-by: Aaron Erickson --- .../managed-startup-application.test.ts | 187 ++++++++++++- .../onboard/managed-startup/application.ts | 258 +++++++++++++++--- 2 files changed, 400 insertions(+), 45 deletions(-) diff --git a/src/lib/onboard/managed-startup-application.test.ts b/src/lib/onboard/managed-startup-application.test.ts index 31d13e32039..59aeb249b5d 100644 --- a/src/lib/onboard/managed-startup-application.test.ts +++ b/src/lib/onboard/managed-startup-application.test.ts @@ -161,13 +161,23 @@ describe("managed startup application", () => { corporateCaB64: string | undefined = corporateCa === null ? undefined : Buffer.from(corporateCa, "utf8").toString("base64"), + ) { + return prepareProfile(profileFor(agent, corporateCa), corporateCaB64); + } + + function prepareProfile( + profile: ManagedStartupProfile, + corporateCaB64: string | undefined = profile.corporateCa.bundleSha256 === null + ? undefined + : Buffer.from(PEM, "utf8").toString("base64"), + targetStateDirectory: string = stateDirectory, ) { return prepareManagedStartupApplication( { - encodedProfile: encodeManagedStartupProfile(profileFor(agent, corporateCa)), - expectedAgent: agent, + encodedProfile: encodeManagedStartupProfile(profile), + expectedAgent: profile.agent, corporateCaB64, - stateDirectory, + stateDirectory: targetStateDirectory, }, runtime, ); @@ -396,6 +406,177 @@ describe("managed startup application", () => { expect(fs.existsSync(path.join(stateDirectory, "committed.json"))).toBe(true); }); + it("recovers a complete generation left before pending-state publication", () => { + const first = prepare("hermes"); + fs.unlinkSync(path.join(stateDirectory, "pending.json")); + + const recovered = prepare("hermes"); + expect(recovered.status).toBe("prepared"); + expect(recovered.generationDirectory).toBe(first.generationDirectory); + expect(() => commitManagedStartupApplication(recovered, runtime)).not.toThrow(); + }); + + it("recovers an atomic-control link left after publication", () => { + const first = prepare("hermes"); + const pending = path.join(stateDirectory, "pending.json"); + const interruptedTemporary = path.join(stateDirectory, `.pending.json-${"a".repeat(24)}.tmp`); + fs.linkSync(pending, interruptedTemporary); + expect(fs.statSync(pending).nlink).toBe(2); + + const recovered = prepare("hermes"); + expect(recovered.fingerprint).toBe(first.fingerprint); + expect(fs.existsSync(interruptedTemporary)).toBe(false); + expect(fs.statSync(pending).nlink).toBe(1); + expect(() => commitManagedStartupApplication(recovered, runtime)).not.toThrow(); + }); + + it("does not let a different profile replace an active pending transaction", () => { + const active = prepare("openclaw"); + const pendingBefore = fs.readFileSync(path.join(stateDirectory, "pending.json"), "utf8"); + const changed = { + ...profileFor("openclaw"), + inference: { + ...profileFor("openclaw").inference, + model: "nvidia/a-competing-model", + primaryModelRef: "inference/nvidia/a-competing-model", + }, + }; + + expect(() => prepareProfile(changed)).toThrow(/different startup profile is already pending/u); + expect(fs.readFileSync(path.join(stateDirectory, "pending.json"), "utf8")).toBe(pendingBefore); + expect(fs.existsSync(active.generationDirectory)).toBe(true); + expect(() => commitManagedStartupApplication(active, runtime)).not.toThrow(); + }); + + it("uses compare-and-swap when two profiles interleave before pending publication", () => { + const changed = { + ...profileFor("openclaw"), + inference: { + ...profileFor("openclaw").inference, + model: "nvidia/a-competing-model", + primaryModelRef: "inference/nvidia/a-competing-model", + }, + }; + const originalRenameSync = fs.renameSync.bind(fs); + const race: { active: ReturnType | null } = { active: null }; + let interleaved = false; + vi.spyOn(fs, "renameSync").mockImplementation((source, destination) => { + originalRenameSync(source, destination); + if ( + !interleaved && + path.dirname(destination.toString()) === stateDirectory && + path.basename(destination.toString()).startsWith("generation-") + ) { + interleaved = true; + race.active = prepare("openclaw"); + } + }); + + expect(() => prepareProfile(changed)).toThrow(/won the pending-state transaction/u); + expect(race.active).not.toBeNull(); + if (race.active === null) throw new Error("interleaved winner was not prepared"); + const winner = race.active; + expect( + JSON.parse(fs.readFileSync(path.join(stateDirectory, "pending.json"), "utf8")), + ).toMatchObject({ fingerprint: winner.fingerprint }); + expect( + fs + .readdirSync(stateDirectory) + .filter((entry) => entry.startsWith("generation-")) + .sort(), + ).toEqual([path.basename(winner.generationDirectory)]); + expect(() => commitManagedStartupApplication(winner, runtime)).not.toThrow(); + }); + + it("rejects a delayed contender after the pending owner commits", () => { + const changed = { + ...profileFor("openclaw"), + inference: { + ...profileFor("openclaw").inference, + model: "nvidia/a-delayed-competing-model", + primaryModelRef: "inference/nvidia/a-delayed-competing-model", + }, + }; + const originalOpenSync = fs.openSync.bind(fs); + const race: { committed: ReturnType | null } = { committed: null }; + let interleaved = false; + vi.spyOn(fs, "openSync").mockImplementation((target, flags, mode) => { + if ( + !interleaved && + path.dirname(target.toString()) === stateDirectory && + /^\.pending\.json-[a-f0-9]{24}\.tmp$/u.test(path.basename(target.toString())) + ) { + interleaved = true; + race.committed = prepare("openclaw"); + commitManagedStartupApplication(race.committed, runtime); + } + return originalOpenSync(target, flags, mode); + }); + + expect(() => prepareProfile(changed)).toThrow( + /different startup profile committed during pending-state publication/u, + ); + expect(race.committed).not.toBeNull(); + if (race.committed === null) throw new Error("interleaved winner was not committed"); + const winner = race.committed; + expect(fs.existsSync(path.join(stateDirectory, "pending.json"))).toBe(false); + expect( + JSON.parse(fs.readFileSync(path.join(stateDirectory, "committed.json"), "utf8")), + ).toMatchObject({ fingerprint: winner.fingerprint }); + expect( + fs + .readdirSync(stateDirectory) + .filter((entry) => entry.startsWith("generation-")) + .sort(), + ).toEqual([path.basename(winner.generationDirectory)]); + }); + + it("makes committed state authoritative for a reader straddling pending publication", () => { + const winner = prepare("openclaw"); + commitManagedStartupApplication(winner, runtime); + const committedPath = path.join(stateDirectory, "committed.json"); + const committedBefore = fs.readFileSync(committedPath, "utf8"); + + const changed = { + ...profileFor("openclaw"), + inference: { + ...profileFor("openclaw").inference, + model: "nvidia/a-straddling-competing-model", + primaryModelRef: "inference/nvidia/a-straddling-competing-model", + }, + }; + const competingStateDirectory = path.join(fixtureRoot, "competing-state"); + const competing = prepareProfile(changed, undefined, competingStateDirectory); + const competingGeneration = path.join( + stateDirectory, + path.basename(competing.generationDirectory), + ); + fs.renameSync(competing.generationDirectory, competingGeneration); + fs.renameSync( + path.join(competingStateDirectory, "pending.json"), + path.join(stateDirectory, "pending.json"), + ); + + const originalLstatSync = fs.lstatSync.bind(fs); + let hidInitialCommittedRead = false; + vi.spyOn(fs, "lstatSync").mockImplementation((target) => { + if (!hidInitialCommittedRead && target.toString() === committedPath) { + hidInitialCommittedRead = true; + throw Object.assign(new Error("simulated pre-commit read"), { code: "ENOENT" }); + } + return originalLstatSync(target); + }); + + expect(() => prepareProfile(changed)).toThrow( + /different startup profile is already committed/u, + ); + expect(hidInitialCommittedRead).toBe(true); + expect(fs.readFileSync(committedPath, "utf8")).toBe(committedBefore); + expect(fs.existsSync(path.join(stateDirectory, "pending.json"))).toBe(false); + expect(fs.existsSync(competingGeneration)).toBe(false); + expect(fs.existsSync(winner.generationDirectory)).toBe(true); + }); + it("never accepts a partial committed generation", () => { const prepared = prepare("langchain-deepagents-code"); commitManagedStartupApplication(prepared, runtime); diff --git a/src/lib/onboard/managed-startup/application.ts b/src/lib/onboard/managed-startup/application.ts index 07e5d9adf19..9a9527ffb19 100644 --- a/src/lib/onboard/managed-startup/application.ts +++ b/src/lib/onboard/managed-startup/application.ts @@ -307,26 +307,45 @@ function parseStateControl( return control; } -function atomicWriteStateControl( +function publishStateControlIfAbsent( stateDirectory: string, basename: "committed.json" | "pending.json", control: StateControl, runtime: ManagedStartupApplicationRuntime, -): void { +): { + readonly control: StateControl; + readonly created: boolean; +} { const target = path.join(stateDirectory, basename); const temporary = path.join(stateDirectory, `.${basename}-${randomToken()}.tmp`); writeSecureNewFile(temporary, serializeStateControl(control), runtime); + try { - fs.renameSync(temporary, target); - syncDirectory(stateDirectory); - } catch { + fs.linkSync(temporary, target); + } catch (error) { try { - fs.unlinkSync(temporary); + unlinkSecureControlOrTemp(temporary, runtime); } catch { - // Preserve the primary atomic-write error. + // Preserve the primary publication error. + } + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + return { + control: parseStateControl(target, runtime), + created: false, + }; } - fail(`could not atomically write ${basename}`); + fail(`could not atomically publish ${basename}`); } + + try { + fs.unlinkSync(temporary); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + fail(`could not finalize atomic publication of ${basename}`); + } + } + syncDirectory(stateDirectory); + return { control, created: true }; } function validateCorporateCaBytes(bytes: Buffer): void { @@ -520,6 +539,20 @@ function discardDirectory(target: string, runtime: ManagedStartupApplicationRunt fs.rmSync(target, { recursive: true }); } +function discardDirectoryIfPresent( + target: string, + runtime: ManagedStartupApplicationRuntime, +): boolean { + try { + fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + fail(`could not inspect disposable generation ${target}`); + } + discardDirectory(target, runtime); + return true; +} + function unlinkSecureControlOrTemp( target: string, runtime: ManagedStartupApplicationRuntime, @@ -540,19 +573,65 @@ function listStateEntries(stateDirectory: string): string[] { return entries; } +function unlinkRecoverableControlTemp( + stateDirectory: string, + entry: string, + runtime: ManagedStartupApplicationRuntime, +): void { + const temporary = path.join(stateDirectory, entry); + const stat = fs.lstatSync(temporary); + if (stat.nlink === 1) { + unlinkSecureControlOrTemp(temporary, runtime); + return; + } + + const basename = entry.startsWith(".committed.json-") + ? "committed.json" + : entry.startsWith(".pending.json-") + ? "pending.json" + : null; + const target = basename === null ? null : path.join(stateDirectory, basename); + let targetStat: fs.Stats | null = null; + try { + targetStat = target === null ? null : fs.lstatSync(target); + } catch { + fail(`refused to remove an unpaired atomic-control temporary file: ${temporary}`); + } + if ( + stat.nlink !== 2 || + targetStat === null || + stat.dev !== targetStat.dev || + stat.ino !== targetStat.ino || + !stat.isFile() || + stat.isSymbolicLink() || + modeOf(stat) !== STATE_FILE_MODE || + stat.size < 1 || + stat.size > MAX_CONTROL_FILE_BYTES + ) { + fail(`refused to remove an unpaired atomic-control temporary file: ${temporary}`); + } + requireOwner(stat, temporary, runtime); + requireOwner(targetStat, target as string, runtime); + fs.unlinkSync(temporary); +} + function cleanAtomicTemps( stateDirectory: string, entries: readonly string[], runtime: ManagedStartupApplicationRuntime, ): void { + let changed = false; for (const entry of entries) { const target = path.join(stateDirectory, entry); if (PREPARE_TEMP_RE.test(entry)) { discardDirectory(target, runtime); + changed = true; } else if (CONTROL_TEMP_RE.test(entry)) { - unlinkSecureControlOrTemp(target, runtime); + unlinkRecoverableControlTemp(stateDirectory, entry, runtime); + changed = true; } } + if (changed) syncDirectory(stateDirectory); } function requireKnownStateEntries(stateDirectory: string, entries: readonly string[]): void { @@ -577,7 +656,7 @@ function discardGenerationsExcept( ): void { for (const entry of listStateEntries(stateDirectory)) { if (GENERATION_RE.test(entry) && entry !== keepGeneration) { - discardDirectory(path.join(stateDirectory, entry), runtime); + discardDirectoryIfPresent(path.join(stateDirectory, entry), runtime); } } } @@ -601,10 +680,37 @@ function removePendingControl( stateDirectory: string, runtime: ManagedStartupApplicationRuntime, ): void { - unlinkSecureControlOrTemp(path.join(stateDirectory, "pending.json"), runtime); + try { + unlinkSecureControlOrTemp(path.join(stateDirectory, "pending.json"), runtime); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } syncDirectory(stateDirectory); } +function stateControlsMatch(left: StateControl, right: StateControl): boolean { + return left.fingerprint === right.fingerprint && left.generation === right.generation; +} + +function recoverCommittedState( + stateDirectory: string, + committedControl: StateControl, + pendingControl: StateControl | null, + requested: StateControl, + expectedAgent: ManagedStartupAgent, + runtime: ManagedStartupApplicationRuntime, +): ValidatedGeneration { + const committed = validateGeneration(stateDirectory, committedControl, runtime, expectedAgent); + if (pendingControl) removePendingControl(stateDirectory, runtime); + discardGenerationsExcept(stateDirectory, committedControl.generation, runtime); + syncDirectory(stateDirectory); + if (!stateControlsMatch(committedControl, requested)) { + fail("a different startup profile is already committed; recreate the sandbox to change it"); + } + return committed; +} + function recoverState( stateDirectory: string, requested: StateControl, @@ -618,44 +724,51 @@ function recoverState( requireKnownStateEntries(stateDirectory, initialEntries); cleanAtomicTemps(stateDirectory, initialEntries, runtime); - const committedControl = optionalStateControl(stateDirectory, "committed.json", runtime); + const initiallyCommittedControl = optionalStateControl(stateDirectory, "committed.json", runtime); const pendingControl = optionalStateControl(stateDirectory, "pending.json", runtime); + const committedAfterPendingRead = optionalStateControl(stateDirectory, "committed.json", runtime); + const committedControl = committedAfterPendingRead ?? initiallyCommittedControl; if (committedControl) { - const committed = validateGeneration(stateDirectory, committedControl, runtime, expectedAgent); - if ( - committedControl.fingerprint !== requested.fingerprint || - committedControl.generation !== requested.generation - ) { - fail("a different startup profile is already committed; recreate the sandbox to change it"); - } - if (pendingControl) { - if ( - pendingControl.fingerprint !== committedControl.fingerprint || - pendingControl.generation !== committedControl.generation - ) { - fail("committed and pending startup state disagree"); - } - removePendingControl(stateDirectory, runtime); - } - discardGenerationsExcept(stateDirectory, committedControl.generation, runtime); - return { committed, pending: null }; + return { + committed: recoverCommittedState( + stateDirectory, + committedControl, + pendingControl, + requested, + expectedAgent, + runtime, + ), + pending: null, + }; } if (pendingControl) { - if ( - pendingControl.fingerprint === requested.fingerprint && - pendingControl.generation === requested.generation - ) { + if (stateControlsMatch(pendingControl, requested)) { const pending = validateGeneration(stateDirectory, pendingControl, runtime, expectedAgent); + const committedAfterPendingValidation = optionalStateControl( + stateDirectory, + "committed.json", + runtime, + ); + if (committedAfterPendingValidation) { + return { + committed: recoverCommittedState( + stateDirectory, + committedAfterPendingValidation, + pendingControl, + requested, + expectedAgent, + runtime, + ), + pending: null, + }; + } discardGenerationsExcept(stateDirectory, pendingControl.generation, runtime); return { committed: null, pending }; } - discardGenerationsExcept(stateDirectory, null, runtime); - removePendingControl(stateDirectory, runtime); - return { committed: null, pending: null }; + fail("a different startup profile is already pending; wait for it to commit or recreate"); } - discardGenerationsExcept(stateDirectory, null, runtime); return { committed: null, pending: null }; } @@ -669,6 +782,7 @@ function createGeneration( const temporaryName = `.prepare-${String(process.pid)}-${randomToken()}`; const temporary = path.join(stateDirectory, temporaryName); const generation = path.join(stateDirectory, control.generation); + let renameAttempted = false; try { fs.mkdirSync(temporary, { mode: STATE_DIRECTORY_MODE }); fs.chownSync(temporary, runtime.rootUid, runtime.rootGid); @@ -678,6 +792,7 @@ function createGeneration( writeSecureNewFile(path.join(temporary, "corporate-ca.pem"), corporateCa, runtime); } syncDirectory(temporary); + renameAttempted = true; fs.renameSync(temporary, generation); syncDirectory(stateDirectory); } catch (error) { @@ -688,6 +803,13 @@ function createGeneration( // Preserve the generation error. } if (error instanceof ManagedStartupApplicationError) throw error; + if ( + renameAttempted && + ((error as NodeJS.ErrnoException).code === "EEXIST" || + (error as NodeJS.ErrnoException).code === "ENOTEMPTY") + ) { + return validateGeneration(stateDirectory, control, runtime); + } fail(`could not atomically prepare generation ${control.generation}`); } return validateGeneration(stateDirectory, control, runtime); @@ -752,8 +874,47 @@ export function prepareManagedStartupApplication( } const generation = createGeneration(stateDirectory, control, profileJson, corporateCa, runtime); - atomicWriteStateControl(stateDirectory, "pending.json", control, runtime); - return toPrepared("prepared", stateDirectory, generation, input.expectedAgent); + const publication = publishStateControlIfAbsent(stateDirectory, "pending.json", control, runtime); + if ( + publication.control.fingerprint !== control.fingerprint || + publication.control.generation !== control.generation + ) { + discardDirectoryIfPresent(generation.directory, runtime); + syncDirectory(stateDirectory); + fail("a different startup profile won the pending-state transaction"); + } + const committedAfterPublication = optionalStateControl(stateDirectory, "committed.json", runtime); + if (committedAfterPublication) { + if ( + committedAfterPublication.fingerprint !== control.fingerprint || + committedAfterPublication.generation !== control.generation + ) { + if (publication.created) { + removePendingControl(stateDirectory, runtime); + discardDirectoryIfPresent(generation.directory, runtime); + syncDirectory(stateDirectory); + } + fail("a different startup profile committed during pending-state publication"); + } + const committedGeneration = validateGeneration( + stateDirectory, + committedAfterPublication, + runtime, + input.expectedAgent, + ); + removePendingControl(stateDirectory, runtime); + discardGenerationsExcept(stateDirectory, committedAfterPublication.generation, runtime); + return toPrepared( + "already-committed", + stateDirectory, + committedGeneration, + input.expectedAgent, + ); + } + const activeGeneration = publication.created + ? generation + : validateGeneration(stateDirectory, publication.control, runtime, input.expectedAgent); + return toPrepared("prepared", stateDirectory, activeGeneration, input.expectedAgent); } function validatePreparedHandle(handle: PreparedManagedStartupApplication): StateControl { @@ -773,7 +934,7 @@ function validatePreparedHandle(handle: PreparedManagedStartupApplication): Stat /** * Mark a prepared profile applied only after every agent-specific adapter has - * completed. The marker rename is the sole commit point. + * completed. Exclusive publication of the marker is the sole commit point. */ export function commitManagedStartupApplication( prepared: PreparedManagedStartupApplication, @@ -817,8 +978,21 @@ export function commitManagedStartupApplication( runtime, prepared.expectedAgent, ); - atomicWriteStateControl(stateDirectory, "committed.json", pendingControl, runtime); + const publication = publishStateControlIfAbsent( + stateDirectory, + "committed.json", + pendingControl, + runtime, + ); + if ( + publication.control.fingerprint !== requested.fingerprint || + publication.control.generation !== requested.generation + ) { + fail("a different startup profile won the committed-state transaction"); + } removePendingControl(stateDirectory, runtime); + discardGenerationsExcept(stateDirectory, publication.control.generation, runtime); + syncDirectory(stateDirectory); return { ...toPrepared("already-committed", stateDirectory, generation, prepared.expectedAgent), status: "committed", From 7d1668bbe6b1758539531db185903fb0256b150a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 21:36:57 -0700 Subject: [PATCH 012/117] test(onboard): keep transaction races branchless Signed-off-by: Aaron Erickson --- .../managed-startup-application.test.ts | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/src/lib/onboard/managed-startup-application.test.ts b/src/lib/onboard/managed-startup-application.test.ts index 59aeb249b5d..9c04795ac8c 100644 --- a/src/lib/onboard/managed-startup-application.test.ts +++ b/src/lib/onboard/managed-startup-application.test.ts @@ -462,20 +462,19 @@ describe("managed startup application", () => { let interleaved = false; vi.spyOn(fs, "renameSync").mockImplementation((source, destination) => { originalRenameSync(source, destination); - if ( - !interleaved && - path.dirname(destination.toString()) === stateDirectory && - path.basename(destination.toString()).startsWith("generation-") - ) { - interleaved = true; - race.active = prepare("openclaw"); - } + void (!interleaved && + path.dirname(destination.toString()) === stateDirectory && + path.basename(destination.toString()).startsWith("generation-") + ? (() => { + interleaved = true; + race.active = prepare("openclaw"); + })() + : undefined); }); expect(() => prepareProfile(changed)).toThrow(/won the pending-state transaction/u); expect(race.active).not.toBeNull(); - if (race.active === null) throw new Error("interleaved winner was not prepared"); - const winner = race.active; + const winner = race.active as ReturnType; expect( JSON.parse(fs.readFileSync(path.join(stateDirectory, "pending.json"), "utf8")), ).toMatchObject({ fingerprint: winner.fingerprint }); @@ -501,15 +500,15 @@ describe("managed startup application", () => { const race: { committed: ReturnType | null } = { committed: null }; let interleaved = false; vi.spyOn(fs, "openSync").mockImplementation((target, flags, mode) => { - if ( - !interleaved && - path.dirname(target.toString()) === stateDirectory && - /^\.pending\.json-[a-f0-9]{24}\.tmp$/u.test(path.basename(target.toString())) - ) { - interleaved = true; - race.committed = prepare("openclaw"); - commitManagedStartupApplication(race.committed, runtime); - } + void (!interleaved && + path.dirname(target.toString()) === stateDirectory && + /^\.pending\.json-[a-f0-9]{24}\.tmp$/u.test(path.basename(target.toString())) + ? (() => { + interleaved = true; + race.committed = prepare("openclaw"); + commitManagedStartupApplication(race.committed, runtime); + })() + : undefined); return originalOpenSync(target, flags, mode); }); @@ -517,8 +516,7 @@ describe("managed startup application", () => { /different startup profile committed during pending-state publication/u, ); expect(race.committed).not.toBeNull(); - if (race.committed === null) throw new Error("interleaved winner was not committed"); - const winner = race.committed; + const winner = race.committed as ReturnType; expect(fs.existsSync(path.join(stateDirectory, "pending.json"))).toBe(false); expect( JSON.parse(fs.readFileSync(path.join(stateDirectory, "committed.json"), "utf8")), @@ -560,11 +558,12 @@ describe("managed startup application", () => { const originalLstatSync = fs.lstatSync.bind(fs); let hidInitialCommittedRead = false; vi.spyOn(fs, "lstatSync").mockImplementation((target) => { - if (!hidInitialCommittedRead && target.toString() === committedPath) { - hidInitialCommittedRead = true; - throw Object.assign(new Error("simulated pre-commit read"), { code: "ENOENT" }); - } - return originalLstatSync(target); + return !hidInitialCommittedRead && target.toString() === committedPath + ? (() => { + hidInitialCommittedRead = true; + throw Object.assign(new Error("simulated pre-commit read"), { code: "ENOENT" }); + })() + : originalLstatSync(target); }); expect(() => prepareProfile(changed)).toThrow( From deb15814281aa534a4828fb3aef62bd387688781 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 22:19:02 -0700 Subject: [PATCH 013/117] chore(stack): refresh PR3.4b validation Trigger exact-head CI after the canonical GitHub bot restack without changing the reviewed tree. Signed-off-by: Aaron Erickson From a0f12f1d8696d4ce541df1d78b056178260a9c4e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 22:20:51 -0700 Subject: [PATCH 014/117] fix(onboard): harden managed startup inputs Signed-off-by: Aaron Erickson --- .../applier/build/messaging-build-applier.mts | 39 +++++- src/lib/onboard/managed-image-catalog.test.ts | 69 +++++++--- src/lib/onboard/managed-image/catalog.ts | 45 +++---- .../managed-startup-application.test.ts | 19 ++- .../managed-startup-coordinator.test.ts | 13 +- .../managed-startup-image-runtime.test.ts | 85 ++++++++++++- .../managed-startup-onboard-profile.test.ts | 38 +++++- .../managed-startup-profile-builder.test.ts | 118 ++++++++++++++---- .../onboard/managed-startup-profile.test.ts | 7 +- .../managed-startup/agent-environment.ts | 3 +- .../onboard/managed-startup/application.ts | 50 +++++++- .../onboard/managed-startup/image-runtime.ts | 22 ++-- .../managed-startup/onboard-profile.ts | 37 ++++-- .../managed-startup/profile-builder.ts | 54 ++++++-- src/lib/onboard/managed-startup/profile.ts | 60 ++++----- src/lib/onboard/workload/runtime.ts | 10 +- src/lib/onboard/workload/source.ts | 13 +- test/messaging-build-applier.test.ts | 64 +++++++++- 18 files changed, 595 insertions(+), 151 deletions(-) diff --git a/src/lib/messaging/applier/build/messaging-build-applier.mts b/src/lib/messaging/applier/build/messaging-build-applier.mts index ea03cea82f5..b74270c4a41 100755 --- a/src/lib/messaging/applier/build/messaging-build-applier.mts +++ b/src/lib/messaging/applier/build/messaging-build-applier.mts @@ -1706,6 +1706,7 @@ function formatError(error: unknown): string { } export type MessagingBuildPhase = "runtime-setup" | "agent-install" | "post-agent-install"; +export type MessagingBuildApplyMode = "apply" | "clear"; export interface MessagingBuildPhaseOptions { /** @@ -1713,6 +1714,8 @@ export interface MessagingBuildPhaseOptions { * the explicit render and build-file plan to its durable home directory. */ readonly managedStartupRuntime?: boolean; + /** Explicit provider-owned intent for managed startup profile application. */ + readonly mode?: MessagingBuildApplyMode; } export function applyMessagingBuildPhase( @@ -1721,11 +1724,24 @@ export function applyMessagingBuildPhase( env: Env = process.env, options: MessagingBuildPhaseOptions = {}, ): readonly string[] { + const mode = options.mode ?? "apply"; + if (mode !== "apply" && mode !== "clear") { + throw new MessagingBuildApplierError("Messaging apply mode must be 'apply' or 'clear'"); + } if (options.managedStartupRuntime && phase !== "post-agent-install") { throw new MessagingBuildApplierError( "Managed startup runtime mode is only valid for post-agent-install", ); } + if (mode === "clear") { + if (plan !== null) { + throw new MessagingBuildApplierError("Messaging clear mode requires an absent plan"); + } + return []; + } + if (options.managedStartupRuntime && plan === null) { + throw new MessagingBuildApplierError("Managed startup apply mode requires a messaging plan"); + } if (phase === "runtime-setup") { const target = writeMessagingRuntimePlanArtifact(plan, messagingRuntimePlanPath(env)); return target ? [target] : []; @@ -1809,13 +1825,13 @@ export function describeMessagingBuildPhase( } export function main(argv: readonly string[] = process.argv.slice(2)): void { - const { agent, phase, dryRun, managedStartupRuntime } = parseMessagingBuildArgs(argv); + const { agent, phase, dryRun, managedStartupRuntime, mode } = parseMessagingBuildArgs(argv); const plan = readMessagingBuildPlanFromEnv(process.env, agent); if (dryRun) { console.log(JSON.stringify(describeMessagingBuildPhase(plan, phase, process.env), null, 2)); return; } - applyMessagingBuildPhase(plan, phase, process.env, { managedStartupRuntime }); + applyMessagingBuildPhase(plan, phase, process.env, { managedStartupRuntime, mode }); } function parseMessagingBuildArgs(argv: readonly string[]): { @@ -1823,11 +1839,13 @@ function parseMessagingBuildArgs(argv: readonly string[]): { readonly phase: MessagingBuildPhase; readonly dryRun: boolean; readonly managedStartupRuntime: boolean; + readonly mode: MessagingBuildApplyMode; } { let agent: MessagingAgentId | undefined; let phase: MessagingBuildPhase | undefined; let dryRun = false; let managedStartupRuntime = false; + let mode: MessagingBuildApplyMode = "apply"; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; @@ -1839,6 +1857,15 @@ function parseMessagingBuildArgs(argv: readonly string[]): { managedStartupRuntime = true; continue; } + if (arg === "--mode") { + mode = readApplyModeArg(argv[index + 1]); + index += 1; + continue; + } + if (arg.startsWith("--mode=")) { + mode = readApplyModeArg(arg.slice("--mode=".length)); + continue; + } if (arg === "--agent") { agent = readAgentArg(argv[index + 1]); index += 1; @@ -1875,9 +1902,17 @@ function parseMessagingBuildArgs(argv: readonly string[]): { phase: resolvedPhase, dryRun, managedStartupRuntime, + mode, }; } +function readApplyModeArg(value: string | undefined): MessagingBuildApplyMode { + if (value === "apply" || value === "clear") { + return value; + } + throw new MessagingBuildApplierError("--mode must be 'apply' or 'clear'"); +} + function readAgentArg(value: string | undefined): MessagingAgentId { if (value === "openclaw" || value === "hermes") { return value; diff --git a/src/lib/onboard/managed-image-catalog.test.ts b/src/lib/onboard/managed-image-catalog.test.ts index 98789a25ebb..e57251aab4d 100644 --- a/src/lib/onboard/managed-image-catalog.test.ts +++ b/src/lib/onboard/managed-image-catalog.test.ts @@ -68,6 +68,27 @@ function jsonResponse(body: Buffer, digestHeader?: `sha256:${string}`): Response } function registryFixture(agent: ShippedManagedImageAgent, options: RegistryFixtureOptions = {}) { + const oversizedRootStream = { + cancelled: false, + contentLength: null as string | null, + pulls: 0, + }; + const oversizedRootResponse = (): Response => { + const response = new Response( + new ReadableStream({ + cancel() { + oversizedRootStream.cancelled = true; + }, + pull(controller) { + oversizedRootStream.pulls += 1; + controller.enqueue(new Uint8Array(1024 * 1024).fill(0x20)); + }, + }), + { headers: { "docker-content-digest": rootDigest } }, + ); + oversizedRootStream.contentLength = response.headers.get("content-length"); + return response; + }; const platform = options.platform ?? TEST_PLATFORM; const architecture = platform.slice("linux/".length); const repository = MANAGED_IMAGE_REPOSITORIES[agent].replace(/^ghcr\.io\//u, ""); @@ -155,18 +176,14 @@ function registryFixture(agent: ShippedManagedImageAgent, options: RegistryFixtu }, }); case url.pathname === `${manifestPrefix}${rootReference}` && Boolean(authorization): { - switch (true) { - case options.missingRoot === true: - return new Response("not found", { status: 404 }); - case options.oversizedRootBody === true: - return new Response(Buffer.alloc(2 * 1024 * 1024 + 1, 0x20), { - headers: { "docker-content-digest": rootDigest }, - }); - } const deliveredRootBody = options.rootBodyMismatch ? Buffer.concat([rootBody, Buffer.from(" ", "utf8")]) : rootBody; - return jsonResponse(deliveredRootBody, rootDigest); + return options.missingRoot === true + ? new Response("not found", { status: 404 }) + : options.oversizedRootBody === true + ? oversizedRootResponse() + : jsonResponse(deliveredRootBody, rootDigest); } case url.pathname === `${manifestPrefix}${platformDigest}` && Boolean(authorization): return jsonResponse(imageManifestBody, platformDigest); @@ -185,27 +202,33 @@ function registryFixture(agent: ShippedManagedImageAgent, options: RegistryFixtu url.pathname === `/ghcrblobs13/blobs/${configDigest}`: expect(authorization).toBeNull(); expect(init?.redirect).toBe("manual"); - switch (options.secondBlobRedirect) { - case true: - return new Response(null, { + return options.secondBlobRedirect === true + ? new Response(null, { status: 307, headers: { location: `https://pkg-containers.githubusercontent.com/ghcrblobs14/blobs/${configDigest}?sig=second`, }, - }); - } - return jsonResponse( - options.configBodyMismatch - ? Buffer.concat([configBody, Buffer.from(" ", "utf8")]) - : configBody, - ); + }) + : jsonResponse( + options.configBodyMismatch + ? Buffer.concat([configBody, Buffer.from(" ", "utf8")]) + : configBody, + ); default: return new Response("not found", { status: 404 }); } }); const fetchImpl = fetchMock as unknown as typeof fetch; - return { configDigest, fetchImpl, fetchMock, platform, platformDigest, rootDigest }; + return { + configDigest, + fetchImpl, + fetchMock, + oversizedRootStream, + platform, + platformDigest, + rootDigest, + }; } function catalogFixture( @@ -322,6 +345,12 @@ describe("managed image GHCR catalog", () => { fetchImpl: fixture.fetchImpl, }), ).rejects.toThrow(/GHCR manifest exceeds the registry response limit/); + expect(fixture.oversizedRootStream).toMatchObject({ + cancelled: true, + contentLength: null, + }); + expect(fixture.oversizedRootStream.pulls).toBeGreaterThanOrEqual(3); + expect(fixture.oversizedRootStream.pulls).toBeLessThan(5); }); it("rejects image-config bytes that do not match the manifest descriptor digest", async () => { diff --git a/src/lib/onboard/managed-image/catalog.ts b/src/lib/onboard/managed-image/catalog.ts index 5d655edb4b6..7971a4d0421 100644 --- a/src/lib/onboard/managed-image/catalog.ts +++ b/src/lib/onboard/managed-image/catalog.ts @@ -588,33 +588,34 @@ export async function resolveManagedImageCatalogFromGhcr(options: { }); const cohortReference = `cohort-${openclaw.source.cohort}`; const dependentResults = await Promise.allSettled( - SHIPPED_MANAGED_IMAGE_AGENTS.filter((agent) => agent !== "openclaw").map(async (agent) => [ - agent, - await resolveManagedImageContractAtReferenceFromGhcr({ - agent, - reference: cohortReference, - release, - platform, - fetchImpl, - expectedCohort: openclaw.source.cohort, - expectedRevision: openclaw.source.revision, - }), - ]), + SHIPPED_MANAGED_IMAGE_AGENTS.filter((agent) => agent !== "openclaw").map( + async (agent) => + [ + agent, + await resolveManagedImageContractAtReferenceFromGhcr({ + agent, + reference: cohortReference, + release, + platform, + fetchImpl, + expectedCohort: openclaw.source.cohort, + expectedRevision: openclaw.source.revision, + }), + ] as const, + ), ); + const dependentEntries: Array = []; + let unavailable: ManagedImageCatalogUnavailableError | undefined; for (const result of dependentResults) { - if ( - result.status === "rejected" && - !(result.reason instanceof ManagedImageCatalogUnavailableError) - ) { + if (result.status === "fulfilled") { + dependentEntries.push(result.value); + } else if (!(result.reason instanceof ManagedImageCatalogUnavailableError)) { throw result.reason; + } else if (unavailable === undefined) { + unavailable = result.reason; } } - for (const result of dependentResults) { - if (result.status === "rejected") throw result.reason; - } - const dependentEntries = dependentResults.flatMap((result) => - result.status === "fulfilled" ? [result.value] : [], - ); + if (unavailable !== undefined) throw unavailable; return Object.fromEntries([["openclaw", openclaw], ...dependentEntries]); }); } diff --git a/src/lib/onboard/managed-startup-application.test.ts b/src/lib/onboard/managed-startup-application.test.ts index 9c04795ac8c..6f4f57d024d 100644 --- a/src/lib/onboard/managed-startup-application.test.ts +++ b/src/lib/onboard/managed-startup-application.test.ts @@ -21,6 +21,7 @@ import { type ManagedStartupAgent, type ManagedStartupAgentConfig, type ManagedStartupProfile, + serializeManagedStartupProfile, } from "./managed-startup/profile"; function sha256(bytes: string | Buffer): string { @@ -211,7 +212,7 @@ describe("managed startup application", () => { expect(fs.existsSync(path.join(stateDirectory, "committed.json"))).toBe(false); expect(fs.existsSync(path.join(stateDirectory, "pending.json"))).toBe(true); expect(fs.readFileSync(prepared.profilePath, "utf8")).toBe( - JSON.stringify(JSON.parse(fs.readFileSync(prepared.profilePath, "utf8"))), + serializeManagedStartupProfile(profileFor(agent)), ); expect(fs.readFileSync(prepared.corporateCaPath as string)).toEqual(Buffer.from(PEM)); @@ -348,7 +349,21 @@ describe("managed startup application", () => { }, { ...runtime, rootUid: runtime.rootUid + 1 }, ), - ).toThrow(/root:root/u); + ).toThrow(/trusted identity|root:root/u); + }); + + it("allows a trusted sticky root but rejects a replaceable writable ancestor", () => { + const stickyRoot = path.join(fixtureRoot, "sticky-root"); + fs.mkdirSync(stickyRoot, { mode: 0o700 }); + fs.chmodSync(stickyRoot, 0o1777); + stateDirectory = path.join(stickyRoot, "trusted-state"); + expect(() => prepare("openclaw")).not.toThrow(); + + const replaceable = path.join(fixtureRoot, "replaceable"); + fs.mkdirSync(replaceable, { mode: 0o700 }); + fs.chmodSync(replaceable, 0o777); + stateDirectory = path.join(replaceable, "untrusted-state"); + expect(() => prepare("openclaw")).toThrow(/replaceable group- or world-writable ancestor/u); }); it("rejects hardlinked generation files before commit", () => { diff --git a/src/lib/onboard/managed-startup-coordinator.test.ts b/src/lib/onboard/managed-startup-coordinator.test.ts index dc99e569ca1..89c4cefa987 100644 --- a/src/lib/onboard/managed-startup-coordinator.test.ts +++ b/src/lib/onboard/managed-startup-coordinator.test.ts @@ -226,12 +226,16 @@ describe("managed startup coordinator", () => { expect(dependencies.commitApplication).toHaveBeenCalledTimes(1); }); - it("reapplies a pending adapter after a crash at the commit boundary", async () => { + it("does not reapply after a durable commit loses its acknowledgement", async () => { const prepared = preparedFor("langchain-deepagents-code"); + const recovered = preparedFor("langchain-deepagents-code", "already-committed"); const dependencies = dependenciesFor(prepared); const { adapters, applyByAgent } = adaptersFor(); + dependencies.prepareApplication + .mockResolvedValueOnce(prepared) + .mockResolvedValueOnce(recovered); dependencies.commitApplication.mockRejectedValueOnce( - new Error("simulated process interruption"), + new Error("simulated lost commit acknowledgement"), ); await expect( @@ -240,7 +244,7 @@ describe("managed startup coordinator", () => { adapters, dependencies, ), - ).rejects.toThrow("simulated process interruption"); + ).rejects.toThrow("simulated lost commit acknowledgement"); const retried = await coordinateManagedStartupApplication( inputFor("langchain-deepagents-code"), @@ -248,7 +252,8 @@ describe("managed startup coordinator", () => { dependencies, ); expect(retried.application.status).toBe("committed"); - expect(applyByAgent["langchain-deepagents-code"]).toHaveBeenCalledTimes(2); + expect(retried.adapterApplied).toBe(false); + expect(applyByAgent["langchain-deepagents-code"]).toHaveBeenCalledTimes(1); expect(dependencies.commitApplication).toHaveBeenCalledTimes(2); }); }); diff --git a/src/lib/onboard/managed-startup-image-runtime.test.ts b/src/lib/onboard/managed-startup-image-runtime.test.ts index e7e2d0e6359..6fb84149ecc 100644 --- a/src/lib/onboard/managed-startup-image-runtime.test.ts +++ b/src/lib/onboard/managed-startup-image-runtime.test.ts @@ -116,10 +116,11 @@ describe("buildManagedStartupImageActionPlan", () => { { action: "messaging-post-agent-install", runAs: "sandbox" }, ]); expect(plan[0]?.argv).toContain("runtime-setup"); + expect(plan[0]?.argv).toContain("apply"); expect(plan[0]?.argv).not.toContain("--managed-startup-runtime"); expect(plan[2]?.argv).toContain("post-agent-install"); + expect(plan[2]?.argv).toContain("apply"); expect(plan[2]?.argv).toContain("--managed-startup-runtime"); - expect(plan.some((command) => command.argv.includes("agent-install"))).toBe(false); expect( plan.some((command) => command.argv.some((argument) => /^(?:npm|npx|pip|pip3|uv)$/u.test(argument)), @@ -156,8 +157,17 @@ describe("buildManagedStartupImageActionPlan", () => { expect(command?.argv.at(-1)).toBe(generator); }); - it("constructs the same reviewed commands for apply and clear messaging intent", () => { - expect(buildManagedStartupImageActionPlan(actionInput("openclaw", "clear"))).toEqual( + it.each([ + "apply", + "clear", + ] as const)("passes explicit %s intent to both messaging phases", (mode) => { + const plan = buildManagedStartupImageActionPlan(actionInput("openclaw", mode)); + expect(plan[0]?.argv).toEqual(expect.arrayContaining(["--mode", mode])); + expect(plan[2]?.argv).toEqual(expect.arrayContaining(["--mode", mode])); + }); + + it("keeps apply and clear as distinct reviewed commands", () => { + expect(buildManagedStartupImageActionPlan(actionInput("openclaw", "clear"))).not.toEqual( buildManagedStartupImageActionPlan(actionInput("openclaw", "apply")), ); }); @@ -258,6 +268,51 @@ describe("buildManagedStartupImageActionPlan", () => { }, /unsupported managed startup construction action/, ], + [ + "missing dashboard", + { + ...actionInput("openclaw"), + actions: actionInput("openclaw").actions.filter( + (action) => action.kind !== "configure-dashboard", + ), + }, + /exactly one dashboard construction action/, + ], + [ + "duplicate dashboard", + { + ...actionInput("hermes"), + actions: [...actionInput("hermes").actions, actionInput("hermes").actions.at(-1)!], + }, + /exactly one dashboard construction action/, + ], + [ + "mismatched dashboard", + { + ...actionInput("openclaw"), + actions: actionInput("openclaw").actions.map((action) => + action.kind === "configure-dashboard" + ? { ...action, dashboard: dashboard("hermes") } + : action, + ), + }, + /dashboard for hermes cannot be used by openclaw/, + ], + [ + "unknown image agent", + { ...actionInput("openclaw"), agent: "unknown-agent" }, + /unsupported agent "unknown-agent"/, + ], + [ + "invalid messaging mode", + { + ...actionInput("openclaw"), + actions: actionInput("openclaw").actions.map((action) => + action.kind === "apply-messaging-plan" ? { ...action, mode: "replace" } : action, + ), + }, + /messaging intent must be apply or clear/, + ], ])("fails closed for an incomplete or mismatched construction contract: %s", (_name, input, message) => { expect(() => buildManagedStartupImageActionPlan(input as ManagedStartupImageActionPlanInput), @@ -447,6 +502,29 @@ describe("managed startup image runtime", () => { expect(coordinatorMock.coordinateManagedStartupApplication).not.toHaveBeenCalled(); }); + it.each([ + ["unknown live agent", "unknown-agent", "openclaw", /unsupported agent "unknown-agent"/u], + [ + "configured and live agent mismatch", + "hermes", + "openclaw", + /managed startup profile targets openclaw, expected hermes/u, + ], + ] as const)("rejects %s before filesystem or coordinator mutation", async (_label, expectedAgent, profileAgent, message) => { + const profile = managedStartupE2eProfile(profileAgent); + const lstat = vi.spyOn(fs, "lstatSync"); + vi.spyOn(process, "geteuid").mockReturnValue(0); + + await expect( + applyManagedStartupImageProfile(expectedAgent, { + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "1", + [MANAGED_STARTUP_PROFILE_ENV]: encodeManagedStartupProfile(profile), + }), + ).rejects.toThrow(message); + expect(lstat).not.toHaveBeenCalled(); + expect(coordinatorMock.coordinateManagedStartupApplication).not.toHaveBeenCalled(); + }); + it("refreshes admitted launch controls on committed replay without changing the profile", async () => { const profile = managedStartupE2eProfile("openclaw"); const encodedProfile = encodeManagedStartupProfile(profile); @@ -549,7 +627,6 @@ describe("managed startup image runtime", () => { ? ["generate-agent-config"] : ["messaging-runtime-setup", "generate-agent-config", "messaging-post-agent-install"], ); - expect(plan.some((command) => command.argv.includes("agent-install"))).toBe(false); }); it.each( diff --git a/src/lib/onboard/managed-startup-onboard-profile.test.ts b/src/lib/onboard/managed-startup-onboard-profile.test.ts index 19400f40cef..25e27a7ee4e 100644 --- a/src/lib/onboard/managed-startup-onboard-profile.test.ts +++ b/src/lib/onboard/managed-startup-onboard-profile.test.ts @@ -5,8 +5,10 @@ import { describe, expect, it } from "vitest"; import { buildManagedStartupOnboardProfile, + ManagedStartupOnboardProfileError, type ManagedStartupOnboardProfileInput, } from "./managed-startup/onboard-profile"; +import { decodeManagedStartupProfile } from "./managed-startup/profile"; const EMPTY_ENVIRONMENT: NodeJS.ProcessEnv = {}; @@ -197,6 +199,20 @@ describe("buildManagedStartupOnboardProfile", () => { ).toThrow(/dashboard bind address must be empty or 0\.0\.0\.0/); }); + it("normalizes a malformed dashboard URL without echoing its input", () => { + const canary = "dashboard-url-secret-canary"; + const invoke = () => + buildManagedStartupOnboardProfile(openClawInput({ chatUiUrl: `http://[${canary}` })); + + expect(invoke).toThrow(ManagedStartupOnboardProfileError); + expect(invoke).toThrow(/chatUiUrl must be a valid HTTP\(S\) URL/u); + try { + invoke(); + } catch (error) { + expect(String(error)).not.toContain(canary); + } + }); + it("maps Hermes with its dashboard disabled and Tavily retained", () => { const built = buildManagedStartupOnboardProfile(hermesInput()); @@ -300,6 +316,23 @@ describe("buildManagedStartupOnboardProfile", () => { }); }); + it.each([ + ["hermes", hermesInput, "text,image"], + ["langchain-deepagents-code", dcodeInput, "text"], + ] as const)("rejects OpenClaw input modalities for %s before filtering ambient input", (agent, input, modalities) => { + expect(() => + buildManagedStartupOnboardProfile( + input({ environment: { NEMOCLAW_INFERENCE_INPUTS: modalities } }), + ), + ).toThrow(new RegExp(`NEMOCLAW_INFERENCE_INPUTS is not supported by ${agent}`, "u")); + }); + + it("rejects DCode messaging intent instead of silently discarding it", () => { + expect(() => + buildManagedStartupOnboardProfile(dcodeInput({ messagingPlan: messagingPlan("openclaw") })), + ).toThrow(/langchain-deepagents-code does not support messaging/u); + }); + it("does not inspect host CA settings during profile construction", () => { const built = buildManagedStartupOnboardProfile( openClawInput({ @@ -383,7 +416,9 @@ describe("buildManagedStartupOnboardProfile", () => { hostNoProxy: [], }); expect(built.credentialProxyReplayRequired).toBe(agent !== "langchain-deepagents-code"); - const serialized = JSON.stringify(built); + const decoded = decodeManagedStartupProfile(built.encodedProfile); + const serialized = JSON.stringify(decoded); + expect(decoded).toEqual(built.profile); expect(serialized).not.toContain("upper-secret"); expect(serialized).not.toContain("lower-secret"); expect(serialized).not.toContain("upper.internal"); @@ -416,6 +451,7 @@ describe("buildManagedStartupOnboardProfile", () => { model: "gpt-5.4", api: "openai-responses", }); + expect(decodeManagedStartupProfile(built.encodedProfile).inference.model).toBe("gpt-5.4"); expect(built.profile.tools.disclosure).toBe("direct"); expect(built.profile.agentConfig).toMatchObject({ agent: "openclaw", diff --git a/src/lib/onboard/managed-startup-profile-builder.test.ts b/src/lib/onboard/managed-startup-profile-builder.test.ts index 64095af3a1c..fd4da929e2b 100644 --- a/src/lib/onboard/managed-startup-profile-builder.test.ts +++ b/src/lib/onboard/managed-startup-profile-builder.test.ts @@ -11,6 +11,7 @@ import { decodeManagedStartupProfile } from "./managed-startup/profile"; import { assertManagedStartupProfileBuilderInventoryCoverage, buildManagedStartupProfile, + ManagedStartupProfileBuilderError, type ManagedStartupProfileBuilderInput, type ValidatedManagedStartupProfileTransport, } from "./managed-startup/profile-builder"; @@ -298,9 +299,7 @@ describe("buildManagedStartupProfile", () => { }); expect(built.corporateCaB64).toBeUndefined(); expect(decodeManagedStartupProfile(built.encodedProfile)).toEqual(built.profile); - expect(built.startupProfileSha256).toBe( - createHash("sha256").update(built.encodedProfile, "utf8").digest("hex"), - ); + expect(built.startupProfileSha256).toMatch(/^[a-f0-9]{64}$/u); }); it("builds Hermes with Tavily, gateway presets, messaging, context, and forwarding", () => { @@ -451,7 +450,7 @@ describe("buildManagedStartupProfile", () => { }), ); - const normalizedPem = PEM.endsWith("\n") ? PEM : `${PEM}\n`; + const normalizedPem = `${PEM.trimEnd()}\n`; expect(built.corporateCaB64).toBe(Buffer.from(normalizedPem, "utf8").toString("base64")); expect(built.profile.corporateCa.bundleSha256).toBe( createHash("sha256").update(normalizedPem, "utf8").digest("hex"), @@ -521,6 +520,30 @@ describe("buildManagedStartupProfile", () => { hermesInput({ environment: { NEMOCLAW_OPENCLAW_OTEL: "1" } }), /not supported by hermes/, ], + [ + "Hermes inference compatibility", + hermesInput({ + inference: { ...hermesInput().inference, compatibility: { strict: true } }, + }), + /does not support inference compatibility/, + ], + [ + "DCode inference compatibility", + dcodeInput({ + inference: { ...dcodeInput().inference, compatibility: { strict: true } }, + }), + /does not support inference compatibility/, + ], + [ + "Hermes input modalities", + hermesInput({ environment: { NEMOCLAW_INFERENCE_INPUTS: "text,image" } }), + /NEMOCLAW_INFERENCE_INPUTS is not supported by hermes/, + ], + [ + "DCode input modalities", + dcodeInput({ environment: { NEMOCLAW_INFERENCE_INPUTS: "text" } }), + /NEMOCLAW_INFERENCE_INPUTS is not supported by langchain-deepagents-code/, + ], ])("rejects unsupported cross-agent intent: %s", (_label, input, message) => { expect(() => buildManagedStartupProfile(input)).toThrow(message); }); @@ -571,29 +594,72 @@ describe("buildManagedStartupProfile", () => { }); it.each([ - openClawInput({ - environment: { HTTP_PROXY: "http://operator:password@proxy.example.test:8080" }, - }), - openClawInput({ - inference: { - ...openClawInput().inference, - model: "sk-proj-secret-material-1234567890", - }, - }), - openClawInput({ - environment: { - NEMOCLAW_EXTRA_AGENTS_JSON: JSON.stringify({ - agents: [ - { - id: "reviewer", - api_key: ["sk", "secret", "material", "1234567890"].join("-"), - }, - ], + [ + "credential-bearing proxy URL", + openClawInput({ + environment: { HTTP_PROXY: "http://operator:password@proxy.example.test:8080" }, + }), + "proxy.hostHttpUrl", + "password", + ], + [ + "secret-shaped model", + openClawInput({ + inference: { + ...openClawInput().inference, + model: "sk-proj-secret-material-1234567890", + }, + }), + "inference.model", + "sk-proj-secret-material-1234567890", + ], + [ + "credential-shaped extra-agent field", + openClawInput({ + environment: { + NEMOCLAW_EXTRA_AGENTS_JSON: JSON.stringify({ + agents: [ + { + id: "reviewer", + api_key: ["sk", "secret", "material", "1234567890"].join("-"), + }, + ], + }), + }, + }), + "agentConfig.extraAgents.agents[0].api_key", + "sk-secret-material-1234567890", + ], + ] as const)("rejects %s with a precise non-secret-bearing domain error", (_label, input, field, secret) => { + let thrown: unknown; + try { + buildManagedStartupProfile(input); + } catch (error) { + thrown = error; + } + expect(thrown).toBeInstanceOf(ManagedStartupProfileBuilderError); + expect(thrown).toHaveProperty("message", expect.stringContaining(field)); + expect(thrown).toHaveProperty("message", expect.not.stringContaining(secret)); + }); + + it.each([ + ["null", "[null]", /NEMOCLAW_EXTRA_AGENTS_JSON\[0\] must be an object/u], + ["unknown primitive", '["unknown"]', /NEMOCLAW_EXTRA_AGENTS_JSON\[0\] must be an object/u], + [ + "unsafe prototype field", + '[{"id":"reviewer","__proto__":{"polluted":true}}]', + /NEMOCLAW_EXTRA_AGENTS_JSON\[0\] contains an unsafe prototype field/u, + ], + ] as const)("rejects a %s top-level extra-agent entry", (_label, value, message) => { + expect(() => + buildManagedStartupProfile( + openClawInput({ + environment: { + NEMOCLAW_EXTRA_AGENTS_JSON: value, + }, }), - }, - }), - ])("rejects secret material instead of serializing it", (input) => { - expect(() => buildManagedStartupProfile(input)).toThrow(); + ), + ).toThrow(message); }); it("rejects malformed or non-CA certificate material", () => { diff --git a/src/lib/onboard/managed-startup-profile.test.ts b/src/lib/onboard/managed-startup-profile.test.ts index 21f60a6a419..4d4819e0ec5 100644 --- a/src/lib/onboard/managed-startup-profile.test.ts +++ b/src/lib/onboard/managed-startup-profile.test.ts @@ -309,8 +309,9 @@ describe("managed startup profile", () => { const validated = validateManagedStartupProfile(profile); const encoded = encodeManagedStartupProfile(profile); - expect(decodeManagedStartupProfile(encoded)).toEqual(validated); - expect(encoded).not.toContain(profile.inference.model); + const decoded = decodeManagedStartupProfile(encoded); + expect(decoded).toEqual(validated); + expect(decoded.inference.model).toBe(profile.inference.model); expect(fingerprintManagedStartupProfile(profile)).toMatch(/^[a-f0-9]{64}$/); }); @@ -882,7 +883,7 @@ describe("managed startup profile", () => { ...OPENCLAW_PROFILE, tools: { ...OPENCLAW_PROFILE.tools, enabledGateways: ["nous-web"] }, }), - ).toThrow(/supported only by hermes/); + ).toThrow(/unsupported value/); }); it("enforces adapter-specific web-search providers", () => { diff --git a/src/lib/onboard/managed-startup/agent-environment.ts b/src/lib/onboard/managed-startup/agent-environment.ts index fe34d8dae73..18910713fa7 100644 --- a/src/lib/onboard/managed-startup/agent-environment.ts +++ b/src/lib/onboard/managed-startup/agent-environment.ts @@ -8,12 +8,13 @@ import { MANAGED_STARTUP_RUNTIME_CLEANUP_OBLIGATIONS, type ManagedStartupAgent, type ManagedStartupDashboard, + type ManagedStartupMessagingAgent, type ManagedStartupProfile, validateManagedStartupProfile, } from "./profile"; export type ManagedStartupConfigAgent = ManagedStartupAgent; -export type ManagedStartupMessagingAgent = "openclaw" | "hermes"; +export type { ManagedStartupMessagingAgent } from "./profile"; export interface ManagedStartupCorporateCaMaterial { readonly kind: "corporate-ca-handoff"; diff --git a/src/lib/onboard/managed-startup/application.ts b/src/lib/onboard/managed-startup/application.ts index 9a9527ffb19..a107bf8b22a 100644 --- a/src/lib/onboard/managed-startup/application.ts +++ b/src/lib/onboard/managed-startup/application.ts @@ -137,17 +137,59 @@ function requireSecureDirectory( if (stat.isSymbolicLink() || !stat.isDirectory()) { fail(`state directory component must be a real directory: ${target}`); } - requireOwner(stat, target, runtime); + const runtimeOwned = stat.uid === runtime.rootUid && stat.gid === runtime.rootGid; + const systemRootOwned = stat.uid === 0 && stat.gid === 0; + if (exactMode) { + requireOwner(stat, target, runtime); + } else if (!runtimeOwned && !systemRootOwned) { + fail(`state directory ancestor is not owned by a trusted identity: ${target}`); + } const mode = modeOf(stat); - if ((exactMode && mode !== STATE_DIRECTORY_MODE) || (!exactMode && (mode & 0o022) !== 0)) { + const writableByUntrustedIdentity = (mode & 0o022) !== 0; + const trustedStickyRoot = (stat.mode & 0o1000) !== 0 && (runtimeOwned || systemRootOwned); + if ( + (exactMode && mode !== STATE_DIRECTORY_MODE) || + (!exactMode && writableByUntrustedIdentity && !trustedStickyRoot) + ) { fail( exactMode ? `${target} must have mode 0700` - : `${target} must not be group- or world-writable`, + : `${target} is a replaceable group- or world-writable ancestor`, ); } } +function requireSecureAncestors(target: string, runtime: ManagedStartupApplicationRuntime): void { + const root = path.parse(target).root; + let current = root; + requireSecureDirectory(current, runtime, false); + for (const segment of path.relative(root, target).split(path.sep).filter(Boolean)) { + current = path.join(current, segment); + let stat: fs.Stats; + try { + stat = fs.lstatSync(current); + } catch { + fail(`state directory component is missing or unreadable: ${current}`); + } + if (stat.isSymbolicLink()) { + const runtimeOwned = stat.uid === runtime.rootUid && stat.gid === runtime.rootGid; + const systemRootOwned = stat.uid === 0 && stat.gid === 0; + if (!runtimeOwned && !systemRootOwned) { + fail(`state directory ancestor is a replaceable symlink: ${current}`); + } + let resolved: string; + try { + resolved = fs.realpathSync(current); + } catch { + fail(`state directory symlink is missing or unreadable: ${current}`); + } + requireSecureAncestors(resolved, runtime); + continue; + } + requireSecureDirectory(current, runtime, false); + } +} + function ensureStateDirectory( rawStateDirectory: string | undefined, runtime: ManagedStartupApplicationRuntime, @@ -158,7 +200,7 @@ function ensureStateDirectory( } const normalized = path.resolve(stateDirectory); const parent = path.dirname(normalized); - requireSecureDirectory(parent, runtime, false); + requireSecureAncestors(parent, runtime); try { fs.mkdirSync(normalized, { mode: STATE_DIRECTORY_MODE }); fs.chownSync(normalized, runtime.rootUid, runtime.rootGid); diff --git a/src/lib/onboard/managed-startup/image-runtime.ts b/src/lib/onboard/managed-startup/image-runtime.ts index c6252ad97c2..2aac60603b5 100644 --- a/src/lib/onboard/managed-startup/image-runtime.ts +++ b/src/lib/onboard/managed-startup/image-runtime.ts @@ -21,8 +21,10 @@ import { decodeManagedStartupProfile, fingerprintManagedStartupProfile, MANAGED_STARTUP_AGENTS, + MANAGED_STARTUP_MESSAGING_AGENTS, type ManagedStartupAgent, type ManagedStartupDashboard, + type ManagedStartupMessagingAgent, type ManagedStartupProfile, } from "./profile"; import { @@ -63,7 +65,6 @@ const MAX_MANAGED_STARTUP_COMPLETION_BYTES = 4096; const MAX_MANAGED_STARTUP_RUNTIME_ENVIRONMENT_BYTES = 512 * 1024; export type ManagedStartupImageIdentity = "root" | "sandbox"; -export type ManagedStartupMessagingAgent = "openclaw" | "hermes"; export interface ManagedStartupGenerateConfigConstructionAction { readonly kind: "generate-agent-config"; @@ -523,6 +524,7 @@ function generatorCommand(agent: ManagedStartupAgent): readonly string[] { function messagingCommand( agent: ManagedStartupMessagingAgent, phase: "runtime-setup" | "post-agent-install", + mode: "apply" | "clear", ): readonly string[] { return [ "/usr/local/bin/node", @@ -532,6 +534,8 @@ function messagingCommand( agent, "--phase", phase, + "--mode", + mode, ...(phase === "post-agent-install" ? ["--managed-startup-runtime"] : []), ]; } @@ -597,7 +601,7 @@ export function buildManagedStartupImageActionPlan( commands.push({ action: "messaging-runtime-setup", runAs: action.runAs, - argv: messagingCommand(action.agent, action.phase), + argv: messagingCommand(action.agent, action.phase, action.mode), }); } else if (action.phase === "post-agent-install") { if (action.runAs !== "sandbox") { @@ -607,7 +611,7 @@ export function buildManagedStartupImageActionPlan( commands.push({ action: "messaging-post-agent-install", runAs: action.runAs, - argv: messagingCommand(action.agent, action.phase), + argv: messagingCommand(action.agent, action.phase, action.mode), }); } else { failActionPlan("unsupported messaging construction phase"); @@ -625,7 +629,10 @@ export function buildManagedStartupImageActionPlan( if (generateActions !== 1) { failActionPlan("exactly one agent config construction action is required"); } - const expectedMessagingActions = inputAgent === "langchain-deepagents-code" ? 0 : 1; + const supportsMessaging = (MANAGED_STARTUP_MESSAGING_AGENTS as readonly string[]).includes( + inputAgent, + ); + const expectedMessagingActions = supportsMessaging ? 1 : 0; if ( runtimeMessagingActions !== expectedMessagingActions || postMessagingActions !== expectedMessagingActions @@ -634,10 +641,9 @@ export function buildManagedStartupImageActionPlan( `${inputAgent} requires ${String(expectedMessagingActions)} action for each messaging phase`, ); } - const expectedOrder = - inputAgent === "langchain-deepagents-code" - ? ["generate-agent-config"] - : ["messaging-runtime-setup", "generate-agent-config", "messaging-post-agent-install"]; + const expectedOrder = supportsMessaging + ? ["messaging-runtime-setup", "generate-agent-config", "messaging-post-agent-install"] + : ["generate-agent-config"]; if (commands.some((command, index) => command.action !== expectedOrder[index])) { failActionPlan(`${inputAgent} image actions are not in the required construction order`); } diff --git a/src/lib/onboard/managed-startup/onboard-profile.ts b/src/lib/onboard/managed-startup/onboard-profile.ts index b7efddbb387..6358ee38b52 100644 --- a/src/lib/onboard/managed-startup/onboard-profile.ts +++ b/src/lib/onboard/managed-startup/onboard-profile.ts @@ -9,12 +9,14 @@ import type { HermesDashboardOnboardState } from "../hermes-dashboard"; import { hasCredentialBearingHostProxyEnvironment } from "../host-proxy-env"; import { MANAGED_STARTUP_AGENTS, + MANAGED_STARTUP_PROFILE_CAPABILITIES, type ManagedStartupAgent, type ManagedStartupDashboard, } from "./profile"; import { type BuiltManagedStartupProfile, buildManagedStartupProfile, + MANAGED_STARTUP_HOST_PROXY_URL_INPUTS, type ManagedStartupResolvedInferenceInput, } from "./profile-builder"; @@ -41,7 +43,6 @@ const PROFILE_ENVIRONMENT_INPUTS = { "langchain-deepagents-code": ["NEMOCLAW_PROXY_HOST", "NEMOCLAW_PROXY_PORT"], } as const satisfies Record; -const HOST_PROXY_URL_INPUTS = ["HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"] as const; const HOST_NO_PROXY_INPUTS = ["NO_PROXY", "no_proxy"] as const; export interface ManagedStartupOnboardProfileInput { @@ -96,6 +97,14 @@ function requireDashboardPort(port: number): number { return port; } +function dashboardHostname(chatUiUrl: string): string { + try { + return new URL(chatUiUrl).hostname; + } catch { + throw new ManagedStartupOnboardProfileError("chatUiUrl must be a valid HTTP(S) URL"); + } +} + function dashboardForInput( agent: ManagedStartupAgent, input: ManagedStartupOnboardProfileInput, @@ -122,7 +131,7 @@ function dashboardForInput( const remote = bindAddress === "0.0.0.0" || input.wslExposure || - !["127.0.0.1", "localhost", "::1", "[::1]"].includes(new URL(input.chatUiUrl).hostname); + !["127.0.0.1", "localhost", "::1", "[::1]"].includes(dashboardHostname(input.chatUiUrl)); return { agent, mode: remote ? "remote" : "loopback", @@ -176,12 +185,14 @@ function profileEnvironment( } const credentialProxy = hasCredentialBearingHostProxyEnvironment(environment); if (!credentialProxy) { - for (const name of HOST_PROXY_URL_INPUTS) { + for (const name of MANAGED_STARTUP_HOST_PROXY_URL_INPUTS) { const value = environment[name]?.trim(); if (!value) continue; selected[name] = value; } - const hasCredentialFreeProxy = HOST_PROXY_URL_INPUTS.some((name) => selected[name]); + const hasCredentialFreeProxy = MANAGED_STARTUP_HOST_PROXY_URL_INPUTS.some( + (name) => selected[name], + ); if (hasCredentialFreeProxy) { for (const name of HOST_NO_PROXY_INPUTS) { const value = environment[name]; @@ -196,11 +207,23 @@ export function buildManagedStartupOnboardProfile( input: ManagedStartupOnboardProfileInput, ): BuiltManagedStartupOnboardProfile { const agent = exactManagedAgent(input.agentName); + const capabilities = MANAGED_STARTUP_PROFILE_CAPABILITIES[agent]; + if ( + agent !== "openclaw" && + typeof input.environment.NEMOCLAW_INFERENCE_INPUTS === "string" && + input.environment.NEMOCLAW_INFERENCE_INPUTS.trim() !== "" + ) { + throw new ManagedStartupOnboardProfileError( + `NEMOCLAW_INFERENCE_INPUTS is not supported by ${agent}`, + ); + } + if (!capabilities.supportsMessaging && input.messagingPlan !== null) { + throw new ManagedStartupOnboardProfileError(`${agent} does not support messaging`); + } const dashboard = dashboardForInput(agent, input); const environment = profileEnvironment(agent, input.environment); const credentialProxyReplayRequired = - agent !== "langchain-deepagents-code" && - hasCredentialBearingHostProxyEnvironment(input.environment); + capabilities.supportsMessaging && hasCredentialBearingHostProxyEnvironment(input.environment); const built = buildManagedStartupProfile({ agent, inference: input.inference, @@ -208,7 +231,7 @@ export function buildManagedStartupOnboardProfile( webSearch: agent === "langchain-deepagents-code" ? null : input.webSearch, toolDisclosure: input.toolDisclosure, hermesToolGateways: agent === "hermes" ? input.hermesToolGateways : [], - messagingPlan: agent === "langchain-deepagents-code" ? null : input.messagingPlan, + messagingPlan: capabilities.supportsMessaging ? input.messagingPlan : null, dcodeAutoApprovalMode: agent === "langchain-deepagents-code" ? input.dcodeAutoApprovalMode : null, observabilityEnabled: agent === "langchain-deepagents-code" ? input.observabilityEnabled : null, diff --git a/src/lib/onboard/managed-startup/profile-builder.ts b/src/lib/onboard/managed-startup/profile-builder.ts index 4f906ba6bc8..0c40b4fb32d 100644 --- a/src/lib/onboard/managed-startup/profile-builder.ts +++ b/src/lib/onboard/managed-startup/profile-builder.ts @@ -46,6 +46,12 @@ const MAX_PROFILE_TUNING_INTEGER = 1_000_000_000; const FALSE_VALUES = new Set(["0", "false", "no", "off"]); const TRUE_VALUES = new Set(["1", "true", "yes", "on"]); const STANDARD_BASE64_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u; +export const MANAGED_STARTUP_HOST_PROXY_URL_INPUTS = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "http_proxy", + "https_proxy", +] as const; /** * These digests deliberately bind the builder to every classified stock @@ -263,6 +269,28 @@ function isPlainObject(value: unknown): value is Record { return prototype === Object.prototype || prototype === null; } +function normalizeExtraAgentList(value: unknown, field: string): ManagedStartupJsonObject[] { + if (!Array.isArray(value) || Object.getPrototypeOf(value) !== Array.prototype) { + fail(`${field} must be an object list`); + } + const normalized: ManagedStartupJsonObject[] = []; + for (let index = 0; index < value.length; index += 1) { + const descriptor = Object.getOwnPropertyDescriptor(value, String(index)); + if (!descriptor || !("value" in descriptor) || !isPlainObject(descriptor.value)) { + fail(`${field}[${String(index)}] must be an object`); + } + if ( + Object.hasOwn(descriptor.value, "__proto__") || + Object.hasOwn(descriptor.value, "prototype") || + Object.hasOwn(descriptor.value, "constructor") + ) { + fail(`${field}[${String(index)}] contains an unsafe prototype field`); + } + normalized.push(descriptor.value as ManagedStartupJsonObject); + } + return normalized; +} + function normalizeExtraAgentsCandidate(value: unknown): ManagedStartupExtraAgents { const emptyDefaults = { subagents: {} }; if (value === null || value === undefined) { @@ -270,7 +298,7 @@ function normalizeExtraAgentsCandidate(value: unknown): ManagedStartupExtraAgent } if (Array.isArray(value)) { return { - agents: value as ManagedStartupJsonObject[], + agents: normalizeExtraAgentList(value, "NEMOCLAW_EXTRA_AGENTS_JSON"), defaults: emptyDefaults, main: {}, }; @@ -289,14 +317,12 @@ function normalizeExtraAgentsCandidate(value: unknown): ManagedStartupExtraAgent const agents = value.agents ?? []; const defaults = value.defaults ?? emptyDefaults; const main = value.main ?? {}; - if (!Array.isArray(agents) || !agents.every((agent) => isPlainObject(agent))) { - fail("NEMOCLAW_EXTRA_AGENTS_JSON.agents must be an object list"); - } + const normalizedAgents = normalizeExtraAgentList(agents, "NEMOCLAW_EXTRA_AGENTS_JSON.agents"); if (!isPlainObject(defaults) || !isPlainObject(main)) { fail("NEMOCLAW_EXTRA_AGENTS_JSON defaults and main must be objects"); } return { - agents: agents as ManagedStartupJsonObject[], + agents: normalizedAgents, defaults: defaults as ManagedStartupJsonObject, main: main as ManagedStartupJsonObject, }; @@ -396,8 +422,16 @@ function resolveHostProxy( _agent: ManagedStartupAgent, environment: NodeJS.ProcessEnv, ): Pick { - const hostHttpUrl = resolveAliasedEnvironmentValue(environment, "HTTP_PROXY", "http_proxy"); - const hostHttpsUrl = resolveAliasedEnvironmentValue(environment, "HTTPS_PROXY", "https_proxy"); + const hostHttpUrl = resolveAliasedEnvironmentValue( + environment, + MANAGED_STARTUP_HOST_PROXY_URL_INPUTS[0], + MANAGED_STARTUP_HOST_PROXY_URL_INPUTS[2], + ); + const hostHttpsUrl = resolveAliasedEnvironmentValue( + environment, + MANAGED_STARTUP_HOST_PROXY_URL_INPUTS[1], + MANAGED_STARTUP_HOST_PROXY_URL_INPUTS[3], + ); const noProxy = resolveAliasedEnvironmentValue(environment, "NO_PROXY", "no_proxy"); if (hostHttpUrl === null && hostHttpsUrl === null) { if (noProxy !== null) { @@ -481,6 +515,9 @@ function assertAgentSpecificInput(input: ManagedStartupProfileBuilderInput): voi return; } if (input.agent === "hermes") { + if (input.inference.compatibility !== null) { + fail("Hermes does not support inference compatibility"); + } if ( input.inference.upstreamEndpointUrl !== null || input.dcodeAutoApprovalMode !== null || @@ -499,6 +536,9 @@ function assertAgentSpecificInput(input: ManagedStartupProfileBuilderInput): voi if (input.messagingPlan !== null) { fail("langchain-deepagents-code messagingPlan must be null"); } + if (input.inference.compatibility !== null) { + fail("langchain-deepagents-code does not support inference compatibility"); + } if ( input.dcodeAutoApprovalMode !== "disabled" && input.dcodeAutoApprovalMode !== "thread-opt-in" diff --git a/src/lib/onboard/managed-startup/profile.ts b/src/lib/onboard/managed-startup/profile.ts index d57722afd4f..32338e00b9d 100644 --- a/src/lib/onboard/managed-startup/profile.ts +++ b/src/lib/onboard/managed-startup/profile.ts @@ -107,6 +107,8 @@ export type ManagedStartupDeviceAuthOptOutSource = "operator" | "managed-onboard export const MANAGED_STARTUP_AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; export type ManagedStartupAgent = (typeof MANAGED_STARTUP_AGENTS)[number]; +export const MANAGED_STARTUP_MESSAGING_AGENTS = ["openclaw", "hermes"] as const; +export type ManagedStartupMessagingAgent = (typeof MANAGED_STARTUP_MESSAGING_AGENTS)[number]; export type ManagedStartupJsonScalar = string | number | boolean | null; export type ManagedStartupJsonValue = @@ -328,12 +330,6 @@ export interface ManagedStartupAgentCapabilities { * does not advertise the requested semantic capability is rejected instead of * silently dropping a field. */ -const VALIDATED_INFERENCE_APIS_BY_AGENT = Object.freeze({ - openclaw: Object.freeze([...MANAGED_STARTUP_INFERENCE_APIS]), - hermes: Object.freeze([...MANAGED_STARTUP_INFERENCE_APIS]), - "langchain-deepagents-code": Object.freeze(["openai-completions"] as const), -}) satisfies Readonly>; - function freezeAgentCapabilities( capabilities: ManagedStartupAgentCapabilities, ): Readonly { @@ -350,7 +346,7 @@ function freezeAgentCapabilities( const PROFILE_CAPABILITIES = { openclaw: { - inferenceApis: [...VALIDATED_INFERENCE_APIS_BY_AGENT.openclaw], + inferenceApis: [...MANAGED_STARTUP_INFERENCE_APIS], dashboardModes: ["loopback", "remote"], inputModalities: ["text", "image"], webSearchProviders: ["brave", "tavily"], @@ -369,7 +365,7 @@ const PROFILE_CAPABILITIES = { supportsMinimalBootstrap: true, }, hermes: { - inferenceApis: [...VALIDATED_INFERENCE_APIS_BY_AGENT.hermes], + inferenceApis: [...MANAGED_STARTUP_INFERENCE_APIS], dashboardModes: ["disabled", "loopback-forwarded"], inputModalities: [], webSearchProviders: ["tavily"], @@ -840,9 +836,7 @@ const DEVICE_AUTH_KEYS = new Set(["disabled", "optOutSource"]); const EXTRA_AGENTS_KEYS = new Set(["agents", "defaults", "main"]); const MANAGED_STARTUP_AGENT_SET = new Set(MANAGED_STARTUP_AGENTS); const DCODE_AUTO_APPROVAL_MODE_SET = new Set(MANAGED_STARTUP_DCODE_AUTO_APPROVAL_MODES); -const INFERENCE_API_SET = new Set(MANAGED_STARTUP_INFERENCE_APIS); const REASONING_EFFORT_SET = new Set(MANAGED_STARTUP_REASONING_EFFORTS); -const HERMES_GATEWAY_SET = new Set(MANAGED_STARTUP_HERMES_TOOL_GATEWAYS); const HERMES_RESERVED_API_PORTS = new Set([8642, 18_642]); function isPlainObject(value: unknown): value is Record { @@ -914,6 +908,14 @@ function invalid(reason: string): never { throw new ManagedStartupProfileError(reason); } +function payloadPath(path: readonly string[]): string { + return path.reduce( + (result, segment) => + segment.startsWith("[") ? `${result}${segment}` : `${result}${result ? "." : ""}${segment}`, + "", + ); +} + function mapArrayByIndex(values: readonly T[], mapper: (value: T, index: number) => U): U[] { const mapped: U[] = []; for (let index = 0; index < values.length; index += 1) { @@ -1241,7 +1243,9 @@ function assertPayloadStructureAndCredentialShapes(root: unknown): void { !isMessagingCredentialPlaceholder(current.path, current.value) && valueLooksLikeSecret(current.value) ) { - invalid("payload contains credential-shaped string data"); + invalid( + `payload field ${payloadPath(current.path)} contains credential-shaped string data`, + ); } if ( RAW_CA_PEM_RE.test(current.value) || @@ -1249,10 +1253,14 @@ function assertPayloadStructureAndCredentialShapes(root: unknown): void { RAW_CA_DER_BASE64_RE.test(current.value) || RAW_CA_DATA_URI_RE.test(current.value) ) { - invalid("payload contains raw certificate data; provide only the CA SHA-256 digest"); + invalid( + `payload field ${payloadPath(current.path)} contains raw certificate data; provide only the CA SHA-256 digest`, + ); } if (containsUrlWithCredentialMaterial(current.value)) { - invalid("payload contains a URL with embedded credentials"); + invalid( + `payload field ${payloadPath(current.path)} contains a URL with embedded credentials`, + ); } continue; } @@ -1279,7 +1287,7 @@ function assertPayloadStructureAndCredentialShapes(root: unknown): void { pending.push({ value: descriptor?.value, depth, - path: current.path, + path: [...current.path, `[${String(index)}]`], }); } continue; @@ -1317,7 +1325,9 @@ function assertPayloadStructureAndCredentialShapes(root: unknown): void { } const child = descriptor.value; if (isCredentialShapedName(key) && !isMessagingCredentialPlaceholder(current.path, child)) { - invalid("payload contains a credential-shaped field name"); + invalid( + `payload field ${payloadPath([...current.path, key])} has a credential-shaped field name`, + ); } pending.push({ value: child, @@ -1357,7 +1367,7 @@ function validateWebSearch(value: unknown, agent: "openclaw" | "hermes"): Manage rejectUnknownKeys(webSearch, WEB_SEARCH_KEYS, "agentConfig.webSearch"); const provider = requireStringEnum( webSearch.provider, - new Set(agent === "openclaw" ? ["brave", "tavily"] : ["tavily"]), + new Set(MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].webSearchProviders), "agentConfig.webSearch.provider", ); return { @@ -1484,7 +1494,7 @@ function validateDashboard( rejectUnknownKeys(dashboard, OPENCLAW_DASHBOARD_KEYS, "dashboard"); const mode = requireStringEnum<"loopback" | "remote">( dashboard.mode, - new Set(["loopback", "remote"]), + new Set(MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].dashboardModes), "dashboard.mode", ); const url = requireHttpUrl(dashboard.url, "dashboard.url"); @@ -1517,7 +1527,7 @@ function validateDashboard( rejectUnknownKeys(dashboard, HERMES_DASHBOARD_KEYS, "dashboard"); const mode = requireStringEnum<"disabled" | "loopback-forwarded">( dashboard.mode, - new Set(["disabled", "loopback-forwarded"]), + new Set(MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].dashboardModes), "dashboard.mode", ); const url = requireHttpUrl(dashboard.url, "dashboard.url"); @@ -1576,17 +1586,9 @@ function validateInference(value: unknown, agent: ManagedStartupAgent): ManagedS const model = requireBoundedString(inference.model, "inference.model", MAX_MODEL_BYTES); const api = requireStringEnum( inference.api, - INFERENCE_API_SET, + new Set(MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].inferenceApis), "inference.api", ); - const supportedInferenceApis = VALIDATED_INFERENCE_APIS_BY_AGENT[agent]; - let apiSupported = false; - for (let index = 0; index < supportedInferenceApis.length; index += 1) { - if (supportedInferenceApis[index] === api) apiSupported = true; - } - if (!apiSupported) { - invalid(`inference.api is not supported by ${agent}`); - } const upstreamEndpointUrl = inference.upstreamEndpointUrl === null ? null @@ -1605,7 +1607,7 @@ function validateInference(value: unknown, agent: ManagedStartupAgent): ManagedS ? null : requireEnumList( inference.inputModalities, - new Set(["text", "image"]), + new Set(MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].inputModalities), "inference.inputModalities", { allowEmpty: false }, ); @@ -1677,7 +1679,7 @@ function validateTools(value: unknown, agent: ManagedStartupAgent): ManagedStart rejectUnknownKeys(tools, TOOLS_KEYS, "tools"); const enabledGateways = requireEnumList( tools.enabledGateways, - HERMES_GATEWAY_SET, + new Set(MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].toolGateways), "tools.enabledGateways", { allowEmpty: true }, ); diff --git a/src/lib/onboard/workload/runtime.ts b/src/lib/onboard/workload/runtime.ts index b87a12b98ff..a6fd8a61e4b 100644 --- a/src/lib/onboard/workload/runtime.ts +++ b/src/lib/onboard/workload/runtime.ts @@ -82,19 +82,21 @@ export function resolveSandboxWorkloadRuntimeCapabilities( nodeArchitecture: string = process.arch, ): SandboxWorkloadRuntimeCapabilities { const profile = Object.hasOwn(profiles, plan.driverName) ? profiles[plan.driverName] : undefined; + const support = profile?.support; const hostPlatform = managedImagePlatformForNodeArchitecture(nodeArchitecture); const supportedHost = hostPlatform !== null && - profile?.support !== null && + support !== undefined && + support !== null && profile?.hostArchitectures.includes(hostOciArchitecture(nodeArchitecture)) === true && - profile?.support.platforms.includes(hostPlatform) === true; + support.platforms.includes(hostPlatform); return { driverName: plan.driverName, managedImageSelectionPolicy: profile?.managedImageSelectionPolicy ?? "require-managed", legacyDockerfileBuilds: profile?.legacyDockerfileBuilds ?? false, managedImages: - profile === undefined || profile.support === null || !supportedHost + support === undefined || support === null || !supportedHost ? null - : cloneRuntimeSupport(profile.support, hostPlatform), + : cloneRuntimeSupport(support, hostPlatform), }; } diff --git a/src/lib/onboard/workload/source.ts b/src/lib/onboard/workload/source.ts index bea8facc308..24e25314a1f 100644 --- a/src/lib/onboard/workload/source.ts +++ b/src/lib/onboard/workload/source.ts @@ -178,13 +178,14 @@ export function resolveSandboxWorkloadSource( ); } + const expectedPlatform = managedImageRuntimePlatform(options.runtime); + if (expectedPlatform === null) { + throw new SandboxWorkloadSourceError( + `Driver '${options.runtime.driverName}' has no unambiguous managed-image host platform.`, + ); + } + try { - const expectedPlatform = managedImageRuntimePlatform(options.runtime); - if (expectedPlatform === null) { - throw new SandboxWorkloadSourceError( - `Driver '${options.runtime.driverName}' has no unambiguous managed-image host platform.`, - ); - } const contract = parseManagedImageContractV1(candidate, options.agentName, expectedPlatform); return { kind: "managed-image", diff --git a/test/messaging-build-applier.test.ts b/test/messaging-build-applier.test.ts index 30e3fdb5775..d56a0dddff3 100644 --- a/test/messaging-build-applier.test.ts +++ b/test/messaging-build-applier.test.ts @@ -419,7 +419,16 @@ describe("messaging-build-applier.mts: agent-install", () => { try { const result = spawnSync( "node", - ["--experimental-strip-types", SCRIPT_PATH, "--agent", agent, "--phase", "runtime-setup"], + [ + "--experimental-strip-types", + SCRIPT_PATH, + "--agent", + agent, + "--phase", + "runtime-setup", + "--mode", + "apply", + ], { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"], @@ -486,6 +495,59 @@ describe("messaging-build-applier.mts: agent-install", () => { } }); + it("honors explicit clear intent without recreating a provider-removed runtime artifact", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-clear-runtime-plan-artifact-")); + const artifactPath = path.join(tmp, "messaging-runtime-plan.json"); + fs.writeFileSync(artifactPath, "stale\n"); + fs.unlinkSync(artifactPath); + + try { + expect( + applyMessagingBuildPhase( + null, + "runtime-setup", + { NEMOCLAW_MESSAGING_RUNTIME_PLAN_PATH: artifactPath }, + { mode: "clear" }, + ), + ).toEqual([]); + expect(fs.existsSync(artifactPath)).toBe(false); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("rejects contradictory managed messaging mode and plan state", () => { + const plan = readMessagingBuildPlanFromEnv( + { + NEMOCLAW_MESSAGING_PLAN_B64: encodePlan({ + schemaVersion: 1, + sandboxName: "test-sandbox", + agent: "hermes", + channels: [], + credentialBindings: [], + agentRender: [], + buildSteps: [], + }), + }, + "hermes", + ); + + expect(() => applyMessagingBuildPhase(plan, "runtime-setup", {}, { mode: "clear" })).toThrow( + /clear mode requires an absent plan/u, + ); + expect(() => + applyMessagingBuildPhase( + null, + "post-agent-install", + {}, + { + managedStartupRuntime: true, + mode: "apply", + }, + ), + ).toThrow(/apply mode requires a messaging plan/u); + }); + it("preserves Hermes runtime env aliases in the reduced runtime plan artifact", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-runtime-plan-artifact-")); const artifactPath = path.join(tmp, "runtime", "messaging-runtime-plan.json"); From 27c72f5f888e9301b3cce1894431ef5de3669425 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 23:11:57 -0700 Subject: [PATCH 015/117] fix(runtime): close prior review debt Signed-off-by: Aaron Erickson --- src/lib/messaging/applier/plan-filter.ts | 43 ++++-- .../post-agent-install-selection.test.ts | 129 ++++++++++++++++++ .../messaging/post-agent-install-selection.ts | 82 +++++------ .../onboard/managed-startup/image-runtime.ts | 2 +- test/entrypoint-env-wrapper.test.ts | 25 +++- 5 files changed, 223 insertions(+), 58 deletions(-) create mode 100644 src/lib/messaging/post-agent-install-selection.test.ts diff --git a/src/lib/messaging/applier/plan-filter.ts b/src/lib/messaging/applier/plan-filter.ts index 93953963327..5c5663e238a 100644 --- a/src/lib/messaging/applier/plan-filter.ts +++ b/src/lib/messaging/applier/plan-filter.ts @@ -1,31 +1,44 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { - MessagingChannelId, - SandboxMessagingChannelPlan, - SandboxMessagingPlan, -} from "../manifest"; +import type { MessagingChannelId, SandboxMessagingChannelPlan } from "../manifest"; -export function enabledPlanChannels(plan: SandboxMessagingPlan): SandboxMessagingChannelPlan[] { +export type EnabledPlanChannel = Pick & + Partial>; + +export interface EnabledPlanSelection { + readonly channels: readonly Channel[]; + readonly disabledChannels?: readonly MessagingChannelId[]; +} + +export function normalizeMessagingChannelId(channelId: MessagingChannelId): MessagingChannelId { + return channelId.trim().toLowerCase(); +} + +export function enabledPlanChannels( + plan: EnabledPlanSelection, +): Channel[] { const disabled = disabledPlanChannelIds(plan); - return plan.channels.filter( - (channel) => channel.active && !channel.disabled && !disabled.has(channel.channelId), - ); + return plan.channels.filter((channel) => { + const channelId = normalizeMessagingChannelId(channel.channelId); + return channelId.length > 0 && channel.active && !channel.disabled && !disabled.has(channelId); + }); } -export function enabledPlanChannelIds(plan: SandboxMessagingPlan): Set { - return new Set(enabledPlanChannels(plan).map((channel) => channel.channelId)); +export function enabledPlanChannelIds(plan: EnabledPlanSelection): Set { + return new Set( + enabledPlanChannels(plan).map((channel) => normalizeMessagingChannelId(channel.channelId)), + ); } export function filterEnabledPlanEntries( - plan: SandboxMessagingPlan, + plan: EnabledPlanSelection, entries: readonly T[], ): T[] { const enabled = enabledPlanChannelIds(plan); - return entries.filter((entry) => enabled.has(entry.channelId)); + return entries.filter((entry) => enabled.has(normalizeMessagingChannelId(entry.channelId))); } -function disabledPlanChannelIds(plan: SandboxMessagingPlan): Set { - return new Set(plan.disabledChannels); +function disabledPlanChannelIds(plan: EnabledPlanSelection): Set { + return new Set((plan.disabledChannels ?? []).map(normalizeMessagingChannelId).filter(Boolean)); } diff --git a/src/lib/messaging/post-agent-install-selection.test.ts b/src/lib/messaging/post-agent-install-selection.test.ts new file mode 100644 index 00000000000..86cdcfad11a --- /dev/null +++ b/src/lib/messaging/post-agent-install-selection.test.ts @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { filterEnabledPlanEntries } from "./applier/plan-filter"; +import type { + SandboxMessagingAgentRenderPlan, + SandboxMessagingBuildStepPlan, + SandboxMessagingChannelPlan, + SandboxMessagingPlan, +} from "./manifest"; +import { + selectActiveMessagingChannelIds, + selectEnabledMessagingAgentRender, + selectEnabledPostAgentInstallBuildFiles, +} from "./post-agent-install-selection"; + +function channel( + channelId: string, + hooks: SandboxMessagingChannelPlan["hooks"] = [], +): SandboxMessagingChannelPlan { + return { + channelId, + displayName: channelId, + authMode: "none", + active: true, + selected: true, + configured: true, + disabled: false, + inputs: [], + hooks, + }; +} + +function plan(overrides: Partial = {}): SandboxMessagingPlan { + return { + schemaVersion: 1, + sandboxName: "selection-test", + agent: "openclaw", + workflow: "onboard", + channels: [], + disabledChannels: [], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + stateUpdates: [], + healthChecks: [], + ...overrides, + }; +} + +function render(channelId: string): SandboxMessagingAgentRenderPlan { + return { + channelId, + kind: "json-fragment", + agent: "openclaw", + target: "/sandbox/.openclaw/openclaw.json", + path: "channels.telegram.enabled", + value: true, + templateRefs: [], + }; +} + +function buildFile( + channelId: string, + outputId: string, + hookId?: string, +): SandboxMessagingBuildStepPlan { + return { + channelId, + kind: "build-file", + hookId, + outputId, + required: true, + value: "fixture", + }; +} + +describe("post-agent-install messaging selection", () => { + it("uses the shared enabled-channel contract and preserves canonical ordering", () => { + const selection = plan({ + channels: [channel(" Discord "), channel(" TeLeGrAm "), channel("DISCORD")], + disabledChannels: [" telegram "], + }); + + expect(selectActiveMessagingChannelIds(selection)).toEqual(["discord"]); + expect( + filterEnabledPlanEntries(selection, [ + { channelId: "DISCORD", value: "kept" }, + { channelId: "telegram", value: "disabled" }, + ]), + ).toEqual([{ channelId: "DISCORD", value: "kept" }]); + }); + + it("matches normalized channel ids across channels, render entries, and build steps", () => { + const selection = plan({ + channels: [ + channel(" Telegram ", [ + { + channelId: " Telegram ", + id: "post-install", + phase: "post-agent-install", + handler: "telegram.post-install", + }, + { + channelId: " Telegram ", + id: "render-only", + phase: "render", + handler: "telegram.render", + }, + ]), + ], + agentRender: [render("TELEGRAM"), render("discord")], + buildSteps: [ + buildFile("telegram", "post-install-file", "post-install"), + buildFile(" TELEGRAM ", "render-file", "render-only"), + buildFile("discord", "unrelated-file"), + ], + }); + + expect(selectEnabledMessagingAgentRender(selection).map(({ channelId }) => channelId)).toEqual([ + "TELEGRAM", + ]); + expect( + selectEnabledPostAgentInstallBuildFiles(selection).map(({ outputId }) => outputId), + ).toEqual(["post-install-file"]); + }); +}); diff --git a/src/lib/messaging/post-agent-install-selection.ts b/src/lib/messaging/post-agent-install-selection.ts index 63c9afb380c..716c4a5283a 100644 --- a/src/lib/messaging/post-agent-install-selection.ts +++ b/src/lib/messaging/post-agent-install-selection.ts @@ -1,76 +1,76 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -interface SelectionHook { - readonly id: string; - readonly phase: string; -} +import { + type EnabledPlanChannel, + type EnabledPlanSelection, + enabledPlanChannels, + normalizeMessagingChannelId, +} from "./applier/plan-filter"; +import type { + MessagingChannelId, + SandboxMessagingAgentRenderPlan, + SandboxMessagingBuildStepPlan, + SandboxMessagingHookReferencePlan, +} from "./manifest"; -interface SelectionChannel { - readonly channelId: string; - readonly active?: boolean; - readonly disabled?: boolean; - readonly hooks?: readonly SelectionHook[]; -} +type SelectionChannel = EnabledPlanChannel & { + readonly hooks?: readonly (Pick & { + readonly phase: string; + })[]; +}; -interface SelectionPlanBase { - readonly channels: readonly SelectionChannel[]; -} +type SelectionRender = Pick; + +type SelectionBuildStep = Pick; /** * Canonical active-channel selection for the image applier. Each selection * consumer must resolve the same active channels and mutable outputs. */ -export function selectActiveMessagingChannelIds(plan: SelectionPlanBase): string[] { +export function selectActiveMessagingChannelIds( + plan: EnabledPlanSelection, +): MessagingChannelId[] { const seen = new Set(); const channels: string[] = []; - for (const item of plan.channels) { - const channel = String(item.channelId || "") - .trim() - .toLowerCase(); + for (const item of enabledPlanChannels(plan)) { + const channel = normalizeMessagingChannelId(item.channelId); if (!channel || seen.has(channel)) continue; - if (item.active === true && item.disabled !== true) { - seen.add(channel); - channels.push(channel); - } + seen.add(channel); + channels.push(channel); } return channels; } -export function selectEnabledMessagingAgentRender< - Render extends { - readonly agent: string; - readonly channelId: string; - }, ->( - plan: SelectionPlanBase & { +export function selectEnabledMessagingAgentRender( + plan: EnabledPlanSelection & { readonly agent: string; readonly agentRender: readonly Render[]; }, ): Render[] { const active = new Set(selectActiveMessagingChannelIds(plan)); return plan.agentRender.filter( - (render) => render.agent === plan.agent && active.has(render.channelId), + (render) => + render.agent === plan.agent && active.has(normalizeMessagingChannelId(render.channelId)), ); } export function selectEnabledPostAgentInstallBuildFiles< - Step extends { - readonly channelId: string; - readonly kind: string; - readonly hookId?: string; - }, + Channel extends SelectionChannel, + Step extends SelectionBuildStep, >( - plan: SelectionPlanBase & { + plan: EnabledPlanSelection & { readonly buildSteps: readonly Step[]; }, -): Step[] { +): Array { const active = new Set(selectActiveMessagingChannelIds(plan)); - return plan.buildSteps.filter((step) => { - if (!active.has(step.channelId) || step.kind !== "build-file") return false; + const channels = enabledPlanChannels(plan); + return plan.buildSteps.filter((step): step is Step & { readonly kind: "build-file" } => { + const channelId = normalizeMessagingChannelId(step.channelId); + if (!active.has(channelId) || step.kind !== "build-file") return false; if (!step.hookId) return true; - const hookPhase = plan.channels - .find((channel) => channel.channelId === step.channelId) + const hookPhase = channels + .find((channel) => normalizeMessagingChannelId(channel.channelId) === channelId) ?.hooks?.find((hook) => hook.id === step.hookId)?.phase; return hookPhase === undefined || hookPhase === "post-agent-install"; }); diff --git a/src/lib/onboard/managed-startup/image-runtime.ts b/src/lib/onboard/managed-startup/image-runtime.ts index 2aac60603b5..f8e5c7f4edd 100644 --- a/src/lib/onboard/managed-startup/image-runtime.ts +++ b/src/lib/onboard/managed-startup/image-runtime.ts @@ -1631,7 +1631,7 @@ export async function main(argv: readonly string[] = process.argv.slice(2)): Pro ); } -if (require.main === module) { +if (typeof require !== "undefined" && typeof module !== "undefined" && require.main === module) { main().catch((error: unknown) => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; diff --git a/test/entrypoint-env-wrapper.test.ts b/test/entrypoint-env-wrapper.test.ts index 8bd74015681..0c9e727bcc8 100644 --- a/test/entrypoint-env-wrapper.test.ts +++ b/test/entrypoint-env-wrapper.test.ts @@ -134,8 +134,13 @@ describe("OCI entrypoint env-wrapper normalization", () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-env-wrapper-")); const fakeBin = path.join(tmpDir, "bin"); const scriptPath = path.join(tmpDir, "run.sh"); + const inheritedDashboardPort = process.env.NEMOCLAW_DASHBOARD_PORT; + const inheritedChatUiUrl = process.env.CHAT_UI_URL; function runScenario(setArgs: string, extraEnv: Record = {}) { + const baseEnv = { ...process.env }; + delete baseEnv.NEMOCLAW_DASHBOARD_PORT; + delete baseEnv.CHAT_UI_URL; const script = [ "#!/usr/bin/env bash", "set -euo pipefail", @@ -156,10 +161,13 @@ describe("OCI entrypoint env-wrapper normalization", () => { return spawnSync("bash", [scriptPath], { encoding: "utf-8", timeout: 5000, - env: { ...process.env, PATH: `${fakeBin}:${process.env.PATH || ""}`, ...extraEnv }, + env: { ...baseEnv, PATH: `${fakeBin}:${process.env.PATH || ""}`, ...extraEnv }, }); } + process.env.NEMOCLAW_DASHBOARD_PORT = "19999"; + process.env.CHAT_UI_URL = "https://ambient.example.test/ui"; + try { fs.mkdirSync(fakeBin); fs.writeFileSync(path.join(fakeBin, "openclaw"), "#!/usr/bin/env bash\nexit 0\n", { @@ -205,6 +213,11 @@ describe("OCI entrypoint env-wrapper normalization", () => { expect(baked.stdout).toContain("OPENCLAW_STATE_DIR=/sandbox/.openclaw"); expect(baked.stdout).toContain("CMD=openclaw agent"); + const defaults = runScenario("set -- nemoclaw-start openclaw agent"); + expect(defaults.status).toBe(0); + expect(defaults.stdout).toContain("CHAT_UI_URL=http://127.0.0.1:18789"); + expect(defaults.stdout).toContain("PUBLIC_PORT=18789"); + const invalidHighPort = runScenario("set -- nemoclaw-start openclaw agent", { NEMOCLAW_DASHBOARD_PORT: "70000", }); @@ -212,6 +225,16 @@ describe("OCI entrypoint env-wrapper normalization", () => { expect(invalidHighPort.stderr).toContain("Invalid NEMOCLAW_DASHBOARD_PORT='70000'"); expect(invalidHighPort.stderr).toContain("must be an integer between 1024 and 65535"); } finally { + if (inheritedDashboardPort === undefined) { + delete process.env.NEMOCLAW_DASHBOARD_PORT; + } else { + process.env.NEMOCLAW_DASHBOARD_PORT = inheritedDashboardPort; + } + if (inheritedChatUiUrl === undefined) { + delete process.env.CHAT_UI_URL; + } else { + process.env.CHAT_UI_URL = inheritedChatUiUrl; + } fs.rmSync(tmpDir, { recursive: true, force: true }); } }); From 1c824cc97ce1264d0c7af9d6404344ef096097e4 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 23:17:08 -0700 Subject: [PATCH 016/117] test(runtime): keep env cleanup branchless Signed-off-by: Aaron Erickson --- test/entrypoint-env-wrapper.test.ts | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/test/entrypoint-env-wrapper.test.ts b/test/entrypoint-env-wrapper.test.ts index 0c9e727bcc8..5e58805d17f 100644 --- a/test/entrypoint-env-wrapper.test.ts +++ b/test/entrypoint-env-wrapper.test.ts @@ -6,7 +6,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { sliceBlock } from "./helpers/corporate-ca-support"; const HELPER = path.join(import.meta.dirname, "..", "scripts", "lib", "entrypoint-env-wrapper.sh"); @@ -134,9 +134,6 @@ describe("OCI entrypoint env-wrapper normalization", () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-env-wrapper-")); const fakeBin = path.join(tmpDir, "bin"); const scriptPath = path.join(tmpDir, "run.sh"); - const inheritedDashboardPort = process.env.NEMOCLAW_DASHBOARD_PORT; - const inheritedChatUiUrl = process.env.CHAT_UI_URL; - function runScenario(setArgs: string, extraEnv: Record = {}) { const baseEnv = { ...process.env }; delete baseEnv.NEMOCLAW_DASHBOARD_PORT; @@ -165,8 +162,8 @@ describe("OCI entrypoint env-wrapper normalization", () => { }); } - process.env.NEMOCLAW_DASHBOARD_PORT = "19999"; - process.env.CHAT_UI_URL = "https://ambient.example.test/ui"; + vi.stubEnv("NEMOCLAW_DASHBOARD_PORT", "19999"); + vi.stubEnv("CHAT_UI_URL", "https://ambient.example.test/ui"); try { fs.mkdirSync(fakeBin); @@ -225,16 +222,7 @@ describe("OCI entrypoint env-wrapper normalization", () => { expect(invalidHighPort.stderr).toContain("Invalid NEMOCLAW_DASHBOARD_PORT='70000'"); expect(invalidHighPort.stderr).toContain("must be an integer between 1024 and 65535"); } finally { - if (inheritedDashboardPort === undefined) { - delete process.env.NEMOCLAW_DASHBOARD_PORT; - } else { - process.env.NEMOCLAW_DASHBOARD_PORT = inheritedDashboardPort; - } - if (inheritedChatUiUrl === undefined) { - delete process.env.CHAT_UI_URL; - } else { - process.env.CHAT_UI_URL = inheritedChatUiUrl; - } + vi.unstubAllEnvs(); fs.rmSync(tmpDir, { recursive: true, force: true }); } }); From 5c6b3f7f0c84532453b434aeb0d480cafa608182 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 23:30:50 -0700 Subject: [PATCH 017/117] fix(runtime): preserve direct image execution Signed-off-by: Aaron Erickson --- src/lib/messaging/applier/plan-filter.ts | 49 +++---------------- .../post-agent-install-selection.test.ts | 18 +++++++ .../messaging/post-agent-install-selection.ts | 45 ++++++++++++++--- 3 files changed, 65 insertions(+), 47 deletions(-) diff --git a/src/lib/messaging/applier/plan-filter.ts b/src/lib/messaging/applier/plan-filter.ts index 5c5663e238a..6bfdec9a422 100644 --- a/src/lib/messaging/applier/plan-filter.ts +++ b/src/lib/messaging/applier/plan-filter.ts @@ -1,44 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { MessagingChannelId, SandboxMessagingChannelPlan } from "../manifest"; - -export type EnabledPlanChannel = Pick & - Partial>; - -export interface EnabledPlanSelection { - readonly channels: readonly Channel[]; - readonly disabledChannels?: readonly MessagingChannelId[]; -} - -export function normalizeMessagingChannelId(channelId: MessagingChannelId): MessagingChannelId { - return channelId.trim().toLowerCase(); -} - -export function enabledPlanChannels( - plan: EnabledPlanSelection, -): Channel[] { - const disabled = disabledPlanChannelIds(plan); - return plan.channels.filter((channel) => { - const channelId = normalizeMessagingChannelId(channel.channelId); - return channelId.length > 0 && channel.active && !channel.disabled && !disabled.has(channelId); - }); -} - -export function enabledPlanChannelIds(plan: EnabledPlanSelection): Set { - return new Set( - enabledPlanChannels(plan).map((channel) => normalizeMessagingChannelId(channel.channelId)), - ); -} - -export function filterEnabledPlanEntries( - plan: EnabledPlanSelection, - entries: readonly T[], -): T[] { - const enabled = enabledPlanChannelIds(plan); - return entries.filter((entry) => enabled.has(normalizeMessagingChannelId(entry.channelId))); -} - -function disabledPlanChannelIds(plan: EnabledPlanSelection): Set { - return new Set((plan.disabledChannels ?? []).map(normalizeMessagingChannelId).filter(Boolean)); -} +export { + type EnabledPlanChannel, + type EnabledPlanSelection, + enabledPlanChannelIds, + enabledPlanChannels, + filterEnabledPlanEntries, + normalizeMessagingChannelId, +} from "../post-agent-install-selection"; diff --git a/src/lib/messaging/post-agent-install-selection.test.ts b/src/lib/messaging/post-agent-install-selection.test.ts index 86cdcfad11a..eebdfd18145 100644 --- a/src/lib/messaging/post-agent-install-selection.test.ts +++ b/src/lib/messaging/post-agent-install-selection.test.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; + import { describe, expect, it } from "vitest"; import { filterEnabledPlanEntries } from "./applier/plan-filter"; import type { @@ -78,6 +80,22 @@ function buildFile( } describe("post-agent-install messaging selection", () => { + it("loads through Node's direct TypeScript ESM path used by image builds", () => { + const moduleUrl = new URL("./post-agent-install-selection.ts", import.meta.url).href; + const result = spawnSync( + process.execPath, + [ + "--experimental-strip-types", + "--input-type=module", + "--eval", + `import(${JSON.stringify(moduleUrl)})`, + ], + { encoding: "utf8" }, + ); + + expect({ status: result.status, signal: result.signal }).toEqual({ status: 0, signal: null }); + }); + it("uses the shared enabled-channel contract and preserves canonical ordering", () => { const selection = plan({ channels: [channel(" Discord "), channel(" TeLeGrAm "), channel("DISCORD")], diff --git a/src/lib/messaging/post-agent-install-selection.ts b/src/lib/messaging/post-agent-install-selection.ts index 716c4a5283a..e7346a64106 100644 --- a/src/lib/messaging/post-agent-install-selection.ts +++ b/src/lib/messaging/post-agent-install-selection.ts @@ -1,19 +1,52 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { - type EnabledPlanChannel, - type EnabledPlanSelection, - enabledPlanChannels, - normalizeMessagingChannelId, -} from "./applier/plan-filter"; import type { MessagingChannelId, SandboxMessagingAgentRenderPlan, SandboxMessagingBuildStepPlan, + SandboxMessagingChannelPlan, SandboxMessagingHookReferencePlan, } from "./manifest"; +export type EnabledPlanChannel = Pick & + Partial>; + +export interface EnabledPlanSelection { + readonly channels: readonly Channel[]; + readonly disabledChannels?: readonly MessagingChannelId[]; +} + +export function normalizeMessagingChannelId(channelId: MessagingChannelId): MessagingChannelId { + return channelId.trim().toLowerCase(); +} + +export function enabledPlanChannels( + plan: EnabledPlanSelection, +): Channel[] { + const disabled = new Set( + (plan.disabledChannels ?? []).map(normalizeMessagingChannelId).filter(Boolean), + ); + return plan.channels.filter((channel) => { + const channelId = normalizeMessagingChannelId(channel.channelId); + return channelId.length > 0 && channel.active && !channel.disabled && !disabled.has(channelId); + }); +} + +export function enabledPlanChannelIds(plan: EnabledPlanSelection): Set { + return new Set( + enabledPlanChannels(plan).map((channel) => normalizeMessagingChannelId(channel.channelId)), + ); +} + +export function filterEnabledPlanEntries( + plan: EnabledPlanSelection, + entries: readonly T[], +): T[] { + const enabled = enabledPlanChannelIds(plan); + return entries.filter((entry) => enabled.has(normalizeMessagingChannelId(entry.channelId))); +} + type SelectionChannel = EnabledPlanChannel & { readonly hooks?: readonly (Pick & { readonly phase: string; From 611fdb61dfb522d095dd83dcc82e5237c190c636 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 23:45:38 -0700 Subject: [PATCH 018/117] fix(messaging): reject ambiguous channel identities Signed-off-by: Aaron Erickson --- src/lib/messaging/plan-validation.test.ts | 10 +++++++ src/lib/messaging/plan-validation.ts | 14 +++++----- .../post-agent-install-selection.test.ts | 26 +++++++++++++++++++ .../messaging/post-agent-install-selection.ts | 8 +++--- 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/src/lib/messaging/plan-validation.test.ts b/src/lib/messaging/plan-validation.test.ts index a0993887ad6..27984dbafcf 100644 --- a/src/lib/messaging/plan-validation.test.ts +++ b/src/lib/messaging/plan-validation.test.ts @@ -237,6 +237,16 @@ describe("parseSandboxMessagingPlan", () => { makePlan({ channels: [makePlan().channels[0], makePlan().channels[0]] }), ), ).toBeNull(); + expect( + parseSandboxMessagingPlan( + makePlan({ + channels: [ + makePlan().channels[0], + { ...makePlan().channels[0], channelId: " TELEGRAM " }, + ], + }), + ), + ).toBeNull(); }); it("rejects any persisted channel when supportedChannelIds: [] is passed (deny-all)", () => { diff --git a/src/lib/messaging/plan-validation.ts b/src/lib/messaging/plan-validation.ts index 6b59cf31f87..101451cd47e 100644 --- a/src/lib/messaging/plan-validation.ts +++ b/src/lib/messaging/plan-validation.ts @@ -13,6 +13,7 @@ import { type MaybeCompactMessagingPlan, normalizePersistedSandboxMessagingPlanShape, } from "./persistence"; +import { normalizeMessagingChannelId } from "./post-agent-install-selection"; export interface SandboxMessagingPlanParseOptions { sandboxName?: string | null; @@ -49,7 +50,8 @@ export function parseSandboxMessagingPlan( const supported = Array.isArray(options.supportedChannelIds) ? new Set(options.supportedChannelIds) : null; - for (const [index, channel] of value.channels.entries()) { + const normalizedChannelIds = new Set(); + for (const channel of value.channels) { if (!isObjectRecord(channel) || typeof channel.channelId !== "string") return null; if (Object.hasOwn(channel, "configured") && typeof channel.configured !== "boolean") { return null; @@ -69,13 +71,9 @@ export function parseSandboxMessagingPlan( return null; } if (supported && !supported.has(channel.channelId)) return null; - if ( - value.channels.findIndex( - (candidate) => isObjectRecord(candidate) && candidate.channelId === channel.channelId, - ) !== index - ) { - return null; - } + const normalizedChannelId = normalizeMessagingChannelId(channel.channelId); + if (!normalizedChannelId || normalizedChannelIds.has(normalizedChannelId)) return null; + normalizedChannelIds.add(normalizedChannelId); } if (!value.disabledChannels.every((channelId) => typeof channelId === "string")) return null; diff --git a/src/lib/messaging/post-agent-install-selection.test.ts b/src/lib/messaging/post-agent-install-selection.test.ts index eebdfd18145..1ecd4298540 100644 --- a/src/lib/messaging/post-agent-install-selection.test.ts +++ b/src/lib/messaging/post-agent-install-selection.test.ts @@ -144,4 +144,30 @@ describe("post-agent-install messaging selection", () => { selectEnabledPostAgentInstallBuildFiles(selection).map(({ outputId }) => outputId), ).toEqual(["post-install-file"]); }); + + it("fails closed instead of selecting a hook phase from ambiguous normalized ids", () => { + const selection = plan({ + channels: [ + channel("telegram", [ + { + channelId: "telegram", + id: "shared-hook", + phase: "post-agent-install", + handler: "telegram.post-install", + }, + ]), + channel(" TELEGRAM ", [ + { + channelId: " TELEGRAM ", + id: "shared-hook", + phase: "render", + handler: "telegram.render", + }, + ]), + ], + buildSteps: [buildFile("TELEGRAM", "ambiguous-file", "shared-hook")], + }); + + expect(selectEnabledPostAgentInstallBuildFiles(selection)).toEqual([]); + }); }); diff --git a/src/lib/messaging/post-agent-install-selection.ts b/src/lib/messaging/post-agent-install-selection.ts index e7346a64106..f866edfb31b 100644 --- a/src/lib/messaging/post-agent-install-selection.ts +++ b/src/lib/messaging/post-agent-install-selection.ts @@ -102,9 +102,11 @@ export function selectEnabledPostAgentInstallBuildFiles< const channelId = normalizeMessagingChannelId(step.channelId); if (!active.has(channelId) || step.kind !== "build-file") return false; if (!step.hookId) return true; - const hookPhase = channels - .find((channel) => normalizeMessagingChannelId(channel.channelId) === channelId) - ?.hooks?.find((hook) => hook.id === step.hookId)?.phase; + const matchingChannels = channels.filter( + (channel) => normalizeMessagingChannelId(channel.channelId) === channelId, + ); + if (matchingChannels.length !== 1) return false; + const hookPhase = matchingChannels[0]?.hooks?.find((hook) => hook.id === step.hookId)?.phase; return hookPhase === undefined || hookPhase === "post-agent-install"; }); } From 76c54d49634d81ede157bc3a5f93f7213e44059d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 00:00:51 -0700 Subject: [PATCH 019/117] fix(messaging): require canonical persisted channel ids Signed-off-by: Aaron Erickson --- src/lib/messaging/plan-validation.test.ts | 92 +++++++++++++++++++++++ src/lib/messaging/plan-validation.ts | 75 ++++++++++++++++-- 2 files changed, 162 insertions(+), 5 deletions(-) diff --git a/src/lib/messaging/plan-validation.test.ts b/src/lib/messaging/plan-validation.test.ts index 27984dbafcf..3879b920dea 100644 --- a/src/lib/messaging/plan-validation.test.ts +++ b/src/lib/messaging/plan-validation.test.ts @@ -249,6 +249,98 @@ describe("parseSandboxMessagingPlan", () => { ).toBeNull(); }); + it("rejects a lone noncanonical channel id even when related ids are canonical", () => { + const source = makePlan(); + expect( + parseSandboxMessagingPlan( + makePlan({ + channels: [{ ...source.channels[0], channelId: " Telegram " }], + disabledChannels: ["telegram"], + credentialBindings: [ + { + channelId: "telegram", + credentialId: "telegramBotToken", + sourceInput: "botToken", + providerName: "sb-telegram-bridge", + providerEnvKey: "TELEGRAM_BOT_TOKEN", + placeholder: "openshell:resolve:env:TELEGRAM_BOT_TOKEN", + credentialAvailable: true, + }, + ], + }), + ), + ).toBeNull(); + }); + + it.each([ + ["disabledChannels", { disabledChannels: [" Telegram "] }], + ["credentialBindings", { credentialBindings: [{ channelId: "Telegram" }] }], + [ + "networkPolicy.entries", + { networkPolicy: { presets: [], entries: [{ channelId: "Telegram" }] } }, + ], + ["agentRender", { agentRender: [{ channelId: "Telegram" }] }], + ["buildSteps", { buildSteps: [{ channelId: "Telegram" }] }], + ["stateUpdates", { stateUpdates: [{ channelId: "Telegram" }] }], + ["healthChecks", { healthChecks: [{ channelId: "Telegram" }] }], + [ + "runtimeSetup.nodePreloads", + { + runtimeSetup: { + nodePreloads: [{ channelId: "Telegram" }], + envAliases: [], + secretScans: [], + }, + }, + ], + [ + "runtimeSetup.envAliases", + { + runtimeSetup: { + nodePreloads: [], + envAliases: [{ channelId: "Telegram" }], + secretScans: [], + }, + }, + ], + [ + "runtimeSetup.secretScans", + { + runtimeSetup: { + nodePreloads: [], + envAliases: [], + secretScans: [{ channelId: "Telegram" }], + }, + }, + ], + ])("rejects noncanonical channel references in %s", (_field, overrides) => { + expect(parseSandboxMessagingPlan({ ...makePlan(), ...overrides })).toBeNull(); + }); + + it.each([ + ["input", { inputs: [{ inputId: "token", channelId: "Telegram" }] }], + ["hook", { hooks: [{ channelId: "Telegram" }] }], + [ + "host forward", + { + hostForward: { + channelId: "Telegram", + port: 3978, + label: "Telegram webhook", + }, + }, + ], + ])("rejects a noncanonical nested %s channel reference", (_field, channelOverrides) => { + const channel = makePlan().channels[0]; + expect( + parseSandboxMessagingPlan( + makePlan({ + channels: [{ ...channel, ...channelOverrides }] as SandboxMessagingPlan["channels"], + }), + ), + ).toBeNull(); + }); + it("rejects any persisted channel when supportedChannelIds: [] is passed (deny-all)", () => { expect(parseSandboxMessagingPlan(makePlan(), { supportedChannelIds: [] })).toBeNull(); }); diff --git a/src/lib/messaging/plan-validation.ts b/src/lib/messaging/plan-validation.ts index 101451cd47e..422d0a61ae3 100644 --- a/src/lib/messaging/plan-validation.ts +++ b/src/lib/messaging/plan-validation.ts @@ -53,6 +53,14 @@ export function parseSandboxMessagingPlan( const normalizedChannelIds = new Set(); for (const channel of value.channels) { if (!isObjectRecord(channel) || typeof channel.channelId !== "string") return null; + const normalizedChannelId = normalizeMessagingChannelId(channel.channelId); + if ( + !normalizedChannelId || + normalizedChannelId !== channel.channelId || + normalizedChannelIds.has(normalizedChannelId) + ) { + return null; + } if (Object.hasOwn(channel, "configured") && typeof channel.configured !== "boolean") { return null; } @@ -63,19 +71,47 @@ export function parseSandboxMessagingPlan( if (Object.hasOwn(channel, "hooks") && !Array.isArray(channel.hooks)) return null; if ( Array.isArray(channel.inputs) && - channel.inputs.some((input) => !isObjectRecord(input) || typeof input.inputId !== "string") + channel.inputs.some( + (input) => + !isObjectRecord(input) || + typeof input.inputId !== "string" || + (Object.hasOwn(input, "channelId") && input.channelId !== normalizedChannelId), + ) ) { return null; } - if (Array.isArray(channel.hooks) && channel.hooks.some((hook) => !isObjectRecord(hook))) { + if ( + Array.isArray(channel.hooks) && + channel.hooks.some( + (hook) => + !isObjectRecord(hook) || + (Object.hasOwn(hook, "channelId") && hook.channelId !== normalizedChannelId), + ) + ) { + return null; + } + if ( + Object.hasOwn(channel, "hostForward") && + isObjectRecord(channel.hostForward) && + channel.hostForward.channelId !== normalizedChannelId + ) { return null; } if (supported && !supported.has(channel.channelId)) return null; - const normalizedChannelId = normalizeMessagingChannelId(channel.channelId); - if (!normalizedChannelId || normalizedChannelIds.has(normalizedChannelId)) return null; normalizedChannelIds.add(normalizedChannelId); } - if (!value.disabledChannels.every((channelId) => typeof channelId === "string")) return null; + if (!value.disabledChannels.every(isCanonicalMessagingChannelId)) return null; + if ( + !hasCanonicalChannelReferences(value.credentialBindings) || + !hasCanonicalChannelReferences(value.agentRender) || + !hasCanonicalChannelReferences(value.buildSteps) || + !hasCanonicalChannelReferences(value.stateUpdates) || + !hasCanonicalChannelReferences(value.healthChecks) || + !hasCanonicalNetworkPolicyReferences(value.networkPolicy) || + !hasCanonicalRuntimeSetupReferences(value.runtimeSetup) + ) { + return null; + } return cloneSandboxMessagingPlan( normalizePersistedSandboxMessagingPlanShape(value as MaybeCompactMessagingPlan), @@ -190,3 +226,32 @@ function isRuntimeSetup(value: unknown): boolean { value.secretScans.every(isObjectRecord) ); } + +function isCanonicalMessagingChannelId(value: unknown): value is string { + return ( + typeof value === "string" && value.length > 0 && normalizeMessagingChannelId(value) === value + ); +} + +function hasCanonicalChannelReferences(value: unknown): boolean { + return ( + value === undefined || + (Array.isArray(value) && + value.every( + (entry) => isObjectRecord(entry) && isCanonicalMessagingChannelId(entry.channelId), + )) + ); +} + +function hasCanonicalNetworkPolicyReferences(value: unknown): boolean { + if (!isObjectRecord(value) || !Object.hasOwn(value, "entries")) return true; + return hasCanonicalChannelReferences(value.entries); +} + +function hasCanonicalRuntimeSetupReferences(value: unknown): boolean { + if (value === undefined) return true; + if (!isObjectRecord(value)) return false; + return ["nodePreloads", "envAliases", "secretScans"].every((field) => + hasCanonicalChannelReferences(value[field]), + ); +} From 5f9d97d5a0a1302c1e14af9bd8f90d529fc3bd69 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 00:06:37 -0700 Subject: [PATCH 020/117] test(e2e): add cross-runtime foundation Signed-off-by: Aaron Erickson (cherry picked from commit f99197be76301cc0534d2d62d6937d49e9119cd6) --- test/e2e/docs/README.md | 42 ++ test/e2e/fixtures/artifacts.ts | 13 + test/e2e/fixtures/e2e-test.ts | 11 + test/e2e/fixtures/runtime-provider.ts | 150 ++++++ test/e2e/live/run-plan.ts | 21 +- test/e2e/registry/builder.ts | 7 + test/e2e/registry/execution-profile.ts | 124 +++++ test/e2e/registry/parity-evidence.ts | 301 +++++++++++ test/e2e/registry/runtime-matrix.ts | 474 ++++++++++++++++++ test/e2e/registry/scenario.ts | 278 ++++++++++ test/e2e/registry/types.ts | 8 + .../cross-runtime-foundation-fixtures.ts | 175 +++++++ .../e2e-cross-runtime-compatibility.test.ts | 51 ++ test/e2e/support/e2e-parity-evidence.test.ts | 197 ++++++++ .../e2e-runtime-foundation-types.test.ts | 125 +++++ test/e2e/support/e2e-runtime-matrix.test.ts | 364 ++++++++++++++ 16 files changed, 2339 insertions(+), 2 deletions(-) create mode 100644 test/e2e/fixtures/runtime-provider.ts create mode 100644 test/e2e/registry/execution-profile.ts create mode 100644 test/e2e/registry/parity-evidence.ts create mode 100644 test/e2e/registry/runtime-matrix.ts create mode 100644 test/e2e/registry/scenario.ts create mode 100644 test/e2e/support/cross-runtime-foundation-fixtures.ts create mode 100644 test/e2e/support/e2e-cross-runtime-compatibility.test.ts create mode 100644 test/e2e/support/e2e-parity-evidence.test.ts create mode 100644 test/e2e/support/e2e-runtime-foundation-types.test.ts create mode 100644 test/e2e/support/e2e-runtime-matrix.test.ts diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index a6388b30fb1..b889fbba258 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -67,6 +67,48 @@ harness or runner. Vitest remains the only test harness. `suiteIds` remain metadata for reporting and migration planning. They do not dispatch shell validation suites. +## Cross-Runtime Foundation + +The registry contains an inert foundation for describing the same behavior on +more than one execution provider: + +- `scenario.ts` owns provider-neutral desired state and explicit support + obligations, an ordered semantic user journey, and normalized assertions. +- `execution-profile.ts` describes provider, host platform and architecture, + root mode, acceleration, capabilities, and bounded runner capacity. Provider + IDs are open; adding one does not require editing a central union. +- `runtime-matrix.ts` binds every scenario obligation to a registered callable + fixture adapter, rejects incompatible capabilities, keeps full-profile + preparation batches atomic, schedules those batches within a host-wide shard + ceiling, and derives isolated resource identities. +- `fixtures/runtime-provider.ts` is the provider-command boundary for + readiness, exact workload identity, obligation execution, lifecycle evidence, + and cleanup. Its fixture-only executor exercises compiled cases without + crossing the legacy Docker phase-fixture path. +- `parity-evidence.ts` compares normalized lifecycle traces, desired-state + fingerprints, terminal outcomes, and user-visible projections. It retains + exact head/base, engine, architecture, workload, managed-image, capability, + and opaque provider receipt evidence without comparing provider internals. + +Compile one registry-wide `RuntimeMatrixDefinition`, then attach only a +`scenarioId`/`profileId` reference with `TargetBuilder.runtimeCase(...)` in fast +compiler tests today. The existing target compiler resolves the reference but +does not dispatch its adapter IDs. Support tests execute the same compiled case +through Docker-shaped and fake-MXC providers; no canonical target, workflow +selector, live scenario, or production runtime registration consumes this +metadata yet. The shared fixture context exposes optional `executionProfile` +and `runtimeProvider` injection points, both defaulting to `undefined`. Existing +legacy Docker command fixtures, their ordering, and their output contracts are +unchanged. +Execution evidence must be published with +`ArtifactSink.writeExecutionEvidence(...)` so normal artifact redaction still +applies. + +When extending the foundation, keep product intent in the scenario, runtime +mechanics in obligation bindings, and support facts in capabilities. A binding +must cover every obligation explicitly; a missing adapter or capability is a +compile error rather than a skip. + ## How To Run ```bash diff --git a/test/e2e/fixtures/artifacts.ts b/test/e2e/fixtures/artifacts.ts index 833f4f823d3..f8c2735d637 100644 --- a/test/e2e/fixtures/artifacts.ts +++ b/test/e2e/fixtures/artifacts.ts @@ -5,6 +5,7 @@ import fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; +import type { ExecutionEvidence } from "../registry/parity-evidence.ts"; import { redactString } from "./redaction.ts"; export type TargetContract = string | readonly string[]; @@ -138,6 +139,18 @@ export class ArtifactSink { async writeJson(relativePath: string, value: unknown): Promise { return this.writeText(relativePath, `${JSON.stringify(value, null, 2)}\n`); } + + async writeExecutionEvidence(resultId: string, evidence: ExecutionEvidence): Promise { + if (!/^[a-z0-9][a-z0-9-]{0,127}$/u.test(resultId)) { + throw new Error(`execution result id is not artifact-safe: ${resultId}`); + } + if (resultId !== evidence.resultId) { + throw new Error( + `execution result id '${resultId}' does not match evidence result '${evidence.resultId}'`, + ); + } + return this.writeJson(path.join("execution", `${resultId}.json`), evidence); + } } export function slugifyArtifactName(name: string): string { diff --git a/test/e2e/fixtures/e2e-test.ts b/test/e2e/fixtures/e2e-test.ts index b036689d3da..550c1c0fbab 100644 --- a/test/e2e/fixtures/e2e-test.ts +++ b/test/e2e/fixtures/e2e-test.ts @@ -11,6 +11,7 @@ import { collectResourceSnapshot, } from "../../../tools/e2e/runner-pressure.mts"; import { renderSnapshotLine } from "../../../tools/e2e/runner-pressure-core.mts"; +import type { ExecutionProfile } from "../registry/execution-profile.ts"; import { type ArtifactSink, createArtifactSink } from "./artifacts.ts"; import { assertCleanupPassed, CleanupRegistry } from "./cleanup.ts"; @@ -37,6 +38,7 @@ import { type TestProgress, type TestProgressOptions, } from "./progress.ts"; +import type { RuntimeProviderFixture } from "./runtime-provider.ts"; import { SecretStore } from "./secrets.ts"; import { ShellProbe } from "./shell-probe.ts"; @@ -63,6 +65,9 @@ export interface E2ETargetFixtures { lifecycle: LifecyclePhaseFixture; runtime: RuntimePhaseFixture; stateValidation: StateValidationPhaseFixture; + /** Inert injection points for a future compiled cross-runtime target. */ + executionProfile: ExecutionProfile | undefined; + runtimeProvider: RuntimeProviderFixture | undefined; progress: TestProgress; } @@ -292,6 +297,12 @@ export const test = base.extend({ stateValidation: async ({ artifacts, host, gateway, sandbox }, use) => { await use(new StateValidationPhaseFixture(host, gateway, sandbox, {}, artifacts)); }, + executionProfile: async ({}, use) => { + await use(undefined); + }, + runtimeProvider: async ({}, use) => { + await use(undefined); + }, }); export { expect }; diff --git a/test/e2e/fixtures/runtime-provider.ts b/test/e2e/fixtures/runtime-provider.ts new file mode 100644 index 00000000000..59eeff068ce --- /dev/null +++ b/test/e2e/fixtures/runtime-provider.ts @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ExecutionCapability, ExecutionProfile } from "../registry/execution-profile.ts"; +import { + buildExecutionEvidence, + type ExecutionEvidence, + type ManagedImageEvidence, + type ProviderReceipt, +} from "../registry/parity-evidence.ts"; +import { + executionPreparationKey, + type ResolvedRuntimeCase, + type RuntimeAdapterRequest, + type RuntimeAdapterRuntime, +} from "../registry/runtime-matrix.ts"; +import type { FsmTransition, JsonValue, TerminalOutcome } from "../registry/scenario.ts"; + +export interface RuntimeReadinessEvidence { + profileId: string; + ready: true; + engineName: string; + engineVersion: string; + capabilities: readonly ExecutionCapability[]; +} + +export interface ExactWorkloadIdentity { + logicalId: string; + providerResourceId: string; + managedImages: readonly ManagedImageEvidence[]; +} + +export interface RuntimeLifecycleRequest { + caseId: string; + workload: ExactWorkloadIdentity; +} + +export interface RuntimeLifecycleEvidence { + desiredState: JsonValue; + fsmTrace: readonly FsmTransition[]; + terminalOutcome: TerminalOutcome; + userVisibleState: JsonValue; + providerReceipts: readonly ProviderReceipt[]; +} + +/** + * Provider commands stop at this seam. Scenario, matrix, and parity code use + * only normalized evidence and exact workload identities. + */ +export interface RuntimeProviderEnvironment { + prepare(): Promise; +} + +export interface RuntimeProviderLifecycle { + executeAdapter(adapterId: string, request: RuntimeAdapterRequest): Promise; + cleanup(identity: ExactWorkloadIdentity): Promise; +} + +export interface RuntimeProviderState { + inspectWorkload(request: { logicalId: string }): Promise; + observe(request: RuntimeLifecycleRequest): Promise; +} + +export interface RuntimeProviderFixture extends RuntimeAdapterRuntime { + readonly profile: ExecutionProfile; + readonly environment: RuntimeProviderEnvironment; + readonly lifecycle: RuntimeProviderLifecycle; + readonly state: RuntimeProviderState; +} + +export interface RuntimeExecutionRequest { + resolved: ResolvedRuntimeCase; + provider: RuntimeProviderFixture; + source: { + headSha: string; + baseSha: string; + }; +} + +/** + * The only executable cross-runtime path in this foundation. It does not use + * the legacy Docker-shaped environment/lifecycle/state fixtures and is not + * selected by any canonical target or workflow. + */ +export async function executeRuntimeCaseThroughProvider( + request: RuntimeExecutionRequest, +): Promise { + const { case: runtimeCase } = request.resolved; + const provider = request.provider; + if ( + provider.profile.id !== runtimeCase.profile.id || + executionPreparationKey(provider.profile) !== runtimeCase.preparationKey + ) { + throw new Error( + `Runtime provider profile '${provider.profile.id}' does not match case '${runtimeCase.id}'`, + ); + } + const readiness = await provider.environment.prepare(); + if (readiness.profileId !== runtimeCase.profile.id) { + throw new Error( + `Runtime readiness profile '${readiness.profileId}' does not match '${runtimeCase.profile.id}'`, + ); + } + const missingCapabilities = runtimeCase.profile.capabilities.filter( + (capability) => !readiness.capabilities.includes(capability), + ); + if (missingCapabilities.length > 0) { + throw new Error( + `Runtime readiness for '${runtimeCase.profile.id}' is missing capabilities: ${missingCapabilities.join(", ")}`, + ); + } + + const workload = await provider.state.inspectWorkload({ + logicalId: runtimeCase.identities.sandbox, + }); + let lifecycle: RuntimeLifecycleEvidence; + let cleanupReceipts: readonly ProviderReceipt[] = []; + try { + for (const binding of runtimeCase.obligationBindings) { + await binding.adapter.execute(provider, { + caseId: runtimeCase.id, + obligationId: binding.obligationId, + workloadId: workload.logicalId, + }); + } + lifecycle = await provider.state.observe({ + caseId: runtimeCase.id, + workload, + }); + } finally { + cleanupReceipts = await provider.lifecycle.cleanup(workload); + } + + return buildExecutionEvidence({ + resolved: request.resolved, + source: request.source, + engine: { + name: readiness.engineName, + version: readiness.engineVersion, + }, + workload, + observed: { + desiredState: lifecycle.desiredState, + fsmTrace: lifecycle.fsmTrace, + terminalOutcome: lifecycle.terminalOutcome, + userVisibleState: lifecycle.userVisibleState, + }, + providerReceipts: [...lifecycle.providerReceipts, ...cleanupReceipts], + }); +} diff --git a/test/e2e/live/run-plan.ts b/test/e2e/live/run-plan.ts index 21e1af9c9ef..a56f58c4ce9 100644 --- a/test/e2e/live/run-plan.ts +++ b/test/e2e/live/run-plan.ts @@ -1,8 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { cloudExperimentalChecksForOnboarding } from "./cloud-experimental-check-list.ts"; +import { + type CompiledRuntimeMatrix, + type ResolvedRuntimeCase, + resolveRuntimeCase, +} from "../registry/runtime-matrix.ts"; import type { TargetDefinition } from "../registry/types.ts"; +import { cloudExperimentalChecksForOnboarding } from "./cloud-experimental-check-list.ts"; export interface LiveTargetRunPlan { targetId: string; @@ -10,10 +15,14 @@ export interface LiveTargetRunPlan { expectedStateId: string | undefined; suiteIds: string[]; phases: string[]; + runtimeCase?: ResolvedRuntimeCase; e2eCloudExperimentalChecks?: string[]; } -export function buildLiveTargetRunPlan(target: TargetDefinition): LiveTargetRunPlan { +export function buildLiveTargetRunPlan( + target: TargetDefinition, + runtimeMatrix?: CompiledRuntimeMatrix, +): LiveTargetRunPlan { const plan: LiveTargetRunPlan = { targetId: target.id, manifestPath: target.manifestPath ?? null, @@ -32,5 +41,13 @@ export function buildLiveTargetRunPlan(target: TargetDefinition): LiveTargetRunP if (cloudExperimentalChecks.length > 0) { plan.e2eCloudExperimentalChecks = [...cloudExperimentalChecks]; } + if (target.runtimeCase) { + if (!runtimeMatrix) { + throw new Error( + `Target '${target.id}' references a runtime case without a compiled runtime matrix`, + ); + } + plan.runtimeCase = resolveRuntimeCase(runtimeMatrix, target.runtimeCase); + } return plan; } diff --git a/test/e2e/registry/builder.ts b/test/e2e/registry/builder.ts index d884f301277..402fb7b7c46 100644 --- a/test/e2e/registry/builder.ts +++ b/test/e2e/registry/builder.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { RuntimeCaseReference } from "./runtime-matrix.ts"; import type { AssertionGroup, TargetDefinition, TargetEnvironment } from "./types.ts"; export class TargetBuilder { @@ -25,6 +26,11 @@ export class TargetBuilder { return this; } + runtimeCase(runtimeCase: RuntimeCaseReference): TargetBuilder { + this.definition.runtimeCase = runtimeCase; + return this; + } + expectedState(expectedStateId: string): TargetBuilder { this.definition.expectedStateId = expectedStateId; return this; @@ -68,6 +74,7 @@ export class TargetBuilder { build(): TargetDefinition { return { ...this.definition, + ...(this.definition.runtimeCase ? { runtimeCase: { ...this.definition.runtimeCase } } : {}), assertionGroups: [...this.definition.assertionGroups], suiteIds: [...(this.definition.suiteIds ?? [])], onboardingAssertionIds: [...(this.definition.onboardingAssertionIds ?? [])], diff --git a/test/e2e/registry/execution-profile.ts b/test/e2e/registry/execution-profile.ts new file mode 100644 index 00000000000..1601cb38a12 --- /dev/null +++ b/test/e2e/registry/execution-profile.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const EXECUTION_FOUNDATION_ID_PATTERN = /^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)*$/u; +export const MAX_HOST_SHARDS = 32; + +declare const executionProviderIdBrand: unique symbol; + +/** + * Open provider identity. A provider becomes executable only when its adapter + * is present in a RuntimeAdapterCatalog; naming one here is not registration or + * a support claim. + */ +export type ExecutionProviderId = string & { + readonly [executionProviderIdBrand]: true; +}; +export type ExecutionPlatform = "linux" | "macos" | "windows"; +export type ExecutionArchitecture = "amd64" | "arm64"; +export type ExecutionRootMode = "rootful" | "rootless"; +export type ExecutionAcceleration = "cpu" | "nvidia-gpu"; + +export type ExecutionCapability = + | "agent.configure" + | "agent.turn" + | "evidence.collect" + | "sandbox.lifecycle" + | "state.observe" + | "transport.docker-socket" + | "transport.socket-free"; + +export interface ExecutionRunner { + /** Stable physical or virtual host identity used to bound shard lanes. */ + hostId: string; + /** Runner label for future consumers. This foundation does not select it. */ + label: string; + /** Maximum serial shard lanes that may be assigned on this host. */ + maxShards: number; +} + +export interface ExecutionProfile { + id: string; + provider: ExecutionProviderId; + platform: ExecutionPlatform; + architecture: ExecutionArchitecture; + rootMode: ExecutionRootMode; + acceleration: ExecutionAcceleration; + capabilities: readonly ExecutionCapability[]; + runner: Readonly; +} + +const PLATFORMS = new Set(["linux", "macos", "windows"]); +const ARCHITECTURES = new Set(["amd64", "arm64"]); +const ROOT_MODES = new Set(["rootful", "rootless"]); +const ACCELERATIONS = new Set(["cpu", "nvidia-gpu"]); +const CAPABILITIES = new Set([ + "agent.configure", + "agent.turn", + "evidence.collect", + "sandbox.lifecycle", + "state.observe", + "transport.docker-socket", + "transport.socket-free", +]); + +export function assertExecutionFoundationId(value: string, label: string): void { + if (!EXECUTION_FOUNDATION_ID_PATTERN.test(value)) { + throw new Error( + `${label} '${value}' must start with a lowercase letter and contain only lowercase letters, digits, dots, or hyphens`, + ); + } +} + +export function executionProviderId(value: string): ExecutionProviderId { + assertExecutionFoundationId(value, "Execution provider id"); + return value as ExecutionProviderId; +} + +function assertEnumValue(values: ReadonlySet, value: T, label: string): void { + if (!values.has(value)) { + throw new Error(`${label} '${value}' is not recognized`); + } +} + +export function defineExecutionProfile(input: ExecutionProfile): ExecutionProfile { + assertExecutionFoundationId(input.id, "Execution profile id"); + const provider = executionProviderId(input.provider); + assertEnumValue(PLATFORMS, input.platform, "Execution platform"); + assertEnumValue(ARCHITECTURES, input.architecture, "Execution architecture"); + assertEnumValue(ROOT_MODES, input.rootMode, "Execution root mode"); + assertEnumValue(ACCELERATIONS, input.acceleration, "Execution acceleration"); + + if (input.capabilities.length === 0) { + throw new Error(`Execution profile '${input.id}' must declare capabilities`); + } + const capabilities = [...input.capabilities]; + for (const capability of capabilities) { + assertEnumValue(CAPABILITIES, capability, "Execution capability"); + } + if (new Set(capabilities).size !== capabilities.length) { + throw new Error(`Execution profile '${input.id}' declares duplicate capabilities`); + } + + assertExecutionFoundationId(input.runner.hostId, "Execution runner host id"); + const runnerLabel = input.runner.label.trim(); + if (!runnerLabel || /[\r\n]/u.test(runnerLabel)) { + throw new Error(`Execution profile '${input.id}' must declare a single-line runner label`); + } + if ( + !Number.isSafeInteger(input.runner.maxShards) || + input.runner.maxShards < 1 || + input.runner.maxShards > MAX_HOST_SHARDS + ) { + throw new Error( + `Execution profile '${input.id}' runner maxShards must be between 1 and ${MAX_HOST_SHARDS}`, + ); + } + + return Object.freeze({ + ...input, + provider, + capabilities: Object.freeze([...capabilities].sort()), + runner: Object.freeze({ ...input.runner, label: runnerLabel }), + }); +} diff --git a/test/e2e/registry/parity-evidence.ts b/test/e2e/registry/parity-evidence.ts new file mode 100644 index 00000000000..4b9c5571f86 --- /dev/null +++ b/test/e2e/registry/parity-evidence.ts @@ -0,0 +1,301 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { + defineExecutionProfile, + type ExecutionAcceleration, + type ExecutionArchitecture, + type ExecutionCapability, + type ExecutionPlatform, + type ExecutionProviderId, + type ExecutionRootMode, +} from "./execution-profile.ts"; +import { assertResolvedRuntimeCase, type ResolvedRuntimeCase } from "./runtime-matrix.ts"; +import { + assertTerminalMatchesTrace, + compareCodeUnits, + type FsmTransition, + freezeJsonValue, + type JsonValue, + normalizeFsmTrace, + normalizeJsonValue, + normalizeTerminalOutcome, + type TerminalOutcome, +} from "./scenario.ts"; + +export interface ManagedImageEvidence { + role: string; + digest: string; +} + +export interface ProviderReceipt { + kind: string; + operationId: string; + /** Provider-defined diagnostic payload; parity comparison never interprets it. */ + value: JsonValue; +} + +export interface NormalizedParityEvidence { + desiredStateFingerprint: string; + fsmTrace: readonly Readonly[]; + terminalOutcome: Readonly; + userVisibleState: JsonValue; +} + +export interface ExecutionEvidence { + schemaVersion: 1; + caseId: string; + resultId: string; + shardId: string; + scenarioId: string; + profileId: string; + source: Readonly<{ + headSha: string; + baseSha: string; + }>; + runtime: Readonly<{ + provider: ExecutionProviderId; + engineName: string; + engineVersion: string; + platform: ExecutionPlatform; + architecture: ExecutionArchitecture; + rootMode: ExecutionRootMode; + acceleration: ExecutionAcceleration; + capabilities: readonly ExecutionCapability[]; + }>; + workload: Readonly<{ + logicalId: string; + providerResourceId: string; + managedImages: readonly Readonly[]; + }>; + parity: Readonly; + providerReceipts: readonly Readonly[]; +} + +export interface ExecutionEvidenceInput { + resolved: ResolvedRuntimeCase; + source: { + headSha: string; + baseSha: string; + }; + engine: { + name: string; + version: string; + }; + workload: { + logicalId: string; + providerResourceId: string; + managedImages: readonly ManagedImageEvidence[]; + }; + observed: { + desiredState: JsonValue; + fsmTrace: readonly FsmTransition[]; + terminalOutcome: TerminalOutcome; + userVisibleState: JsonValue; + }; + providerReceipts: readonly ProviderReceipt[]; +} + +export interface ParityMismatch { + field: + | "scenarioId" + | "desiredStateFingerprint" + | "fsmTrace" + | "terminalOutcome" + | "userVisibleState"; + expected: unknown; + actual: unknown; +} + +const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const IMAGE_DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/u; +const RECEIPT_ID_PATTERN = /^[a-z][a-z0-9]*(?:[./_-][a-z0-9]+)*$/u; + +function normalizeSingleLine(value: string, fieldPath: string): string { + const normalized = value.trim(); + if (!normalized || /[\r\n]/u.test(normalized)) { + throw new Error(`${fieldPath} must be a non-empty single-line string`); + } + return normalized; +} + +function stableJson(value: JsonValue): string { + return JSON.stringify(normalizeJsonValue(value)); +} + +export function fingerprintDesiredState(desiredState: JsonValue): string { + return createHash("sha256").update(stableJson(desiredState)).digest("hex"); +} + +function normalizeSha(value: string, fieldPath: string): string { + const normalized = value.toLowerCase(); + if (!SHA_PATTERN.test(normalized)) { + throw new Error(`${fieldPath} must be a 40-character commit SHA`); + } + return normalized; +} + +function normalizeManagedImages( + images: readonly ManagedImageEvidence[], +): readonly Readonly[] { + if (images.length === 0) { + throw new Error("workload.managedImages must contain at least one exact image digest"); + } + const roles = new Set(); + return Object.freeze( + images + .map((image) => { + const role = normalizeSingleLine(image.role, "workload.managedImages.role"); + if (roles.has(role)) { + throw new Error(`workload.managedImages repeats role '${role}'`); + } + roles.add(role); + const digest = image.digest.toLowerCase(); + if (!IMAGE_DIGEST_PATTERN.test(digest)) { + throw new Error(`workload.managedImages '${role}' must use an exact sha256 digest`); + } + return Object.freeze({ role, digest }); + }) + .sort((left, right) => compareCodeUnits(left.role, right.role)), + ); +} + +function normalizeProviderReceipts( + receipts: readonly ProviderReceipt[], +): readonly Readonly[] { + if (receipts.length === 0) { + throw new Error("providerReceipts must contain at least one provider receipt"); + } + const operationIds = new Set(); + return Object.freeze( + receipts.map((receipt) => { + if (!RECEIPT_ID_PATTERN.test(receipt.kind)) { + throw new Error(`provider receipt kind '${receipt.kind}' is invalid`); + } + if (!RECEIPT_ID_PATTERN.test(receipt.operationId)) { + throw new Error(`provider receipt operationId '${receipt.operationId}' is invalid`); + } + if (operationIds.has(receipt.operationId)) { + throw new Error(`provider receipt operationId '${receipt.operationId}' is duplicated`); + } + operationIds.add(receipt.operationId); + return Object.freeze({ + ...receipt, + value: freezeJsonValue( + normalizeJsonValue(receipt.value, `providerReceipts.${receipt.operationId}.value`), + ), + }); + }), + ); +} + +export function buildExecutionEvidence(input: ExecutionEvidenceInput): ExecutionEvidence { + assertResolvedRuntimeCase(input.resolved); + const { case: runtimeCase, shard } = input.resolved; + const profile = defineExecutionProfile(runtimeCase.profile); + const logicalId = normalizeSingleLine(input.workload.logicalId, "workload.logicalId"); + if (logicalId !== runtimeCase.identities.sandbox) { + throw new Error( + `workload.logicalId '${logicalId}' does not match case sandbox identity '${runtimeCase.identities.sandbox}'`, + ); + } + const providerResourceId = normalizeSingleLine( + input.workload.providerResourceId, + "workload.providerResourceId", + ); + const fsmTrace = normalizeFsmTrace(input.observed.fsmTrace); + const terminalOutcome = normalizeTerminalOutcome(input.observed.terminalOutcome); + assertTerminalMatchesTrace(fsmTrace, terminalOutcome, "observed.terminalOutcome"); + const desiredState = normalizeJsonValue(input.observed.desiredState, "observed.desiredState"); + const userVisibleState = normalizeJsonValue( + input.observed.userVisibleState, + "observed.userVisibleState", + ); + const assertions = runtimeCase.scenario.assertions; + const observedContract = { + desiredState, + fsmTrace, + terminalOutcome, + userVisibleState, + }; + for (const field of Object.keys(observedContract) as Array) { + if (!sameJson(observedContract[field], assertions[field])) { + throw new Error( + `Runtime case '${runtimeCase.id}' observed ${field} does not satisfy its scenario assertion`, + ); + } + } + + return Object.freeze({ + schemaVersion: 1, + caseId: runtimeCase.id, + resultId: runtimeCase.identities.result, + shardId: shard.id, + scenarioId: runtimeCase.scenario.id, + profileId: profile.id, + source: Object.freeze({ + headSha: normalizeSha(input.source.headSha, "source.headSha"), + baseSha: normalizeSha(input.source.baseSha, "source.baseSha"), + }), + runtime: Object.freeze({ + provider: profile.provider, + engineName: normalizeSingleLine(input.engine.name, "engine.name"), + engineVersion: normalizeSingleLine(input.engine.version, "engine.version"), + platform: profile.platform, + architecture: profile.architecture, + rootMode: profile.rootMode, + acceleration: profile.acceleration, + capabilities: profile.capabilities, + }), + workload: Object.freeze({ + logicalId, + providerResourceId, + managedImages: normalizeManagedImages(input.workload.managedImages), + }), + parity: Object.freeze({ + desiredStateFingerprint: fingerprintDesiredState(desiredState), + fsmTrace, + terminalOutcome, + userVisibleState: freezeJsonValue(userVisibleState), + }), + providerReceipts: normalizeProviderReceipts(input.providerReceipts), + }); +} + +function sameJson(left: unknown, right: unknown): boolean { + return JSON.stringify(left) === JSON.stringify(right); +} + +export function compareParityEvidence( + expected: ExecutionEvidence, + actual: ExecutionEvidence, +): ParityMismatch[] { + const comparable = { + scenarioId: [expected.scenarioId, actual.scenarioId], + desiredStateFingerprint: [ + expected.parity.desiredStateFingerprint, + actual.parity.desiredStateFingerprint, + ], + fsmTrace: [expected.parity.fsmTrace, actual.parity.fsmTrace], + terminalOutcome: [expected.parity.terminalOutcome, actual.parity.terminalOutcome], + userVisibleState: [expected.parity.userVisibleState, actual.parity.userVisibleState], + } satisfies Record; + + return (Object.keys(comparable) as ParityMismatch["field"][]).flatMap((field) => { + const [expectedValue, actualValue] = comparable[field]; + return sameJson(expectedValue, actualValue) + ? [] + : [{ field, expected: expectedValue, actual: actualValue }]; + }); +} + +export function assertParityEvidence(expected: ExecutionEvidence, actual: ExecutionEvidence): void { + const mismatches = compareParityEvidence(expected, actual); + if (mismatches.length > 0) { + throw new Error( + `Runtime parity mismatch in: ${mismatches.map((mismatch) => mismatch.field).join(", ")}`, + ); + } +} diff --git a/test/e2e/registry/runtime-matrix.ts b/test/e2e/registry/runtime-matrix.ts new file mode 100644 index 00000000000..36ea682444c --- /dev/null +++ b/test/e2e/registry/runtime-matrix.ts @@ -0,0 +1,474 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { + assertExecutionFoundationId, + defineExecutionProfile, + type ExecutionProfile, + type ExecutionProviderId, + executionProviderId, +} from "./execution-profile.ts"; +import { + compareCodeUnits, + defineRuntimeScenario, + type RuntimeNeutralScenario, + type ScenarioSupportObligation, +} from "./scenario.ts"; + +export interface ObligationBinding { + obligationId: string; + /** Registry-local executable adapter identity. */ + adapterId: string; +} + +export interface RuntimeBindingSpec { + scenarioId: string; + profileId: string; + obligationBindings: readonly ObligationBinding[]; +} + +export interface RuntimeAdapterRequest { + caseId: string; + obligationId: string; + workloadId: string; +} + +export interface RuntimeAdapterRuntime { + readonly profile: ExecutionProfile; + readonly lifecycle: { + executeAdapter(adapterId: string, request: RuntimeAdapterRequest): Promise; + }; +} + +export interface RuntimeAdapterRegistration { + id: string; + provider: ExecutionProviderId; + scenarioId: string; + obligationId: string; + execute(runtime: RuntimeAdapterRuntime, request: RuntimeAdapterRequest): Promise; +} + +export interface RuntimeCaseReference { + scenarioId: string; + profileId: string; +} + +export interface RuntimeMatrixDefinition { + scenarios: readonly RuntimeNeutralScenario[]; + profiles: readonly ExecutionProfile[]; + adapterCatalog: readonly RuntimeAdapterRegistration[]; + bindings: readonly RuntimeBindingSpec[]; +} + +export interface RuntimeResourceIdentities { + sandbox: string; + artifact: string; + cleanup: string; + result: string; +} + +export interface RuntimeMatrixCase { + id: string; + preparationKey: string; + scenario: RuntimeNeutralScenario; + profile: ExecutionProfile; + obligationBindings: readonly Readonly[]; + identities: Readonly; +} + +export interface CompiledObligationBinding extends ObligationBinding { + adapter: Readonly; +} + +export interface RuntimePreparationBatch { + preparationKey: string; + profile: ExecutionProfile; + cases: readonly RuntimeMatrixCase[]; +} + +export interface RuntimeHostShard { + id: string; + runner: ExecutionProfile["runner"]; + index: number; + count: number; + preparations: readonly Readonly[]; + cases: readonly RuntimeMatrixCase[]; +} + +declare const compiledRuntimeMatrixBrand: unique symbol; +declare const resolvedRuntimeCaseBrand: unique symbol; + +export interface CompiledRuntimeMatrix { + readonly [compiledRuntimeMatrixBrand]: true; + cases: readonly RuntimeMatrixCase[]; + shards: readonly Readonly[]; +} + +export interface ResolvedRuntimeCase { + readonly [resolvedRuntimeCaseBrand]: true; + case: RuntimeMatrixCase; + shard: Readonly; +} + +const ADAPTER_ID_PATTERN = /^[a-z][a-z0-9]*(?:[./_-][a-z0-9]+)*$/u; +const compiledRuntimeMatrices = new WeakSet(); +const resolvedRuntimeCases = new WeakSet(); + +function indexById(values: readonly T[], label: string): Map { + const index = new Map(); + for (const value of values) { + if (index.has(value.id)) { + throw new Error(`Duplicate ${label} id '${value.id}'`); + } + index.set(value.id, value); + } + return index; +} + +function assertConsistentHosts(profiles: readonly ExecutionProfile[]): void { + const runnersByHost = new Map(); + for (const profile of profiles) { + const existing = runnersByHost.get(profile.runner.hostId); + if ( + existing && + (existing.maxShards !== profile.runner.maxShards || existing.label !== profile.runner.label) + ) { + throw new Error( + `Execution host '${profile.runner.hostId}' has conflicting runner label or maxShards`, + ); + } + runnersByHost.set(profile.runner.hostId, profile.runner); + } +} + +function compileObligationBindings( + scenario: RuntimeNeutralScenario, + profile: ExecutionProfile, + bindings: readonly ObligationBinding[], + adapterCatalog: ReadonlyMap>, +): readonly Readonly[] { + const declared = new Map(); + for (const binding of bindings) { + assertExecutionFoundationId(binding.obligationId, "Bound obligation id"); + if (!ADAPTER_ID_PATTERN.test(binding.adapterId)) { + throw new Error( + `Runtime binding '${scenario.id}' -> '${profile.id}' has invalid adapter id '${binding.adapterId}'`, + ); + } + const adapter = adapterCatalog.get(binding.adapterId); + if (!adapter) { + throw new Error( + `Runtime binding '${scenario.id}' -> '${profile.id}' references unregistered adapter '${binding.adapterId}'`, + ); + } + if (adapter.provider !== profile.provider) { + throw new Error( + `Runtime adapter '${binding.adapterId}' belongs to provider '${adapter.provider}', not '${profile.provider}'`, + ); + } + if (adapter.scenarioId !== scenario.id) { + throw new Error( + `Runtime adapter '${binding.adapterId}' belongs to scenario '${adapter.scenarioId}', not '${scenario.id}'`, + ); + } + if (adapter.obligationId !== binding.obligationId) { + throw new Error( + `Runtime adapter '${binding.adapterId}' implements obligation '${adapter.obligationId}', not '${binding.obligationId}'`, + ); + } + if (declared.has(binding.obligationId)) { + throw new Error( + `Runtime binding '${scenario.id}' -> '${profile.id}' repeats obligation '${binding.obligationId}'`, + ); + } + declared.set(binding.obligationId, binding); + } + + const expected = new Map( + scenario.supportObligations.map((obligation) => [obligation.id, obligation]), + ); + const missing = [...expected.keys()].filter((id) => !declared.has(id)).sort(); + if (missing.length > 0) { + throw new Error( + `Runtime binding '${scenario.id}' -> '${profile.id}' is missing obligations: ${missing.join(", ")}`, + ); + } + const unknown = [...declared.keys()].filter((id) => !expected.has(id)).sort(); + if (unknown.length > 0) { + throw new Error( + `Runtime binding '${scenario.id}' -> '${profile.id}' has unknown obligations: ${unknown.join(", ")}`, + ); + } + + const capabilities = new Set(profile.capabilities); + for (const obligation of expected.values()) { + assertCompatibleObligation(scenario, profile, obligation, capabilities); + } + return Object.freeze( + scenario.supportObligations.map((obligation) => { + const binding = declared.get(obligation.id) as ObligationBinding; + return Object.freeze({ + ...binding, + adapter: adapterCatalog.get(binding.adapterId) as Readonly, + }); + }), + ); +} + +function compileAdapterCatalog( + registrations: readonly RuntimeAdapterRegistration[], +): ReadonlyMap> { + const catalog = new Map>(); + for (const registration of registrations) { + if (!ADAPTER_ID_PATTERN.test(registration.id)) { + throw new Error(`Runtime adapter id '${registration.id}' is invalid`); + } + const provider = executionProviderId(registration.provider); + assertExecutionFoundationId(registration.scenarioId, "Runtime adapter scenario id"); + assertExecutionFoundationId(registration.obligationId, "Runtime adapter obligation id"); + if (typeof registration.execute !== "function") { + throw new Error(`Runtime adapter '${registration.id}' has no executable implementation`); + } + if (catalog.has(registration.id)) { + throw new Error(`Duplicate runtime adapter id '${registration.id}'`); + } + catalog.set( + registration.id, + Object.freeze({ + ...registration, + provider, + }), + ); + } + return catalog; +} + +function assertCompatibleObligation( + scenario: RuntimeNeutralScenario, + profile: ExecutionProfile, + obligation: ScenarioSupportObligation, + capabilities: ReadonlySet, +): void { + const missing = obligation.requiredCapabilities.filter( + (capability) => !capabilities.has(capability), + ); + if (missing.length > 0) { + throw new Error( + `Runtime binding '${scenario.id}' -> '${profile.id}' cannot satisfy obligation '${obligation.id}'; missing capabilities: ${missing.join(", ")}`, + ); + } +} + +function digest(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function resourceIdentities(bindingId: string): RuntimeResourceIdentities { + const token = digest(`nemoclaw-runtime-binding-v1\0${bindingId}`).slice(0, 16); + const readable = bindingId + .replaceAll(".", "-") + .replaceAll("--", "-") + .slice(0, 28) + .replace(/-+$/u, ""); + return Object.freeze({ + sandbox: `e2e-${readable}-${token}`, + artifact: `runtime-artifact-${token}`, + cleanup: `runtime-cleanup-${token}`, + result: `runtime-result-${token}`, + }); +} + +export function executionPreparationKey(profile: ExecutionProfile): string { + const serialized = JSON.stringify({ + id: profile.id, + provider: profile.provider, + platform: profile.platform, + architecture: profile.architecture, + rootMode: profile.rootMode, + acceleration: profile.acceleration, + capabilities: profile.capabilities, + runner: { + hostId: profile.runner.hostId, + label: profile.runner.label, + maxShards: profile.runner.maxShards, + }, + }); + return `runtime-preparation-${digest(serialized).slice(0, 16)}`; +} + +function assertUniqueResourceIdentities(cases: readonly RuntimeMatrixCase[]): void { + const identities = cases.flatMap((entry) => Object.values(entry.identities)); + if (new Set(identities).size !== identities.length) { + throw new Error("Runtime matrix generated colliding resource identities"); + } +} + +function compileHostShards( + profiles: readonly ExecutionProfile[], + cases: readonly RuntimeMatrixCase[], +): readonly Readonly[] { + const shards: RuntimeHostShard[] = []; + const preparationsByHost = new Map(); + for (const profile of [...profiles].sort((left, right) => compareCodeUnits(left.id, right.id))) { + const profileCases = cases.filter((entry) => entry.profile.id === profile.id); + if (profileCases.length === 0) continue; + const preparations = preparationsByHost.get(profile.runner.hostId) ?? []; + preparations.push({ + preparationKey: executionPreparationKey(profile), + profile, + cases: Object.freeze(profileCases), + }); + preparationsByHost.set(profile.runner.hostId, preparations); + } + + for (const [hostId, preparations] of [...preparationsByHost.entries()].sort(([left], [right]) => + compareCodeUnits(left, right), + )) { + preparations.sort((left, right) => compareCodeUnits(left.preparationKey, right.preparationKey)); + const maxShards = preparations[0]?.profile.runner.maxShards ?? 0; + const count = Math.min(maxShards, preparations.length); + const lanes = Array.from({ length: count }, (): RuntimePreparationBatch[] => []); + preparations.forEach((preparation, index) => lanes[index % count]?.push(preparation)); + lanes.forEach((lane, index) => { + const shardIndex = index + 1; + const preparationKeys = lane.map((entry) => entry.preparationKey); + const laneIdentity = digest(JSON.stringify(preparationKeys)).slice(0, 16); + const runner = lane[0]?.profile.runner as ExecutionProfile["runner"]; + shards.push( + Object.freeze({ + id: `${hostId}-runtime-lane-${laneIdentity}-${shardIndex}-of-${count}`, + runner, + index: shardIndex, + count, + preparations: Object.freeze( + lane.map((entry) => + Object.freeze({ + ...entry, + }), + ), + ), + cases: Object.freeze(lane.flatMap((entry) => entry.cases)), + }), + ); + }); + } + if (new Set(shards.map((shard) => shard.id)).size !== shards.length) { + throw new Error("Runtime matrix generated colliding shard identities"); + } + return Object.freeze(shards); +} + +/** + * Compiles inert registry metadata. It selects no runner and executes no + * fixture; a future live consumer must opt in explicitly. + */ +export function compileRuntimeMatrix(definition: RuntimeMatrixDefinition): CompiledRuntimeMatrix { + if (definition.scenarios.length === 0) { + throw new Error("Runtime matrix must declare at least one scenario"); + } + if (definition.profiles.length === 0) { + throw new Error("Runtime matrix must declare at least one execution profile"); + } + + const scenarios = definition.scenarios.map(defineRuntimeScenario); + const profiles = definition.profiles.map(defineExecutionProfile); + const adapterCatalog = compileAdapterCatalog(definition.adapterCatalog); + const scenariosById = indexById(scenarios, "runtime scenario"); + const profilesById = indexById(profiles, "execution profile"); + assertConsistentHosts(profiles); + + const boundScenarios = new Set(); + const bindingIds = new Set(); + const cases = [...definition.bindings] + .sort( + (left, right) => + compareCodeUnits(left.scenarioId, right.scenarioId) || + compareCodeUnits(left.profileId, right.profileId), + ) + .map((binding): RuntimeMatrixCase => { + const scenario = scenariosById.get(binding.scenarioId); + if (!scenario) { + throw new Error(`Runtime binding references unknown scenario '${binding.scenarioId}'`); + } + const profile = profilesById.get(binding.profileId); + if (!profile) { + throw new Error(`Runtime binding references unknown profile '${binding.profileId}'`); + } + const id = `${scenario.id}--${profile.id}`; + if (bindingIds.has(id)) { + throw new Error(`Duplicate runtime binding '${id}'`); + } + bindingIds.add(id); + boundScenarios.add(scenario.id); + const missingScenarioCapabilities = scenario.requiredCapabilities.filter( + (capability) => !profile.capabilities.includes(capability), + ); + if (missingScenarioCapabilities.length > 0) { + throw new Error( + `Runtime binding '${scenario.id}' -> '${profile.id}' is incompatible; missing scenario capabilities: ${missingScenarioCapabilities.join(", ")}`, + ); + } + return Object.freeze({ + id, + preparationKey: executionPreparationKey(profile), + scenario, + profile, + obligationBindings: compileObligationBindings( + scenario, + profile, + binding.obligationBindings, + adapterCatalog, + ), + identities: resourceIdentities(id), + }); + }); + + const unboundScenarios = scenarios + .map((scenario) => scenario.id) + .filter((id) => !boundScenarios.has(id)); + if (unboundScenarios.length > 0) { + throw new Error( + `Runtime scenarios have no explicit binding: ${unboundScenarios.sort().join(", ")}`, + ); + } + assertUniqueResourceIdentities(cases); + const matrix = Object.freeze({ + cases: Object.freeze(cases), + shards: compileHostShards(profiles, cases), + }) as CompiledRuntimeMatrix; + compiledRuntimeMatrices.add(matrix); + return matrix; +} + +export function resolveRuntimeCase( + matrix: CompiledRuntimeMatrix, + reference: RuntimeCaseReference, +): ResolvedRuntimeCase { + if (!compiledRuntimeMatrices.has(matrix)) { + throw new Error("Runtime matrix was not issued by compileRuntimeMatrix"); + } + const id = `${reference.scenarioId}--${reference.profileId}`; + const runtimeCase = matrix.cases.find((entry) => entry.id === id); + if (!runtimeCase) { + throw new Error(`Runtime case '${id}' is not present in the compiled matrix`); + } + const shard = matrix.shards.find((entry) => + entry.cases.some((candidate) => candidate.id === runtimeCase.id), + ); + if (!shard) { + throw new Error(`Runtime case '${id}' has no compiled host shard`); + } + const resolved = Object.freeze({ case: runtimeCase, shard }) as ResolvedRuntimeCase; + resolvedRuntimeCases.add(resolved); + return resolved; +} + +export function assertResolvedRuntimeCase( + resolved: ResolvedRuntimeCase, +): asserts resolved is ResolvedRuntimeCase { + if (!resolvedRuntimeCases.has(resolved)) { + throw new Error("Runtime case resolution was not issued by resolveRuntimeCase"); + } +} diff --git a/test/e2e/registry/scenario.ts b/test/e2e/registry/scenario.ts new file mode 100644 index 00000000000..53f167642ad --- /dev/null +++ b/test/e2e/registry/scenario.ts @@ -0,0 +1,278 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { assertExecutionFoundationId, type ExecutionCapability } from "./execution-profile.ts"; + +export type RuntimeAgent = "openclaw" | "hermes" | "dcode"; + +export type JsonPrimitive = boolean | number | string | null; +export type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue }; + +export interface FsmTransition { + from: string; + event: string; + to: string; +} + +export interface TerminalOutcome { + status: "succeeded" | "failed"; + state: string; + failureClass?: string; +} + +export interface ScenarioJourneyStep { + id: string; + /** Stable semantic action. Never a provider command or shell fragment. */ + action: string; +} + +export interface ScenarioAssertionContract { + desiredState: JsonValue; + fsmTrace: readonly FsmTransition[]; + terminalOutcome: TerminalOutcome; + userVisibleState: JsonValue; +} + +export interface ScenarioSupportObligation { + id: string; + description: string; + requiredCapabilities: readonly ExecutionCapability[]; +} + +/** + * Product intent shared by runtime providers. Provider-specific setup and + * evidence adapters belong to RuntimeBindingSpec, never in the scenario. + */ +export interface RuntimeNeutralScenario { + id: string; + agent: RuntimeAgent; + description: string; + journey: readonly Readonly[]; + requiredCapabilities: readonly ExecutionCapability[]; + assertions: Readonly; + supportObligations: readonly Readonly[]; +} + +const AGENTS = new Set(["openclaw", "hermes", "dcode"]); + +/** Locale-independent ordering for canonical evidence and registry identities. */ +export function compareCodeUnits(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +export function normalizeJsonValue( + value: unknown, + fieldPath = "$", + ancestors = new Set(), +): JsonValue { + if (value === null || typeof value === "string" || typeof value === "boolean") { + return value; + } + if (typeof value === "number") { + if (!Number.isFinite(value)) { + throw new Error(`${fieldPath} must contain only finite JSON numbers`); + } + return Object.is(value, -0) ? 0 : value; + } + if (!value || typeof value !== "object") { + throw new Error(`${fieldPath} must be JSON-serializable`); + } + if (ancestors.has(value)) { + throw new Error(`${fieldPath} must not contain cyclic values`); + } + + ancestors.add(value); + try { + if (Array.isArray(value)) { + return value.map((entry, index) => + normalizeJsonValue(entry, `${fieldPath}[${index}]`, ancestors), + ); + } + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + throw new Error(`${fieldPath} must contain only plain JSON objects`); + } + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => compareCodeUnits(left, right)) + .map(([key, entry]) => [key, normalizeJsonValue(entry, `${fieldPath}.${key}`, ancestors)]), + ); + } finally { + ancestors.delete(value); + } +} + +export function freezeJsonValue(value: JsonValue): JsonValue { + if (Array.isArray(value)) { + return Object.freeze(value.map(freezeJsonValue)) as unknown as JsonValue; + } + if (value && typeof value === "object") { + return Object.freeze( + Object.fromEntries( + Object.entries(value).map(([key, entry]) => [key, freezeJsonValue(entry)]), + ), + ); + } + return value; +} + +function normalizeLabel(value: string, fieldPath: string): string { + const normalized = value.trim().replace(/\s+/gu, " "); + if (!normalized) { + throw new Error(`${fieldPath} must be a non-empty string`); + } + return normalized; +} + +export function normalizeFsmTrace( + transitions: readonly FsmTransition[], +): readonly Readonly[] { + if (transitions.length === 0) { + throw new Error("FSM trace must contain at least one transition"); + } + const normalized = transitions.map((transition, index) => + Object.freeze({ + from: normalizeLabel(transition.from, `fsmTrace[${index}].from`), + event: normalizeLabel(transition.event, `fsmTrace[${index}].event`), + to: normalizeLabel(transition.to, `fsmTrace[${index}].to`), + }), + ); + for (let index = 1; index < normalized.length; index += 1) { + if (normalized[index - 1]?.to !== normalized[index]?.from) { + throw new Error(`FSM trace is discontinuous between transitions ${index - 1} and ${index}`); + } + } + return Object.freeze(normalized); +} + +export function normalizeTerminalOutcome(outcome: TerminalOutcome): Readonly { + const state = normalizeLabel(outcome.state, "terminalOutcome.state"); + if (outcome.status === "succeeded") { + if (outcome.failureClass !== undefined) { + throw new Error("Successful terminal outcome must not declare failureClass"); + } + return Object.freeze({ status: outcome.status, state }); + } + if (outcome.status !== "failed") { + throw new Error(`Terminal outcome status '${String(outcome.status)}' is not recognized`); + } + if (outcome.failureClass === undefined) { + throw new Error("Failed terminal outcome must declare failureClass"); + } + return Object.freeze({ + status: outcome.status, + state, + failureClass: normalizeLabel(outcome.failureClass, "terminalOutcome.failureClass"), + }); +} + +export function assertTerminalMatchesTrace( + trace: readonly Readonly[], + outcome: Readonly, + fieldPath = "terminalOutcome", +): void { + const terminalState = trace.at(-1)?.to; + if (terminalState !== outcome.state) { + throw new Error( + `${fieldPath}.state '${outcome.state}' does not match FSM terminal state '${terminalState ?? ""}'`, + ); + } +} + +function normalizeDescription(value: string, fieldPath: string): string { + const description = value.trim(); + if (!description || /[\r\n]/u.test(description)) { + throw new Error(`${fieldPath} must be a single-line description`); + } + return description; +} + +function normalizeCapabilities( + values: readonly ExecutionCapability[], + fieldPath: string, +): readonly ExecutionCapability[] { + if (values.length === 0) { + throw new Error(`${fieldPath} must declare capabilities`); + } + if (new Set(values).size !== values.length) { + throw new Error(`${fieldPath} repeats a capability`); + } + return Object.freeze([...values].sort()); +} + +function assertUniqueIds( + values: readonly { id: string }[], + scenarioId: string, + label: string, +): void { + const ids = new Set(); + for (const value of values) { + assertExecutionFoundationId(value.id, `${label} id`); + if (ids.has(value.id)) { + throw new Error(`Runtime scenario '${scenarioId}' declares duplicate ${label} '${value.id}'`); + } + ids.add(value.id); + } +} + +export function defineRuntimeScenario(input: RuntimeNeutralScenario): RuntimeNeutralScenario { + assertExecutionFoundationId(input.id, "Runtime scenario id"); + if (!AGENTS.has(input.agent)) { + throw new Error(`Runtime scenario '${input.id}' agent '${input.agent}' is not recognized`); + } + const description = normalizeDescription(input.description, `Runtime scenario '${input.id}'`); + if (input.journey.length === 0) { + throw new Error(`Runtime scenario '${input.id}' must declare a user journey`); + } + if (!input.assertions) { + throw new Error(`Runtime scenario '${input.id}' must declare normalized assertions`); + } + if (input.supportObligations.length === 0) { + throw new Error(`Runtime scenario '${input.id}' must declare support obligations`); + } + + assertUniqueIds(input.journey, input.id, "journey step"); + const journey = input.journey.map((step) => { + assertExecutionFoundationId(step.action, "Journey action"); + return Object.freeze({ ...step }); + }); + + assertUniqueIds(input.supportObligations, input.id, "obligation"); + const supportObligations = input.supportObligations.map((obligation) => + Object.freeze({ + ...obligation, + description: normalizeDescription( + obligation.description, + `Runtime scenario '${input.id}' obligation '${obligation.id}'`, + ), + requiredCapabilities: normalizeCapabilities( + obligation.requiredCapabilities, + `Runtime scenario '${input.id}' obligation '${obligation.id}'`, + ), + }), + ); + const fsmTrace = normalizeFsmTrace(input.assertions.fsmTrace); + const terminalOutcome = normalizeTerminalOutcome(input.assertions.terminalOutcome); + assertTerminalMatchesTrace(fsmTrace, terminalOutcome, "assertions.terminalOutcome"); + + return Object.freeze({ + ...input, + description, + journey: Object.freeze(journey), + requiredCapabilities: normalizeCapabilities( + input.requiredCapabilities, + `Runtime scenario '${input.id}'`, + ), + assertions: Object.freeze({ + desiredState: freezeJsonValue( + normalizeJsonValue(input.assertions.desiredState, "assertions.desiredState"), + ), + fsmTrace, + terminalOutcome, + userVisibleState: freezeJsonValue( + normalizeJsonValue(input.assertions.userVisibleState, "assertions.userVisibleState"), + ), + }), + supportObligations: Object.freeze(supportObligations), + }); +} diff --git a/test/e2e/registry/types.ts b/test/e2e/registry/types.ts index 4761ec3f393..dd10bd3d626 100644 --- a/test/e2e/registry/types.ts +++ b/test/e2e/registry/types.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { RuntimeCaseReference } from "./runtime-matrix.ts"; + export type PhaseName = "environment" | "onboarding" | "state-validation" | "lifecycle" | "runtime"; // Synthetic phase appended by the target runner when a target @@ -196,6 +198,12 @@ export interface TargetDefinition { description?: string; manifestPath?: string; environment?: TargetEnvironment; + /** + * Optional reference into the registry-wide cross-runtime catalog. Canonical + * targets do not declare this until a live consumer and support policy land + * separately. + */ + runtimeCase?: RuntimeCaseReference; assertionGroups: AssertionGroup[]; expectedStateId?: string; suiteIds?: string[]; diff --git a/test/e2e/support/cross-runtime-foundation-fixtures.ts b/test/e2e/support/cross-runtime-foundation-fixtures.ts new file mode 100644 index 00000000000..756f026eb5d --- /dev/null +++ b/test/e2e/support/cross-runtime-foundation-fixtures.ts @@ -0,0 +1,175 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + defineExecutionProfile, + type ExecutionCapability, + type ExecutionProfile, + executionProviderId, +} from "../registry/execution-profile.ts"; +import type { + RuntimeAdapterRegistration, + RuntimeBindingSpec, + RuntimeMatrixDefinition, +} from "../registry/runtime-matrix.ts"; +import { + defineRuntimeScenario, + type RuntimeAgent, + type RuntimeNeutralScenario, +} from "../registry/scenario.ts"; + +const COMMON_CAPABILITIES = [ + "agent.configure", + "agent.turn", + "evidence.collect", + "sandbox.lifecycle", + "state.observe", +] as const satisfies readonly ExecutionCapability[]; + +export function foundationScenarios(): RuntimeNeutralScenario[] { + return (["openclaw", "hermes", "dcode"] as const).map((agent) => + defineRuntimeScenario({ + id: `${agent}-smoke`, + agent, + description: `Runtime-neutral ${agent} lifecycle and turn`, + journey: [ + { id: "provision", action: "sandbox.provision" }, + { id: "configure", action: "agent.configure" }, + { id: "turn", action: "agent.turn" }, + { id: "observe", action: "state.observe" }, + ], + requiredCapabilities: [...COMMON_CAPABILITIES], + assertions: { + desiredState: { + agent, + inference: { model: "fixture-model", provider: "fixture-inference" }, + }, + fsmTrace: [ + { from: "requested", event: "provision", to: "ready" }, + { from: "ready", event: "complete turn", to: "completed" }, + ], + terminalOutcome: { status: "succeeded", state: "completed" }, + userVisibleState: { status: "ready", response: "fixture response" }, + }, + supportObligations: [ + { + id: "provision", + description: "Create and own the isolated sandbox", + requiredCapabilities: ["sandbox.lifecycle"], + }, + { + id: "configure", + description: "Apply the requested agent configuration", + requiredCapabilities: ["agent.configure"], + }, + { + id: "turn", + description: "Complete one agent turn", + requiredCapabilities: ["agent.turn"], + }, + { + id: "observe", + description: "Observe final state and collect evidence", + requiredCapabilities: ["state.observe", "evidence.collect"], + }, + ], + }), + ); +} + +export function foundationProfiles(): ExecutionProfile[] { + return [ + defineExecutionProfile({ + id: "docker-linux-amd64", + provider: executionProviderId("docker"), + platform: "linux", + architecture: "amd64", + rootMode: "rootful", + acceleration: "cpu", + capabilities: [...COMMON_CAPABILITIES, "transport.docker-socket"], + runner: { + hostId: "fixture-host", + label: "fixture-linux", + maxShards: 2, + }, + }), + defineExecutionProfile({ + id: "test-mxc-linux-arm64", + provider: executionProviderId("test-mxc"), + platform: "linux", + architecture: "arm64", + rootMode: "rootless", + acceleration: "cpu", + capabilities: [...COMMON_CAPABILITIES, "transport.socket-free"], + runner: { + hostId: "fixture-host", + label: "fixture-linux", + maxShards: 2, + }, + }), + ]; +} + +export function obligationBindings( + scenario: RuntimeNeutralScenario, + profile: ExecutionProfile, +): RuntimeBindingSpec["obligationBindings"] { + return scenario.supportObligations.map((obligation) => ({ + obligationId: obligation.id, + adapterId: `${profile.provider}.${scenario.agent}.${obligation.id}`, + })); +} + +export function foundationBindings( + scenarios: readonly RuntimeNeutralScenario[], + profiles: readonly ExecutionProfile[], +): RuntimeBindingSpec[] { + return scenarios.flatMap((scenario) => + profiles.map((profile) => ({ + scenarioId: scenario.id, + profileId: profile.id, + obligationBindings: obligationBindings(scenario, profile), + })), + ); +} + +export function foundationAdapterCatalog( + scenarios: readonly RuntimeNeutralScenario[], + profiles: readonly ExecutionProfile[], +): RuntimeAdapterRegistration[] { + const adapters = new Map(); + for (const scenario of scenarios) { + for (const profile of profiles) { + for (const binding of obligationBindings(scenario, profile)) { + const adapterId = binding.adapterId; + adapters.set(binding.adapterId, { + id: adapterId, + provider: profile.provider, + scenarioId: scenario.id, + obligationId: binding.obligationId, + execute(runtime, request) { + return runtime.lifecycle.executeAdapter(adapterId, request); + }, + }); + } + } + } + return [...adapters.values()]; +} + +export function foundationDefinition(): RuntimeMatrixDefinition { + const scenarios = foundationScenarios(); + const profiles = foundationProfiles(); + return { + scenarios, + profiles, + adapterCatalog: foundationAdapterCatalog(scenarios, profiles), + bindings: foundationBindings(scenarios, profiles), + }; +} + +export function scenarioFor(agent: RuntimeAgent): RuntimeNeutralScenario { + const scenario = foundationScenarios().find((entry) => entry.agent === agent); + if (!scenario) throw new Error(`Missing fixture scenario for ${agent}`); + return scenario; +} diff --git a/test/e2e/support/e2e-cross-runtime-compatibility.test.ts b/test/e2e/support/e2e-cross-runtime-compatibility.test.ts new file mode 100644 index 00000000000..ced45f0d2e2 --- /dev/null +++ b/test/e2e/support/e2e-cross-runtime-compatibility.test.ts @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { describe, expect, it } from "vitest"; + +import { buildRiskPlan } from "../../../tools/advisors/risk-plan.mts"; +import { buildE2eWorkflowPlan } from "../../../tools/e2e/workflow-plan.mts"; +import { buildLiveTargetMatrix } from "../registry/run.ts"; + +function digestOutput(value: unknown): string { + return createHash("sha256") + .update(`${JSON.stringify(value)}\n`) + .digest("hex"); +} + +describe("cross-runtime foundation compatibility", () => { + it("preserves the exact canonical Docker live matrix output", () => { + expect(digestOutput(buildLiveTargetMatrix())).toBe( + "1e9b8aa3f3435e32398f8a6e13b8daf97e6fa89be263d9b9f99d13694c143f1b", + ); + expect( + digestOutput( + buildLiveTargetMatrix([ + "ubuntu-repo-cloud-langchain-deepagents-code", + "ubuntu-repo-docker-post-reboot-recovery", + ]), + ), + ).toBe("6272aab16cf4b9555bdc4b3f4c0cdd24b5faa55118cbd61cbb4b30a3d418a63a"); + expect(digestOutput(buildE2eWorkflowPlan())).toBe( + "9c391dfd06884da3898cc5590e7b8d4f43f1a2ad5ebb0c141da5f6383021aeb5", + ); + }); + + it("preserves exact risk-plan outputs for established policy cases", () => { + const headSha = "0123456789abcdef0123456789abcdef01234567"; + const cases = [ + { headSha, changedFiles: [] }, + { headSha, changedFiles: ["test/e2e/registry/run.ts"] }, + { + headSha, + changedFiles: ["src/lib/onboard.ts", "src/lib/inference/foo.ts"], + }, + ]; + + expect(digestOutput(cases.map(buildRiskPlan))).toBe( + "70fdce2f30eaf736c7cb54638ec617412580751710a058da545af841bef5ccbe", + ); + }); +}); diff --git a/test/e2e/support/e2e-parity-evidence.test.ts b/test/e2e/support/e2e-parity-evidence.test.ts new file mode 100644 index 00000000000..85a761c9f3c --- /dev/null +++ b/test/e2e/support/e2e-parity-evidence.test.ts @@ -0,0 +1,197 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { ArtifactSink } from "../fixtures/artifacts.ts"; +import { + assertParityEvidence, + buildExecutionEvidence, + compareParityEvidence, + type ExecutionEvidenceInput, + fingerprintDesiredState, +} from "../registry/parity-evidence.ts"; +import { + compileRuntimeMatrix, + type ResolvedRuntimeCase, + resolveRuntimeCase, +} from "../registry/runtime-matrix.ts"; +import { foundationDefinition } from "./cross-runtime-foundation-fixtures.ts"; + +const HEAD_SHA = "0123456789abcdef0123456789abcdef01234567"; +const BASE_SHA = "89abcdef0123456789abcdef0123456789abcdef"; +const IMAGE_DIGEST = `sha256:${"a".repeat(64)}`; + +function evidenceInput(provider: "docker" | "test-mxc"): ExecutionEvidenceInput { + const matrix = compileRuntimeMatrix(foundationDefinition()); + const runtimeCase = matrix.cases.find( + (entry) => entry.scenario.agent === "openclaw" && entry.profile.provider === provider, + ); + if (!runtimeCase) throw new Error(`Missing fixture runtime case for ${provider}`); + const resolved = resolveRuntimeCase(matrix, { + scenarioId: runtimeCase.scenario.id, + profileId: runtimeCase.profile.id, + }); + + return { + resolved, + source: { headSha: HEAD_SHA, baseSha: BASE_SHA }, + engine: { + name: provider === "docker" ? "docker-engine" : "fake-mxc-engine", + version: provider === "docker" ? "29.1.0" : "0.0.0-fixture", + }, + workload: { + logicalId: runtimeCase.identities.sandbox, + providerResourceId: `${provider}://fixture/${runtimeCase.identities.sandbox}`, + managedImages: [{ role: "agent", digest: IMAGE_DIGEST }], + }, + observed: { + desiredState: runtimeCase.scenario.assertions.desiredState, + fsmTrace: runtimeCase.scenario.assertions.fsmTrace, + terminalOutcome: runtimeCase.scenario.assertions.terminalOutcome, + userVisibleState: runtimeCase.scenario.assertions.userVisibleState, + }, + providerReceipts: [ + { + kind: "prepare", + operationId: `${provider}.prepare`, + value: { diagnostic: `${provider}-private` }, + }, + ], + }; +} + +describe("cross-runtime parity evidence", () => { + it("normalizes desired-state fingerprints and exact execution evidence", () => { + expect(fingerprintDesiredState({ b: 2, a: { y: 2, x: 1 } })).toBe( + fingerprintDesiredState({ a: { x: 1, y: 2 }, b: 2 }), + ); + + const evidence = buildExecutionEvidence(evidenceInput("docker")); + expect(evidence).toMatchObject({ + source: { headSha: HEAD_SHA, baseSha: BASE_SHA }, + runtime: { + provider: "docker", + architecture: "amd64", + capabilities: expect.arrayContaining(["transport.docker-socket"]), + }, + workload: { + logicalId: expect.stringMatching(/^e2e-/), + providerResourceId: expect.stringMatching(/^docker:/), + managedImages: [{ role: "agent", digest: IMAGE_DIGEST }], + }, + parity: { + terminalOutcome: { status: "succeeded", state: "completed" }, + }, + }); + }); + + it("ignores provider runtime identity and opaque receipts when comparing parity", () => { + const docker = buildExecutionEvidence(evidenceInput("docker")); + const mxc = buildExecutionEvidence(evidenceInput("test-mxc")); + + expect(compareParityEvidence(docker, mxc)).toEqual([]); + expect(() => assertParityEvidence(docker, mxc)).not.toThrow(); + }); + + it("rejects executions that do not satisfy the scenario before parity comparison", () => { + const mismatchInput = evidenceInput("test-mxc"); + mismatchInput.observed.userVisibleState = { + status: "degraded", + response: "fixture response", + }; + expect(() => buildExecutionEvidence(mismatchInput)).toThrow( + /observed userVisibleState does not satisfy its scenario assertion/, + ); + + const wrongTrace = evidenceInput("docker"); + wrongTrace.observed.fsmTrace = [ + { from: "requested", event: "skip provision", to: "ready" }, + { from: "ready", event: "complete turn", to: "completed" }, + ]; + expect(() => buildExecutionEvidence(wrongTrace)).toThrow( + /observed fsmTrace does not satisfy its scenario assertion/, + ); + }); + + it("rejects incomplete source, image, and provider receipt evidence", () => { + const badSource = evidenceInput("docker"); + badSource.source.headSha = "not-a-sha"; + expect(() => buildExecutionEvidence(badSource)).toThrow(/40-character commit SHA/); + + const badImage = evidenceInput("docker"); + badImage.workload.managedImages = [{ role: "agent", digest: "latest" }]; + expect(() => buildExecutionEvidence(badImage)).toThrow(/exact sha256 digest/); + + const missingReceipt = evidenceInput("docker"); + missingReceipt.providerReceipts = []; + expect(() => buildExecutionEvidence(missingReceipt)).toThrow(/provider receipt/); + + const wrongWorkload = evidenceInput("docker"); + wrongWorkload.workload.logicalId = "some-other-sandbox"; + expect(() => buildExecutionEvidence(wrongWorkload)).toThrow( + /does not match case sandbox identity/, + ); + + const wrongTerminal = evidenceInput("docker"); + wrongTerminal.observed.terminalOutcome = { + status: "succeeded", + state: "not-the-trace-terminal", + }; + expect(() => buildExecutionEvidence(wrongTerminal)).toThrow( + /does not match FSM terminal state/, + ); + + const wrongDesiredState = evidenceInput("docker"); + wrongDesiredState.observed.desiredState = { configured: false }; + expect(() => buildExecutionEvidence(wrongDesiredState)).toThrow( + /observed desiredState does not satisfy its scenario assertion/, + ); + + const forgedResolution = evidenceInput("docker"); + forgedResolution.resolved = { + case: forgedResolution.resolved.case, + shard: { + ...forgedResolution.resolved.shard, + id: "attacker-chosen-shard-id", + }, + } as unknown as ResolvedRuntimeCase; + expect(() => buildExecutionEvidence(forgedResolution)).toThrow( + /resolution was not issued by resolveRuntimeCase/, + ); + }); + + it("publishes provider receipts through the redacting artifact boundary", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-execution-evidence-")); + try { + const secret = "provider-private-secret"; + const input = evidenceInput("test-mxc"); + input.providerReceipts = [ + { + kind: "prepare", + operationId: "test-mxc.prepare", + value: { diagnostic: secret }, + }, + ]; + const sink = new ArtifactSink(root, [secret]); + const evidence = buildExecutionEvidence(input); + const file = await sink.writeExecutionEvidence( + input.resolved.case.identities.result, + evidence, + ); + + expect(file).toBe( + path.join(sink.rootDir, "execution", `${input.resolved.case.identities.result}.json`), + ); + const published = fs.readFileSync(file, "utf8"); + expect(published).toContain("[REDACTED]"); + expect(published).not.toContain(secret); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/test/e2e/support/e2e-runtime-foundation-types.test.ts b/test/e2e/support/e2e-runtime-foundation-types.test.ts new file mode 100644 index 00000000000..e535c8cc5f7 --- /dev/null +++ b/test/e2e/support/e2e-runtime-foundation-types.test.ts @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { test as e2eFixtureTest } from "../fixtures/e2e-test.ts"; +import { + EnvironmentPhaseFixture, + LifecyclePhaseFixture, + StateValidationPhaseFixture, +} from "../fixtures/phases/index.ts"; +import { defineExecutionProfile, executionProviderId } from "../registry/execution-profile.ts"; +import { defineRuntimeScenario } from "../registry/scenario.ts"; +import { foundationProfiles, foundationScenarios } from "./cross-runtime-foundation-fixtures.ts"; + +describe("cross-runtime foundation types", () => { + it("models Docker and socket-free fake MXC profiles without registering either", () => { + const [docker, mxc] = foundationProfiles(); + + expect(docker).toMatchObject({ + provider: "docker", + architecture: "amd64", + rootMode: "rootful", + acceleration: "cpu", + }); + expect(docker?.capabilities).toContain("transport.docker-socket"); + expect(mxc).toMatchObject({ + provider: "test-mxc", + architecture: "arm64", + rootMode: "rootless", + acceleration: "cpu", + }); + expect(mxc?.capabilities).toContain("transport.socket-free"); + expect(mxc?.capabilities).not.toContain("transport.docker-socket"); + }); + + it("keeps OpenClaw, Hermes, and DCode scenarios provider-neutral and obligation-explicit", () => { + const scenarios = foundationScenarios(); + + expect(scenarios.map((scenario) => scenario.agent)).toEqual(["openclaw", "hermes", "dcode"]); + for (const scenario of scenarios) { + expect(scenario).not.toHaveProperty("provider"); + expect(scenario.journey.map((step) => step.action)).toEqual([ + "sandbox.provision", + "agent.configure", + "agent.turn", + "state.observe", + ]); + expect(scenario.assertions.terminalOutcome).toEqual({ + status: "succeeded", + state: "completed", + }); + expect(scenario.supportObligations.map((obligation) => obligation.id)).toEqual([ + "provision", + "configure", + "turn", + "observe", + ]); + } + }); + + it("keeps provider ids open while rejecting invalid profiles", () => { + const valid = foundationProfiles()[0]!; + expect( + defineExecutionProfile({ + ...valid, + id: "future-provider-profile", + provider: executionProviderId("future-provider"), + }).provider, + ).toBe("future-provider"); + expect(() => executionProviderId("../unsafe")).toThrow(/provider id/); + expect(() => + defineExecutionProfile({ + ...valid, + runner: { ...valid.runner, maxShards: 0 }, + }), + ).toThrow(/maxShards must be between/); + expect(() => + defineExecutionProfile({ + ...valid, + capabilities: [...valid.capabilities, valid.capabilities[0]!], + }), + ).toThrow(/duplicate capabilities/); + }); + + it("rejects missing and duplicate support obligations", () => { + const valid = foundationScenarios()[0]!; + expect(() => + defineRuntimeScenario({ + ...valid, + supportObligations: [], + }), + ).toThrow(/must declare support obligations/); + expect(() => + defineRuntimeScenario({ + ...valid, + supportObligations: [valid.supportObligations[0]!, valid.supportObligations[0]!], + }), + ).toThrow(/duplicate obligation/); + }); + + it("rejects scenarios without a user journey or normalized assertions", () => { + const valid = foundationScenarios()[0]!; + expect(() => defineRuntimeScenario({ ...valid, journey: [] })).toThrow( + /must declare a user journey/, + ); + expect(() => + defineRuntimeScenario({ + ...valid, + assertions: undefined, + } as unknown as typeof valid), + ).toThrow(/must declare normalized assertions/); + }); +}); + +e2eFixtureTest( + "keeps cross-runtime injection inert beside the existing Docker phase fixtures", + async ({ environment, executionProfile, lifecycle, runtimeProvider, stateValidation }) => { + expect(executionProfile).toBeUndefined(); + expect(runtimeProvider).toBeUndefined(); + expect(environment).toBeInstanceOf(EnvironmentPhaseFixture); + expect(lifecycle).toBeInstanceOf(LifecyclePhaseFixture); + expect(stateValidation).toBeInstanceOf(StateValidationPhaseFixture); + }, +); diff --git a/test/e2e/support/e2e-runtime-matrix.test.ts b/test/e2e/support/e2e-runtime-matrix.test.ts new file mode 100644 index 00000000000..cf0b7ffcdfa --- /dev/null +++ b/test/e2e/support/e2e-runtime-matrix.test.ts @@ -0,0 +1,364 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + executeRuntimeCaseThroughProvider, + type RuntimeProviderFixture, +} from "../fixtures/runtime-provider.ts"; +import { buildLiveTargetRunPlan } from "../live/run-plan.ts"; +import { target } from "../registry/builder.ts"; +import { + defineExecutionProfile, + type ExecutionProfile, + executionProviderId, +} from "../registry/execution-profile.ts"; +import { + compileRuntimeMatrix, + executionPreparationKey, + type RuntimeMatrixDefinition, + resolveRuntimeCase, +} from "../registry/runtime-matrix.ts"; +import { + foundationDefinition, + foundationProfiles, + obligationBindings, +} from "./cross-runtime-foundation-fixtures.ts"; + +function fakeRuntimeProvider( + profile: ExecutionProfile, + adapterCalls: string[] = [], +): RuntimeProviderFixture { + const imageDigest = `sha256:${"b".repeat(64)}`; + return { + profile, + environment: { + async prepare() { + return { + profileId: profile.id, + ready: true, + engineName: `${profile.provider}-fixture`, + engineVersion: "0.0.0-fixture", + capabilities: profile.capabilities, + }; + }, + }, + lifecycle: { + async executeAdapter(adapterId, request) { + if ( + !adapterId.startsWith(`${profile.provider}.`) || + !adapterId.endsWith(`.${request.obligationId}`) + ) { + throw new Error(`Fixture provider cannot execute ${adapterId}`); + } + adapterCalls.push(adapterId); + }, + async cleanup() { + return [ + { + kind: "cleanup", + operationId: `${profile.provider}.cleanup`, + value: { fixture: true }, + }, + ]; + }, + }, + state: { + async inspectWorkload({ logicalId }) { + return { + logicalId, + providerResourceId: `${profile.provider}://fixture/${logicalId}`, + managedImages: [{ role: "agent", digest: imageDigest }], + }; + }, + async observe({ caseId }) { + const agent = caseId.split("-smoke--", 1)[0]!; + return { + desiredState: { + agent, + inference: { model: "fixture-model", provider: "fixture-inference" }, + }, + fsmTrace: [ + { from: "requested", event: "provision", to: "ready" }, + { from: "ready", event: "complete turn", to: "completed" }, + ], + terminalOutcome: { status: "succeeded", state: "completed" }, + userVisibleState: { status: "ready", response: "fixture response" }, + providerReceipts: [ + { + kind: "lifecycle", + operationId: `${profile.provider}.lifecycle`, + value: { fixture: true }, + }, + ], + }; + }, + }, + }; +} + +describe("cross-runtime E2E matrix compiler", () => { + it("binds OpenClaw, Hermes, and DCode explicitly on Docker and fake MXC", () => { + const { cases, shards } = compileRuntimeMatrix(foundationDefinition()); + + expect(cases).toHaveLength(6); + expect(cases.map((entry) => `${entry.scenario.agent}:${entry.profile.provider}`)).toEqual([ + "dcode:docker", + "dcode:test-mxc", + "hermes:docker", + "hermes:test-mxc", + "openclaw:docker", + "openclaw:test-mxc", + ]); + for (const entry of cases) { + expect(entry.obligationBindings).toHaveLength(entry.scenario.supportObligations.length); + } + expect(shards).toHaveLength(2); + expect(shards.some((shard) => shard.cases.length > 1)).toBe(true); + for (const shard of shards) { + expect(shard.preparations).toHaveLength(1); + expect(shard.cases).toEqual(shard.preparations.flatMap((entry) => entry.cases)); + } + }); + + it("fails incompatible profile bindings before producing a matrix", () => { + const definition = foundationDefinition(); + const scenario = definition.scenarios[0]!; + const baseProfile = definition.profiles[0]!; + const incompatible = defineExecutionProfile({ + ...baseProfile, + id: "docker-without-turn", + capabilities: baseProfile.capabilities.filter((capability) => capability !== "agent.turn"), + }); + + expect(() => + compileRuntimeMatrix({ + scenarios: [scenario], + profiles: [incompatible], + adapterCatalog: definition.adapterCatalog, + bindings: [ + { + scenarioId: scenario.id, + profileId: incompatible.id, + obligationBindings: obligationBindings(scenario, incompatible), + }, + ], + }), + ).toThrow(/incompatible.*agent\.turn/); + }); + + it("fails bindings that omit a scenario support obligation", () => { + const definition = foundationDefinition(); + const scenario = definition.scenarios[0]!; + const profile = definition.profiles[0]!; + + expect(() => + compileRuntimeMatrix({ + scenarios: [scenario], + profiles: [profile], + adapterCatalog: definition.adapterCatalog, + bindings: [ + { + scenarioId: scenario.id, + profileId: profile.id, + obligationBindings: obligationBindings(scenario, profile).slice(1), + }, + ], + }), + ).toThrow(/missing obligations: provision/); + }); + + it("assigns deterministic bounded host shards and isolated resource identities", () => { + const definition = foundationDefinition(); + const first = compileRuntimeMatrix(definition); + const reversed: RuntimeMatrixDefinition = { + scenarios: [...definition.scenarios].reverse(), + profiles: [...definition.profiles].reverse(), + adapterCatalog: [...definition.adapterCatalog].reverse(), + bindings: [...definition.bindings].reverse(), + }; + const second = compileRuntimeMatrix(reversed); + + expect(second).toEqual(first); + expect(first.shards).toHaveLength(2); + expect(new Set(first.shards.map((shard) => shard.runner.hostId))).toEqual( + new Set(["fixture-host"]), + ); + const preparationKeys = first.shards.flatMap((shard) => + shard.preparations.map((preparation) => preparation.preparationKey), + ); + expect(new Set(preparationKeys).size).toBe(2); + for (const shard of first.shards) { + expect(shard.index).toBeGreaterThanOrEqual(1); + expect(shard.index).toBeLessThanOrEqual(shard.count); + expect(shard.count).toBeLessThanOrEqual(shard.runner.maxShards); + expect(shard.id).toContain("runtime-lane"); + expect(shard.cases).toHaveLength(3); + for (const preparation of shard.preparations) { + expect(new Set(preparation.cases.map((entry) => entry.preparationKey))).toEqual( + new Set([preparation.preparationKey]), + ); + } + } + const allIdentities = first.cases.flatMap((entry) => Object.values(entry.identities)); + expect(new Set(allIdentities).size).toBe(first.cases.length * 4); + }); + + it("derives preparation identity from every profile and runner dimension", () => { + const profile = foundationProfiles()[0]!; + const variants = [ + { ...profile, id: "docker-linux-amd64-second" }, + { ...profile, provider: executionProviderId("future-provider") }, + { ...profile, platform: "macos" as const }, + { ...profile, architecture: "arm64" as const }, + { ...profile, rootMode: "rootless" as const }, + { ...profile, acceleration: "nvidia-gpu" as const }, + { ...profile, capabilities: [...profile.capabilities, "transport.socket-free" as const] }, + { ...profile, runner: { ...profile.runner, hostId: "fixture-host-second" } }, + { ...profile, runner: { ...profile.runner, label: "fixture-linux-second" } }, + { ...profile, runner: { ...profile.runner, maxShards: 1 } }, + ].map(defineExecutionProfile); + + expect(new Set([profile, ...variants].map(executionPreparationKey)).size).toBe( + variants.length + 1, + ); + }); + + it("rejects obligation bindings whose adapter is not registered", () => { + const definition = foundationDefinition(); + const binding = definition.bindings[0]!; + expect(() => + compileRuntimeMatrix({ + ...definition, + bindings: [ + { + ...binding, + obligationBindings: binding.obligationBindings.map((entry, index) => + index === 0 ? { ...entry, adapterId: "docker.missing.adapter" } : entry, + ), + }, + ...definition.bindings.slice(1), + ], + }), + ).toThrow(/unregistered adapter/); + }); + + it("schedules preparation-atomic work within the shared host maxShards ceiling", () => { + const definition = foundationDefinition(); + const scenario = definition.scenarios[0]!; + const docker = definition.profiles[0]!; + const mxc = definition.profiles[1]!; + const secondDockerPreparation = defineExecutionProfile({ + ...docker, + id: "docker-linux-amd64-rootless", + rootMode: "rootless", + }); + + const compiled = compileRuntimeMatrix({ + scenarios: [scenario], + profiles: [docker, mxc, secondDockerPreparation], + adapterCatalog: definition.adapterCatalog, + bindings: [docker, mxc, secondDockerPreparation].map((profile) => ({ + scenarioId: scenario.id, + profileId: profile.id, + obligationBindings: obligationBindings(scenario, profile), + })), + }); + + expect(compiled.shards).toHaveLength(2); + expect(compiled.shards.flatMap((shard) => shard.preparations)).toHaveLength(3); + expect(compiled.shards.some((shard) => shard.preparations.length === 2)).toBe(true); + expect( + new Set( + compiled.shards.flatMap((shard) => + shard.preparations.map((preparation) => preparation.preparationKey), + ), + ).size, + ).toBe(3); + }); + + it("keeps Docker and socket-free fake MXC commands behind one provider fixture seam", async () => { + const providers = foundationProfiles().map((profile) => fakeRuntimeProvider(profile)); + + for (const provider of providers) { + const ready = await provider.environment.prepare(); + const workload = await provider.state.inspectWorkload({ logicalId: "fixture-workload" }); + await provider.lifecycle.executeAdapter(`${provider.profile.provider}.openclaw.provision`, { + caseId: "fixture-case", + obligationId: "provision", + workloadId: workload.logicalId, + }); + const lifecycle = await provider.state.observe({ + caseId: "fixture-case", + workload, + }); + const cleanup = await provider.lifecycle.cleanup(workload); + + expect(ready.profileId).toBe(provider.profile.id); + expect(workload.providerResourceId).toMatch(new RegExp(`^${provider.profile.provider}:`)); + expect(lifecycle.terminalOutcome.status).toBe("succeeded"); + expect(cleanup).toHaveLength(1); + } + expect(providers[1]?.profile.capabilities).toContain("transport.socket-free"); + expect(providers[1]?.profile.capabilities).not.toContain("transport.docker-socket"); + }); + + it("executes compiled cases only through the provider-neutral seam", async () => { + const matrix = compileRuntimeMatrix(foundationDefinition()); + for (const runtimeCase of matrix.cases.filter((entry) => entry.scenario.agent === "openclaw")) { + const shard = matrix.shards.find((entry) => entry.cases.includes(runtimeCase)); + if (!shard) throw new Error(`Missing shard for ${runtimeCase.id}`); + const adapterCalls: string[] = []; + const resolved = resolveRuntimeCase(matrix, { + scenarioId: runtimeCase.scenario.id, + profileId: runtimeCase.profile.id, + }); + expect(resolved.shard).toBe(shard); + const evidence = await executeRuntimeCaseThroughProvider({ + resolved, + provider: fakeRuntimeProvider(runtimeCase.profile, adapterCalls), + source: { + headSha: "0123456789abcdef0123456789abcdef01234567", + baseSha: "89abcdef0123456789abcdef0123456789abcdef", + }, + }); + expect(evidence.caseId).toBe(runtimeCase.id); + expect(evidence.runtime.provider).toBe(runtimeCase.profile.provider); + expect(evidence.providerReceipts.map((receipt) => receipt.kind)).toEqual([ + "lifecycle", + "cleanup", + ]); + expect(adapterCalls).toEqual( + ["provision", "configure", "turn", "observe"].map( + (obligation) => + `${runtimeCase.profile.provider}.${runtimeCase.scenario.agent}.${obligation}`, + ), + ); + } + }); + + it("compiles optional runtime metadata through the existing target run-plan path", () => { + const runtimeMatrix = foundationDefinition(); + const compiled = compileRuntimeMatrix(runtimeMatrix); + const registered = target("synthetic-cross-runtime") + .manifest("test/e2e/manifests/openclaw-nvidia.yaml") + .environment({ + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + }) + .expectedState("cloud-openclaw-ready") + .runtimeCase({ + scenarioId: "openclaw-smoke", + profileId: "docker-linux-amd64", + }) + .build(); + + const plan = buildLiveTargetRunPlan(registered, compiled); + expect(plan.runtimeCase?.case.id).toBe("openclaw-smoke--docker-linux-amd64"); + expect(plan.runtimeCase?.shard.cases).toContain(plan.runtimeCase?.case); + expect(plan.phases).toEqual(["environment", "onboarding", "state-validation"]); + }); +}); From abf45b3796b2f3411bfb0aaba337c3eb4eeda0b2 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 00:36:44 -0700 Subject: [PATCH 021/117] test(e2e): keep runtime foundation branchless Signed-off-by: Aaron Erickson --- test/e2e/support/e2e-parity-evidence.test.ts | 3 ++- test/e2e/support/e2e-runtime-matrix.test.ts | 15 ++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/test/e2e/support/e2e-parity-evidence.test.ts b/test/e2e/support/e2e-parity-evidence.test.ts index 85a761c9f3c..123b17044be 100644 --- a/test/e2e/support/e2e-parity-evidence.test.ts +++ b/test/e2e/support/e2e-parity-evidence.test.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -31,7 +32,7 @@ function evidenceInput(provider: "docker" | "test-mxc"): ExecutionEvidenceInput const runtimeCase = matrix.cases.find( (entry) => entry.scenario.agent === "openclaw" && entry.profile.provider === provider, ); - if (!runtimeCase) throw new Error(`Missing fixture runtime case for ${provider}`); + assert.ok(runtimeCase, `Missing fixture runtime case for ${provider}`); const resolved = resolveRuntimeCase(matrix, { scenarioId: runtimeCase.scenario.id, profileId: runtimeCase.profile.id, diff --git a/test/e2e/support/e2e-runtime-matrix.test.ts b/test/e2e/support/e2e-runtime-matrix.test.ts index cf0b7ffcdfa..0892f7688d5 100644 --- a/test/e2e/support/e2e-runtime-matrix.test.ts +++ b/test/e2e/support/e2e-runtime-matrix.test.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import assert from "node:assert/strict"; + import { describe, expect, it } from "vitest"; import { @@ -46,12 +48,11 @@ function fakeRuntimeProvider( }, lifecycle: { async executeAdapter(adapterId, request) { - if ( - !adapterId.startsWith(`${profile.provider}.`) || - !adapterId.endsWith(`.${request.obligationId}`) - ) { - throw new Error(`Fixture provider cannot execute ${adapterId}`); - } + assert.ok( + adapterId.startsWith(`${profile.provider}.`) && + adapterId.endsWith(`.${request.obligationId}`), + `Fixture provider cannot execute ${adapterId}`, + ); adapterCalls.push(adapterId); }, async cleanup() { @@ -308,7 +309,7 @@ describe("cross-runtime E2E matrix compiler", () => { const matrix = compileRuntimeMatrix(foundationDefinition()); for (const runtimeCase of matrix.cases.filter((entry) => entry.scenario.agent === "openclaw")) { const shard = matrix.shards.find((entry) => entry.cases.includes(runtimeCase)); - if (!shard) throw new Error(`Missing shard for ${runtimeCase.id}`); + assert.ok(shard, `Missing shard for ${runtimeCase.id}`); const adapterCalls: string[] = []; const resolved = resolveRuntimeCase(matrix, { scenarioId: runtimeCase.scenario.id, From ed479aeb6e0ee0f628dd583c19a31dca9d994424 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 22:21:24 -0700 Subject: [PATCH 022/117] refactor(runtime): unify provider bundle registry Signed-off-by: Aaron Erickson (cherry picked from commit 7436f351a3628908d82d5b67f2377f8f38222ff8) --- ci/source-architecture-budget.json | 4 +- ci/source-shape-test-budget.json | 5 + .../inference-set-failure-handling.test.ts | 14 + src/lib/actions/inference-set-provider.ts | 17 + src/lib/actions/inference-set.ts | 12 +- src/lib/actions/sandbox/destroy-execution.ts | 39 +- src/lib/actions/sandbox/destroy-flow.test.ts | 13 + src/lib/actions/sandbox/destroy.ts | 68 ++- .../actions/sandbox/doctor-system-checks.ts | 15 +- src/lib/actions/sandbox/doctor.ts | 46 +- .../sandbox/runtime/lifecycle-runtime.ts | 84 +++ src/lib/actions/sandbox/start.test.ts | 47 +- src/lib/actions/sandbox/start.ts | 107 +--- src/lib/actions/sandbox/stop.test.ts | 49 +- src/lib/actions/sandbox/stop.ts | 203 ++----- src/lib/onboard/compute/plan.ts | 39 +- src/lib/onboard/runtime-provider/access.ts | 24 + src/lib/onboard/runtime-provider/contract.ts | 256 +++++++++ src/lib/onboard/runtime-provider/current.ts | 26 + src/lib/onboard/runtime-provider/docker.ts | 425 +++++++++++++++ src/lib/onboard/runtime-provider/registry.ts | 513 ++++++++++++++++++ .../runtime-provider-contract.test.ts | 495 +++++++++++++++++ src/lib/onboard/sandbox-registration.test.ts | 52 +- src/lib/onboard/sandbox-registration.ts | 27 + .../sandbox-workload-preparation.test.ts | 36 +- .../onboard/sandbox-workload-runtime.test.ts | 41 +- src/lib/onboard/workload/runtime.ts | 76 +-- src/lib/state/registry.ts | 3 + src/lib/state/registry/persistence.ts | 7 + src/lib/state/registry/types.ts | 40 ++ src/lib/state/registry/workload.ts | 116 ++++ test/helpers/destroy-flow-test-harness.ts | 2 + test/helpers/runtime-provider-bundle.ts | 171 ++++++ test/image-cleanup.test.ts | 59 +- test/runtime-provider-source-shape.test.ts | 43 ++ 35 files changed, 2764 insertions(+), 410 deletions(-) create mode 100644 src/lib/actions/sandbox/runtime/lifecycle-runtime.ts create mode 100644 src/lib/onboard/runtime-provider/access.ts create mode 100644 src/lib/onboard/runtime-provider/contract.ts create mode 100644 src/lib/onboard/runtime-provider/current.ts create mode 100644 src/lib/onboard/runtime-provider/docker.ts create mode 100644 src/lib/onboard/runtime-provider/registry.ts create mode 100644 src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts create mode 100644 src/lib/state/registry/workload.ts create mode 100644 test/helpers/runtime-provider-bundle.ts create mode 100644 test/runtime-provider-source-shape.test.ts diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 3168eb519ba..38b8f1a63d3 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -5,13 +5,13 @@ "maxByFile": { "src/lib/actions/sandbox/mcp-bridge-contracts.ts": 26, "src/lib/actions/sandbox/process-recovery.ts": 26, - "src/lib/adapters/docker/index.ts": 45, + "src/lib/adapters/docker/index.ts": 43, "src/lib/adapters/openshell/client.ts": 22, "src/lib/adapters/openshell/resolve.ts": 28, "src/lib/adapters/openshell/runtime.ts": 50, "src/lib/adapters/openshell/timeouts.ts": 36, "src/lib/agent/defs.ts": 32, - "src/lib/cli/branding.ts": 85, + "src/lib/cli/branding.ts": 84, "src/lib/cli/nemoclaw-oclif-command.ts": 103, "src/lib/cli/terminal-style.ts": 45, "src/lib/core/json-types.ts": 37, diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 3dd45f3afe5..8a22d0441c2 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -506,6 +506,11 @@ "test": "requires one advisor lane to publish the PR comment", "category": "security" }, + { + "file": "test/runtime-provider-source-shape.test.ts", + "test": "keeps provider identities and implementations behind the one bundle composition", + "category": "compatibility" + }, { "file": "test/pr-limit-policy.test.ts", "test": "keeps contributor guidance aligned with the enforced maintainer exemption", diff --git a/src/lib/actions/inference-set-failure-handling.test.ts b/src/lib/actions/inference-set-failure-handling.test.ts index 8813deec77a..b646dc71905 100644 --- a/src/lib/actions/inference-set-failure-handling.test.ts +++ b/src/lib/actions/inference-set-failure-handling.test.ts @@ -6,6 +6,20 @@ import { InferenceSetError, runInferenceSet } from "./inference-set"; import { createDeps } from "./inference-set.test-support"; describe("runInferenceSet failure handling", () => { + it("fails before OpenShell or config mutation for an unknown durable runtime provider", async () => { + const deps = createDeps({ + config: {}, + entry: { name: "alpha", agent: "openclaw", openshellDriver: "unknown-runtime" }, + }); + + await expect( + runInferenceSet({ provider: "nvidia-prod", model: "nvidia/model-a" }, deps), + ).rejects.toThrow(/unknown-runtime.*not registered/u); + expect(deps.calls.prepareRunOpenshell).not.toHaveBeenCalled(); + expect(deps.calls.captureOpenshell).not.toHaveBeenCalled(); + expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled(); + }); + it("resolves the OpenShell runner before entering the async mutation lock", async () => { const deps = createDeps({ config: { agents: { defaults: { model: { primary: "inference/nvidia/model-a" } } } }, diff --git a/src/lib/actions/inference-set-provider.ts b/src/lib/actions/inference-set-provider.ts index 022526999e4..0efdacd7058 100644 --- a/src/lib/actions/inference-set-provider.ts +++ b/src/lib/actions/inference-set-provider.ts @@ -6,6 +6,13 @@ import { matchesGatewayProviderBinding, parseGatewayProviderMetadata, } from "../onboard/gateway-provider-metadata"; +import { + CURRENT_RUNTIME_PROVIDER_BUNDLES, + type RuntimeProviderBundleRegistry, + requireRuntimeProviderBundleForSandbox, + requireRuntimeProviderMutationAuthority, +} from "../onboard/runtime-provider/access"; +import type { SandboxEntry } from "../state/registry"; import { InferenceSetError, OPEN_SHELL_FAILURE_CAPTURE_MAX_BUFFER, @@ -13,6 +20,16 @@ import { } from "./inference-set-error"; import type { InferenceSetProviderBinding } from "./inference-set-route-containment"; +export type { RuntimeProviderBundleRegistry }; + +export function requireInferenceSetRuntimeAuthority( + entry: SandboxEntry, + providers: RuntimeProviderBundleRegistry = CURRENT_RUNTIME_PROVIDER_BUNDLES, +): void { + const runtimeProvider = requireRuntimeProviderBundleForSandbox(entry, providers); + requireRuntimeProviderMutationAuthority(runtimeProvider, "inference-set"); +} + type CaptureProviderCommand = ( args: string[], options: { diff --git a/src/lib/actions/inference-set.ts b/src/lib/actions/inference-set.ts index 5c56e0906d5..080db2f82db 100644 --- a/src/lib/actions/inference-set.ts +++ b/src/lib/actions/inference-set.ts @@ -70,7 +70,11 @@ import { type InferenceMutation, readPreviousOpenClawInferenceApi, } from "./inference-set-gateway-restart"; -import { prepareInferenceSetProviderBinding } from "./inference-set-provider"; +import { + prepareInferenceSetProviderBinding, + requireInferenceSetRuntimeAuthority, + type RuntimeProviderBundleRegistry, +} from "./inference-set-provider"; import { buildInferenceSetFailure } from "./inference-set-provider-diagnostics"; import { applyOpenClawAnthropicReplyBudget, @@ -134,6 +138,7 @@ export interface InferenceSetDeps extends InferenceGatewayRestartDeps { target: AgentConfigTarget, config: ConfigObject, ) => void; + runtimeProviders?: RuntimeProviderBundleRegistry; recomputeSandboxConfigHash: (sandboxName: string, target: AgentConfigTarget) => void; seedHermesDashboardConfig: ( sandboxName: string, @@ -1336,6 +1341,11 @@ export async function runInferenceSet( // missing-binary path exits the process, which cannot be deferred safely by // an async lock. The inner resolution still validates the live registry entry. const selected = resolveTargetSandbox(options.sandboxName, deps); + try { + requireInferenceSetRuntimeAuthority(selected.entry, deps.runtimeProviders); + } catch (error) { + throw new InferenceSetError(error instanceof Error ? error.message : String(error), 2); + } deps.prepareRunOpenshell(); return withSandboxMutationLock(selected.sandboxName, async () => { const lockedSelection = resolveTargetSandbox(selected.sandboxName, deps); diff --git a/src/lib/actions/sandbox/destroy-execution.ts b/src/lib/actions/sandbox/destroy-execution.ts index 5bfb0d7cc96..036cfc5d9de 100644 --- a/src/lib/actions/sandbox/destroy-execution.ts +++ b/src/lib/actions/sandbox/destroy-execution.ts @@ -7,6 +7,13 @@ import { type DetachSandboxProvidersResult, runSandboxProviderPreDeleteCleanup, } from "../../onboard/sandbox-provider-cleanup"; +import { + CURRENT_RUNTIME_PROVIDER_BUNDLES, + RuntimeProviderBundle, + RuntimeProviderBundleRegistry, + requireRuntimeProviderBundleForSandbox, + requireRuntimeProviderMutationAuthority, +} from "../../onboard/runtime-provider/access"; import { redact } from "../../security/redact"; import { withTimerBoundShieldsMutationLockAsync } from "../../shields/timer-bound-lock"; import { readTimerMarker } from "../../shields/timer-control"; @@ -28,6 +35,7 @@ type SandboxDestroyExecutionInput = { sandbox: SandboxEntry | null; sandboxConfirmedAbsent: boolean; sandboxName: string; + runtimeProviders?: RuntimeProviderBundleRegistry; }; export type SandboxDestroyExecutionResult = @@ -179,8 +187,33 @@ export async function executeSandboxDestroy({ sandbox, sandboxConfirmedAbsent, sandboxName, + runtimeProviders = CURRENT_RUNTIME_PROVIDER_BUNDLES, }: SandboxDestroyExecutionInput): Promise { return withTimerBoundShieldsMutationLockAsync(sandboxName, "destroy sandbox", async () => { + let runtimeProvider: RuntimeProviderBundle | null = null; + if (sandbox) { + try { + runtimeProvider = requireRuntimeProviderBundleForSandbox(sandbox, runtimeProviders); + requireRuntimeProviderMutationAuthority(runtimeProvider, "provider-cleanup"); + requireRuntimeProviderMutationAuthority(runtimeProvider, "destroy"); + if (runtimeProvider.cleanup.supported !== true) { + throw new Error( + `Runtime provider '${runtimeProvider.identity.id}' has no cleanup implementation.`, + ); + } + } catch (error) { + return { + ok: false as const, + deleteOutput: + error instanceof Error + ? error.message + : `Runtime provider authority could not be proven: ${String(error)}`, + exitCode: 1, + gatewayUnreachable: false, + mcpOwnershipRequiresGateway: false, + }; + } + } const mcpPreparation = await prepareMcpDestroy( sandboxName, sandbox, @@ -192,9 +225,13 @@ export async function executeSandboxDestroy({ // provider ownership manifest and must survive an unconfirmed delete. const hasMcpOwnership = mcpPreparation.entries.length > 0; const hardened = wipeAndHardenLiveSandbox(sandboxName, sandboxConfirmedAbsent); + const detachProviders = (): DetachSandboxProvidersResult => + runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); const detachOutcome: DetachSandboxProvidersResult = sandboxConfirmedAbsent ? { detached: [], failures: [] } - : runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); + : runtimeProvider?.cleanup.supported === true && sandbox + ? runtimeProvider.cleanup.prepareDestroy({ sandbox, sandboxName }, { detachProviders }) + : detachProviders(); const deleteResult = runOpenshell(["sandbox", "delete", sandboxName], { ignoreError: true, stdio: ["ignore", "pipe", "pipe"], diff --git a/src/lib/actions/sandbox/destroy-flow.test.ts b/src/lib/actions/sandbox/destroy-flow.test.ts index 0f34afa94d5..6e52882c97d 100644 --- a/src/lib/actions/sandbox/destroy-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-flow.test.ts @@ -104,6 +104,19 @@ describe("destroySandbox flow", () => { expectFailedDeletePreservesHostState(harness, exitSpy); }); + it("preserves provider and registry ownership when runtime authority is unknown", async () => { + const harness = createDestroyHarness({ openshellDriver: "unknown-runtime" }); + + await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow("process.exit(1)"); + + expect( + harness.runOpenshellSpy.mock.calls.some( + ([args]) => Array.isArray(args) && args[0] === "sandbox" && args[1] === "delete", + ), + ).toBe(false); + expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); + }); + it("refuses shields-up Hermes MCP destroy before stopping services or preparing MCP state", async () => { const harness = createDestroyHarness({ agent: "hermes", diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 78a9483f8cc..5ae88a0896f 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -23,6 +23,13 @@ import { emitProviderDetachResidualHint, SANDBOX_PROVIDER_SUFFIXES, } from "../../onboard/sandbox-provider-cleanup"; +import { + CURRENT_RUNTIME_PROVIDER_BUNDLES, + RuntimeProviderBundleRegistry, + RuntimeProviderWorkloadCleanupResult, + requireRuntimeProviderBundleForSandbox, + requireRuntimeProviderMutationAuthority, +} from "../../onboard/runtime-provider/access"; import { validateName } from "../../runner"; import { killTimer as defaultKillShieldsTimer } from "../../shields/timer-control"; import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock"; @@ -39,20 +46,20 @@ import { type WipeSandboxStateDeps, wipeSandboxState } from "./wipe-state"; export { classifyDestroySandboxPresence } from "./destroy-presence"; -type DockerRmi = (tag: string, opts?: { ignoreError?: boolean }) => { status: number | null }; - type RemoveSandboxImageDeps = { getSandbox?: typeof registry.getSandbox; - dockerRmi?: DockerRmi; + runtimeProviders?: RuntimeProviderBundleRegistry; + log?: (message: string) => void; + warn?: (message: string) => void; }; type RemoveSandboxRegistryEntryDeps = { - removeImage?: (sandboxName: string) => void; + removeImage?: (sandboxName: string) => RuntimeProviderWorkloadCleanupResult | void; removeSandbox?: typeof registry.removeSandbox; }; type RemoveSandboxRegistryEntryWithReceiptDeps = { - removeImage?: (sandboxName: string) => void; + removeImage?: (sandboxName: string) => RuntimeProviderWorkloadCleanupResult | void; removeSandboxWithReceipt?: typeof registry.removeSandboxWithReceipt; }; @@ -216,23 +223,42 @@ export function removeShieldsState( } /** - * Remove the host-side Docker image that was built for a sandbox during onboard. + * Remove only a provider-owned per-sandbox workload image. Shared managed + * cohorts and ambiguous ownership are never deleted. * Must be called before registry.removeSandbox() since the imageTag is stored there. */ -export function removeSandboxImage(sandboxName: string, deps: RemoveSandboxImageDeps = {}): void { +export function removeSandboxImage( + sandboxName: string, + deps: RemoveSandboxImageDeps = {}, +): RuntimeProviderWorkloadCleanupResult { const getSandbox = deps.getSandbox ?? registry.getSandbox; - const removeImage = - deps.dockerRmi ?? (require("../../adapters/docker") as { dockerRmi: DockerRmi }).dockerRmi; const sb = getSandbox(sandboxName); - if (!sb?.imageTag) return; - const result = removeImage(sb.imageTag, { ignoreError: true }); - if (result.status === 0) { - console.log(` Removed Docker image ${sb.imageTag}`); - } else { - console.warn( - ` ${YW}⚠${R} Failed to remove Docker image ${sb.imageTag}; run '${CLI_NAME} gc' to clean up.`, + if (!sb) return { status: "skipped", reason: "no-owned-image" }; + let result: RuntimeProviderWorkloadCleanupResult; + try { + const provider = requireRuntimeProviderBundleForSandbox( + sb, + deps.runtimeProviders ?? CURRENT_RUNTIME_PROVIDER_BUNDLES, ); + requireRuntimeProviderMutationAuthority(provider, "workload-cleanup"); + if (provider.cleanup.supported !== true) { + return { status: "skipped", reason: "authority-unproven" }; + } + result = provider.cleanup.removeOwnedWorkload({ sandbox: sb, sandboxName }); + } catch { + return { status: "skipped", reason: "authority-unproven" }; } + const log = deps.log ?? console.log; + const warn = deps.warn ?? console.warn; + if (result.status === "removed") { + log(` Removed ${result.engineDisplayName} image ${result.reference}`); + } else if (result.status === "failed") { + warn( + ` ${YW}⚠${R} Failed to remove ${result.engineDisplayName} image ${result.reference}; ` + + `run '${CLI_NAME} gc' to clean up.`, + ); + } + return result; } export function removeSandboxRegistryEntry( @@ -241,7 +267,10 @@ export function removeSandboxRegistryEntry( ): boolean { const removeImage = deps.removeImage ?? removeSandboxImage; const removeSandbox = deps.removeSandbox ?? registry.removeSandbox; - removeImage(sandboxName); + const imageResult = removeImage(sandboxName); + if (imageResult?.status === "skipped" && imageResult.reason === "authority-unproven") { + return false; + } return removeSandbox(sandboxName); } @@ -252,7 +281,10 @@ export function removeSandboxRegistryEntryWithReceipt( const removeImage = deps.removeImage ?? removeSandboxImage; const removeSandboxWithReceipt = deps.removeSandboxWithReceipt ?? registry.removeSandboxWithReceipt; - removeImage(sandboxName); + const imageResult = removeImage(sandboxName); + if (imageResult?.status === "skipped" && imageResult.reason === "authority-unproven") { + return null; + } return removeSandboxWithReceipt(sandboxName); } diff --git a/src/lib/actions/sandbox/doctor-system-checks.ts b/src/lib/actions/sandbox/doctor-system-checks.ts index 1d3fb564e1a..14843f2c385 100644 --- a/src/lib/actions/sandbox/doctor-system-checks.ts +++ b/src/lib/actions/sandbox/doctor-system-checks.ts @@ -6,7 +6,11 @@ import { buildValidatedCurlCommandArgs } from "../../adapters/http/curl-args"; import { stripAnsi } from "../../adapters/openshell/client"; import { CLI_NAME } from "../../cli/branding"; import { GATEWAY_PORT, OLLAMA_PORT } from "../../core/ports"; -import { isLinuxDockerDriverGatewayEnabled } from "../../onboard/docker-driver-platform"; +import { + CURRENT_RUNTIME_PROVIDER_BUNDLES, + resolveCurrentRuntimeProviderBundle, + resolveRuntimeProviderBundle, +} from "../../onboard/runtime-provider/access"; import type { SandboxEntry } from "../../state/registry"; import { readCloudflaredState } from "../../tunnel/services"; import { @@ -193,8 +197,9 @@ export function ollamaDoctorCheck(currentProvider: string): DoctorCheck { * Prefer the recorded driver and use platform detection for older entries. */ export function shouldInspectLegacyGatewayContainer(sb: SandboxEntry | null | undefined): boolean { - const driver = sb?.openshellDriver; - if (driver === "docker" || driver === "vm") return false; - if (driver === "kubernetes") return true; - return !isLinuxDockerDriverGatewayEnabled(); + const recorded = sb?.openshellDriver?.trim(); + const provider = recorded + ? resolveRuntimeProviderBundle(recorded, CURRENT_RUNTIME_PROVIDER_BUNDLES) + : resolveCurrentRuntimeProviderBundle(); + return provider?.gateway.inspectLegacyContainer === true; } diff --git a/src/lib/actions/sandbox/doctor.ts b/src/lib/actions/sandbox/doctor.ts index 8c07a63f1bb..108448d93ae 100644 --- a/src/lib/actions/sandbox/doctor.ts +++ b/src/lib/actions/sandbox/doctor.ts @@ -18,6 +18,12 @@ import { import { parseGatewayInference } from "../../inference/config"; import { shouldManageDashboardForAgent } from "../../onboard/dashboard-runtime"; import { resolveGatewayName, resolveSandboxGatewayName } from "../../onboard/gateway-binding"; +import { + CURRENT_RUNTIME_PROVIDER_BUNDLES, + requireRuntimeProviderBundle, + resolveCurrentRuntimeProviderBundle, + RuntimeProviderSelectionError, +} from "../../onboard/runtime-provider/access"; import { executeSandboxCommandForVerification } from "../../onboard/sandbox-verification-exec"; import { getBaselineExclusionRuntimeStatus } from "../../policy"; import { @@ -32,7 +38,6 @@ import type { SandboxEntry } from "../../state/registry"; import * as registry from "../../state/registry"; import { runSandboxAutoPairApprovalPass } from "./auto-pair-approval"; import { buildConfigPermsCheck } from "./doctor-config-perms"; -import { captureHostCommand } from "./doctor-host-command"; import { collectInferenceChecks, type DoctorInferenceRoute, @@ -125,29 +130,36 @@ function cliBuildCheck(): DoctorCheck { }; } -function collectHostChecks(): { +function collectHostChecks(sb: SandboxEntry | null | undefined): { checks: DoctorCheck[]; openshellBin: ReturnType; } { const cli = cliBuildCheck(); - const dockerInfo = captureHostCommand("docker", ["info", "--format", "{{.ServerVersion}}"], 8000); const openshellBin = resolveOpenshell(); + let runtimeCheck: DoctorCheck; + try { + const recorded = sb?.openshellDriver?.trim(); + const provider = recorded + ? requireRuntimeProviderBundle(recorded, CURRENT_RUNTIME_PROVIDER_BUNDLES) + : resolveCurrentRuntimeProviderBundle(); + runtimeCheck = provider.preflightDoctor.inspectHost(); + } catch (error) { + const detail = + error instanceof RuntimeProviderSelectionError + ? error.message + : `Runtime provider inspection failed: ${error instanceof Error ? error.message : String(error)}`; + runtimeCheck = { + group: "Host", + label: "Runtime provider", + status: "fail", + detail, + hint: "restore a supported durable runtime provider identity before retrying", + }; + } return { checks: [ cli, - { - group: "Host", - label: "Docker daemon", - status: dockerInfo.status === 0 ? "ok" : "fail", - detail: - dockerInfo.status === 0 - ? `server ${dockerInfo.stdout.trim() || "unknown"}` - : oneLine(dockerInfo.stderr || dockerInfo.error?.message || "docker info failed"), - hint: - dockerInfo.status === 0 - ? undefined - : "start Docker and verify your user can access the daemon", - }, + runtimeCheck, { group: "Host", label: "OpenShell CLI", @@ -506,7 +518,7 @@ async function collectDoctorChecks( gatewayName: string | null, intent: DoctorIntent, ): Promise { - const host = collectHostChecks(); + const host = collectHostChecks(sb); const gateway: GatewayProbe = gatewayName ? await collectGatewayChecks(gatewayName, sb, host.openshellBin, !intent.asJson) : { diff --git a/src/lib/actions/sandbox/runtime/lifecycle-runtime.ts b/src/lib/actions/sandbox/runtime/lifecycle-runtime.ts new file mode 100644 index 00000000000..3c43780b617 --- /dev/null +++ b/src/lib/actions/sandbox/runtime/lifecycle-runtime.ts @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + RuntimeProviderBundle, + RuntimeProviderBundleRegistry, + normalizeRuntimeProviderIdentity, + requireRuntimeProviderMutationAuthority, + resolveRuntimeProviderBundle, + RuntimeProviderSelectionError, +} from "../../../onboard/runtime-provider/access"; +import type { + RuntimeProviderLifecycleAction, + RuntimeProviderLifecycleResult, +} from "../../../onboard/runtime-provider/contract"; +import { cliName } from "../../../onboard/branding"; +import type { SandboxEntry } from "../../../state/registry/types"; + +export type { RuntimeProviderLifecycleResult as SandboxLifecycleResult }; + +export type SandboxLifecycleProviderResolution = + | { + readonly ok: true; + readonly bundle: RuntimeProviderBundle; + readonly lifecycle: Extract; + } + | { + readonly ok: false; + readonly result: RuntimeProviderLifecycleResult; + }; + +/** + * Resolve the exact provider recorded on the sandbox. The action layer never + * infers lifecycle behavior from a gateway launcher or container-engine name. + */ +export function resolveSandboxLifecycleProvider( + sandboxName: string, + sandbox: SandboxEntry | null, + action: RuntimeProviderLifecycleAction, + providers: RuntimeProviderBundleRegistry, +): SandboxLifecycleProviderResolution { + if (!sandbox) { + return { + ok: false, + result: { + exitCode: 1, + message: + ` Sandbox '${sandboxName}' is not registered. ` + + `Run '${cliName()} list' to see registered sandboxes.`, + }, + }; + } + const providerId = normalizeRuntimeProviderIdentity(sandbox.openshellDriver); + const bundle = resolveRuntimeProviderBundle(providerId, providers); + if (!bundle) { + return { + ok: false, + result: { + exitCode: 1, + message: + ` '${cliName()} ${sandboxName} ${action}' has no registered lifecycle ` + + `provider for '${providerId}'.`, + }, + }; + } + try { + requireRuntimeProviderMutationAuthority(bundle, action); + } catch (error) { + if (!(error instanceof RuntimeProviderSelectionError)) throw error; + return { ok: false, result: { exitCode: 1, message: ` ${error.message}` } }; + } + if (bundle.lifecycle.supported !== true) { + return { + ok: false, + result: { + exitCode: 1, + message: + ` '${cliName()} ${sandboxName} ${action}' is unavailable for runtime provider ` + + `'${providerId}': ${bundle.lifecycle.reason}`, + }, + }; + } + return { ok: true, bundle, lifecycle: bundle.lifecycle }; +} diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index ed1f97f1397..f3eb3a9bc3c 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -3,6 +3,12 @@ import { describe, expect, it, vi } from "vitest"; +import { + createDockerRuntimeProviderBundle, + createKubernetesRuntimeProviderBundle, + type DockerRuntimeProviderDependencies, +} from "../../onboard/runtime-provider/docker"; +import { createRuntimeProviderBundleRegistry } from "../../onboard/runtime-provider/registry"; import type { SandboxEntry } from "../../state/registry"; import { type SandboxStartDeps, startSandbox } from "./start"; @@ -12,35 +18,44 @@ function sandbox(values: Partial = {}): SandboxEntry { function harness(overrides: Partial = {}) { const getSandbox = vi.fn>(() => sandbox()); - const isDockerRuntimeDown = vi.fn>( + const isDockerRuntimeDown = vi.fn( () => false, ); const printDockerRuntimeDownGuidance = - vi.fn>(); + vi.fn(); const findLabeledSandboxContainers = vi.fn< - NonNullable + DockerRuntimeProviderDependencies["findLabeledSandboxContainers"] >(() => [{ name: "openshell-my-sandbox", status: "Exited (0) 2 hours ago", running: false }]); - const recoverDockerDriverSandbox = vi.fn< - NonNullable - >(() => ({ - recovered: true, - via: "started-stopped-original", - containerName: "openshell-my-sandbox", - })); - const dockerUnpause = vi.fn>(() => ({ + const recoverDockerDriverSandbox = vi.fn( + () => ({ + recovered: true, + via: "started-stopped-original", + containerName: "openshell-my-sandbox", + }), + ); + const dockerUnpause = vi.fn(() => ({ status: 0, })); const probeSandbox = vi.fn>(() => Promise.resolve(), ); const log = vi.fn<(message: string) => void>(); + const runtimeProviders = createRuntimeProviderBundleRegistry([ + [ + "docker", + createDockerRuntimeProviderBundle({ + findLabeledSandboxContainers, + isRuntimeDown: isDockerRuntimeDown, + printRuntimeDownGuidance: printDockerRuntimeDownGuidance, + recoverSandbox: recoverDockerDriverSandbox, + unpauseContainer: dockerUnpause, + }), + ], + ["kubernetes", createKubernetesRuntimeProviderBundle()], + ]); const deps: SandboxStartDeps = { getSandbox, - isDockerRuntimeDown, - printDockerRuntimeDownGuidance, - findLabeledSandboxContainers, - recoverDockerDriverSandbox, - dockerUnpause, + runtimeProviders, probeSandbox, log, ...overrides, diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index 0f6b05884b3..972c0d64502 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -1,22 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { CLI_NAME } from "../../cli/branding"; import { - findLabeledSandboxContainers, - recoverDockerDriverSandbox, -} from "../../onboard/docker-driver-sandbox-recovery"; + CURRENT_RUNTIME_PROVIDER_BUNDLES, + type RuntimeProviderBundleRegistry, +} from "../../onboard/runtime-provider/access"; import * as registry from "../../state/registry"; import { - gateDirectDriverLifecycle, - gateDockerRuntimeUp, + resolveSandboxLifecycleProvider, type SandboxLifecycleResult, - type SandboxStopDeps, -} from "./stop"; +} from "./runtime/lifecycle-runtime"; -// Lazy requires keep the heavy connect module (and the docker adapter's -// transitive imports) out of this module's load path; tests inject -// `deps.probeSandbox` / `deps.dockerUnpause`. +// Lazy require keeps the heavy connect module out of this module's load path; +// tests inject `deps.probeSandbox`. function loadConnectProbe(): (sandboxName: string) => Promise { const { connectSandbox } = require("./connect") as { connectSandbox: (sandboxName: string, options?: { probeOnly?: boolean }) => Promise; @@ -24,94 +20,43 @@ function loadConnectProbe(): (sandboxName: string) => Promise { return (sandboxName) => connectSandbox(sandboxName, { probeOnly: true }); } -type DockerOpResult = { status?: number | null }; -type DockerUnpauseFn = (name: string, opts?: Record) => DockerOpResult; - -function loadDockerUnpause(): DockerUnpauseFn { - return (require("../../adapters/docker") as { dockerUnpause: DockerUnpauseFn }).dockerUnpause; -} - -const DOCKER_UNPAUSE_TIMEOUT_MS = 30_000; - -// Paused containers report `Up N minutes (Paused)` from `docker ps`, so the -// recovery classifier counts them as running and would no-op — while -// `docker start` on them fails outright. `docker unpause` is the only verb -// that resumes them (#6026). -function isPausedStatus(status: string): boolean { - return status.startsWith("Up") && status.endsWith("(Paused)"); -} - -export interface SandboxStartDeps - extends Pick { +export interface SandboxStartDeps { + environment?: NodeJS.ProcessEnv; getSandbox?: typeof registry.getSandbox; - findLabeledSandboxContainers?: typeof findLabeledSandboxContainers; - recoverDockerDriverSandbox?: typeof recoverDockerDriverSandbox; - dockerUnpause?: DockerUnpauseFn; + runtimeProviders?: RuntimeProviderBundleRegistry; /** Gateway/forward health probe; defaults to the `recover` action body. */ probeSandbox?: (sandboxName: string) => Promise; log?: (message: string) => void; } /** - * Restart a stopped sandbox container and bring its gateway and host - * forwards back up (#6026). Counterpart to `stopSandbox`. - * - * Container restart reuses the #4423 recovery module (handles the stopped - * original and the gpu-backup-sibling rename) plus a paused-container - * unpause branch; the health probe reuses the `recover` action body so - * forwards and the in-sandbox gateway come back exactly as they would after - * `nemoclaw recover`. + * Restart a stopped sandbox through the lifecycle facet bound to its durable + * provider identity, then restore gateway health and host forwards. */ export async function startSandbox( sandboxName: string, deps: SandboxStartDeps = {}, ): Promise { const log = deps.log ?? console.log; - - const gate = gateDirectDriverLifecycle( + const sandbox = (deps.getSandbox ?? registry.getSandbox)(sandboxName); + const resolved = resolveSandboxLifecycleProvider( sandboxName, + sandbox, "start", - deps.getSandbox ?? registry.getSandbox, + deps.runtimeProviders ?? CURRENT_RUNTIME_PROVIDER_BUNDLES, ); - if (gate) return gate; - - const runtimeGate = gateDockerRuntimeUp(sandboxName, "start", deps); - if (runtimeGate) return runtimeGate; + if (!resolved.ok) return resolved.result; - const containers = (deps.findLabeledSandboxContainers ?? findLabeledSandboxContainers)( + const input = { + environment: deps.environment ?? process.env, + log, + sandbox: sandbox!, sandboxName, - ); - const paused = containers.find((container) => isPausedStatus(container.status)); - if (paused) { - const dockerUnpause = deps.dockerUnpause ?? ((name, opts) => loadDockerUnpause()(name, opts)); - const result = dockerUnpause(paused.name, { - ignoreError: true, - timeout: DOCKER_UNPAUSE_TIMEOUT_MS, - }); - if (result.status !== 0) { - return { - exitCode: 1, - message: ` docker unpause ${paused.name} failed (exit ${result.status ?? "unknown"}).`, - }; - } - log(` Container '${paused.name}' unpaused.`); - } else { - const recovery = (deps.recoverDockerDriverSandbox ?? recoverDockerDriverSandbox)(sandboxName); - if (!recovery.recovered) { - return { - exitCode: 1, - message: - ` Could not start sandbox '${sandboxName}': ${recovery.detail ?? "unknown failure"}. ` + - `If the container was removed, run '${CLI_NAME} ${sandboxName} rebuild' to recreate it.`, - }; - } - - if (recovery.via === "started-running-original") { - log(` Sandbox '${sandboxName}' is already running.`); - } else { - log(` Container '${recovery.containerName ?? sandboxName}' started.`); - } - } + }; + const preflight = resolved.bundle.preflightDoctor.preflightLifecycle("start", input); + if (preflight) return preflight; + const result = resolved.lifecycle.start(input); + if (result.exitCode !== 0) return result; log(" Checking gateway health and host forwards…"); await (deps.probeSandbox ?? loadConnectProbe())(sandboxName); diff --git a/src/lib/actions/sandbox/stop.test.ts b/src/lib/actions/sandbox/stop.test.ts index 77ec0b8519e..d3b750c8b26 100644 --- a/src/lib/actions/sandbox/stop.test.ts +++ b/src/lib/actions/sandbox/stop.test.ts @@ -3,6 +3,12 @@ import { describe, expect, it, vi } from "vitest"; +import { + createDockerRuntimeProviderBundle, + createKubernetesRuntimeProviderBundle, + type DockerRuntimeProviderDependencies, +} from "../../onboard/runtime-provider/docker"; +import { createRuntimeProviderBundleRegistry } from "../../onboard/runtime-provider/registry"; import type { SandboxEntry } from "../../state/registry"; import { teardownSandboxDashboardForward } from "./forward-recovery"; import { type SandboxStopDeps, stopSandbox } from "./stop"; @@ -15,33 +21,54 @@ function container(name: string, running: boolean) { return { name, status: running ? "Up 5 minutes" : "Exited (0) 2 hours ago", running }; } -function harness(overrides: Partial = {}) { +type StopHarnessOverrides = Partial & { + dockerStop?: DockerRuntimeProviderDependencies["stopContainer"]; + findLabeledSandboxContainers?: DockerRuntimeProviderDependencies["findLabeledSandboxContainers"]; +}; + +function harness(overrides: StopHarnessOverrides = {}) { + const { + dockerStop: dockerStopOverride, + findLabeledSandboxContainers: findContainersOverride, + ...actionOverrides + } = overrides; const getSandbox = vi.fn>(() => sandbox()); - const isDockerRuntimeDown = vi.fn>( + const isDockerRuntimeDown = vi.fn( () => false, ); const printDockerRuntimeDownGuidance = - vi.fn>(); + vi.fn(); const findLabeledSandboxContainers = vi.fn< - NonNullable - >(() => [container("openshell-my-sandbox", true)]); + DockerRuntimeProviderDependencies["findLabeledSandboxContainers"] + >(findContainersOverride ?? (() => [container("openshell-my-sandbox", true)])); const stopSandboxChannels = vi.fn>(); - const dockerStop = vi.fn>(() => ({ status: 0 })); + const dockerStop = vi.fn( + dockerStopOverride ?? (() => ({ status: 0 })), + ); const teardownSandboxDashboardForward = vi.fn>(); const log = vi.fn<(message: string) => void>(); const warn = vi.fn<(message: string) => void>(); + const runtimeProviders = createRuntimeProviderBundleRegistry([ + [ + "docker", + createDockerRuntimeProviderBundle({ + findLabeledSandboxContainers, + isRuntimeDown: isDockerRuntimeDown, + printRuntimeDownGuidance: printDockerRuntimeDownGuidance, + stopContainer: dockerStop, + }), + ], + ["kubernetes", createKubernetesRuntimeProviderBundle()], + ]); const deps: SandboxStopDeps = { getSandbox, - isDockerRuntimeDown, - printDockerRuntimeDownGuidance, - findLabeledSandboxContainers, + runtimeProviders, stopSandboxChannels, teardownSandboxDashboardForward, - dockerStop, log, warn, - ...overrides, + ...actionOverrides, }; return { deps, diff --git a/src/lib/actions/sandbox/stop.ts b/src/lib/actions/sandbox/stop.ts index 53cc1489ecd..1163d8e3a9f 100644 --- a/src/lib/actions/sandbox/stop.ts +++ b/src/lib/actions/sandbox/stop.ts @@ -2,32 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 import { CLI_NAME } from "../../cli/branding"; -import { findLabeledSandboxContainers } from "../../onboard/docker-driver-sandbox-recovery"; +import { + CURRENT_RUNTIME_PROVIDER_BUNDLES, + type RuntimeProviderBundleRegistry, +} from "../../onboard/runtime-provider/access"; import * as registry from "../../state/registry"; import { stopSandboxChannels } from "../../tunnel/sandbox-gateway-stop"; import { teardownSandboxDashboardForward } from "./forward-recovery"; -import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "./gateway-failure-classifier"; - -// Lazy adapter accessor, same pattern as docker-driver-sandbox-recovery.ts: -// tests inject `deps.dockerStop` so the lazy require never fires. -type DockerOpResult = { status?: number | null }; -type DockerStopFn = (name: string, opts?: Record) => DockerOpResult; - -function loadDockerStop(): DockerStopFn { - return (require("../../adapters/docker") as { dockerStop: DockerStopFn }).dockerStop; -} - -const DOCKER_STOP_TIMEOUT_MS = 30_000; - -// Docker `Status` strings for containers that hold no resources: nothing to -// stop. Everything else — `Up`, `Up … (Paused)`, and crash-looping -// `Restarting (N) …` — is stoppable; `docker stop` also disarms an armed -// restart policy, which is exactly how a crash loop is silenced (#6026). -const AT_REST_STATUS_PREFIXES = ["Exited", "Created", "Dead"] as const; - -function isAtRest(status: string): boolean { - return AT_REST_STATUS_PREFIXES.some((prefix) => status.startsWith(prefix)); -} +import { + resolveSandboxLifecycleProvider, + type SandboxLifecycleResult, +} from "./runtime/lifecycle-runtime"; function teardownDashboardForwardBestEffort( sandboxName: string, @@ -42,84 +27,21 @@ function teardownDashboardForwardBestEffort( } } -export type SandboxLifecycleResult = { - exitCode: number; - message?: string; -}; +export type { SandboxLifecycleResult } from "./runtime/lifecycle-runtime"; export interface SandboxStopDeps { + environment?: NodeJS.ProcessEnv; getSandbox?: typeof registry.getSandbox; - isDockerRuntimeDown?: typeof isDockerRuntimeDown; - printDockerRuntimeDownGuidance?: typeof printDockerRuntimeDownGuidance; - findLabeledSandboxContainers?: typeof findLabeledSandboxContainers; + runtimeProviders?: RuntimeProviderBundleRegistry; stopSandboxChannels?: typeof stopSandboxChannels; teardownSandboxDashboardForward?: typeof teardownSandboxDashboardForward; - dockerStop?: DockerStopFn; log?: (message: string) => void; warn?: (message: string) => void; } -function normalizeDriver(driver: unknown): string | null { - return typeof driver === "string" && driver.trim() ? driver.trim().toLowerCase() : null; -} - /** - * Refuse lifecycle control for sandboxes we cannot reach through the local - * Docker daemon. Mirrors the gate `privilegedSandboxExecArgv` applies before - * direct-container mutation (src/lib/sandbox/privileged-exec.ts). - */ -export function gateDirectDriverLifecycle( - sandboxName: string, - action: "stop" | "start", - getSandbox: typeof registry.getSandbox, -): SandboxLifecycleResult | null { - const entry = getSandbox(sandboxName); - if (!entry) { - return { - exitCode: 1, - message: - ` Sandbox '${sandboxName}' is not registered. ` + - `Run '${CLI_NAME} list' to see registered sandboxes.`, - }; - } - const driver = normalizeDriver(entry.openshellDriver); - if (driver !== null && driver !== "docker" && driver !== "vm") { - return { - exitCode: 1, - message: - ` '${CLI_NAME} ${sandboxName} ${action}' controls the local Docker container ` + - `directly and is unavailable for driver '${driver}'.`, - }; - } - return null; -} - -/** - * Fail fast with the shared #4428 outage guidance when the Docker daemon is - * unreachable. Without this preflight an empty `docker ps` result is - * indistinguishable from "no containers", and stop/start would misreport a - * daemon outage as a removed container and steer the user toward `rebuild` — - * exactly the guidance printDockerRuntimeDownGuidance exists to prevent. - */ -export function gateDockerRuntimeUp( - sandboxName: string, - retryCommand: "stop" | "start", - deps: Pick, -): SandboxLifecycleResult | null { - if (!(deps.isDockerRuntimeDown ?? isDockerRuntimeDown)(sandboxName)) return null; - (deps.printDockerRuntimeDownGuidance ?? printDockerRuntimeDownGuidance)(sandboxName, { - retryCommand, - }); - return { exitCode: 1 }; -} - -/** - * Stop a sandbox's Docker container while preserving every piece of state - * destroy would remove: the workspace volume, registry entry, OpenShell - * sandbox record, credentials, and images all stay in place (#6026). - * - * The shared host gateway, tunnel, and any NIM inference container are - * gateway-scoped and serve other sandboxes — deliberately untouched. + * Stop the selected provider workload while preserving registry, workspace, + * credentials, and shared gateway state. */ export function stopSandbox( sandboxName: string, @@ -127,36 +49,44 @@ export function stopSandbox( ): SandboxLifecycleResult { const log = deps.log ?? console.log; const warn = deps.warn ?? console.warn; - - const gate = gateDirectDriverLifecycle( + const sandbox = (deps.getSandbox ?? registry.getSandbox)(sandboxName); + const resolved = resolveSandboxLifecycleProvider( sandboxName, + sandbox, "stop", - deps.getSandbox ?? registry.getSandbox, + deps.runtimeProviders ?? CURRENT_RUNTIME_PROVIDER_BUNDLES, ); - if (gate) return gate; - - const runtimeGate = gateDockerRuntimeUp(sandboxName, "stop", deps); - if (runtimeGate) return runtimeGate; + if (!resolved.ok) return resolved.result; - const containers = (deps.findLabeledSandboxContainers ?? findLabeledSandboxContainers)( + const input = { + environment: deps.environment ?? process.env, + log, + sandbox: sandbox!, sandboxName, - ); - if (containers.length === 0) { - return { - exitCode: 1, - message: - ` No Docker container found for sandbox '${sandboxName}'. ` + - `If the container was removed, run '${CLI_NAME} ${sandboxName} rebuild' to recreate it.`, - }; - } + }; + const preflight = resolved.bundle.preflightDoctor.preflightLifecycle("stop", input); + if (preflight) return preflight; + + let channelsStopped = false; + const outcome = resolved.lifecycle.stop(input, { + beforeStop() { + if (channelsStopped) return; + channelsStopped = true; + try { + (deps.stopSandboxChannels ?? stopSandboxChannels)(sandboxName, { + info: (message) => log(` ${message}`), + warn: (message) => warn(` ${message}`), + }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + warn(` Warning: could not stop in-sandbox channels gracefully: ${detail}`); + } + }, + }); + if (outcome.exitCode !== 0) return outcome; - const stoppable = containers.filter((container) => !isAtRest(container.status)); - if (stoppable.length === 0) { + if (outcome.state === "already-stopped") { log(` Sandbox '${sandboxName}' is already stopped.`); - // Idempotent teardown: an earlier stop may have left the dashboard forward - // alive (e.g. openshell was unreachable then, or the forward was orphaned by - // a raw `docker stop`). Release it here too so a repeated stop always - // converges on no leftover listener (#7227). teardownDashboardForwardBestEffort( sandboxName, deps.teardownSandboxDashboardForward ?? teardownSandboxDashboardForward, @@ -166,56 +96,11 @@ export function stopSandbox( return { exitCode: 0 }; } - // Graceful in-sandbox gateway shutdown first, so channels disconnect - // cleanly instead of dying with the container's SIGTERM. Best-effort: - // a stop must still free resources when the gateway is unreachable. - // Agent-managed gateways (e.g. Hermes) are supervised inside the sandbox - // and shut down with the container's stop signal instead. - try { - (deps.stopSandboxChannels ?? stopSandboxChannels)(sandboxName, { - info: (message) => log(` ${message}`), - warn: (message) => warn(` ${message}`), - }); - } catch (error) { - const detail = error instanceof Error ? error.message : String(error); - warn(` Warning: could not stop in-sandbox channels gracefully: ${detail}`); - } - - // Attempt every stoppable container even if one fails: a sandbox can have - // more than one labeled container (e.g. a gpu-backup sibling), and aborting - // on the first failure would leave the rest running — the opposite of what - // stop is for. Collect failures and report them together. - const dockerStop = deps.dockerStop ?? ((name, opts) => loadDockerStop()(name, opts)); - const failures: string[] = []; - for (const container of stoppable) { - log(` Stopping container '${container.name}'…`); - const result = dockerStop(container.name, { - ignoreError: true, - timeout: DOCKER_STOP_TIMEOUT_MS, - }); - if (result.status !== 0) { - failures.push(`${container.name} (exit ${result.status ?? "unknown"})`); - } - } - - if (failures.length > 0) { - return { - exitCode: 1, - message: ` docker stop failed for: ${failures.join(", ")}.`, - }; - } - - // Release the host-side dashboard port-forward this sandbox created. Without - // this, the `ssh -L` listener stays alive after the container is stopped, so - // `status` misreports the cleanly-stopped sandbox as a foreign - // `sandbox_dashboard_port_conflict` and `start`/`recover` contend with the - // still-held port (#7227). Best-effort — the container is already stopped. teardownDashboardForwardBestEffort( sandboxName, deps.teardownSandboxDashboardForward ?? teardownSandboxDashboardForward, warn, ); - log(` Sandbox '${sandboxName}' stopped. Workspace state is preserved.`); log(` Start it again with '${CLI_NAME} ${sandboxName} start'.`); return { exitCode: 0 }; diff --git a/src/lib/onboard/compute/plan.ts b/src/lib/onboard/compute/plan.ts index 1e6079d086b..fae373961b3 100644 --- a/src/lib/onboard/compute/plan.ts +++ b/src/lib/onboard/compute/plan.ts @@ -1,7 +1,17 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -export type OpenShellGatewayLauncher = "nemoclaw" | "openshell"; +import { + CURRENT_RUNTIME_PROVIDER_BUNDLES, + RuntimeProviderBundle, + RuntimeProviderBundleRegistry, + RuntimeProviderGatewayLauncher, + resolveRuntimeProviderBundle, + resolveCurrentRuntimeProviderBundle, + runtimeProviderContainerEngineIdentity, +} from "../runtime-provider/access"; + +export type OpenShellGatewayLauncher = RuntimeProviderGatewayLauncher; /** * Keeps OpenShell driver identity separate from the component that launches @@ -13,6 +23,15 @@ export interface OpenShellComputePlan { readonly gatewayLauncher: OpenShellGatewayLauncher; } +export function projectRuntimeProviderComputePlan( + bundle: RuntimeProviderBundle, +): OpenShellComputePlan { + return { + driverName: bundle.identity.id, + gatewayLauncher: bundle.plan.gatewayLauncher, + }; +} + /** * Describes the behavior NemoClaw uses today. Driver selection will move behind * this seam without changing the existing Docker and Kubernetes paths first. @@ -21,16 +40,20 @@ export function resolveCurrentOpenShellComputePlan( platform: NodeJS.Platform = process.platform, arch: NodeJS.Architecture = process.arch, ): OpenShellComputePlan { - const managedDockerGateway = platform === "linux" || (platform === "darwin" && arch === "arm64"); - - return { - driverName: managedDockerGateway ? "docker" : "kubernetes", - gatewayLauncher: managedDockerGateway ? "nemoclaw" : "openshell", - }; + return projectRuntimeProviderComputePlan(resolveCurrentRuntimeProviderBundle(platform, arch)); } export function usesManagedDockerGateway( plan: Pick, + providers: RuntimeProviderBundleRegistry = CURRENT_RUNTIME_PROVIDER_BUNDLES, ): boolean { - return plan.driverName === "docker" && plan.gatewayLauncher === "nemoclaw"; + const bundle = resolveRuntimeProviderBundle(plan.driverName, providers); + const engine = bundle + ? runtimeProviderContainerEngineIdentity(bundle, "gateway-inspection") + : null; + return ( + bundle?.gateway.launcher === plan.gatewayLauncher && + plan.gatewayLauncher === "nemoclaw" && + engine?.engineId === "docker" + ); } diff --git a/src/lib/onboard/runtime-provider/access.ts b/src/lib/onboard/runtime-provider/access.ts new file mode 100644 index 00000000000..f8fe5e63cb1 --- /dev/null +++ b/src/lib/onboard/runtime-provider/access.ts @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type { + RuntimeProviderBundle, + RuntimeProviderBundleRegistry, + RuntimeProviderGatewayLauncher, + RuntimeProviderManagedImageSupport, + RuntimeProviderWorkloadProfile, + RuntimeProviderWorkloadCleanupResult, +} from "./contract"; +export { + CURRENT_RUNTIME_PROVIDER_BUNDLES, + resolveCurrentRuntimeProviderBundle, +} from "./current"; +export { + normalizeRuntimeProviderIdentity, + requireRuntimeProviderBundle, + requireRuntimeProviderBundleForSandbox, + requireRuntimeProviderMutationAuthority, + resolveRuntimeProviderBundle, + RuntimeProviderSelectionError, + runtimeProviderContainerEngineIdentity, +} from "./registry"; diff --git a/src/lib/onboard/runtime-provider/contract.ts b/src/lib/onboard/runtime-provider/contract.ts new file mode 100644 index 00000000000..38403f2d1e6 --- /dev/null +++ b/src/lib/onboard/runtime-provider/contract.ts @@ -0,0 +1,256 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; +import type { ManagedImageSelectionPolicy } from "../workload/source"; + +export const RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION = 1 as const; + +export type RuntimeProviderGatewayLauncher = "nemoclaw" | "openshell"; +export type RuntimeProviderLifecycleAction = "start" | "stop"; +export type RuntimeProviderMutationOperation = + | "registration" + | "start" + | "stop" + | "inference-set" + | "rebuild" + | "provider-cleanup" + | "destroy" + | "workload-cleanup"; +export type RuntimeProviderContainerEngineOperation = + | "host-doctor" + | "gateway-inspection" + | "sandbox-lifecycle" + | "workload-cleanup"; + +export interface RuntimeProviderIdentity { + readonly contractVersion: typeof RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION; + readonly id: string; + readonly displayName: string; +} + +export interface RuntimeProviderBoundSurface { + readonly providerId: string; + readonly supported: boolean; +} + +export interface RuntimeProviderUnsupportedSurface extends RuntimeProviderBoundSurface { + readonly supported: false; + readonly reason: string; +} + +export type RuntimeProviderSupportedSurface = Readonly< + RuntimeProviderBoundSurface & { + readonly supported: true; + } & T +>; + +export interface RuntimeProviderPlanDefinition { + readonly gatewayLauncher: RuntimeProviderGatewayLauncher; +} + +export interface RuntimeProviderNormalizedCapabilities { + readonly hostLocalInference: boolean; + readonly directLifecycle: boolean; + readonly legacyGatewayContainerInspection: boolean; + readonly workloadImageCleanup: boolean; +} + +export type RuntimeProviderManagedImageSupport = { + readonly exactDigestReferences: boolean; + readonly platforms: readonly ("linux/amd64" | "linux/arm64")[]; + readonly startupProfileContractVersions: readonly number[]; + readonly capabilityContractVersions: readonly number[]; +}; + +export interface RuntimeProviderWorkloadProfile { + readonly support: RuntimeProviderManagedImageSupport | null; + readonly hostArchitectures: readonly string[]; + readonly managedImageSelectionPolicy: ManagedImageSelectionPolicy; + readonly legacyDockerfileBuilds: boolean; +} + +export type RuntimeProviderDoctorCheck = { + readonly group: "Host"; + readonly label: string; + readonly status: "ok" | "warn" | "fail" | "info"; + readonly detail: string; + readonly hint?: string; +}; + +export type RuntimeProviderCommandCapture = { + readonly status: number; + readonly stdout: string; + readonly stderr: string; + readonly error?: Error; +}; + +export interface RuntimeProviderLifecycleInput { + readonly environment: NodeJS.ProcessEnv; + readonly log: (message: string) => void; + readonly sandbox: SandboxEntry; + readonly sandboxName: string; +} + +export type RuntimeProviderLifecycleResult = { + readonly exitCode: number; + readonly message?: string; +}; + +export type RuntimeProviderLifecycleStopOutcome = RuntimeProviderLifecycleResult & { + readonly state?: "already-stopped" | "stopped"; +}; + +export interface RuntimeProviderLifecycleStopHooks { + readonly beforeStop: () => void; +} + +export type RuntimeProviderProviderDetachResult = { + readonly detached: string[]; + readonly failures: Array<{ readonly name: string; readonly output: string }>; +}; + +export interface RuntimeProviderCleanupInput { + readonly sandbox: SandboxEntry; + readonly sandboxName: string; +} + +export type RuntimeProviderWorkloadCleanupResult = + | { + readonly status: "skipped"; + readonly reason: "no-owned-image" | "shared-image" | "authority-unproven"; + } + | { + readonly status: "removed"; + readonly engineDisplayName: string; + readonly reference: string; + } + | { + readonly status: "failed"; + readonly engineDisplayName: string; + readonly reference: string; + }; + +export interface RuntimeProviderCleanupOperations { + readonly detachProviders: (sandboxName: string) => RuntimeProviderProviderDetachResult; +} + +/** + * Provider-neutral, bounded state that later snapshot work may persist. + * Provider handles remain opaque strings; acceleration is normalized so no + * action module needs a Docker-, CDI-, or device-specific DTO. + */ +export interface RuntimeProviderRuntimeReceipt { + readonly schemaVersion: 1; + readonly providerId: string; + readonly runtime: { + readonly kind: string; + readonly handle: string; + }; + readonly acceleration: + | { + readonly kind: "none"; + } + | { + readonly kind: "gpu"; + readonly vendor: string; + readonly devices: readonly string[]; + }; +} + +export type RuntimeProviderPreflightDoctorSurface = RuntimeProviderSupportedSurface<{ + inspectHost(): RuntimeProviderDoctorCheck; + preflightLifecycle( + action: RuntimeProviderLifecycleAction, + input: RuntimeProviderLifecycleInput, + ): RuntimeProviderLifecycleResult | null; +}>; + +export type RuntimeProviderGatewaySurface = RuntimeProviderSupportedSurface<{ + readonly launcher: RuntimeProviderGatewayLauncher; + readonly inspectLegacyContainer: boolean; +}>; + +export type RuntimeProviderWorkloadSurface = RuntimeProviderSupportedSurface<{ + readonly profile: RuntimeProviderWorkloadProfile; + acceptsReceipt(receipt: SandboxWorkloadReceipt | undefined): boolean; +}>; + +export type RuntimeProviderLifecycleSurface = + | RuntimeProviderSupportedSurface<{ + readonly channelStopTransport: "docker-kubectl-first" | "openshell"; + start(input: RuntimeProviderLifecycleInput): RuntimeProviderLifecycleResult; + stop( + input: RuntimeProviderLifecycleInput, + hooks: RuntimeProviderLifecycleStopHooks, + ): RuntimeProviderLifecycleStopOutcome; + }> + | RuntimeProviderUnsupportedSurface; + +export type RuntimeProviderMutationAuthoritySurface = + | RuntimeProviderSupportedSurface<{ + readonly operations: readonly RuntimeProviderMutationOperation[]; + }> + | RuntimeProviderUnsupportedSurface; + +export type RuntimeProviderBootstrapSurface = + | RuntimeProviderSupportedSurface<{ + prepare(sandbox: SandboxEntry): unknown; + }> + | RuntimeProviderUnsupportedSurface; + +export type RuntimeProviderSnapshotSurface = + | RuntimeProviderSupportedSurface<{ + capture(sandbox: SandboxEntry): RuntimeProviderRuntimeReceipt; + restore(sandbox: SandboxEntry, receipt: RuntimeProviderRuntimeReceipt): void; + }> + | RuntimeProviderUnsupportedSurface; + +export type RuntimeProviderRecoverySurface = + | RuntimeProviderSupportedSurface<{ + recover(sandbox: SandboxEntry): RuntimeProviderLifecycleResult; + }> + | RuntimeProviderUnsupportedSurface; + +export type RuntimeProviderCleanupSurface = + | RuntimeProviderSupportedSurface<{ + prepareDestroy( + input: RuntimeProviderCleanupInput, + operations: RuntimeProviderCleanupOperations, + ): RuntimeProviderProviderDetachResult; + removeOwnedWorkload(input: RuntimeProviderCleanupInput): RuntimeProviderWorkloadCleanupResult; + }> + | RuntimeProviderUnsupportedSurface; + +export type RuntimeProviderContainerEngineSurface = + | RuntimeProviderSupportedSurface<{ + readonly identities: readonly { + readonly operation: RuntimeProviderContainerEngineOperation; + readonly engineId: string; + readonly displayName: string; + }[]; + }> + | RuntimeProviderUnsupportedSurface; + +/** + * The sole registration unit for a runtime provider. Every surface is present + * and bound to the same opaque identity; future work extends this object + * instead of creating another independently populated registry. + */ +export interface RuntimeProviderBundle { + readonly identity: RuntimeProviderIdentity; + readonly plan: RuntimeProviderSupportedSurface; + readonly capabilities: RuntimeProviderSupportedSurface; + readonly preflightDoctor: RuntimeProviderPreflightDoctorSurface; + readonly gateway: RuntimeProviderGatewaySurface; + readonly workload: RuntimeProviderWorkloadSurface; + readonly lifecycle: RuntimeProviderLifecycleSurface; + readonly mutationAuthority: RuntimeProviderMutationAuthoritySurface; + readonly bootstrap: RuntimeProviderBootstrapSurface; + readonly snapshot: RuntimeProviderSnapshotSurface; + readonly recovery: RuntimeProviderRecoverySurface; + readonly cleanup: RuntimeProviderCleanupSurface; + readonly containerEngine: RuntimeProviderContainerEngineSurface; +} + +export type RuntimeProviderBundleRegistry = Readonly>; diff --git a/src/lib/onboard/runtime-provider/current.ts b/src/lib/onboard/runtime-provider/current.ts new file mode 100644 index 00000000000..3f812630d69 --- /dev/null +++ b/src/lib/onboard/runtime-provider/current.ts @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { RuntimeProviderBundle, RuntimeProviderBundleRegistry } from "./contract"; +import { createDockerRuntimeProviderBundle, createKubernetesRuntimeProviderBundle } from "./docker"; +import { createRuntimeProviderBundleRegistry, requireRuntimeProviderBundle } from "./registry"; + +/** + * The production-selectable set remains intentionally limited to the two + * providers NemoClaw already ships. Future providers must land as one complete + * bundle and separately pass their activation gate. + */ +export const CURRENT_RUNTIME_PROVIDER_BUNDLES: RuntimeProviderBundleRegistry = + createRuntimeProviderBundleRegistry([ + ["docker", createDockerRuntimeProviderBundle()], + ["kubernetes", createKubernetesRuntimeProviderBundle()], + ]); + +export function resolveCurrentRuntimeProviderBundle( + platform: NodeJS.Platform = process.platform, + arch: NodeJS.Architecture = process.arch, + providers: RuntimeProviderBundleRegistry = CURRENT_RUNTIME_PROVIDER_BUNDLES, +): RuntimeProviderBundle { + const managedLocalGateway = platform === "linux" || (platform === "darwin" && arch === "arm64"); + return requireRuntimeProviderBundle(managedLocalGateway ? "docker" : "kubernetes", providers); +} diff --git a/src/lib/onboard/runtime-provider/docker.ts b/src/lib/onboard/runtime-provider/docker.ts new file mode 100644 index 00000000000..2f3e5fa21a3 --- /dev/null +++ b/src/lib/onboard/runtime-provider/docker.ts @@ -0,0 +1,425 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { captureHostCommand } from "../../actions/sandbox/doctor-host-command"; +import { + isDockerRuntimeDown, + printDockerRuntimeDownGuidance, +} from "../../actions/sandbox/gateway-failure-classifier"; +import { cliName } from "../branding"; +import { + findLabeledSandboxContainers, + recoverDockerDriverSandbox, +} from "../docker-driver-sandbox-recovery"; +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORMS, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, +} from "../managed-image/contract"; +import { + RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, + type RuntimeProviderBundle, + type RuntimeProviderCleanupInput, + type RuntimeProviderCommandCapture, + type RuntimeProviderDoctorCheck, + type RuntimeProviderLifecycleInput, + type RuntimeProviderLifecycleResult, + type RuntimeProviderLifecycleStopHooks, + type RuntimeProviderLifecycleStopOutcome, + type RuntimeProviderWorkloadCleanupResult, + type RuntimeProviderWorkloadProfile, +} from "./contract"; + +type DockerOpResult = { status?: number | null }; +type DockerStop = (name: string, options?: Record) => DockerOpResult; +type DockerUnpause = (name: string, options?: Record) => DockerOpResult; +type DockerRemoveImage = ( + reference: string, + options?: { ignoreError?: boolean }, +) => { status: number | null }; + +export interface DockerRuntimeProviderDependencies { + readonly captureHostCommand: ( + command: string, + args: string[], + timeout?: number, + ) => RuntimeProviderCommandCapture; + readonly findLabeledSandboxContainers: typeof findLabeledSandboxContainers; + readonly isRuntimeDown: typeof isDockerRuntimeDown; + readonly printRuntimeDownGuidance: typeof printDockerRuntimeDownGuidance; + readonly recoverSandbox: typeof recoverDockerDriverSandbox; + readonly removeImage: DockerRemoveImage; + readonly stopContainer: DockerStop; + readonly unpauseContainer: DockerUnpause; +} + +const DOCKER_OPERATION_TIMEOUT_MS = 30_000; +const AT_REST_STATUS_PREFIXES = ["Exited", "Created", "Dead"] as const; + +function loadDockerStop(): DockerStop { + return (require("../../adapters/docker") as { dockerStop: DockerStop }).dockerStop; +} + +function loadDockerUnpause(): DockerUnpause { + return (require("../../adapters/docker") as { dockerUnpause: DockerUnpause }).dockerUnpause; +} + +function loadDockerRemoveImage(): DockerRemoveImage { + return (require("../../adapters/docker") as { dockerRmi: DockerRemoveImage }).dockerRmi; +} + +function resolveDependencies( + overrides: Partial = {}, +): DockerRuntimeProviderDependencies { + return { + captureHostCommand: + overrides.captureHostCommand ?? + ((command, args, timeout) => captureHostCommand(command, args, timeout)), + findLabeledSandboxContainers: + overrides.findLabeledSandboxContainers ?? findLabeledSandboxContainers, + isRuntimeDown: overrides.isRuntimeDown ?? isDockerRuntimeDown, + printRuntimeDownGuidance: overrides.printRuntimeDownGuidance ?? printDockerRuntimeDownGuidance, + recoverSandbox: overrides.recoverSandbox ?? recoverDockerDriverSandbox, + removeImage: + overrides.removeImage ?? + ((reference, options) => loadDockerRemoveImage()(reference, options)), + stopContainer: overrides.stopContainer ?? ((name, options) => loadDockerStop()(name, options)), + unpauseContainer: + overrides.unpauseContainer ?? ((name, options) => loadDockerUnpause()(name, options)), + }; +} + +function oneLine(value = ""): string { + return value.replace(/\s+/gu, " ").trim(); +} + +function inspectDockerHost(deps: DockerRuntimeProviderDependencies): RuntimeProviderDoctorCheck { + const result = deps.captureHostCommand( + "docker", + ["info", "--format", "{{.ServerVersion}}"], + 8000, + ); + return { + group: "Host", + label: "Docker daemon", + status: result.status === 0 ? "ok" : "fail", + detail: + result.status === 0 + ? `server ${result.stdout.trim() || "unknown"}` + : oneLine(result.stderr || result.error?.message || "docker info failed"), + hint: + result.status === 0 ? undefined : "start Docker and verify your user can access the daemon", + }; +} + +function dockerLifecyclePreflight( + action: "start" | "stop", + input: RuntimeProviderLifecycleInput, + deps: DockerRuntimeProviderDependencies, +): RuntimeProviderLifecycleResult | null { + if (!deps.isRuntimeDown(input.sandboxName)) return null; + deps.printRuntimeDownGuidance(input.sandboxName, { retryCommand: action }); + return { exitCode: 1 }; +} + +function isPausedStatus(status: string): boolean { + return status.startsWith("Up") && status.endsWith("(Paused)"); +} + +function isAtRestStatus(status: string): boolean { + return AT_REST_STATUS_PREFIXES.some((prefix) => status.startsWith(prefix)); +} + +function startDockerSandbox( + input: RuntimeProviderLifecycleInput, + deps: DockerRuntimeProviderDependencies, +): RuntimeProviderLifecycleResult { + const containers = deps.findLabeledSandboxContainers(input.sandboxName); + const paused = containers.find((container) => isPausedStatus(container.status)); + if (paused) { + const result = deps.unpauseContainer(paused.name, { + ignoreError: true, + timeout: DOCKER_OPERATION_TIMEOUT_MS, + }); + if (result.status !== 0) { + return { + exitCode: 1, + message: ` docker unpause ${paused.name} failed (exit ${result.status ?? "unknown"}).`, + }; + } + input.log(` Container '${paused.name}' unpaused.`); + return { exitCode: 0 }; + } + + const recovery = deps.recoverSandbox(input.sandboxName); + if (!recovery.recovered) { + return { + exitCode: 1, + message: + ` Could not start sandbox '${input.sandboxName}': ${recovery.detail ?? "unknown failure"}. ` + + `If the container was removed, run '${cliName()} ${input.sandboxName} rebuild' to recreate it.`, + }; + } + if (recovery.via === "started-running-original") { + input.log(` Sandbox '${input.sandboxName}' is already running.`); + } else { + input.log(` Container '${recovery.containerName ?? input.sandboxName}' started.`); + } + return { exitCode: 0 }; +} + +function stopDockerSandbox( + input: RuntimeProviderLifecycleInput, + hooks: RuntimeProviderLifecycleStopHooks, + deps: DockerRuntimeProviderDependencies, +): RuntimeProviderLifecycleStopOutcome { + const containers = deps.findLabeledSandboxContainers(input.sandboxName); + if (containers.length === 0) { + return { + exitCode: 1, + message: + ` No Docker container found for sandbox '${input.sandboxName}'. ` + + `If the container was removed, run '${cliName()} ${input.sandboxName} rebuild' to recreate it.`, + }; + } + + const stoppable = containers.filter((container) => !isAtRestStatus(container.status)); + if (stoppable.length === 0) return { exitCode: 0, state: "already-stopped" }; + + hooks.beforeStop(); + const failures: string[] = []; + for (const container of stoppable) { + input.log(` Stopping container '${container.name}'…`); + const result = deps.stopContainer(container.name, { + ignoreError: true, + timeout: DOCKER_OPERATION_TIMEOUT_MS, + }); + if (result.status !== 0) { + failures.push(`${container.name} (exit ${result.status ?? "unknown"})`); + } + } + if (failures.length > 0) { + return { + exitCode: 1, + message: ` docker stop failed for: ${failures.join(", ")}.`, + }; + } + return { exitCode: 0, state: "stopped" }; +} + +function removeOwnedDockerWorkload( + input: RuntimeProviderCleanupInput, + deps: DockerRuntimeProviderDependencies, +): RuntimeProviderWorkloadCleanupResult { + const { imageTag, workload } = input.sandbox; + if (workload?.shared === true) return { status: "skipped", reason: "shared-image" }; + if (!imageTag) return { status: "skipped", reason: "no-owned-image" }; + if ( + workload?.kind === "legacy-dockerfile" && + workload.reference !== null && + workload.reference !== imageTag + ) { + return { status: "skipped", reason: "authority-unproven" }; + } + const result = deps.removeImage(imageTag, { ignoreError: true }); + return { + status: result.status === 0 ? "removed" : "failed", + engineDisplayName: "Docker", + reference: imageTag, + }; +} + +const COMPLETE_MANAGED_IMAGE_V1_PROFILE = { + support: { + exactDigestReferences: true, + platforms: MANAGED_IMAGE_PLATFORMS, + startupProfileContractVersions: [MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION], + capabilityContractVersions: [MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION], + }, + hostArchitectures: ["amd64", "arm64"], + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: true, +} as const satisfies RuntimeProviderWorkloadProfile; + +function acceptsReceipt( + profile: RuntimeProviderWorkloadProfile, + receipt: RuntimeProviderCleanupInput["sandbox"]["workload"], +): boolean { + if (!receipt) return true; + if (receipt.kind === "legacy-dockerfile") return profile.legacyDockerfileBuilds; + if (receipt.platform === undefined) return false; + return ( + profile.support !== null && + profile.support.platforms.includes(receipt.platform) && + profile.support.capabilityContractVersions.includes(receipt.capabilityContractVersion) && + profile.support.startupProfileContractVersions.includes(receipt.startupProfileContractVersion) + ); +} + +function unsupported(providerId: string, reason: string) { + return { providerId, supported: false as const, reason }; +} + +export function createDockerRuntimeProviderBundle( + overrides: Partial = {}, +): RuntimeProviderBundle { + const providerId = "docker"; + const deps = resolveDependencies(overrides); + const futureReason = "This operation is intentionally deferred to a later provider slice."; + return { + identity: { + contractVersion: RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, + id: providerId, + displayName: "Docker", + }, + plan: { providerId, supported: true, gatewayLauncher: "nemoclaw" }, + capabilities: { + providerId, + supported: true, + hostLocalInference: true, + directLifecycle: true, + legacyGatewayContainerInspection: false, + workloadImageCleanup: true, + }, + preflightDoctor: { + providerId, + supported: true, + inspectHost: () => inspectDockerHost(deps), + preflightLifecycle: (action, input) => dockerLifecyclePreflight(action, input, deps), + }, + gateway: { + providerId, + supported: true, + launcher: "nemoclaw", + inspectLegacyContainer: false, + }, + workload: { + providerId, + supported: true, + profile: COMPLETE_MANAGED_IMAGE_V1_PROFILE, + acceptsReceipt: (receipt) => acceptsReceipt(COMPLETE_MANAGED_IMAGE_V1_PROFILE, receipt), + }, + lifecycle: { + providerId, + supported: true, + channelStopTransport: "docker-kubectl-first", + start: (input) => startDockerSandbox(input, deps), + stop: (input, hooks) => stopDockerSandbox(input, hooks, deps), + }, + mutationAuthority: { + providerId, + supported: true, + operations: [ + "registration", + "start", + "stop", + "inference-set", + "rebuild", + "provider-cleanup", + "destroy", + "workload-cleanup", + ], + }, + bootstrap: unsupported(providerId, futureReason), + snapshot: unsupported(providerId, futureReason), + recovery: unsupported(providerId, futureReason), + cleanup: { + providerId, + supported: true, + prepareDestroy: (input, operations) => operations.detachProviders(input.sandboxName), + removeOwnedWorkload: (input) => removeOwnedDockerWorkload(input, deps), + }, + containerEngine: { + providerId, + supported: true, + identities: [ + { operation: "host-doctor", engineId: "docker", displayName: "Docker" }, + { operation: "gateway-inspection", engineId: "docker", displayName: "Docker" }, + { operation: "sandbox-lifecycle", engineId: "docker", displayName: "Docker" }, + { operation: "workload-cleanup", engineId: "docker", displayName: "Docker" }, + ], + }, + }; +} + +export function createKubernetesRuntimeProviderBundle( + overrides: Partial = {}, +): RuntimeProviderBundle { + const providerId = "kubernetes"; + const deps = resolveDependencies(overrides); + const futureReason = "This operation is intentionally deferred to a later provider slice."; + const profile = { + support: null, + hostArchitectures: [], + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: true, + } as const satisfies RuntimeProviderWorkloadProfile; + return { + identity: { + contractVersion: RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, + id: providerId, + displayName: "Kubernetes", + }, + plan: { providerId, supported: true, gatewayLauncher: "openshell" }, + capabilities: { + providerId, + supported: true, + hostLocalInference: true, + directLifecycle: false, + legacyGatewayContainerInspection: true, + workloadImageCleanup: true, + }, + preflightDoctor: { + providerId, + supported: true, + inspectHost: () => inspectDockerHost(deps), + preflightLifecycle: () => null, + }, + gateway: { + providerId, + supported: true, + launcher: "openshell", + inspectLegacyContainer: true, + }, + workload: { + providerId, + supported: true, + profile, + acceptsReceipt: (receipt) => acceptsReceipt(profile, receipt), + }, + lifecycle: unsupported( + providerId, + "Direct local lifecycle control is unavailable for the Kubernetes provider.", + ), + mutationAuthority: { + providerId, + supported: true, + operations: [ + "registration", + "inference-set", + "rebuild", + "provider-cleanup", + "destroy", + "workload-cleanup", + ], + }, + bootstrap: unsupported(providerId, futureReason), + snapshot: unsupported(providerId, futureReason), + recovery: unsupported(providerId, futureReason), + cleanup: { + providerId, + supported: true, + prepareDestroy: (input, operations) => operations.detachProviders(input.sandboxName), + removeOwnedWorkload: (input) => removeOwnedDockerWorkload(input, deps), + }, + containerEngine: { + providerId, + supported: true, + identities: [ + { operation: "host-doctor", engineId: "docker", displayName: "Docker" }, + { operation: "gateway-inspection", engineId: "docker", displayName: "Docker" }, + { operation: "workload-cleanup", engineId: "docker", displayName: "Docker" }, + ], + }, + }; +} diff --git a/src/lib/onboard/runtime-provider/registry.ts b/src/lib/onboard/runtime-provider/registry.ts new file mode 100644 index 00000000000..04572271ba9 --- /dev/null +++ b/src/lib/onboard/runtime-provider/registry.ts @@ -0,0 +1,513 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxEntry } from "../../state/registry/types"; +import { + RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, + type RuntimeProviderBundle, + type RuntimeProviderBundleRegistry, + type RuntimeProviderContainerEngineOperation, + type RuntimeProviderMutationOperation, + type RuntimeProviderRuntimeReceipt, +} from "./contract"; + +const PROVIDER_ID_PATTERN = /^[a-z][a-z0-9-]{0,62}$/u; +const RESERVED_PROVIDER_IDS = new Set(["constructor", "prototype"]); +const BUNDLE_SURFACES = [ + "plan", + "capabilities", + "preflightDoctor", + "gateway", + "workload", + "lifecycle", + "mutationAuthority", + "bootstrap", + "snapshot", + "recovery", + "cleanup", + "containerEngine", +] as const; +const MAX_RECEIPT_HANDLE_BYTES = 4096; +const MAX_RECEIPT_DEVICES = 64; +const GATEWAY_LAUNCHERS = new Set(["nemoclaw", "openshell"]); +const CHANNEL_STOP_TRANSPORTS = new Set(["docker-kubectl-first", "openshell"]); +const MANAGED_IMAGE_SELECTION_POLICIES = new Set(["prefer-managed", "require-managed"]); +const MANAGED_IMAGE_PLATFORMS = new Set(["linux/amd64", "linux/arm64"]); +const MUTATION_OPERATIONS = new Set([ + "registration", + "start", + "stop", + "inference-set", + "rebuild", + "provider-cleanup", + "destroy", + "workload-cleanup", +]); +const CONTAINER_ENGINE_OPERATIONS = new Set([ + "host-doctor", + "gateway-inspection", + "sandbox-lifecycle", + "workload-cleanup", +]); + +export class RuntimeProviderRegistrationError extends Error { + constructor(message: string) { + super(`Invalid runtime provider registration: ${message}`); + this.name = "RuntimeProviderRegistrationError"; + } +} + +export class RuntimeProviderSelectionError extends Error { + constructor(message: string) { + super(message); + this.name = "RuntimeProviderSelectionError"; + } +} + +function isPlainRecord(value: unknown): value is Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function validProviderId(value: unknown): value is string { + return ( + typeof value === "string" && + PROVIDER_ID_PATTERN.test(value) && + !RESERVED_PROVIDER_IDS.has(value) + ); +} + +function cloneAndFreeze(value: T, seen = new WeakMap()): T { + if ((typeof value !== "object" && typeof value !== "function") || value === null) return value; + const object = value as object; + const existing = seen.get(object); + if (existing) return existing as T; + if (typeof value === "function") { + const original = value as (...args: unknown[]) => unknown; + const copy = function (this: unknown, ...args: unknown[]) { + return Reflect.apply(original, this, args); + }; + seen.set(object, copy); + const mutableCopy = copy as unknown as Record; + const originalProperties = value as unknown as Record; + for (const key of Object.keys(originalProperties)) { + mutableCopy[key] = cloneAndFreeze(originalProperties[key], seen); + } + Object.freeze(copy.prototype); + return Object.freeze(copy) as T; + } + if (Array.isArray(value)) { + const copy: unknown[] = []; + seen.set(object, copy); + for (const item of value) copy.push(cloneAndFreeze(item, seen)); + return Object.freeze(copy) as T; + } + if (!isPlainRecord(value)) { + throw new RuntimeProviderRegistrationError("bundle values must be plain records or arrays"); + } + const copy: Record = Object.create(null); + seen.set(object, copy); + for (const key of Object.keys(value)) { + copy[key] = cloneAndFreeze(value[key], seen); + } + return Object.freeze(copy) as T; +} + +function requireOwnRecord(owner: Record, field: string): Record { + if (!Object.hasOwn(owner, field) || !isPlainRecord(owner[field])) { + throw new RuntimeProviderRegistrationError(`missing ${field} surface`); + } + return owner[field]; +} + +function validateBoundSurface( + providerId: string, + name: string, + surface: Record, +): void { + if (surface.providerId !== providerId) { + throw new RuntimeProviderRegistrationError( + `${name} identity '${String(surface.providerId)}' does not match '${providerId}'`, + ); + } + if (typeof surface.supported !== "boolean") { + throw new RuntimeProviderRegistrationError(`${name}.supported must be a boolean`); + } + if ( + surface.supported === false && + (typeof surface.reason !== "string" || surface.reason.trim() === "") + ) { + throw new RuntimeProviderRegistrationError(`${name} must explain why it is unsupported`); + } +} + +function requireSupported(name: string, surface: Record): void { + if (surface.supported !== true) { + throw new RuntimeProviderRegistrationError(`${name} must be supported`); + } +} + +function requireBoolean( + surface: Record, + field: string, + surfaceName: string, +): void { + if (typeof surface[field] !== "boolean") { + throw new RuntimeProviderRegistrationError(`${surfaceName}.${field} must be a boolean`); + } +} + +function requireFunction( + surface: Record, + field: string, + surfaceName: string, +): void { + if (typeof surface[field] !== "function") { + throw new RuntimeProviderRegistrationError(`${surfaceName}.${field} must be a function`); + } +} + +function requireNonEmptyString( + surface: Record, + field: string, + surfaceName: string, +): void { + if (typeof surface[field] !== "string" || surface[field].trim() === "") { + throw new RuntimeProviderRegistrationError( + `${surfaceName}.${field} must be a non-empty string`, + ); + } +} + +function validateWorkloadProfile(providerId: string, surface: Record): void { + requireFunction(surface, "acceptsReceipt", "workload"); + const profile = requireOwnRecord(surface, "profile"); + if (!MANAGED_IMAGE_SELECTION_POLICIES.has(String(profile.managedImageSelectionPolicy))) { + throw new RuntimeProviderRegistrationError( + `workload profile for '${providerId}' has an invalid selection policy`, + ); + } + if (typeof profile.legacyDockerfileBuilds !== "boolean") { + throw new RuntimeProviderRegistrationError( + `workload profile for '${providerId}' must declare legacyDockerfileBuilds`, + ); + } + if ( + !Array.isArray(profile.hostArchitectures) || + profile.hostArchitectures.some( + (architecture) => typeof architecture !== "string" || architecture.trim() === "", + ) || + new Set(profile.hostArchitectures).size !== profile.hostArchitectures.length + ) { + throw new RuntimeProviderRegistrationError( + `workload profile for '${providerId}' has invalid host architectures`, + ); + } + if (profile.support === null) return; + if (!isPlainRecord(profile.support)) { + throw new RuntimeProviderRegistrationError( + `workload profile for '${providerId}' has invalid managed-image support`, + ); + } + const support = profile.support; + if ( + typeof support.exactDigestReferences !== "boolean" || + !Array.isArray(support.platforms) || + support.platforms.length === 0 || + support.platforms.some((platform) => !MANAGED_IMAGE_PLATFORMS.has(String(platform))) || + new Set(support.platforms).size !== support.platforms.length + ) { + throw new RuntimeProviderRegistrationError( + `workload profile for '${providerId}' has invalid managed-image platforms`, + ); + } + for (const field of ["startupProfileContractVersions", "capabilityContractVersions"] as const) { + const versions = support[field]; + if ( + !Array.isArray(versions) || + versions.length === 0 || + versions.some((version) => !Number.isSafeInteger(version) || Number(version) <= 0) || + new Set(versions).size !== versions.length + ) { + throw new RuntimeProviderRegistrationError( + `workload profile for '${providerId}' has invalid ${field}`, + ); + } + } +} + +function validateSupportedSurfaceSchemas( + providerId: string, + surfaces: Record<(typeof BUNDLE_SURFACES)[number], Record>, +): void { + requireSupported("plan", surfaces.plan); + if (!GATEWAY_LAUNCHERS.has(String(surfaces.plan.gatewayLauncher))) { + throw new RuntimeProviderRegistrationError(`plan for '${providerId}' has an invalid launcher`); + } + + requireSupported("capabilities", surfaces.capabilities); + for (const field of [ + "hostLocalInference", + "directLifecycle", + "legacyGatewayContainerInspection", + "workloadImageCleanup", + ] as const) { + requireBoolean(surfaces.capabilities, field, "capabilities"); + } + + requireSupported("preflightDoctor", surfaces.preflightDoctor); + requireFunction(surfaces.preflightDoctor, "inspectHost", "preflightDoctor"); + requireFunction(surfaces.preflightDoctor, "preflightLifecycle", "preflightDoctor"); + + requireSupported("gateway", surfaces.gateway); + if (!GATEWAY_LAUNCHERS.has(String(surfaces.gateway.launcher))) { + throw new RuntimeProviderRegistrationError( + `gateway for '${providerId}' has an invalid launcher`, + ); + } + requireBoolean(surfaces.gateway, "inspectLegacyContainer", "gateway"); + + requireSupported("workload", surfaces.workload); + validateWorkloadProfile(providerId, surfaces.workload); + + if (surfaces.lifecycle.supported === true) { + if (!CHANNEL_STOP_TRANSPORTS.has(String(surfaces.lifecycle.channelStopTransport))) { + throw new RuntimeProviderRegistrationError( + `lifecycle for '${providerId}' has an invalid channel-stop transport`, + ); + } + requireFunction(surfaces.lifecycle, "start", "lifecycle"); + requireFunction(surfaces.lifecycle, "stop", "lifecycle"); + } + if (surfaces.mutationAuthority.supported === true) { + const operations = surfaces.mutationAuthority.operations; + if ( + !Array.isArray(operations) || + operations.length === 0 || + operations.some((operation) => !MUTATION_OPERATIONS.has(operation)) || + new Set(operations).size !== operations.length + ) { + throw new RuntimeProviderRegistrationError( + `mutationAuthority for '${providerId}' must list unique valid operations`, + ); + } + } + if (surfaces.bootstrap.supported === true) { + requireFunction(surfaces.bootstrap, "prepare", "bootstrap"); + } + if (surfaces.snapshot.supported === true) { + requireFunction(surfaces.snapshot, "capture", "snapshot"); + requireFunction(surfaces.snapshot, "restore", "snapshot"); + } + if (surfaces.recovery.supported === true) { + requireFunction(surfaces.recovery, "recover", "recovery"); + } + if (surfaces.cleanup.supported === true) { + requireFunction(surfaces.cleanup, "prepareDestroy", "cleanup"); + requireFunction(surfaces.cleanup, "removeOwnedWorkload", "cleanup"); + } + if (surfaces.containerEngine.supported === true) { + const identities = surfaces.containerEngine.identities; + if (!Array.isArray(identities)) { + throw new RuntimeProviderRegistrationError( + `containerEngine for '${providerId}' must list operation-scoped identities`, + ); + } + const operations: unknown[] = []; + for (const identity of identities) { + if (!isPlainRecord(identity)) { + throw new RuntimeProviderRegistrationError( + `containerEngine for '${providerId}' has an invalid identity`, + ); + } + if ( + !CONTAINER_ENGINE_OPERATIONS.has( + identity.operation as RuntimeProviderContainerEngineOperation, + ) + ) { + throw new RuntimeProviderRegistrationError( + `containerEngine for '${providerId}' has an invalid operation`, + ); + } + requireNonEmptyString(identity, "engineId", "containerEngine identity"); + requireNonEmptyString(identity, "displayName", "containerEngine identity"); + operations.push(identity.operation); + } + if (new Set(operations).size !== operations.length) { + throw new RuntimeProviderRegistrationError( + `containerEngine for '${providerId}' has duplicate operation identities`, + ); + } + } + + if (surfaces.plan.gatewayLauncher !== surfaces.gateway.launcher) { + throw new RuntimeProviderRegistrationError( + `plan and gateway launcher disagree for '${providerId}'`, + ); + } + if ( + surfaces.capabilities.directLifecycle !== (surfaces.lifecycle.supported === true) || + surfaces.capabilities.workloadImageCleanup !== (surfaces.cleanup.supported === true) || + surfaces.capabilities.legacyGatewayContainerInspection !== + surfaces.gateway.inspectLegacyContainer + ) { + throw new RuntimeProviderRegistrationError( + `capabilities disagree with registered surfaces for '${providerId}'`, + ); + } +} + +function validateBundle(key: string, value: RuntimeProviderBundle): void { + if (!validProviderId(key)) { + throw new RuntimeProviderRegistrationError(`unsupported provider key '${key}'`); + } + if (!isPlainRecord(value)) { + throw new RuntimeProviderRegistrationError(`bundle '${key}' must be a plain record`); + } + const identity = requireOwnRecord(value, "identity"); + if ( + identity.contractVersion !== RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION || + identity.id !== key || + typeof identity.displayName !== "string" || + identity.displayName.trim() === "" + ) { + throw new RuntimeProviderRegistrationError( + `bundle key '${key}' does not match a valid contract-v1 identity`, + ); + } + const surfaces = {} as Record<(typeof BUNDLE_SURFACES)[number], Record>; + for (const name of BUNDLE_SURFACES) { + const surface = requireOwnRecord(value, name); + validateBoundSurface(key, name, surface); + surfaces[name] = surface; + } + validateSupportedSurfaceSchemas(key, surfaces); +} + +export function createRuntimeProviderBundleRegistry( + entries: readonly (readonly [string, RuntimeProviderBundle])[], +): RuntimeProviderBundleRegistry { + const registry: Record = Object.create(null); + for (const [key, bundle] of entries) { + if (Object.hasOwn(registry, key)) { + throw new RuntimeProviderRegistrationError(`duplicate provider identity '${key}'`); + } + validateBundle(key, bundle); + registry[key] = cloneAndFreeze(bundle); + } + return Object.freeze(registry); +} + +export function normalizeRuntimeProviderIdentity(driverName: string | null | undefined): string { + const normalized = driverName?.trim().toLowerCase(); + return !normalized || normalized === "vm" ? "docker" : normalized; +} + +export function resolveRuntimeProviderBundle( + driverName: string | null | undefined, + providers: RuntimeProviderBundleRegistry, +): RuntimeProviderBundle | null { + const providerId = normalizeRuntimeProviderIdentity(driverName); + if (!validProviderId(providerId) || !Object.hasOwn(providers, providerId)) return null; + const bundle = providers[providerId]; + if (!bundle) return null; + validateBundle(providerId, bundle); + return bundle; +} + +export function requireRuntimeProviderBundle( + driverName: string | null | undefined, + providers: RuntimeProviderBundleRegistry, +): RuntimeProviderBundle { + const providerId = normalizeRuntimeProviderIdentity(driverName); + const bundle = resolveRuntimeProviderBundle(providerId, providers); + if (!bundle) { + throw new RuntimeProviderSelectionError( + `Runtime provider '${providerId}' is not registered for this operation.`, + ); + } + return bundle; +} + +export function requireRuntimeProviderBundleForSandbox( + sandbox: Pick, + providers: RuntimeProviderBundleRegistry, +): RuntimeProviderBundle { + return requireRuntimeProviderBundle(sandbox.openshellDriver, providers); +} + +export function requireRuntimeProviderMutationAuthority( + bundle: RuntimeProviderBundle, + operation: RuntimeProviderMutationOperation, +): void { + const authority = bundle.mutationAuthority; + if (authority.supported !== true || !authority.operations.includes(operation)) { + throw new RuntimeProviderSelectionError( + `Runtime provider '${bundle.identity.id}' does not authorize '${operation}' mutation.`, + ); + } +} + +export function runtimeProviderContainerEngineIdentity( + bundle: RuntimeProviderBundle, + operation: RuntimeProviderContainerEngineOperation, +): { readonly engineId: string; readonly displayName: string } | null { + if (bundle.containerEngine.supported !== true) return null; + const identity = bundle.containerEngine.identities.find( + (candidate) => candidate.operation === operation, + ); + return identity ? { engineId: identity.engineId, displayName: identity.displayName } : null; +} + +function boundedString(value: unknown, maxBytes: number): value is string { + return ( + typeof value === "string" && value.trim() !== "" && Buffer.byteLength(value, "utf8") <= maxBytes + ); +} + +export function normalizeRuntimeProviderRuntimeReceipt( + value: unknown, +): RuntimeProviderRuntimeReceipt | null { + if (!isPlainRecord(value) || value.schemaVersion !== 1 || !validProviderId(value.providerId)) { + return null; + } + if (!isPlainRecord(value.runtime)) return null; + if ( + !boundedString(value.runtime.kind, 128) || + !boundedString(value.runtime.handle, MAX_RECEIPT_HANDLE_BYTES) + ) { + return null; + } + if (!isPlainRecord(value.acceleration)) return null; + const runtime = { kind: value.runtime.kind, handle: value.runtime.handle }; + if (value.acceleration.kind === "none") { + return { + schemaVersion: 1, + providerId: value.providerId, + runtime, + acceleration: { kind: "none" }, + }; + } + if ( + value.acceleration.kind !== "gpu" || + !boundedString(value.acceleration.vendor, 128) || + !Array.isArray(value.acceleration.devices) || + value.acceleration.devices.length === 0 || + value.acceleration.devices.length > MAX_RECEIPT_DEVICES || + !value.acceleration.devices.every((device) => boundedString(device, 512)) || + new Set(value.acceleration.devices).size !== value.acceleration.devices.length + ) { + return null; + } + return { + schemaVersion: 1, + providerId: value.providerId, + runtime, + acceleration: { + kind: "gpu", + vendor: value.acceleration.vendor, + devices: [...value.acceleration.devices], + }, + }; +} diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts new file mode 100644 index 00000000000..12d4592be3e --- /dev/null +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -0,0 +1,495 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; +import { + createInMemoryRuntimeProviderBundle, + type InMemoryRuntimeProviderBundle, +} from "../../../../test/helpers/runtime-provider-bundle"; +import { removeSandboxImage } from "../../actions/sandbox/destroy"; +import { startSandbox } from "../../actions/sandbox/start"; +import { stopSandbox } from "../../actions/sandbox/stop"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; +import { cloneSandboxWorkloadReceipt } from "../../state/registry/workload"; +import type { RuntimeProviderBundle, RuntimeProviderWorkloadProfile } from "./contract"; +import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "./current"; +import { + createRuntimeProviderBundleRegistry, + normalizeRuntimeProviderRuntimeReceipt, + RuntimeProviderRegistrationError, + resolveRuntimeProviderBundle, +} from "./registry"; + +const PORTABLE_PROFILE = { + support: { + exactDigestReferences: true, + platforms: ["linux/amd64", "linux/arm64"], + startupProfileContractVersions: [1], + capabilityContractVersions: [1], + }, + hostArchitectures: ["amd64", "arm64"], + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: true, +} as const satisfies RuntimeProviderWorkloadProfile; + +const ENCODED_PROFILE = Buffer.from('{"schemaVersion":1}', "utf8").toString("base64url"); +const PROFILE_SHA256 = createHash("sha256").update(ENCODED_PROFILE, "utf8").digest("hex"); +const MANAGED_RECEIPT = { + schemaVersion: 1, + kind: "managed-image", + reference: `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`, + platform: "linux/arm64", + release: "v0.0.97", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: ENCODED_PROFILE, + startupProfileSha256: PROFILE_SHA256, + credentialProxyReplayRequired: false, + shared: true, +} as const satisfies SandboxWorkloadReceipt; + +function mxcBundle(): InMemoryRuntimeProviderBundle { + return createInMemoryRuntimeProviderBundle({ + providerId: "mxc", + workloadProfile: PORTABLE_PROFILE, + }); +} + +function replaceSurface( + bundle: RuntimeProviderBundle, + surface: keyof RuntimeProviderBundle, + value: unknown, +): RuntimeProviderBundle { + return { ...bundle, [surface]: value } as RuntimeProviderBundle; +} + +function expectSupportedSurface( + surface: T, +): asserts surface is Extract { + expect(surface.supported).toBe(true); +} + +describe("RuntimeProviderBundle registry contract", () => { + it("keeps the production selectable set limited to complete Docker and Kubernetes bundles", () => { + expect(Object.keys(CURRENT_RUNTIME_PROVIDER_BUNDLES)).toEqual(["docker", "kubernetes"]); + for (const [providerId, bundle] of Object.entries(CURRENT_RUNTIME_PROVIDER_BUNDLES)) { + expect(bundle.identity.id).toBe(providerId); + expect( + [ + bundle.plan, + bundle.capabilities, + bundle.preflightDoctor, + bundle.gateway, + bundle.workload, + bundle.lifecycle, + bundle.mutationAuthority, + bundle.bootstrap, + bundle.snapshot, + bundle.recovery, + bundle.cleanup, + bundle.containerEngine, + ].every((surface) => surface.providerId === providerId), + ).toBe(true); + expect(bundle.bootstrap).toMatchObject({ supported: false }); + expect(bundle.snapshot).toMatchObject({ supported: false }); + expect(bundle.recovery).toMatchObject({ supported: false }); + } + }); + + it("deeply clones and freezes every registered nested value", () => { + const source = mxcBundle(); + const registry = createRuntimeProviderBundleRegistry([["mxc", source]]); + const registered = registry.mxc!; + + expect(Object.isFrozen(registry)).toBe(true); + expect(Object.isFrozen(registered)).toBe(true); + expect(Object.isFrozen(registered.workload.profile)).toBe(true); + expect(Object.isFrozen(registered.workload.profile.support?.platforms)).toBe(true); + expectSupportedSurface(registered.lifecycle); + expect(Object.isFrozen(registered.lifecycle.start)).toBe(true); + expect(registered).not.toBe(source); + expect(registered.lifecycle.start).not.toBe(source.lifecycle.start); + expect(() => { + (registered.workload.profile.hostArchitectures as string[]).push("s390x"); + }).toThrow(TypeError); + expect(() => { + (registered.capabilities as { directLifecycle: boolean }).directLifecycle = false; + }).toThrow(TypeError); + }); + + it("rejects an omitted managed platform without changing legacy receipt acceptance", () => { + const { platform: _omittedPlatform, ...managedWithoutPlatform } = MANAGED_RECEIPT; + const persistedManaged = cloneSandboxWorkloadReceipt(managedWithoutPlatform); + const legacy = { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "owned:tag", + shared: false, + } as const satisfies SandboxWorkloadReceipt; + const docker = CURRENT_RUNTIME_PROVIDER_BUNDLES.docker!; + const inMemory = mxcBundle(); + + expect(persistedManaged).toEqual(managedWithoutPlatform); + expect(docker.workload.acceptsReceipt(persistedManaged)).toBe(false); + expect(inMemory.workload.acceptsReceipt(persistedManaged)).toBe(false); + expect(docker.workload.acceptsReceipt(legacy)).toBe(true); + expect(inMemory.workload.acceptsReceipt(legacy)).toBe(true); + }); + + it("rejects duplicates, key/identity mismatch, inherited keys, and unknown durable identity", () => { + const bundle = mxcBundle(); + expect(() => + createRuntimeProviderBundleRegistry([ + ["mxc", bundle], + ["mxc", bundle], + ]), + ).toThrow(/duplicate provider identity/u); + expect(() => createRuntimeProviderBundleRegistry([["other", bundle]])).toThrow( + /does not match/u, + ); + expect(() => createRuntimeProviderBundleRegistry([["constructor", bundle]])).toThrow( + /unsupported provider key/u, + ); + const registry = createRuntimeProviderBundleRegistry([["mxc", bundle]]); + expect(resolveRuntimeProviderBundle("toString", registry)).toBeNull(); + expect(resolveRuntimeProviderBundle("future-runtime", registry)).toBeNull(); + }); + + it("rejects a missing surface and every surface identity mismatch", () => { + const bundle = mxcBundle(); + const { cleanup: _cleanup, ...missingCleanup } = bundle; + expect(() => + createRuntimeProviderBundleRegistry([["mxc", missingCleanup as RuntimeProviderBundle]]), + ).toThrow(/missing cleanup surface/u); + + for (const surface of [ + "plan", + "capabilities", + "preflightDoctor", + "gateway", + "workload", + "lifecycle", + "mutationAuthority", + "bootstrap", + "snapshot", + "recovery", + "cleanup", + "containerEngine", + ] as const) { + expect(() => + createRuntimeProviderBundleRegistry([ + [ + "mxc", + replaceSurface(bundle, surface, { + ...bundle[surface], + providerId: "other", + }), + ], + ]), + ).toThrow(new RegExp(`${surface} identity`, "u")); + } + }); + + it.each([ + [ + "plan", + (bundle: RuntimeProviderBundle) => ({ + ...bundle.plan, + gatewayLauncher: "invalid", + }), + ], + [ + "capabilities", + (bundle: RuntimeProviderBundle) => { + const { directLifecycle: _directLifecycle, ...incomplete } = bundle.capabilities; + return incomplete; + }, + ], + [ + "preflightDoctor", + (bundle: RuntimeProviderBundle) => { + const { inspectHost: _inspectHost, ...incomplete } = bundle.preflightDoctor; + return incomplete; + }, + ], + [ + "gateway", + (bundle: RuntimeProviderBundle) => ({ + ...bundle.gateway, + launcher: "invalid", + }), + ], + [ + "workload", + (bundle: RuntimeProviderBundle) => ({ + ...bundle.workload, + profile: { ...bundle.workload.profile, hostArchitectures: ["amd64", "amd64"] }, + }), + ], + [ + "lifecycle", + (_bundle: RuntimeProviderBundle) => { + const { stop: _stop, ...incomplete } = mxcBundle().lifecycle; + return incomplete; + }, + ], + [ + "mutationAuthority", + (bundle: RuntimeProviderBundle) => ({ + ...bundle.mutationAuthority, + operations: ["not-an-operation"], + }), + ], + [ + "bootstrap", + (bundle: RuntimeProviderBundle) => ({ + ...bundle.bootstrap, + supported: true, + reason: undefined, + }), + ], + [ + "snapshot", + (bundle: RuntimeProviderBundle) => ({ + ...bundle.snapshot, + supported: true, + capture: () => undefined, + }), + ], + [ + "recovery", + (bundle: RuntimeProviderBundle) => ({ + ...bundle.recovery, + supported: true, + }), + ], + [ + "cleanup", + (_bundle: RuntimeProviderBundle) => { + const { removeOwnedWorkload: _removeOwnedWorkload, ...incomplete } = mxcBundle().cleanup; + return incomplete; + }, + ], + [ + "containerEngine", + (bundle: RuntimeProviderBundle) => ({ + ...bundle.containerEngine, + identities: [ + { + operation: "invalid-operation", + engineId: "", + displayName: "Broken", + }, + ], + }), + ], + ] as const)("rejects a runtime-cast incomplete or invalid supported %s surface", (surface, mutate) => { + const bundle = mxcBundle(); + expect(() => + createRuntimeProviderBundleRegistry([ + ["mxc", replaceSurface(bundle, surface, mutate(bundle))], + ]), + ).toThrow(RuntimeProviderRegistrationError); + }); + + it("rejects capability/surface drift and duplicate operation-scoped engine identities", () => { + const bundle = mxcBundle(); + expect(() => + createRuntimeProviderBundleRegistry([ + [ + "mxc", + replaceSurface(bundle, "capabilities", { + ...bundle.capabilities, + directLifecycle: false, + }), + ], + ]), + ).toThrow(/capabilities disagree/u); + expect(() => + createRuntimeProviderBundleRegistry([ + [ + "mxc", + replaceSurface(bundle, "containerEngine", { + ...bundle.containerEngine, + identities: [ + ...bundle.containerEngine.identities, + bundle.containerEngine.identities[0], + ], + }), + ], + ]), + ).toThrow(/duplicate operation identities/u); + }); + + it("normalizes bounded opaque runtime receipts and rejects duplicate GPU devices", () => { + const receipt = { + schemaVersion: 1, + providerId: "mxc", + runtime: { kind: "sandbox", handle: "opaque-123" }, + acceleration: { kind: "gpu", vendor: "test", devices: ["gpu0", "gpu1"] }, + }; + expect(normalizeRuntimeProviderRuntimeReceipt(receipt)).toEqual(receipt); + expect( + normalizeRuntimeProviderRuntimeReceipt({ + ...receipt, + acceleration: { ...receipt.acceleration, devices: ["gpu0", "gpu0"] }, + }), + ).toBeNull(); + expect( + normalizeRuntimeProviderRuntimeReceipt({ + ...receipt, + runtime: { ...receipt.runtime, handle: "x".repeat(4097) }, + }), + ).toBeNull(); + }); +}); + +describe("sandbox workload ownership receipt", () => { + it("clones the complete immutable managed-image ownership identity", () => { + const cloned = cloneSandboxWorkloadReceipt(MANAGED_RECEIPT); + + expect(cloned).toEqual(MANAGED_RECEIPT); + expect(cloned).not.toBe(MANAGED_RECEIPT); + }); + + it.each([ + { sourceCohort: "run-123456" }, + { reference: "ghcr.io/nvidia/nemoclaw/openclaw-sandbox:latest" }, + { platform: "linux/s390x" }, + { release: "latest" }, + { capabilityContractVersion: 2 }, + { startupProfileContractVersion: 2 }, + { startupProfileSha256: "not-a-digest" }, + { encodedProfile: `${ENCODED_PROFILE}=` }, + { encodedProfile: Buffer.from("different", "utf8").toString("base64url") }, + { corporateCaB64: "not canonical base64" }, + { shared: false }, + ])("drops malformed managed ownership evidence: %o", (drift) => { + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + ...drift, + } as unknown as SandboxWorkloadReceipt), + ).toBeUndefined(); + }); + + it("retains an owned legacy image receipt independently from managed cohorts", () => { + expect( + cloneSandboxWorkloadReceipt({ + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "nemoclaw-sandbox-local:build-123", + shared: false, + }), + ).toEqual({ + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "nemoclaw-sandbox-local:build-123", + shared: false, + }); + }); + + it("rejects an empty or falsely shared legacy ownership receipt", () => { + expect( + cloneSandboxWorkloadReceipt({ + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "", + shared: false, + }), + ).toBeUndefined(); + expect( + cloneSandboxWorkloadReceipt({ + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "owned:tag", + shared: true, + } as unknown as SandboxWorkloadReceipt), + ).toBeUndefined(); + }); +}); + +describe("socket-free MXC action contract", () => { + const agents = ["openclaw", "hermes", "langchain-deepagents-code"] as const; + + it.each(agents)("routes %s lifecycle and cleanup through one injected bundle", async (agent) => { + const state = { + events: [] as string[], + running: new Set(), + workloads: new Set(), + }; + const bundle = createInMemoryRuntimeProviderBundle({ + providerId: "mxc", + workloadProfile: PORTABLE_PROFILE, + state, + }); + const providers = createRuntimeProviderBundleRegistry([["mxc", bundle]]); + const sandboxName = `${agent}-sandbox`; + const imageTag = `mxc-memory:${agent}`; + const entry: SandboxEntry = { + name: sandboxName, + agent, + openshellDriver: "mxc", + imageTag, + workload: { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: imageTag, + shared: false, + }, + }; + state.workloads.add(imageTag); + const getSandbox = vi.fn(() => entry); + const probeSandbox = vi.fn(async () => undefined); + const stopSandboxChannels = vi.fn(); + const teardownSandboxDashboardForward = vi.fn(); + + await expect( + startSandbox(sandboxName, { + getSandbox, + runtimeProviders: providers, + probeSandbox, + log: vi.fn(), + }), + ).resolves.toEqual({ exitCode: 0 }); + expect( + stopSandbox(sandboxName, { + getSandbox, + runtimeProviders: providers, + stopSandboxChannels, + teardownSandboxDashboardForward, + log: vi.fn(), + warn: vi.fn(), + }), + ).toEqual({ exitCode: 0 }); + expect( + removeSandboxImage(sandboxName, { + getSandbox, + runtimeProviders: providers, + log: vi.fn(), + warn: vi.fn(), + }), + ).toEqual({ + status: "removed", + engineDisplayName: "In-memory", + reference: imageTag, + }); + + expect(probeSandbox).toHaveBeenCalledWith(sandboxName); + expect(stopSandboxChannels).toHaveBeenCalledWith( + sandboxName, + expect.objectContaining({ info: expect.any(Function), warn: expect.any(Function) }), + ); + expect(state.events).toEqual([ + `start:${sandboxName}`, + `stop:${sandboxName}`, + `cleanup:${sandboxName}`, + ]); + expect(state.running).not.toContain(sandboxName); + expect(state.workloads).not.toContain(imageTag); + }); +}); diff --git a/src/lib/onboard/sandbox-registration.test.ts b/src/lib/onboard/sandbox-registration.test.ts index 12f31c6ac50..ef14dc58bb2 100644 --- a/src/lib/onboard/sandbox-registration.test.ts +++ b/src/lib/onboard/sandbox-registration.test.ts @@ -449,7 +449,7 @@ describe("registerCreatedSandbox", () => { it("passes the built entry to the supplied registry writer", () => { const registerSandbox = vi.fn(); - const entry = registerCreatedSandbox({ + const input = { sandboxName: "demo", inferenceSelection: { model: "llama", @@ -465,6 +465,12 @@ describe("registerCreatedSandbox", () => { agent: null, agentVersionKnown: true, imageTag: null, + workload: { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: null, + shared: false, + }, openclawImagePluginInstalls: [], appliedPolicies: [], plannedMessagingState: undefined, @@ -474,10 +480,52 @@ describe("registerCreatedSandbox", () => { gatewayName: "nemoclaw", gatewayPort: 8080, registerSandbox, - }); + } satisfies Parameters[0]; + const entry = registerCreatedSandbox(input); expect(registerSandbox).toHaveBeenCalledWith(entry); expect(entry.name).toBe("demo"); expect(entry.openclawImagePluginInstalls).toEqual([]); + expect(entry.workload).toEqual(input.workload); + expect(() => + registerCreatedSandbox({ + ...input, + workload: { ...input.workload, reference: "" }, + }), + ).toThrow(/workload ownership receipt failed closed validation/u); + expect(registerSandbox).toHaveBeenCalledTimes(1); + }); + + it("fails before registry mutation for an unknown durable provider identity", () => { + const registerSandbox = vi.fn(); + + expect(() => + registerCreatedSandbox({ + sandboxName: "demo", + inferenceSelection: { + model: "llama", + provider: "openai-compatible", + endpointUrl: null, + credentialEnv: null, + preferredInferenceApi: null, + compatibleEndpointReasoning: null, + compatibleEndpointReasoningEffort: null, + nimContainer: null, + }, + runtimeFields: { ...runtimeFields, openshellDriver: "unknown-runtime" }, + agent: null, + agentVersionKnown: true, + imageTag: null, + appliedPolicies: [], + plannedMessagingState: undefined, + hermesToolGateways: [], + hermesDashboardState: { enabled: false, config: null }, + dashboardPort: 18789, + gatewayName: "nemoclaw", + gatewayPort: 8080, + registerSandbox, + }), + ).toThrow(/not registered/u); + expect(registerSandbox).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/onboard/sandbox-registration.ts b/src/lib/onboard/sandbox-registration.ts index 86bfe430a13..a4d6b181e78 100644 --- a/src/lib/onboard/sandbox-registration.ts +++ b/src/lib/onboard/sandbox-registration.ts @@ -16,8 +16,16 @@ import type { SandboxMessagingState, } from "../state/registry"; import * as registry from "../state/registry"; +import { cloneSandboxWorkloadReceipt } from "../state/registry/workload"; import { DEFAULT_TOOL_DISCLOSURE, type ToolDisclosure } from "../tool-disclosure"; import type { DcodeAutoApprovalMode } from "./dcode-auto-approval"; +import { + CURRENT_RUNTIME_PROVIDER_BUNDLES, + RuntimeProviderBundleRegistry, + requireRuntimeProviderBundleForSandbox, + requireRuntimeProviderMutationAuthority, + RuntimeProviderSelectionError, +} from "./runtime-provider/access"; import { getHermesDashboardRegistryFields, type HermesDashboardOnboardState, @@ -43,6 +51,7 @@ export interface CreatedSandboxRegistryEntryInput { agent: AgentDefinition | null | undefined; agentVersionKnown: boolean; imageTag: string | null; + workload?: SandboxEntry["workload"]; openclawImagePluginInstalls?: readonly OpenClawImagePluginInstall[]; appliedPolicies: string[]; toolDisclosure?: ToolDisclosure; @@ -72,6 +81,7 @@ export interface CreatedSandboxRegistryEntryInput { export interface CreatedSandboxRegistrationInput extends CreatedSandboxRegistryEntryInput { registerSandbox?(entry: SandboxEntry): void; + runtimeProviders?: RuntimeProviderBundleRegistry; } export function creationFidelity( @@ -166,6 +176,12 @@ export function buildCreatedSandboxRegistryEntry( input.plannedMessagingState?.plan.sandboxName === input.sandboxName ? input.plannedMessagingState : undefined; + const workload = cloneSandboxWorkloadReceipt(input.workload); + if (input.workload !== undefined && workload === undefined) { + throw new RuntimeProviderSelectionError( + "Sandbox workload ownership receipt failed closed validation.", + ); + } return { name: input.sandboxName, @@ -173,6 +189,7 @@ export function buildCreatedSandboxRegistryEntry( ...input.runtimeFields, ...getSandboxAgentRegistryFields(input.agent, input.agentVersionKnown), imageTag: input.imageTag, + workload, ...(input.openclawImagePluginInstalls !== undefined ? { openclawImagePluginInstalls: input.openclawImagePluginInstalls.map((install) => ({ @@ -210,6 +227,16 @@ export function buildCreatedSandboxRegistryEntry( export function registerCreatedSandbox(input: CreatedSandboxRegistrationInput): SandboxEntry { const entry = buildCreatedSandboxRegistryEntry(input); + const provider = requireRuntimeProviderBundleForSandbox( + entry, + input.runtimeProviders ?? CURRENT_RUNTIME_PROVIDER_BUNDLES, + ); + requireRuntimeProviderMutationAuthority(provider, "registration"); + if (!provider.workload.acceptsReceipt(entry.workload)) { + throw new RuntimeProviderSelectionError( + `Runtime provider '${provider.identity.id}' does not accept the registered workload receipt.`, + ); + } (input.registerSandbox ?? registry.registerSandbox)(entry); return entry; } diff --git a/src/lib/onboard/sandbox-workload-preparation.test.ts b/src/lib/onboard/sandbox-workload-preparation.test.ts index bdfdbad4e6f..a85056b3983 100644 --- a/src/lib/onboard/sandbox-workload-preparation.test.ts +++ b/src/lib/onboard/sandbox-workload-preparation.test.ts @@ -19,12 +19,14 @@ import { SHIPPED_MANAGED_IMAGE_AGENTS, type ShippedManagedImageAgent, } from "./managed-image/contract"; +import { createRuntimeProviderBundleRegistry } from "./runtime-provider/registry"; import { prepareSandboxWorkloadSource, SandboxWorkloadPreparationError, } from "./workload/preparation"; import { resolveSandboxWorkloadRuntimeCapabilities } from "./workload/runtime"; import type { SandboxWorkloadRuntimeCapabilities } from "./workload/source"; +import { createInMemoryRuntimeProviderBundle } from "../../../test/helpers/runtime-provider-bundle"; const RELEASE = "v0.0.97"; const MANAGED_IMAGE_PLATFORM = MANAGED_IMAGE_PLATFORMS[0]; @@ -192,26 +194,30 @@ describe("sandbox workload preparation", () => { expect(resolveCatalog).not.toHaveBeenCalled(); }); - it.each([ - "podman", - "mxc", - ])("rejects custom Dockerfile preparation for buildless %s before catalog access (#7744)", async (driverName) => { + it("rejects custom Dockerfile preparation for a buildless provider before catalog access (#7744)", async () => { + const driverName = "buildless-test"; const resolveCatalog = vi.fn(async () => CATALOG); - const profiles = { - [driverName]: { - support: runtime(driverName).managedImages!, - hostArchitectures: ["amd64"], - managedImageSelectionPolicy: "require-managed" as const, - legacyDockerfileBuilds: false, - }, - }; + const providers = createRuntimeProviderBundleRegistry([ + [ + driverName, + createInMemoryRuntimeProviderBundle({ + providerId: driverName, + workloadProfile: { + support: runtime(driverName).managedImages!, + hostArchitectures: ["amd64"], + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + }, + }), + ], + ]); await expect( prepareSandboxWorkloadSource( { ...input("openclaw"), customDockerfilePath: "/workspace/CustomDockerfile", - runtime: resolveSandboxWorkloadRuntimeCapabilities({ driverName }, profiles, "x64"), + runtime: resolveSandboxWorkloadRuntimeCapabilities({ driverName }, providers, "x64"), }, { resolveCatalog }, ), @@ -227,7 +233,7 @@ describe("sandbox workload preparation", () => { { ...input("langchain-deepagents-code"), runtime: { - driverName: "podman", + driverName: "buildless-test", managedImageSelectionPolicy: "require-managed", legacyDockerfileBuilds: false, managedImages: null, @@ -367,7 +373,7 @@ describe("sandbox workload preparation", () => { it("prepares a managed image for an independently registered MXC-shaped runtime without branching (#7744)", async () => { const prepared = await prepareSandboxWorkloadSource( - { ...input("hermes"), runtime: runtime("mxc") }, + { ...input("hermes"), runtime: runtime("portable-test") }, { resolveCatalog: async () => CATALOG }, ); diff --git a/src/lib/onboard/sandbox-workload-runtime.test.ts b/src/lib/onboard/sandbox-workload-runtime.test.ts index ed51852d83d..cc5d6ff9122 100644 --- a/src/lib/onboard/sandbox-workload-runtime.test.ts +++ b/src/lib/onboard/sandbox-workload-runtime.test.ts @@ -7,11 +7,10 @@ import { MANAGED_IMAGE_PLATFORMS, MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, } from "./managed-image/contract"; -import { - CURRENT_MANAGED_IMAGE_RUNTIME_PROFILES, - type ManagedImageRuntimeProfileRegistry, - resolveSandboxWorkloadRuntimeCapabilities, -} from "./workload/runtime"; +import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "./runtime-provider/current"; +import { createRuntimeProviderBundleRegistry } from "./runtime-provider/registry"; +import { resolveSandboxWorkloadRuntimeCapabilities } from "./workload/runtime"; +import { createInMemoryRuntimeProviderBundle } from "../../../test/helpers/runtime-provider-bundle"; const AMD64_MANAGED_IMAGE_V1_SUPPORT = { exactDigestReferences: true, @@ -93,21 +92,25 @@ describe("sandbox workload runtime capabilities", () => { }); }); - it.each([ - "podman", - "mxc", - ])("lets the %s driver register the same portable contract without Docker coupling (#7744)", (driverName) => { - const profiles: ManagedImageRuntimeProfileRegistry = { - ...CURRENT_MANAGED_IMAGE_RUNTIME_PROFILES, - [driverName]: { - support: COMPLETE_MANAGED_IMAGE_V1_SUPPORT, - hostArchitectures: ["amd64"], - managedImageSelectionPolicy: "require-managed", - legacyDockerfileBuilds: false, - }, - }; + it("projects a complete portable bundle into workload capabilities (#7744)", () => { + const driverName = "portable-test"; + const providers = createRuntimeProviderBundleRegistry([ + ...Object.entries(CURRENT_RUNTIME_PROVIDER_BUNDLES), + [ + driverName, + createInMemoryRuntimeProviderBundle({ + providerId: driverName, + workloadProfile: { + support: COMPLETE_MANAGED_IMAGE_V1_SUPPORT, + hostArchitectures: ["amd64"], + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + }, + }), + ], + ]); - expect(resolveSandboxWorkloadRuntimeCapabilities({ driverName }, profiles, "x64")).toEqual({ + expect(resolveSandboxWorkloadRuntimeCapabilities({ driverName }, providers, "x64")).toEqual({ driverName, managedImageSelectionPolicy: "require-managed", legacyDockerfileBuilds: false, diff --git a/src/lib/onboard/workload/runtime.ts b/src/lib/onboard/workload/runtime.ts index a6fd8a61e4b..da63b28a9fd 100644 --- a/src/lib/onboard/workload/runtime.ts +++ b/src/lib/onboard/workload/runtime.ts @@ -2,62 +2,46 @@ // SPDX-License-Identifier: Apache-2.0 import type { OpenShellComputePlan } from "../compute/plan"; +import { managedImagePlatformForNodeArchitecture } from "../managed-image/contract"; import { - MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, - MANAGED_IMAGE_PLATFORMS, - MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, - managedImagePlatformForNodeArchitecture, -} from "../managed-image/contract"; -import type { ManagedImageSelectionPolicy, SandboxWorkloadRuntimeCapabilities } from "./source"; + CURRENT_RUNTIME_PROVIDER_BUNDLES, + RuntimeProviderBundleRegistry, + RuntimeProviderManagedImageSupport, + RuntimeProviderWorkloadProfile, + resolveRuntimeProviderBundle, +} from "../runtime-provider/access"; +import type { SandboxWorkloadRuntimeCapabilities } from "./source"; -export type ManagedImageRuntimeSupport = NonNullable< - SandboxWorkloadRuntimeCapabilities["managedImages"] ->; - -export interface ManagedImageRuntimeProfile { - readonly support: ManagedImageRuntimeSupport | null; - /** OCI host architectures for which the published image cohort is complete. */ - readonly hostArchitectures: readonly string[]; - readonly managedImageSelectionPolicy: ManagedImageSelectionPolicy; - readonly legacyDockerfileBuilds: boolean; -} +export type ManagedImageRuntimeSupport = RuntimeProviderManagedImageSupport; +export type ManagedImageRuntimeProfile = RuntimeProviderWorkloadProfile; /** * Managed-image capabilities are registered by OpenShell compute-driver - * identity instead of inferred from the gateway launcher. Podman and a future - * MXC runtime can register this contract without inheriting Docker lifecycle + * identity instead of inferred from the gateway launcher. A future provider + * can register this contract without inheriting another provider's lifecycle * code. */ export type ManagedImageRuntimeProfileRegistry = Readonly< Record >; -const COMPLETE_MANAGED_IMAGE_V1_SUPPORT = { - exactDigestReferences: true, - platforms: MANAGED_IMAGE_PLATFORMS, - startupProfileContractVersions: [MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION], - capabilityContractVersions: [MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION], -} as const satisfies ManagedImageRuntimeSupport; +/** Compatibility view only; RuntimeProviderBundle is the registration source. */ +export function projectRuntimeProviderWorkloadProfiles( + providers: RuntimeProviderBundleRegistry, +): ManagedImageRuntimeProfileRegistry { + return Object.freeze( + Object.fromEntries( + Object.keys(providers).map((providerId) => [ + providerId, + providers[providerId]?.workload.profile, + ]), + ), + ); +} -/** - * PR #7747 established Docker as the first explicit OpenShell compute-driver - * identity. Additional runtimes can register the same workload contract here; - * selection remains independent from driver lifecycle. - */ -export const CURRENT_MANAGED_IMAGE_RUNTIME_PROFILES = { - docker: { - support: COMPLETE_MANAGED_IMAGE_V1_SUPPORT, - hostArchitectures: ["amd64", "arm64"], - managedImageSelectionPolicy: "require-managed", - legacyDockerfileBuilds: true, - }, - kubernetes: { - support: null, - hostArchitectures: [], - managedImageSelectionPolicy: "prefer-managed", - legacyDockerfileBuilds: true, - }, -} as const satisfies ManagedImageRuntimeProfileRegistry; +export const CURRENT_MANAGED_IMAGE_RUNTIME_PROFILES = projectRuntimeProviderWorkloadProfiles( + CURRENT_RUNTIME_PROVIDER_BUNDLES, +); function hostOciArchitecture(nodeArchitecture: string): string { if (nodeArchitecture === "x64") return "amd64"; @@ -78,10 +62,10 @@ function cloneRuntimeSupport( export function resolveSandboxWorkloadRuntimeCapabilities( plan: Pick, - profiles: ManagedImageRuntimeProfileRegistry = CURRENT_MANAGED_IMAGE_RUNTIME_PROFILES, + providers: RuntimeProviderBundleRegistry = CURRENT_RUNTIME_PROVIDER_BUNDLES, nodeArchitecture: string = process.arch, ): SandboxWorkloadRuntimeCapabilities { - const profile = Object.hasOwn(profiles, plan.driverName) ? profiles[plan.driverName] : undefined; + const profile = resolveRuntimeProviderBundle(plan.driverName, providers)?.workload.profile; const support = profile?.support; const hostPlatform = managedImagePlatformForNodeArchitecture(nodeArchitecture); const supportedHost = diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index f7809ad28ee..a4736e443f2 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -16,6 +16,7 @@ import { } from "./extra-providers"; import { withLock } from "./registry/lock"; import { load, save } from "./registry/persistence"; +import { cloneSandboxWorkloadReceipt } from "./registry/workload"; import { normalizeSandboxMcpState } from "./registry-mcp"; import { normalizeBaselineExclusions, @@ -70,6 +71,7 @@ export type { SandboxGpuProofResult, SandboxGpuProofStatus, SandboxRegistry, + SandboxWorkloadReceipt, } from "./registry/types"; export type { McpBridgeEntry, SandboxMcpState } from "./registry-mcp"; @@ -162,6 +164,7 @@ export function registerSandbox(entry: SandboxEntry): void { ? entry.hermesAuthMethod : null, imageTag: entry.imageTag || null, + workload: cloneSandboxWorkloadReceipt(entry.workload), messaging: cloneSandboxMessagingState(entry.messaging), mcp: normalizeSandboxMcpState(entry.mcp), hermesToolGateways: diff --git a/src/lib/state/registry/persistence.ts b/src/lib/state/registry/persistence.ts index 7566520f816..d836bf3b3ac 100644 --- a/src/lib/state/registry/persistence.ts +++ b/src/lib/state/registry/persistence.ts @@ -20,6 +20,7 @@ import { import * as reversibleRemoval from "../registry-reversible-removal"; import { nemoclawStateRoot } from "../state-root"; import type { SandboxEntry, SandboxRegistry } from "./types"; +import { cloneSandboxWorkloadReceipt } from "./workload"; export const REGISTRY_FILE = path.join( nemoclawStateRoot(process.env.HOME || "/tmp", GATEWAY_PORT), @@ -83,6 +84,7 @@ function serializeRegistryForDisk(data: SandboxRegistry): SandboxRegistry { function normalizeSandboxEntryForRuntime(entry: SandboxEntry): SandboxEntry { const messaging = cloneSandboxMessagingState(entry.messaging); + const workload = cloneSandboxWorkloadReceipt(entry.workload); const mcp = normalizeSandboxMcpState(entry.mcp); const baselineExclusions = normalizeBaselineExclusions(entry.baselineExclusions); const baselineExclusionTransition = normalizeBaselineExclusionTransition( @@ -90,6 +92,7 @@ function normalizeSandboxEntryForRuntime(entry: SandboxEntry): SandboxEntry { ); const { messaging: _messaging, + workload: _workload, mcp: _mcp, baselineExclusions: _baselineExclusions, baselineExclusionTransition: _baselineExclusionTransition, @@ -97,6 +100,7 @@ function normalizeSandboxEntryForRuntime(entry: SandboxEntry): SandboxEntry { } = entry; return { ...rest, + ...(workload ? { workload } : {}), ...(messaging ? { messaging } : {}), ...(mcp ? { mcp } : {}), ...(baselineExclusions ? { baselineExclusions } : {}), @@ -125,6 +129,7 @@ function serializeSandboxEntryForDisk(entry: SandboxEntry): SandboxEntry { providerCredentialHashes?: unknown; }; const messaging = serializeSandboxMessagingStateForDisk(durable.messaging); + const workload = cloneSandboxWorkloadReceipt(durable.workload); const mcp = serializeSandboxMcpStateForDisk(durable.mcp); const baselineExclusions = normalizeBaselineExclusions(durable.baselineExclusions); const baselineExclusionTransition = normalizeBaselineExclusionTransition( @@ -132,6 +137,7 @@ function serializeSandboxEntryForDisk(entry: SandboxEntry): SandboxEntry { ); const { messaging: _messaging, + workload: _workload, mcp: _mcp, baselineExclusions: _baselineExclusions, baselineExclusionTransition: _baselineExclusionTransition, @@ -140,6 +146,7 @@ function serializeSandboxEntryForDisk(entry: SandboxEntry): SandboxEntry { return { ...rest, ...(rest.dashboardPort === 0 ? { dashboardPort: null } : {}), + ...(workload ? { workload } : {}), ...(messaging ? { messaging } : {}), ...(mcp ? { mcp } : {}), ...(baselineExclusions ? { baselineExclusions } : {}), diff --git a/src/lib/state/registry/types.ts b/src/lib/state/registry/types.ts index 97b35871987..ac7ffa3dc69 100644 --- a/src/lib/state/registry/types.ts +++ b/src/lib/state/registry/types.ts @@ -119,6 +119,12 @@ export interface SandboxEntry extends Partial { fromDockerfile?: string | null; hermesAuthMethod?: "oauth" | "api_key" | null; imageTag?: string | null; + /** + * Durable source and ownership receipt for the workload behind imageTag. + * Managed images are immutable shared release artifacts and must never flow + * through per-sandbox image deletion. + */ + workload?: SandboxWorkloadReceipt; messaging?: SandboxMessagingState; mcp?: SandboxMcpState; hermesToolGateways?: string[]; @@ -141,6 +147,40 @@ export interface SandboxEntry extends Partial { gatewayPort?: number | null; } +export type SandboxWorkloadReceipt = + | { + readonly schemaVersion: 1; + readonly kind: "managed-image"; + readonly reference: string; + /** + * Exact OCI platform selected from the publication index. Receipts + * created before multi-architecture managed images may omit this field, + * but runtime providers must reject the ambiguous receipt rather than + * infer a platform. + */ + readonly platform?: "linux/amd64" | "linux/arm64"; + readonly release: string; + readonly sourceRevision: string; + /** Exact all-agent publication cohort that produced the immutable image. */ + readonly sourceCohort: string; + readonly capabilityContractVersion: number; + readonly startupProfileContractVersion: number; + /** Canonical, secret-free base64url profile transport used to start this image. */ + readonly encodedProfile: string; + readonly startupProfileSha256: string; + /** Re-acquire launch-only proxy credentials from the operator environment when cloning. */ + readonly credentialProxyReplayRequired: boolean; + /** Optional canonical standard-base64 public CA bundle bound by the profile digest. */ + readonly corporateCaB64?: string; + readonly shared: true; + } + | { + readonly schemaVersion: 1; + readonly kind: "legacy-dockerfile"; + readonly reference: string | null; + readonly shared: false; + }; + export interface SandboxRegistry { sandboxes: Record; defaultSandbox: string | null; diff --git a/src/lib/state/registry/workload.ts b/src/lib/state/registry/workload.ts new file mode 100644 index 00000000000..3520c249220 --- /dev/null +++ b/src/lib/state/registry/workload.ts @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; + +import type { SandboxWorkloadReceipt } from "./types"; + +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const REVISION_PATTERN = /^[0-9a-f]{40}$/u; +const COHORT_PATTERN = /^ghrun-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$/u; +const MANAGED_REFERENCE_PATTERN = + /^ghcr[.]io\/nvidia\/nemoclaw\/(?:openclaw|hermes|langchain-deepagents-code)-sandbox@sha256:[0-9a-f]{64}$/u; +const MANAGED_PLATFORMS = new Set(["linux/amd64", "linux/arm64"]); +const RELEASE_PATTERN = /^v[0-9]+(?:[.][0-9]+){1,3}(?:[-.][0-9A-Za-z][0-9A-Za-z.-]*)?$/u; +const MAX_COHORT_BYTES = 128; +const BASE64URL_PATTERN = /^[A-Za-z0-9_-]+$/u; +const STANDARD_BASE64_PATTERN = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u; +const MAX_PROFILE_BYTES = 64 * 1024; +const MAX_PROFILE_ENCODED_BYTES = Math.ceil(MAX_PROFILE_BYTES / 3) * 4; +const MAX_CORPORATE_CA_BYTES = 128 * 1024; +const MAX_CORPORATE_CA_ENCODED_BYTES = Math.ceil(MAX_CORPORATE_CA_BYTES / 3) * 4; + +function nonEmptyString(value: unknown): value is string { + return typeof value === "string" && value.trim() !== ""; +} + +function decodeCanonicalBase64Url(value: unknown): Buffer | null { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > MAX_PROFILE_ENCODED_BYTES || + value.length % 4 === 1 || + !BASE64URL_PATTERN.test(value) + ) { + return null; + } + const decoded = Buffer.from(value, "base64url"); + return decoded.length > 0 && + decoded.length <= MAX_PROFILE_BYTES && + decoded.toString("base64url") === value + ? decoded + : null; +} + +function decodeCanonicalStandardBase64(value: unknown): Buffer | null { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > MAX_CORPORATE_CA_ENCODED_BYTES || + !STANDARD_BASE64_PATTERN.test(value) + ) { + return null; + } + const decoded = Buffer.from(value, "base64"); + return decoded.length > 0 && + decoded.length <= MAX_CORPORATE_CA_BYTES && + decoded.toString("base64") === value + ? decoded + : null; +} + +export function cloneSandboxWorkloadReceipt( + value: SandboxWorkloadReceipt | undefined, +): SandboxWorkloadReceipt | undefined { + if (!value || value.schemaVersion !== 1) return undefined; + if (value.kind === "legacy-dockerfile") { + if (value.shared !== false || (value.reference !== null && !nonEmptyString(value.reference))) { + return undefined; + } + return { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: value.reference, + shared: false, + }; + } + if ( + value.kind !== "managed-image" || + value.shared !== true || + !MANAGED_REFERENCE_PATTERN.test(value.reference) || + (value.platform !== undefined && !MANAGED_PLATFORMS.has(value.platform)) || + !RELEASE_PATTERN.test(value.release) || + !REVISION_PATTERN.test(value.sourceRevision) || + typeof value.sourceCohort !== "string" || + Buffer.byteLength(value.sourceCohort, "utf8") > MAX_COHORT_BYTES || + !COHORT_PATTERN.test(value.sourceCohort) || + value.capabilityContractVersion !== 1 || + value.startupProfileContractVersion !== 1 || + !SHA256_PATTERN.test(value.startupProfileSha256) || + typeof value.credentialProxyReplayRequired !== "boolean" || + decodeCanonicalBase64Url(value.encodedProfile) === null || + createHash("sha256").update(value.encodedProfile, "utf8").digest("hex") !== + value.startupProfileSha256 || + (value.corporateCaB64 !== undefined && + decodeCanonicalStandardBase64(value.corporateCaB64) === null) + ) { + return undefined; + } + return { + schemaVersion: 1, + kind: "managed-image", + reference: value.reference, + ...(value.platform === undefined ? {} : { platform: value.platform }), + release: value.release, + sourceRevision: value.sourceRevision, + sourceCohort: value.sourceCohort, + capabilityContractVersion: value.capabilityContractVersion, + startupProfileContractVersion: value.startupProfileContractVersion, + encodedProfile: value.encodedProfile, + startupProfileSha256: value.startupProfileSha256, + credentialProxyReplayRequired: value.credentialProxyReplayRequired, + ...(value.corporateCaB64 === undefined ? {} : { corporateCaB64: value.corporateCaB64 }), + shared: true, + }; +} diff --git a/test/helpers/destroy-flow-test-harness.ts b/test/helpers/destroy-flow-test-harness.ts index 1d16cc11b02..5ede3c4de62 100644 --- a/test/helpers/destroy-flow-test-harness.ts +++ b/test/helpers/destroy-flow-test-harness.ts @@ -50,6 +50,7 @@ type DestroyHarnessOptions = { liveListOutput?: string; mcpAddState?: "prepared"; mcpServers?: string[]; + openshellDriver?: string; promptResponses?: string[]; registeredSandboxCount?: number; restoreMcpError?: string; @@ -137,6 +138,7 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr vi.spyOn(registry, "getSandbox").mockReturnValue({ ...sandboxEntry, agent: options.agent ?? sandboxEntry.agent, + ...(options.openshellDriver ? { openshellDriver: options.openshellDriver } : {}), ...(options.endpointUrl ? { endpointUrl: options.endpointUrl } : {}), ...(options.mcpServers?.length ? { diff --git a/test/helpers/runtime-provider-bundle.ts b/test/helpers/runtime-provider-bundle.ts new file mode 100644 index 00000000000..048a22e883b --- /dev/null +++ b/test/helpers/runtime-provider-bundle.ts @@ -0,0 +1,171 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, + type RuntimeProviderBundle, + type RuntimeProviderCleanupInput, + type RuntimeProviderLifecycleInput, + type RuntimeProviderLifecycleStopHooks, + type RuntimeProviderWorkloadProfile, +} from "../../src/lib/onboard/runtime-provider/contract"; + +export interface InMemoryRuntimeProviderState { + readonly events: string[]; + readonly running: Set; + readonly workloads: Set; +} + +export type InMemoryRuntimeProviderBundle = RuntimeProviderBundle & { + readonly lifecycle: Extract; + readonly cleanup: Extract; + readonly containerEngine: Extract< + RuntimeProviderBundle["containerEngine"], + { readonly supported: true } + >; +}; + +type InMemoryRuntimeProviderOptions = { + readonly providerId: string; + readonly workloadProfile: RuntimeProviderWorkloadProfile; + readonly state?: InMemoryRuntimeProviderState; + readonly gatewayLauncher?: "nemoclaw" | "openshell"; +}; + +function unsupported(providerId: string, reason: string) { + return { providerId, supported: false as const, reason }; +} + +/** + * Pure test fixture: no host process, socket, environment, or container + * runtime dependency. Tests opt a provider into the complete bundle contract + * without adding it to the production registry. + */ +export function createInMemoryRuntimeProviderBundle({ + providerId, + workloadProfile, + state = { events: [], running: new Set(), workloads: new Set() }, + gatewayLauncher = "nemoclaw", +}: InMemoryRuntimeProviderOptions): InMemoryRuntimeProviderBundle { + const futureReason = "Unsupported by this in-memory contract fixture."; + const event = (kind: string, sandboxName: string) => state.events.push(`${kind}:${sandboxName}`); + return { + identity: { + contractVersion: RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, + id: providerId, + displayName: `In-memory ${providerId}`, + }, + plan: { providerId, supported: true, gatewayLauncher }, + capabilities: { + providerId, + supported: true, + hostLocalInference: false, + directLifecycle: true, + legacyGatewayContainerInspection: false, + workloadImageCleanup: true, + }, + preflightDoctor: { + providerId, + supported: true, + inspectHost: () => ({ + group: "Host", + label: "In-memory runtime", + status: "ok", + detail: "ready", + }), + preflightLifecycle: () => null, + }, + gateway: { + providerId, + supported: true, + launcher: gatewayLauncher, + inspectLegacyContainer: false, + }, + workload: { + providerId, + supported: true, + profile: workloadProfile, + acceptsReceipt(receipt) { + return receipt === undefined + ? true + : receipt.kind === "legacy-dockerfile" + ? workloadProfile.legacyDockerfileBuilds + : receipt.platform !== undefined && + workloadProfile.support?.platforms.includes(receipt.platform) === true; + }, + }, + lifecycle: { + providerId, + supported: true, + channelStopTransport: "openshell", + start(input: RuntimeProviderLifecycleInput) { + state.running.add(input.sandboxName); + event("start", input.sandboxName); + input.log(` In-memory workload '${input.sandboxName}' started.`); + return { exitCode: 0 }; + }, + stop(input: RuntimeProviderLifecycleInput, hooks: RuntimeProviderLifecycleStopHooks) { + const wasRunning = state.running.delete(input.sandboxName); + const beforeStop = wasRunning ? hooks.beforeStop : () => undefined; + const recordStop = wasRunning ? () => event("stop", input.sandboxName) : () => undefined; + beforeStop(); + recordStop(); + return { + exitCode: 0, + state: wasRunning ? "stopped" : "already-stopped", + }; + }, + }, + mutationAuthority: { + providerId, + supported: true, + operations: [ + "registration", + "start", + "stop", + "inference-set", + "rebuild", + "provider-cleanup", + "destroy", + "workload-cleanup", + ], + }, + bootstrap: unsupported(providerId, futureReason), + snapshot: unsupported(providerId, futureReason), + recovery: unsupported(providerId, futureReason), + cleanup: { + providerId, + supported: true, + prepareDestroy(input: RuntimeProviderCleanupInput, operations) { + event("prepare-destroy", input.sandboxName); + return operations.detachProviders(input.sandboxName); + }, + removeOwnedWorkload(input: RuntimeProviderCleanupInput) { + const reference = input.sandbox.imageTag; + const remove = (ownedReference: string) => { + state.workloads.delete(ownedReference); + event("cleanup", input.sandboxName); + return { + status: "removed" as const, + engineDisplayName: "In-memory", + reference: ownedReference, + }; + }; + return input.sandbox.workload?.shared === true + ? { status: "skipped", reason: "shared-image" } + : reference && state.workloads.has(reference) + ? remove(reference) + : { status: "skipped", reason: "no-owned-image" }; + }, + }, + containerEngine: { + providerId, + supported: true, + identities: [ + { operation: "host-doctor", engineId: "memory", displayName: "In-memory" }, + { operation: "sandbox-lifecycle", engineId: "memory", displayName: "In-memory" }, + { operation: "workload-cleanup", engineId: "memory", displayName: "In-memory" }, + ], + }, + }; +} diff --git a/test/image-cleanup.test.ts b/test/image-cleanup.test.ts index 89f0afec445..a97c3642bba 100644 --- a/test/image-cleanup.test.ts +++ b/test/image-cleanup.test.ts @@ -21,13 +21,17 @@ import { resolveNemoclawStateDir } from "../src/lib/state/paths"; import { help as renderRootHelp } from "../src/lib/actions/root-help"; import { COMMANDS, globalCommandTokens } from "../src/lib/cli/command-registry"; import { getRegisteredOclifCommandMetadata } from "../src/lib/cli/oclif-metadata"; +import { createDockerRuntimeProviderBundle } from "../src/lib/onboard/runtime-provider/docker"; +import { createRuntimeProviderBundleRegistry } from "../src/lib/onboard/runtime-provider/registry"; describe("image cleanup: sandbox destroy removes Docker image (#2086)", () => { it("removes sandbox images before deleting the registry entry", () => { const calls: string[] = []; const removed = removeSandboxRegistryEntry("alpha", { - removeImage: (sandboxName) => calls.push(`image:${sandboxName}`), + removeImage: (sandboxName) => { + calls.push(`image:${sandboxName}`); + }, removeSandbox: (sandboxName) => { calls.push(`registry:${sandboxName}`); return true; @@ -40,13 +44,21 @@ describe("image cleanup: sandbox destroy removes Docker image (#2086)", () => { it("removeSandboxImage calls docker rmi for recorded image tags", () => { const removedTags: string[] = []; + const runtimeProviders = createRuntimeProviderBundleRegistry([ + [ + "docker", + createDockerRuntimeProviderBundle({ + removeImage: (tag) => { + removedTags.push(tag); + return { status: 0 }; + }, + }), + ], + ]); removeSandboxImage("alpha", { getSandbox: () => ({ name: "alpha", imageTag: "openshell/sandbox-from:123" }) as any, - dockerRmi: (tag) => { - removedTags.push(tag); - return { status: 0 } as any; - }, + runtimeProviders, }); expect(removedTags).toEqual(["openshell/sandbox-from:123"]); @@ -54,18 +66,47 @@ describe("image cleanup: sandbox destroy removes Docker image (#2086)", () => { it("removeSandboxImage gracefully handles missing imageTag", () => { const removedTags: string[] = []; + const runtimeProviders = createRuntimeProviderBundleRegistry([ + [ + "docker", + createDockerRuntimeProviderBundle({ + removeImage: (tag) => { + removedTags.push(tag); + return { status: 0 }; + }, + }), + ], + ]); removeSandboxImage("alpha", { getSandbox: () => ({ name: "alpha", imageTag: null }) as any, - dockerRmi: (tag) => { - removedTags.push(tag); - return { status: 0 } as any; - }, + runtimeProviders, }); expect(removedTags).toEqual([]); }); + it("never deletes a shared managed workload image", () => { + const removeImage = vi.fn(() => ({ status: 0 })); + const runtimeProviders = createRuntimeProviderBundleRegistry([ + ["docker", createDockerRuntimeProviderBundle({ removeImage })], + ]); + + const result = removeSandboxImage("alpha", { + getSandbox: () => + ({ + name: "alpha", + openshellDriver: "docker", + imageTag: `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`, + workload: { shared: true, kind: "managed-image" }, + }) as any, + runtimeProviders, + }); + + expect(result).toEqual({ status: "skipped", reason: "shared-image" }); + expect(removeImage).not.toHaveBeenCalled(); + }); + it("treats missing sandbox delete results as already gone", () => { expect( getSandboxDeleteOutcome({ status: 1, stderr: "Error: sandbox alpha not found" }), diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts new file mode 100644 index 00000000000..e738d687d59 --- /dev/null +++ b/test/runtime-provider-source-shape.test.ts @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const repoRoot = join(import.meta.dirname, ".."); + +describe("runtime provider central source boundary", () => { + // source-shape-contract: compatibility -- Central non-snapshot actions must stay provider-neutral while production selection excludes unqualified future providers and managed-bootstrap dependencies + it("keeps provider identities and implementations behind the one bundle composition", () => { + const centralConsumers = [ + readFileSync(join(repoRoot, "src/lib/actions/inference-set.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/actions/sandbox/destroy-execution.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/actions/sandbox/destroy.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/actions/sandbox/runtime/lifecycle-runtime.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/actions/sandbox/start.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/actions/sandbox/stop.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/compute/plan.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/sandbox-registration.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/workload/runtime.ts"), "utf8"), + ]; + const nonSnapshotActions = centralConsumers.slice(0, 6); + const providerContract = [ + readFileSync(join(repoRoot, "src/lib/onboard/runtime-provider/contract.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/runtime-provider/current.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/runtime-provider/docker.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/runtime-provider/registry.ts"), "utf8"), + ]; + + for (const source of nonSnapshotActions) { + expect(source).not.toMatch(/\b(?:docker|podman)\b/iu); + expect(source).not.toMatch(/(?:adapters\/docker|docker-driver-sandbox-recovery)/u); + } + for (const source of centralConsumers) { + expect(source).not.toMatch(/\b(?:openshellDriver|driverName)\s*={2,3}\s*["'][^"']+["']/u); + expect(source).not.toMatch(/switch\s*\([^)]*\b(?:openshellDriver|driverName)\b[^)]*\)/u); + } + expect(providerContract.join("\n")).not.toMatch(/managed-bootstrap/u); + expect(providerContract[1]).not.toMatch(/\b(?:podman|mxc)\b/iu); + }); +}); From 005b3a7323b6563ed96b55d56c881a0b973c8640 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 30 Jul 2026 22:46:20 -0700 Subject: [PATCH 023/117] fix(runtime): harden provider contract boundaries Signed-off-by: Aaron Erickson (cherry picked from commit 98f17dbb8409f4320763c4ad653bcda6b554e601) --- .../inference-set-failure-handling.test.ts | 30 ++++++- src/lib/actions/inference-set.ts | 20 +++-- src/lib/actions/sandbox/stop.test.ts | 6 +- src/lib/actions/sandbox/stop.ts | 1 + .../runtime-provider-contract.test.ts | 90 ++++++++++++++++++- src/lib/state/registry/workload.ts | 31 ++++++- src/lib/tunnel/sandbox-gateway-stop.test.ts | 19 ++++ src/lib/tunnel/sandbox-gateway-stop.ts | 17 ++-- 8 files changed, 193 insertions(+), 21 deletions(-) diff --git a/src/lib/actions/inference-set-failure-handling.test.ts b/src/lib/actions/inference-set-failure-handling.test.ts index b646dc71905..d4c80a517bf 100644 --- a/src/lib/actions/inference-set-failure-handling.test.ts +++ b/src/lib/actions/inference-set-failure-handling.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { InferenceSetError, runInferenceSet } from "./inference-set"; import { createDeps } from "./inference-set.test-support"; @@ -20,6 +20,34 @@ describe("runInferenceSet failure handling", () => { expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled(); }); + it("rechecks live runtime authority inside the sandbox mutation lock", async () => { + const initial = { name: "alpha", agent: "openclaw", openshellDriver: "docker" } as const; + const changed = { + name: "alpha", + agent: "openclaw", + openshellDriver: "unknown-runtime", + } as const; + const deps = createDeps({ config: {}, entry: initial }); + deps.getSandbox = vi.fn().mockReturnValueOnce(initial).mockReturnValue(changed); + + await expect( + runInferenceSet( + { + provider: "nvidia-prod", + model: "nvidia/model-a", + sandboxName: "alpha", + }, + deps, + ), + ).rejects.toThrow(/unknown-runtime.*not registered/u); + + expect(deps.calls.prepareRunOpenshell).toHaveBeenCalledOnce(); + expect(deps.calls.withGatewayRouteMutationLock).not.toHaveBeenCalled(); + expect(deps.calls.captureOpenshell).not.toHaveBeenCalled(); + expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled(); + expect(deps.calls.updateSandbox).not.toHaveBeenCalled(); + }); + it("resolves the OpenShell runner before entering the async mutation lock", async () => { const deps = createDeps({ config: { agents: { defaults: { model: { primary: "inference/nvidia/model-a" } } } }, diff --git a/src/lib/actions/inference-set.ts b/src/lib/actions/inference-set.ts index 080db2f82db..7f23328588c 100644 --- a/src/lib/actions/inference-set.ts +++ b/src/lib/actions/inference-set.ts @@ -72,8 +72,8 @@ import { } from "./inference-set-gateway-restart"; import { prepareInferenceSetProviderBinding, - requireInferenceSetRuntimeAuthority, type RuntimeProviderBundleRegistry, + requireInferenceSetRuntimeAuthority, } from "./inference-set-provider"; import { buildInferenceSetFailure } from "./inference-set-provider-diagnostics"; import { @@ -288,6 +288,17 @@ function assertSupportedProvider(provider: string, model: string): void { ); } +function assertInferenceSetRuntimeAuthority( + entry: SandboxEntry, + providers: RuntimeProviderBundleRegistry | undefined, +): void { + try { + requireInferenceSetRuntimeAuthority(entry, providers); + } catch (error) { + throw new InferenceSetError(error instanceof Error ? error.message : String(error), 2); + } +} + function normalizeSandboxAgent(agentName: string | null | undefined): string { const trimmed = typeof agentName === "string" ? agentName.trim() : ""; return (trimmed || "openclaw").toLowerCase(); @@ -1341,14 +1352,11 @@ export async function runInferenceSet( // missing-binary path exits the process, which cannot be deferred safely by // an async lock. The inner resolution still validates the live registry entry. const selected = resolveTargetSandbox(options.sandboxName, deps); - try { - requireInferenceSetRuntimeAuthority(selected.entry, deps.runtimeProviders); - } catch (error) { - throw new InferenceSetError(error instanceof Error ? error.message : String(error), 2); - } + assertInferenceSetRuntimeAuthority(selected.entry, deps.runtimeProviders); deps.prepareRunOpenshell(); return withSandboxMutationLock(selected.sandboxName, async () => { const lockedSelection = resolveTargetSandbox(selected.sandboxName, deps); + assertInferenceSetRuntimeAuthority(lockedSelection.entry, deps.runtimeProviders); const gatewayName = resolveSandboxGatewayName(lockedSelection.entry); const mutation = await deps.withGatewayRouteMutationLock(gatewayName, () => withTimerBoundShieldsMutationLockAsync(selected.sandboxName, "inference set", () => diff --git a/src/lib/actions/sandbox/stop.test.ts b/src/lib/actions/sandbox/stop.test.ts index d3b750c8b26..41d444ee3ad 100644 --- a/src/lib/actions/sandbox/stop.test.ts +++ b/src/lib/actions/sandbox/stop.test.ts @@ -165,7 +165,11 @@ describe("stopSandbox", () => { expect(result.exitCode).toBe(0); expect(h.stopSandboxChannels).toHaveBeenCalledWith( "my-sandbox", - expect.objectContaining({ info: expect.any(Function), warn: expect.any(Function) }), + expect.objectContaining({ + channelStopTransport: "docker-kubectl-first", + info: expect.any(Function), + warn: expect.any(Function), + }), ); expect(h.dockerStop).toHaveBeenCalledWith("openshell-my-sandbox", { ignoreError: true, diff --git a/src/lib/actions/sandbox/stop.ts b/src/lib/actions/sandbox/stop.ts index 1163d8e3a9f..903a595ec4d 100644 --- a/src/lib/actions/sandbox/stop.ts +++ b/src/lib/actions/sandbox/stop.ts @@ -74,6 +74,7 @@ export function stopSandbox( channelsStopped = true; try { (deps.stopSandboxChannels ?? stopSandboxChannels)(sandboxName, { + channelStopTransport: resolved.lifecycle.channelStopTransport, info: (message) => log(` ${message}`), warn: (message) => warn(` ${message}`), }); diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index 12d4592be3e..ef9682320c5 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -4,6 +4,10 @@ import { Buffer } from "node:buffer"; import { createHash } from "node:crypto"; import { describe, expect, it, vi } from "vitest"; +import { + MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, + managedStartupE2eProfile, +} from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; import { createInMemoryRuntimeProviderBundle, type InMemoryRuntimeProviderBundle, @@ -13,6 +17,11 @@ import { startSandbox } from "../../actions/sandbox/start"; import { stopSandbox } from "../../actions/sandbox/stop"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; import { cloneSandboxWorkloadReceipt } from "../../state/registry/workload"; +import { MANAGED_IMAGE_REPOSITORIES } from "../managed-image/contract"; +import { + encodeManagedStartupProfile, + type ManagedStartupProfile, +} from "../managed-startup/profile"; import type { RuntimeProviderBundle, RuntimeProviderWorkloadProfile } from "./contract"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "./current"; import { @@ -34,7 +43,7 @@ const PORTABLE_PROFILE = { legacyDockerfileBuilds: true, } as const satisfies RuntimeProviderWorkloadProfile; -const ENCODED_PROFILE = Buffer.from('{"schemaVersion":1}', "utf8").toString("base64url"); +const ENCODED_PROFILE = encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")); const PROFILE_SHA256 = createHash("sha256").update(ENCODED_PROFILE, "utf8").digest("hex"); const MANAGED_RECEIPT = { schemaVersion: 1, @@ -52,6 +61,22 @@ const MANAGED_RECEIPT = { shared: true, } as const satisfies SandboxWorkloadReceipt; +type ManagedWorkloadReceipt = Extract; + +function receiptForProfile( + profile: ManagedStartupProfile, + corporateCaB64?: string, +): ManagedWorkloadReceipt { + const encodedProfile = encodeManagedStartupProfile(profile); + return { + ...MANAGED_RECEIPT, + reference: `${MANAGED_IMAGE_REPOSITORIES[profile.agent]}@sha256:${"a".repeat(64)}`, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + ...(corporateCaB64 === undefined ? {} : { corporateCaB64 }), + }; +} + function mxcBundle(): InMemoryRuntimeProviderBundle { return createInMemoryRuntimeProviderBundle({ providerId: "mxc", @@ -377,6 +402,63 @@ describe("sandbox workload ownership receipt", () => { ).toBeUndefined(); }); + it("rejects a canonical transport containing credential-shaped profile data", () => { + const profile = managedStartupE2eProfile("openclaw"); + const canonicalProfile = Buffer.from(ENCODED_PROFILE, "base64url").toString("utf8"); + const encodedProfile = Buffer.from( + canonicalProfile.replace( + `"model":${JSON.stringify(profile.inference.model)}`, + `"model":"nvapi-${"a".repeat(32)}"`, + ), + "utf8", + ).toString("base64url"); + + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + }), + ).toBeUndefined(); + }); + + it("binds the decoded startup-profile agent to the immutable image repository", () => { + const hermesReceipt = receiptForProfile(managedStartupE2eProfile("hermes")); + + expect( + cloneSandboxWorkloadReceipt({ + ...hermesReceipt, + reference: MANAGED_RECEIPT.reference, + }), + ).toBeUndefined(); + }); + + it("binds corporate CA presence and exact bytes to the decoded profile digest", () => { + const corporateCaB64 = Buffer.from(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, "utf8").toString( + "base64", + ); + const receipt = receiptForProfile( + managedStartupE2eProfile("openclaw", false, true), + corporateCaB64, + ); + + expect(cloneSandboxWorkloadReceipt(receipt)).toEqual(receipt); + expect( + cloneSandboxWorkloadReceipt({ + ...receipt, + corporateCaB64: Buffer.from("different-ca", "utf8").toString("base64"), + }), + ).toBeUndefined(); + const { corporateCaB64: _omittedCa, ...missingCa } = receipt; + expect(cloneSandboxWorkloadReceipt(missingCa)).toBeUndefined(); + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + corporateCaB64, + }), + ).toBeUndefined(); + }); + it("retains an owned legacy image receipt independently from managed cohorts", () => { expect( cloneSandboxWorkloadReceipt({ @@ -482,7 +564,11 @@ describe("socket-free MXC action contract", () => { expect(probeSandbox).toHaveBeenCalledWith(sandboxName); expect(stopSandboxChannels).toHaveBeenCalledWith( sandboxName, - expect.objectContaining({ info: expect.any(Function), warn: expect.any(Function) }), + expect.objectContaining({ + channelStopTransport: "openshell", + info: expect.any(Function), + warn: expect.any(Function), + }), ); expect(state.events).toEqual([ `start:${sandboxName}`, diff --git a/src/lib/state/registry/workload.ts b/src/lib/state/registry/workload.ts index 3520c249220..fdca4897d04 100644 --- a/src/lib/state/registry/workload.ts +++ b/src/lib/state/registry/workload.ts @@ -4,6 +4,8 @@ import { Buffer } from "node:buffer"; import { createHash } from "node:crypto"; +import { MANAGED_IMAGE_REPOSITORIES } from "../../onboard/managed-image/contract"; +import { decodeManagedStartupProfile } from "../../onboard/managed-startup/profile"; import type { SandboxWorkloadReceipt } from "./types"; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; @@ -75,6 +77,7 @@ export function cloneSandboxWorkloadReceipt( shared: false, }; } + const encodedProfileBytes = decodeCanonicalBase64Url(value.encodedProfile); if ( value.kind !== "managed-image" || value.shared !== true || @@ -89,11 +92,31 @@ export function cloneSandboxWorkloadReceipt( value.startupProfileContractVersion !== 1 || !SHA256_PATTERN.test(value.startupProfileSha256) || typeof value.credentialProxyReplayRequired !== "boolean" || - decodeCanonicalBase64Url(value.encodedProfile) === null || + encodedProfileBytes === null || createHash("sha256").update(value.encodedProfile, "utf8").digest("hex") !== - value.startupProfileSha256 || - (value.corporateCaB64 !== undefined && - decodeCanonicalStandardBase64(value.corporateCaB64) === null) + value.startupProfileSha256 + ) { + return undefined; + } + let profile: ReturnType; + try { + profile = decodeManagedStartupProfile(value.encodedProfile); + } catch { + return undefined; + } + if (!value.reference.startsWith(`${MANAGED_IMAGE_REPOSITORIES[profile.agent]}@sha256:`)) { + return undefined; + } + const corporateCaBytes = + value.corporateCaB64 === undefined ? null : decodeCanonicalStandardBase64(value.corporateCaB64); + if (value.corporateCaB64 !== undefined && corporateCaBytes === null) { + return undefined; + } + const expectedCorporateCaSha256 = profile.corporateCa.bundleSha256; + if ( + (expectedCorporateCaSha256 === null) !== (corporateCaBytes === null) || + (corporateCaBytes !== null && + createHash("sha256").update(corporateCaBytes).digest("hex") !== expectedCorporateCaSha256) ) { return undefined; } diff --git a/src/lib/tunnel/sandbox-gateway-stop.test.ts b/src/lib/tunnel/sandbox-gateway-stop.test.ts index 4178109e63c..ee9d13ad4b9 100644 --- a/src/lib/tunnel/sandbox-gateway-stop.test.ts +++ b/src/lib/tunnel/sandbox-gateway-stop.test.ts @@ -115,6 +115,25 @@ describe("stopSandboxChannels", () => { ); }); + it("uses the OpenShell transport without invoking Docker", () => { + const h = harness(); + + stopSandboxChannels("my-sandbox", { + ...h.deps, + channelStopTransport: "openshell", + }); + + expect(h.runDocker).not.toHaveBeenCalled(); + expect(h.runProcess).toHaveBeenCalledWith( + "/usr/local/bin/openshell", + ["sandbox", "exec", "--name", "my-sandbox", "--gateway", "nemoclaw", "--", "sh", "-s"], + expect.objectContaining({ + input: expect.stringContaining("find_gateway_pids"), + timeout: 20000, + }), + ); + }); + it("selects the exact generated sandbox pod and excludes overlapping names", () => { const h = harness(); h.runDocker diff --git a/src/lib/tunnel/sandbox-gateway-stop.ts b/src/lib/tunnel/sandbox-gateway-stop.ts index 7176783c7b1..e1e4e1b4c53 100644 --- a/src/lib/tunnel/sandbox-gateway-stop.ts +++ b/src/lib/tunnel/sandbox-gateway-stop.ts @@ -24,6 +24,7 @@ type ProcessRunner = ( ) => SpawnSyncReturns; export type SandboxGatewayStopDeps = { + channelStopTransport?: "docker-kubectl-first" | "openshell"; getSandbox?: typeof registry.getSandbox; getRegisteredAgent?: typeof agentRuntime.getRegisteredAgent; getAgentDisplayName?: typeof agentRuntime.getAgentDisplayName; @@ -105,13 +106,15 @@ export function stopSandboxChannels(sandboxName: string, deps: SandboxGatewaySto const gatewayLabel = `${agentDisplayName} gateway`; info(`Stopping in-sandbox ${gatewayLabel} (sandbox: ${validatedSandboxName})...`); - const privilegedResult = stopSandboxChannelsViaKubectl( - validatedSandboxName, - gatewayName, - GATEWAY_STOP_SCRIPT, - deps.runDocker ?? dockerSpawnSync, - ); - if (reportStopResult(privilegedResult, gatewayLabel, info, warn)) return; + if (deps.channelStopTransport !== "openshell") { + const privilegedResult = stopSandboxChannelsViaKubectl( + validatedSandboxName, + gatewayName, + GATEWAY_STOP_SCRIPT, + deps.runDocker ?? dockerSpawnSync, + ); + if (reportStopResult(privilegedResult, gatewayLabel, info, warn)) return; + } const openshell = (deps.resolveOpenshell ?? resolveOpenshell)(); if (!openshell) { From 05cf2a5e613ebd331cdc314da0724e42de926a74 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 00:48:21 -0700 Subject: [PATCH 024/117] fix(runtime): fail closed on managed image cleanup Signed-off-by: Aaron Erickson --- src/lib/onboard/runtime-provider/docker.ts | 11 +++++++++++ test/image-cleanup.test.ts | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/lib/onboard/runtime-provider/docker.ts b/src/lib/onboard/runtime-provider/docker.ts index 2f3e5fa21a3..75f7373de36 100644 --- a/src/lib/onboard/runtime-provider/docker.ts +++ b/src/lib/onboard/runtime-provider/docker.ts @@ -14,6 +14,7 @@ import { import { MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, MANAGED_IMAGE_PLATFORMS, + MANAGED_IMAGE_REPOSITORIES, MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, } from "../managed-image/contract"; import { @@ -214,6 +215,16 @@ function removeOwnedDockerWorkload( const { imageTag, workload } = input.sandbox; if (workload?.shared === true) return { status: "skipped", reason: "shared-image" }; if (!imageTag) return { status: "skipped", reason: "no-owned-image" }; + if ( + Object.values(MANAGED_IMAGE_REPOSITORIES).some( + (repository) => + imageTag === repository || + imageTag.startsWith(`${repository}@`) || + imageTag.startsWith(`${repository}:`), + ) + ) { + return { status: "skipped", reason: "authority-unproven" }; + } if ( workload?.kind === "legacy-dockerfile" && workload.reference !== null && diff --git a/test/image-cleanup.test.ts b/test/image-cleanup.test.ts index a97c3642bba..ee191c2ca1d 100644 --- a/test/image-cleanup.test.ts +++ b/test/image-cleanup.test.ts @@ -107,6 +107,27 @@ describe("image cleanup: sandbox destroy removes Docker image (#2086)", () => { expect(removeImage).not.toHaveBeenCalled(); }); + it("fails closed when a managed workload receipt was dropped as malformed", () => { + const removeImage = vi.fn(() => ({ status: 0 })); + const runtimeProviders = createRuntimeProviderBundleRegistry([ + ["docker", createDockerRuntimeProviderBundle({ removeImage })], + ]); + + const result = removeSandboxImage("alpha", { + getSandbox: () => + ({ + name: "alpha", + openshellDriver: "docker", + imageTag: `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`, + workload: undefined, + }) as any, + runtimeProviders, + }); + + expect(result).toEqual({ status: "skipped", reason: "authority-unproven" }); + expect(removeImage).not.toHaveBeenCalled(); + }); + it("treats missing sandbox delete results as already gone", () => { expect( getSandboxDeleteOutcome({ status: 1, stderr: "Error: sandbox alpha not found" }), From 5c29046940b0a65b322081a60847f4bbbdebcc8b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 01:06:56 -0700 Subject: [PATCH 025/117] fix(messaging): reject unresolved build hooks Signed-off-by: Aaron Erickson --- .../post-agent-install-selection.test.ts | 18 ++++++++++++++++++ .../messaging/post-agent-install-selection.ts | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/lib/messaging/post-agent-install-selection.test.ts b/src/lib/messaging/post-agent-install-selection.test.ts index 1ecd4298540..34e8c3b23f9 100644 --- a/src/lib/messaging/post-agent-install-selection.test.ts +++ b/src/lib/messaging/post-agent-install-selection.test.ts @@ -170,4 +170,22 @@ describe("post-agent-install messaging selection", () => { expect(selectEnabledPostAgentInstallBuildFiles(selection)).toEqual([]); }); + + it("fails closed when a build step references a missing hook", () => { + const selection = plan({ + channels: [ + channel("telegram", [ + { + channelId: "telegram", + id: "post-install", + phase: "post-agent-install", + handler: "telegram.post-install", + }, + ]), + ], + buildSteps: [buildFile("telegram", "stale-hook-file", "missing-hook")], + }); + + expect(selectEnabledPostAgentInstallBuildFiles(selection)).toEqual([]); + }); }); diff --git a/src/lib/messaging/post-agent-install-selection.ts b/src/lib/messaging/post-agent-install-selection.ts index f866edfb31b..8c88f212fc5 100644 --- a/src/lib/messaging/post-agent-install-selection.ts +++ b/src/lib/messaging/post-agent-install-selection.ts @@ -106,7 +106,7 @@ export function selectEnabledPostAgentInstallBuildFiles< (channel) => normalizeMessagingChannelId(channel.channelId) === channelId, ); if (matchingChannels.length !== 1) return false; - const hookPhase = matchingChannels[0]?.hooks?.find((hook) => hook.id === step.hookId)?.phase; - return hookPhase === undefined || hookPhase === "post-agent-install"; + const matchedHook = matchingChannels[0]?.hooks?.find((hook) => hook.id === step.hookId); + return matchedHook !== undefined && matchedHook.phase === "post-agent-install"; }); } From 534773b4384bdeb37d8f3075653431e822281246 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 01:08:08 -0700 Subject: [PATCH 026/117] fix(runtime): close provider lifecycle authority gaps Signed-off-by: Aaron Erickson --- ci/source-shape-test-budget.json | 2 +- src/lib/actions/sandbox/destroy-execution.ts | 20 ++-- src/lib/actions/sandbox/start.test.ts | 24 ++--- src/lib/actions/sandbox/start.ts | 13 +-- src/lib/onboard/runtime-provider/contract.ts | 4 + src/lib/onboard/runtime-provider/docker.ts | 3 +- src/lib/onboard/runtime-provider/registry.ts | 1 + .../runtime-provider-contract.test.ts | 97 +++++++++++++++++-- test/helpers/runtime-provider-bundle.ts | 3 + test/image-cleanup.test.ts | 24 +++-- test/runtime-provider-source-shape.test.ts | 5 +- 11 files changed, 148 insertions(+), 48 deletions(-) diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 8a22d0441c2..c5f842289eb 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -508,7 +508,7 @@ }, { "file": "test/runtime-provider-source-shape.test.ts", - "test": "keeps provider identities and implementations behind the one bundle composition", + "test": "keeps migrated provider identities and implementations behind the one bundle composition", "category": "compatibility" }, { diff --git a/src/lib/actions/sandbox/destroy-execution.ts b/src/lib/actions/sandbox/destroy-execution.ts index 036cfc5d9de..09f68f435ca 100644 --- a/src/lib/actions/sandbox/destroy-execution.ts +++ b/src/lib/actions/sandbox/destroy-execution.ts @@ -3,10 +3,6 @@ import { R, YW } from "../../cli/terminal-style"; import { getSandboxDeleteOutcome } from "../../domain/sandbox/destroy"; -import { - type DetachSandboxProvidersResult, - runSandboxProviderPreDeleteCleanup, -} from "../../onboard/sandbox-provider-cleanup"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, RuntimeProviderBundle, @@ -14,6 +10,10 @@ import { requireRuntimeProviderBundleForSandbox, requireRuntimeProviderMutationAuthority, } from "../../onboard/runtime-provider/access"; +import { + type DetachSandboxProvidersResult, + runSandboxProviderPreDeleteCleanup, +} from "../../onboard/sandbox-provider-cleanup"; import { redact } from "../../security/redact"; import { withTimerBoundShieldsMutationLockAsync } from "../../shields/timer-bound-lock"; import { readTimerMarker } from "../../shields/timer-control"; @@ -36,6 +36,10 @@ type SandboxDestroyExecutionInput = { sandboxConfirmedAbsent: boolean; sandboxName: string; runtimeProviders?: RuntimeProviderBundleRegistry; + deps?: { + readTimerMarker?: typeof readTimerMarker; + wipeSandboxState?: typeof wipeSandboxState; + }; }; export type SandboxDestroyExecutionResult = @@ -94,13 +98,14 @@ async function prepareMcpDestroy( function wipeAndHardenLiveSandbox( sandboxName: string, sandboxConfirmedAbsent: boolean, + deps: NonNullable = {}, ): HardenedDeleteState { if (sandboxConfirmedAbsent) return { hardenedForDelete: false }; // Wipe before delete while the retained volume is still mounted. The caller // holds the timer-bound lock across this phase and all following teardown. - wipeSandboxState(sandboxName); - const timerMarker = readTimerMarker(sandboxName); + (deps.wipeSandboxState ?? wipeSandboxState)(sandboxName); + const timerMarker = (deps.readTimerMarker ?? readTimerMarker)(sandboxName); if (!timerMarker) return { hardenedForDelete: false }; const timerProcessToken = /^[0-9a-f]{32}$/.test(timerMarker.processToken ?? "") @@ -188,6 +193,7 @@ export async function executeSandboxDestroy({ sandboxConfirmedAbsent, sandboxName, runtimeProviders = CURRENT_RUNTIME_PROVIDER_BUNDLES, + deps = {}, }: SandboxDestroyExecutionInput): Promise { return withTimerBoundShieldsMutationLockAsync(sandboxName, "destroy sandbox", async () => { let runtimeProvider: RuntimeProviderBundle | null = null; @@ -224,7 +230,7 @@ export async function executeSandboxDestroy({ // discarded during preparation. Remaining entries are the durable exact // provider ownership manifest and must survive an unconfirmed delete. const hasMcpOwnership = mcpPreparation.entries.length > 0; - const hardened = wipeAndHardenLiveSandbox(sandboxName, sandboxConfirmedAbsent); + const hardened = wipeAndHardenLiveSandbox(sandboxName, sandboxConfirmedAbsent, deps); const detachProviders = (): DetachSandboxProvidersResult => runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); const detachOutcome: DetachSandboxProvidersResult = sandboxConfirmedAbsent diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index f3eb3a9bc3c..a994c575fbc 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -36,7 +36,7 @@ function harness(overrides: Partial = {}) { const dockerUnpause = vi.fn(() => ({ status: 0, })); - const probeSandbox = vi.fn>(() => + const verifyGateway = vi.fn>(() => Promise.resolve(), ); const log = vi.fn<(message: string) => void>(); @@ -56,7 +56,7 @@ function harness(overrides: Partial = {}) { const deps: SandboxStartDeps = { getSandbox, runtimeProviders, - probeSandbox, + verifyGateway, log, ...overrides, }; @@ -68,7 +68,7 @@ function harness(overrides: Partial = {}) { isDockerRuntimeDown, log, printDockerRuntimeDownGuidance, - probeSandbox, + verifyGateway, recoverDockerDriverSandbox, }; } @@ -81,9 +81,9 @@ describe("startSandbox", () => { expect(result.exitCode).toBe(0); expect(h.recoverDockerDriverSandbox).toHaveBeenCalledWith("my-sandbox"); - expect(h.probeSandbox).toHaveBeenCalledWith("my-sandbox"); + expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); expect(h.recoverDockerDriverSandbox.mock.invocationCallOrder[0]).toBeLessThan( - h.probeSandbox.mock.invocationCallOrder[0], + h.verifyGateway.mock.invocationCallOrder[0], ); }); @@ -110,7 +110,7 @@ describe("startSandbox", () => { const result = await startSandbox("my-sandbox", h.deps); expect(result.exitCode).toBe(0); - expect(h.probeSandbox).toHaveBeenCalledWith("my-sandbox"); + expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); const output = h.log.mock.calls.map(([line]) => line).join("\n"); expect(output).toContain("already running"); }); @@ -129,7 +129,7 @@ describe("startSandbox", () => { timeout: 30_000, }); expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled(); - expect(h.probeSandbox).toHaveBeenCalledWith("my-sandbox"); + expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); const output = h.log.mock.calls.map(([line]) => line).join("\n"); expect(output).toContain("unpaused"); }); @@ -146,7 +146,7 @@ describe("startSandbox", () => { expect(result.exitCode).toBe(1); expect(result.message).toContain("openshell-my-sandbox"); expect(result.message).toContain("125"); - expect(h.probeSandbox).not.toHaveBeenCalled(); + expect(h.verifyGateway).not.toHaveBeenCalled(); }); it("restores a gpu-backup sibling through the recovery rename path (#6026)", async () => { @@ -160,7 +160,7 @@ describe("startSandbox", () => { const result = await startSandbox("my-sandbox", h.deps); expect(result.exitCode).toBe(0); - expect(h.probeSandbox).toHaveBeenCalledWith("my-sandbox"); + expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); }); it("names the Docker daemon outage instead of claiming the container was removed (#6026)", async () => { @@ -175,7 +175,7 @@ describe("startSandbox", () => { retryCommand: "start", }); expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled(); - expect(h.probeSandbox).not.toHaveBeenCalled(); + expect(h.verifyGateway).not.toHaveBeenCalled(); }); it("fails with the recovery detail and a rebuild hint when no container exists (#6026)", async () => { @@ -192,7 +192,7 @@ describe("startSandbox", () => { expect(result.exitCode).toBe(1); expect(result.message).toContain("no Docker container labeled"); expect(result.message).toContain("rebuild"); - expect(h.probeSandbox).not.toHaveBeenCalled(); + expect(h.verifyGateway).not.toHaveBeenCalled(); }); it("refuses an unregistered sandbox (#6026)", async () => { @@ -232,7 +232,7 @@ describe("startSandbox", () => { it("propagates a probe rejection instead of reporting success (#6026)", async () => { const h = harness(); - h.probeSandbox.mockRejectedValue(new Error("probe exploded")); + h.verifyGateway.mockRejectedValue(new Error("probe exploded")); await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow("probe exploded"); }); diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index 972c0d64502..3905de3e7f0 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -11,21 +11,18 @@ import { type SandboxLifecycleResult, } from "./runtime/lifecycle-runtime"; -// Lazy require keeps the heavy connect module out of this module's load path; -// tests inject `deps.probeSandbox`. -function loadConnectProbe(): (sandboxName: string) => Promise { +function verifyGateway(sandboxName: string): Promise { const { connectSandbox } = require("./connect") as { - connectSandbox: (sandboxName: string, options?: { probeOnly?: boolean }) => Promise; + connectSandbox: (name: string, options?: { probeOnly?: boolean }) => Promise; }; - return (sandboxName) => connectSandbox(sandboxName, { probeOnly: true }); + return connectSandbox(sandboxName, { probeOnly: true }); } export interface SandboxStartDeps { environment?: NodeJS.ProcessEnv; getSandbox?: typeof registry.getSandbox; runtimeProviders?: RuntimeProviderBundleRegistry; - /** Gateway/forward health probe; defaults to the `recover` action body. */ - probeSandbox?: (sandboxName: string) => Promise; + verifyGateway?: (sandboxName: string) => Promise; log?: (message: string) => void; } @@ -59,6 +56,6 @@ export async function startSandbox( if (result.exitCode !== 0) return result; log(" Checking gateway health and host forwards…"); - await (deps.probeSandbox ?? loadConnectProbe())(sandboxName); + await resolved.lifecycle.verifyStarted(input, deps.verifyGateway ?? verifyGateway); return { exitCode: 0 }; } diff --git a/src/lib/onboard/runtime-provider/contract.ts b/src/lib/onboard/runtime-provider/contract.ts index 38403f2d1e6..04a3e266fd3 100644 --- a/src/lib/onboard/runtime-provider/contract.ts +++ b/src/lib/onboard/runtime-provider/contract.ts @@ -180,6 +180,10 @@ export type RuntimeProviderLifecycleSurface = | RuntimeProviderSupportedSurface<{ readonly channelStopTransport: "docker-kubectl-first" | "openshell"; start(input: RuntimeProviderLifecycleInput): RuntimeProviderLifecycleResult; + verifyStarted( + input: RuntimeProviderLifecycleInput, + verifyGateway: (sandboxName: string) => Promise, + ): Promise; stop( input: RuntimeProviderLifecycleInput, hooks: RuntimeProviderLifecycleStopHooks, diff --git a/src/lib/onboard/runtime-provider/docker.ts b/src/lib/onboard/runtime-provider/docker.ts index 75f7373de36..66c239d1c05 100644 --- a/src/lib/onboard/runtime-provider/docker.ts +++ b/src/lib/onboard/runtime-provider/docker.ts @@ -223,7 +223,7 @@ function removeOwnedDockerWorkload( imageTag.startsWith(`${repository}:`), ) ) { - return { status: "skipped", reason: "authority-unproven" }; + return { status: "skipped", reason: "shared-image" }; } if ( workload?.kind === "legacy-dockerfile" && @@ -315,6 +315,7 @@ export function createDockerRuntimeProviderBundle( supported: true, channelStopTransport: "docker-kubectl-first", start: (input) => startDockerSandbox(input, deps), + verifyStarted: (input, verifyGateway) => verifyGateway(input.sandboxName), stop: (input, hooks) => stopDockerSandbox(input, hooks, deps), }, mutationAuthority: { diff --git a/src/lib/onboard/runtime-provider/registry.ts b/src/lib/onboard/runtime-provider/registry.ts index 04572271ba9..069cd9fc666 100644 --- a/src/lib/onboard/runtime-provider/registry.ts +++ b/src/lib/onboard/runtime-provider/registry.ts @@ -278,6 +278,7 @@ function validateSupportedSurfaceSchemas( ); } requireFunction(surfaces.lifecycle, "start", "lifecycle"); + requireFunction(surfaces.lifecycle, "verifyStarted", "lifecycle"); requireFunction(surfaces.lifecycle, "stop", "lifecycle"); } if (surfaces.mutationAuthority.supported === true) { diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index ef9682320c5..fdf512d5b23 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -12,9 +12,12 @@ import { createInMemoryRuntimeProviderBundle, type InMemoryRuntimeProviderBundle, } from "../../../../test/helpers/runtime-provider-bundle"; +import { requireInferenceSetRuntimeAuthority } from "../../actions/inference-set-provider"; import { removeSandboxImage } from "../../actions/sandbox/destroy"; +import { executeSandboxDestroy } from "../../actions/sandbox/destroy-execution"; import { startSandbox } from "../../actions/sandbox/start"; import { stopSandbox } from "../../actions/sandbox/stop"; +import { loadAgent } from "../../agent/defs"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; import { cloneSandboxWorkloadReceipt } from "../../state/registry/workload"; import { MANAGED_IMAGE_REPOSITORIES } from "../managed-image/contract"; @@ -22,6 +25,7 @@ import { encodeManagedStartupProfile, type ManagedStartupProfile, } from "../managed-startup/profile"; +import { registerCreatedSandbox } from "../sandbox-registration"; import type { RuntimeProviderBundle, RuntimeProviderWorkloadProfile } from "./contract"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "./current"; import { @@ -136,8 +140,10 @@ describe("RuntimeProviderBundle registry contract", () => { expect(Object.isFrozen(registered.workload.profile.support?.platforms)).toBe(true); expectSupportedSurface(registered.lifecycle); expect(Object.isFrozen(registered.lifecycle.start)).toBe(true); + expect(Object.isFrozen(registered.lifecycle.verifyStarted)).toBe(true); expect(registered).not.toBe(source); expect(registered.lifecycle.start).not.toBe(source.lifecycle.start); + expect(registered.lifecycle.verifyStarted).not.toBe(source.lifecycle.verifyStarted); expect(() => { (registered.workload.profile.hostArchitectures as string[]).push("s390x"); }).toThrow(TypeError); @@ -321,6 +327,18 @@ describe("RuntimeProviderBundle registry contract", () => { ).toThrow(RuntimeProviderRegistrationError); }); + it("rejects a lifecycle surface without provider-owned post-start verification", () => { + const bundle = mxcBundle(); + expectSupportedSurface(bundle.lifecycle); + const { verifyStarted: _verifyStarted, ...incomplete } = bundle.lifecycle; + + expect(() => + createRuntimeProviderBundleRegistry([ + ["mxc", replaceSurface(bundle, "lifecycle", incomplete)], + ]), + ).toThrow(/lifecycle\.verifyStarted must be a function/u); + }); + it("rejects capability/surface drift and duplicate operation-scoped engine identities", () => { const bundle = mxcBundle(); expect(() => @@ -498,7 +516,9 @@ describe("sandbox workload ownership receipt", () => { describe("socket-free MXC action contract", () => { const agents = ["openclaw", "hermes", "langchain-deepagents-code"] as const; - it.each(agents)("routes %s lifecycle and cleanup through one injected bundle", async (agent) => { + it.each( + agents, + )("routes %s registration, lifecycle, inference authority, destroy, and cleanup through one injected bundle", async (agent) => { const state = { events: [] as string[], running: new Set(), @@ -512,10 +532,31 @@ describe("socket-free MXC action contract", () => { const providers = createRuntimeProviderBundleRegistry([["mxc", bundle]]); const sandboxName = `${agent}-sandbox`; const imageTag = `mxc-memory:${agent}`; - const entry: SandboxEntry = { - name: sandboxName, - agent, - openshellDriver: "mxc", + const registerSandbox = vi.fn(); + const entry = registerCreatedSandbox({ + sandboxName, + inferenceSelection: { + model: "test/model", + provider: "nvidia-prod", + endpointUrl: null, + endpointSource: null, + credentialEnv: null, + preferredInferenceApi: null, + compatibleEndpointReasoning: null, + compatibleEndpointReasoningEffort: null, + nimContainer: null, + }, + runtimeFields: { + gpuEnabled: false, + hostGpuDetected: false, + sandboxGpuEnabled: false, + sandboxGpuMode: "auto", + sandboxGpuDevice: null, + openshellDriver: "mxc", + openshellVersion: "test", + }, + agent: loadAgent(agent), + agentVersionKnown: false, imageTag, workload: { schemaVersion: 1, @@ -523,18 +564,32 @@ describe("socket-free MXC action contract", () => { reference: imageTag, shared: false, }, - }; + appliedPolicies: [], + plannedMessagingState: undefined, + hermesToolGateways: [], + hermesDashboardState: { enabled: false, config: null }, + dashboardPort: 18789, + gatewayName: "nemoclaw", + gatewayPort: 8080, + registerSandbox, + runtimeProviders: providers, + }); state.workloads.add(imageTag); const getSandbox = vi.fn(() => entry); - const probeSandbox = vi.fn(async () => undefined); const stopSandboxChannels = vi.fn(); const teardownSandboxDashboardForward = vi.fn(); + const cleanupShieldsArtifacts = vi.fn(); + const runOpenshell = vi.fn((args: string[]) => { + if (args[0] === "sandbox" && args[1] === "delete") { + state.events.push(`delete:${sandboxName}`); + } + return { status: 0, stdout: "", stderr: "" }; + }); await expect( startSandbox(sandboxName, { getSandbox, runtimeProviders: providers, - probeSandbox, log: vi.fn(), }), ).resolves.toEqual({ exitCode: 0 }); @@ -548,6 +603,22 @@ describe("socket-free MXC action contract", () => { warn: vi.fn(), }), ).toEqual({ exitCode: 0 }); + expect(() => requireInferenceSetRuntimeAuthority(entry, providers)).not.toThrow(); + await expect( + executeSandboxDestroy({ + cleanupShieldsArtifacts, + force: false, + runOpenshell, + sandbox: entry, + sandboxConfirmedAbsent: false, + sandboxName, + runtimeProviders: providers, + deps: { + readTimerMarker: () => null, + wipeSandboxState: vi.fn(), + }, + }), + ).resolves.toMatchObject({ ok: true }); expect( removeSandboxImage(sandboxName, { getSandbox, @@ -561,7 +632,12 @@ describe("socket-free MXC action contract", () => { reference: imageTag, }); - expect(probeSandbox).toHaveBeenCalledWith(sandboxName); + expect(registerSandbox).toHaveBeenCalledWith(entry); + expect(runOpenshell).toHaveBeenCalledWith(["sandbox", "delete", sandboxName], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + expect(cleanupShieldsArtifacts).toHaveBeenCalledWith(sandboxName); expect(stopSandboxChannels).toHaveBeenCalledWith( sandboxName, expect.objectContaining({ @@ -572,7 +648,10 @@ describe("socket-free MXC action contract", () => { ); expect(state.events).toEqual([ `start:${sandboxName}`, + `verify-started:${sandboxName}`, `stop:${sandboxName}`, + `prepare-destroy:${sandboxName}`, + `delete:${sandboxName}`, `cleanup:${sandboxName}`, ]); expect(state.running).not.toContain(sandboxName); diff --git a/test/helpers/runtime-provider-bundle.ts b/test/helpers/runtime-provider-bundle.ts index 048a22e883b..5c24bb5f69c 100644 --- a/test/helpers/runtime-provider-bundle.ts +++ b/test/helpers/runtime-provider-bundle.ts @@ -104,6 +104,9 @@ export function createInMemoryRuntimeProviderBundle({ input.log(` In-memory workload '${input.sandboxName}' started.`); return { exitCode: 0 }; }, + async verifyStarted(input: RuntimeProviderLifecycleInput) { + event("verify-started", input.sandboxName); + }, stop(input: RuntimeProviderLifecycleInput, hooks: RuntimeProviderLifecycleStopHooks) { const wasRunning = state.running.delete(input.sandboxName); const beforeStop = wasRunning ? hooks.beforeStop : () => undefined; diff --git a/test/image-cleanup.test.ts b/test/image-cleanup.test.ts index ee191c2ca1d..c7848f046aa 100644 --- a/test/image-cleanup.test.ts +++ b/test/image-cleanup.test.ts @@ -4,25 +4,24 @@ // Verify that sandbox lifecycle operations clean up host-side Docker images. // See: https://github.com/NVIDIA/NemoClaw/issues/2086 -import { describe, it, expect, vi } from "vitest"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; - +import { describe, expect, it, vi } from "vitest"; +import { help as renderRootHelp } from "../src/lib/actions/root-help"; import { cleanupShieldsDestroyArtifacts, removeSandboxImage, removeSandboxRegistryEntry, removeShieldsState, } from "../src/lib/actions/sandbox/destroy"; -import { getSandboxDeleteOutcome } from "../src/lib/domain/sandbox/destroy"; -import { normalizeGarbageCollectImagesOptions } from "../src/lib/domain/lifecycle/options"; -import { resolveNemoclawStateDir } from "../src/lib/state/paths"; -import { help as renderRootHelp } from "../src/lib/actions/root-help"; import { COMMANDS, globalCommandTokens } from "../src/lib/cli/command-registry"; import { getRegisteredOclifCommandMetadata } from "../src/lib/cli/oclif-metadata"; +import { normalizeGarbageCollectImagesOptions } from "../src/lib/domain/lifecycle/options"; +import { getSandboxDeleteOutcome } from "../src/lib/domain/sandbox/destroy"; import { createDockerRuntimeProviderBundle } from "../src/lib/onboard/runtime-provider/docker"; import { createRuntimeProviderBundleRegistry } from "../src/lib/onboard/runtime-provider/registry"; +import { resolveNemoclawStateDir } from "../src/lib/state/paths"; describe("image cleanup: sandbox destroy removes Docker image (#2086)", () => { it("removes sandbox images before deleting the registry entry", () => { @@ -107,7 +106,7 @@ describe("image cleanup: sandbox destroy removes Docker image (#2086)", () => { expect(removeImage).not.toHaveBeenCalled(); }); - it("fails closed when a managed workload receipt was dropped as malformed", () => { + it("protects a managed image and removes its registry row when its receipt was dropped", () => { const removeImage = vi.fn(() => ({ status: 0 })); const runtimeProviders = createRuntimeProviderBundleRegistry([ ["docker", createDockerRuntimeProviderBundle({ removeImage })], @@ -124,8 +123,17 @@ describe("image cleanup: sandbox destroy removes Docker image (#2086)", () => { runtimeProviders, }); - expect(result).toEqual({ status: "skipped", reason: "authority-unproven" }); + expect(result).toEqual({ status: "skipped", reason: "shared-image" }); expect(removeImage).not.toHaveBeenCalled(); + + const removeSandbox = vi.fn(() => true); + expect( + removeSandboxRegistryEntry("alpha", { + removeImage: () => result, + removeSandbox, + }), + ).toBe(true); + expect(removeSandbox).toHaveBeenCalledWith("alpha"); }); it("treats missing sandbox delete results as already gone", () => { diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index e738d687d59..47682ce4357 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -8,8 +8,8 @@ import { describe, expect, it } from "vitest"; const repoRoot = join(import.meta.dirname, ".."); describe("runtime provider central source boundary", () => { - // source-shape-contract: compatibility -- Central non-snapshot actions must stay provider-neutral while production selection excludes unqualified future providers and managed-bootstrap dependencies - it("keeps provider identities and implementations behind the one bundle composition", () => { + // source-shape-contract: compatibility -- Migrated lifecycle and mutation consumers must stay provider-neutral while production selection excludes unqualified future providers and managed-bootstrap dependencies + it("keeps migrated provider identities and implementations behind the one bundle composition", () => { const centralConsumers = [ readFileSync(join(repoRoot, "src/lib/actions/inference-set.ts"), "utf8"), readFileSync(join(repoRoot, "src/lib/actions/sandbox/destroy-execution.ts"), "utf8"), @@ -37,6 +37,7 @@ describe("runtime provider central source boundary", () => { expect(source).not.toMatch(/\b(?:openshellDriver|driverName)\s*={2,3}\s*["'][^"']+["']/u); expect(source).not.toMatch(/switch\s*\([^)]*\b(?:openshellDriver|driverName)\b[^)]*\)/u); } + expect(centralConsumers[4]).toMatch(/resolved\.lifecycle\.verifyStarted\(/u); expect(providerContract.join("\n")).not.toMatch(/managed-bootstrap/u); expect(providerContract[1]).not.toMatch(/\b(?:podman|mxc)\b/iu); }); From 77559f7512b565225a6cf5a316d31a0de52301a5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 01:16:16 -0700 Subject: [PATCH 027/117] test(runtime): keep provider parity branchless Signed-off-by: Aaron Erickson --- .../runtime-provider-contract.test.ts | 13 ++++++------- test/helpers/runtime-provider-bundle.ts | 4 +++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index fdf512d5b23..d6362c75510 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -524,10 +524,12 @@ describe("socket-free MXC action contract", () => { running: new Set(), workloads: new Set(), }; + const recordEvent = vi.fn((value: string) => state.events.push(value)); const bundle = createInMemoryRuntimeProviderBundle({ providerId: "mxc", workloadProfile: PORTABLE_PROFILE, state, + recordEvent, }); const providers = createRuntimeProviderBundleRegistry([["mxc", bundle]]); const sandboxName = `${agent}-sandbox`; @@ -579,12 +581,7 @@ describe("socket-free MXC action contract", () => { const stopSandboxChannels = vi.fn(); const teardownSandboxDashboardForward = vi.fn(); const cleanupShieldsArtifacts = vi.fn(); - const runOpenshell = vi.fn((args: string[]) => { - if (args[0] === "sandbox" && args[1] === "delete") { - state.events.push(`delete:${sandboxName}`); - } - return { status: 0, stdout: "", stderr: "" }; - }); + const runOpenshell = vi.fn(() => ({ status: 0, stdout: "", stderr: "" })); await expect( startSandbox(sandboxName, { @@ -637,6 +634,9 @@ describe("socket-free MXC action contract", () => { ignoreError: true, stdio: ["ignore", "pipe", "pipe"], }); + expect(recordEvent.mock.invocationCallOrder[3]).toBeLessThan( + runOpenshell.mock.invocationCallOrder.at(-1)!, + ); expect(cleanupShieldsArtifacts).toHaveBeenCalledWith(sandboxName); expect(stopSandboxChannels).toHaveBeenCalledWith( sandboxName, @@ -651,7 +651,6 @@ describe("socket-free MXC action contract", () => { `verify-started:${sandboxName}`, `stop:${sandboxName}`, `prepare-destroy:${sandboxName}`, - `delete:${sandboxName}`, `cleanup:${sandboxName}`, ]); expect(state.running).not.toContain(sandboxName); diff --git a/test/helpers/runtime-provider-bundle.ts b/test/helpers/runtime-provider-bundle.ts index 5c24bb5f69c..07b9a1ad3b4 100644 --- a/test/helpers/runtime-provider-bundle.ts +++ b/test/helpers/runtime-provider-bundle.ts @@ -30,6 +30,7 @@ type InMemoryRuntimeProviderOptions = { readonly workloadProfile: RuntimeProviderWorkloadProfile; readonly state?: InMemoryRuntimeProviderState; readonly gatewayLauncher?: "nemoclaw" | "openshell"; + readonly recordEvent?: (event: string) => void; }; function unsupported(providerId: string, reason: string) { @@ -46,9 +47,10 @@ export function createInMemoryRuntimeProviderBundle({ workloadProfile, state = { events: [], running: new Set(), workloads: new Set() }, gatewayLauncher = "nemoclaw", + recordEvent = (value) => state.events.push(value), }: InMemoryRuntimeProviderOptions): InMemoryRuntimeProviderBundle { const futureReason = "Unsupported by this in-memory contract fixture."; - const event = (kind: string, sandboxName: string) => state.events.push(`${kind}:${sandboxName}`); + const event = (kind: string, sandboxName: string) => recordEvent(`${kind}:${sandboxName}`); return { identity: { contractVersion: RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, From a2afe77ac318ee1d6b4e20e81d6b108e0a4b090f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 01:31:32 -0700 Subject: [PATCH 028/117] fix(e2e): harden runtime foundation contracts Signed-off-by: Aaron Erickson --- test/e2e/fixtures/runtime-provider.ts | 34 ++++++-- test/e2e/registry/parity-evidence.ts | 61 ++++++++------ test/e2e/registry/scenario.ts | 7 +- test/e2e/support/e2e-parity-evidence.test.ts | 49 +++++++++++- .../e2e-runtime-foundation-types.test.ts | 48 ++++++++++- test/e2e/support/e2e-runtime-matrix.test.ts | 80 +++++++++++++++++++ 6 files changed, 242 insertions(+), 37 deletions(-) diff --git a/test/e2e/fixtures/runtime-provider.ts b/test/e2e/fixtures/runtime-provider.ts index 59eeff068ce..98827c329ed 100644 --- a/test/e2e/fixtures/runtime-provider.ts +++ b/test/e2e/fixtures/runtime-provider.ts @@ -6,7 +6,7 @@ import { buildExecutionEvidence, type ExecutionEvidence, type ManagedImageEvidence, - type ProviderReceipt, + type NonEmptyProviderReceipts, } from "../registry/parity-evidence.ts"; import { executionPreparationKey, @@ -40,7 +40,7 @@ export interface RuntimeLifecycleEvidence { fsmTrace: readonly FsmTransition[]; terminalOutcome: TerminalOutcome; userVisibleState: JsonValue; - providerReceipts: readonly ProviderReceipt[]; + providerReceipts: NonEmptyProviderReceipts; } /** @@ -53,7 +53,7 @@ export interface RuntimeProviderEnvironment { export interface RuntimeProviderLifecycle { executeAdapter(adapterId: string, request: RuntimeAdapterRequest): Promise; - cleanup(identity: ExactWorkloadIdentity): Promise; + cleanup(identity: ExactWorkloadIdentity): Promise; } export interface RuntimeProviderState { @@ -77,6 +77,23 @@ export interface RuntimeExecutionRequest { }; } +async function cleanupAfterExecutionFailure( + provider: RuntimeProviderFixture, + workload: ExactWorkloadIdentity, + executionError: unknown, +): Promise { + try { + await provider.lifecycle.cleanup(workload); + } catch (cleanupError) { + throw new AggregateError( + [executionError, cleanupError], + "Runtime case execution failed and provider cleanup also failed", + { cause: executionError }, + ); + } + throw executionError; +} + /** * The only executable cross-runtime path in this foundation. It does not use * the legacy Docker-shaped environment/lifecycle/state fixtures and is not @@ -114,8 +131,12 @@ export async function executeRuntimeCaseThroughProvider( logicalId: runtimeCase.identities.sandbox, }); let lifecycle: RuntimeLifecycleEvidence; - let cleanupReceipts: readonly ProviderReceipt[] = []; try { + if (workload.logicalId !== runtimeCase.identities.sandbox) { + throw new Error( + `workload.logicalId '${workload.logicalId}' does not match case sandbox identity '${runtimeCase.identities.sandbox}'`, + ); + } for (const binding of runtimeCase.obligationBindings) { await binding.adapter.execute(provider, { caseId: runtimeCase.id, @@ -127,9 +148,10 @@ export async function executeRuntimeCaseThroughProvider( caseId: runtimeCase.id, workload, }); - } finally { - cleanupReceipts = await provider.lifecycle.cleanup(workload); + } catch (executionError) { + return cleanupAfterExecutionFailure(provider, workload, executionError); } + const cleanupReceipts = await provider.lifecycle.cleanup(workload); return buildExecutionEvidence({ resolved: request.resolved, diff --git a/test/e2e/registry/parity-evidence.ts b/test/e2e/registry/parity-evidence.ts index 4b9c5571f86..eb222d643b6 100644 --- a/test/e2e/registry/parity-evidence.ts +++ b/test/e2e/registry/parity-evidence.ts @@ -37,6 +37,13 @@ export interface ProviderReceipt { value: JsonValue; } +export type NonEmptyProviderReceipts = readonly [ProviderReceipt, ...ProviderReceipt[]]; + +type NormalizedProviderReceipts = readonly [ + Readonly, + ...Readonly[], +]; + export interface NormalizedParityEvidence { desiredStateFingerprint: string; fsmTrace: readonly Readonly[]; @@ -71,7 +78,7 @@ export interface ExecutionEvidence { managedImages: readonly Readonly[]; }>; parity: Readonly; - providerReceipts: readonly Readonly[]; + providerReceipts: NormalizedProviderReceipts; } export interface ExecutionEvidenceInput { @@ -95,7 +102,7 @@ export interface ExecutionEvidenceInput { terminalOutcome: TerminalOutcome; userVisibleState: JsonValue; }; - providerReceipts: readonly ProviderReceipt[]; + providerReceipts: NonEmptyProviderReceipts; } export interface ParityMismatch { @@ -162,33 +169,37 @@ function normalizeManagedImages( ); } -function normalizeProviderReceipts( - receipts: readonly ProviderReceipt[], -): readonly Readonly[] { +function normalizeProviderReceipts(receipts: NonEmptyProviderReceipts): NormalizedProviderReceipts { if (receipts.length === 0) { throw new Error("providerReceipts must contain at least one provider receipt"); } const operationIds = new Set(); - return Object.freeze( - receipts.map((receipt) => { - if (!RECEIPT_ID_PATTERN.test(receipt.kind)) { - throw new Error(`provider receipt kind '${receipt.kind}' is invalid`); - } - if (!RECEIPT_ID_PATTERN.test(receipt.operationId)) { - throw new Error(`provider receipt operationId '${receipt.operationId}' is invalid`); - } - if (operationIds.has(receipt.operationId)) { - throw new Error(`provider receipt operationId '${receipt.operationId}' is duplicated`); - } - operationIds.add(receipt.operationId); - return Object.freeze({ - ...receipt, - value: freezeJsonValue( - normalizeJsonValue(receipt.value, `providerReceipts.${receipt.operationId}.value`), - ), - }); - }), - ); + const normalizeReceipt = (receipt: ProviderReceipt): Readonly => { + if (typeof receipt.kind !== "string") { + throw new Error("provider receipt kind must be a string"); + } + if (typeof receipt.operationId !== "string") { + throw new Error("provider receipt operationId must be a string"); + } + if (!RECEIPT_ID_PATTERN.test(receipt.kind)) { + throw new Error(`provider receipt kind '${receipt.kind}' is invalid`); + } + if (!RECEIPT_ID_PATTERN.test(receipt.operationId)) { + throw new Error(`provider receipt operationId '${receipt.operationId}' is invalid`); + } + if (operationIds.has(receipt.operationId)) { + throw new Error(`provider receipt operationId '${receipt.operationId}' is duplicated`); + } + operationIds.add(receipt.operationId); + return Object.freeze({ + kind: receipt.kind, + operationId: receipt.operationId, + value: freezeJsonValue( + normalizeJsonValue(receipt.value, `providerReceipts.${receipt.operationId}.value`), + ), + }); + }; + return Object.freeze([normalizeReceipt(receipts[0]), ...receipts.slice(1).map(normalizeReceipt)]); } export function buildExecutionEvidence(input: ExecutionEvidenceInput): ExecutionEvidence { diff --git a/test/e2e/registry/scenario.ts b/test/e2e/registry/scenario.ts index 53f167642ad..b6b284b7129 100644 --- a/test/e2e/registry/scenario.ts +++ b/test/e2e/registry/scenario.ts @@ -234,13 +234,13 @@ export function defineRuntimeScenario(input: RuntimeNeutralScenario): RuntimeNeu assertUniqueIds(input.journey, input.id, "journey step"); const journey = input.journey.map((step) => { assertExecutionFoundationId(step.action, "Journey action"); - return Object.freeze({ ...step }); + return Object.freeze({ id: step.id, action: step.action }); }); assertUniqueIds(input.supportObligations, input.id, "obligation"); const supportObligations = input.supportObligations.map((obligation) => Object.freeze({ - ...obligation, + id: obligation.id, description: normalizeDescription( obligation.description, `Runtime scenario '${input.id}' obligation '${obligation.id}'`, @@ -256,7 +256,8 @@ export function defineRuntimeScenario(input: RuntimeNeutralScenario): RuntimeNeu assertTerminalMatchesTrace(fsmTrace, terminalOutcome, "assertions.terminalOutcome"); return Object.freeze({ - ...input, + id: input.id, + agent: input.agent, description, journey: Object.freeze(journey), requiredCapabilities: normalizeCapabilities( diff --git a/test/e2e/support/e2e-parity-evidence.test.ts b/test/e2e/support/e2e-parity-evidence.test.ts index 123b17044be..0025deaac4a 100644 --- a/test/e2e/support/e2e-parity-evidence.test.ts +++ b/test/e2e/support/e2e-parity-evidence.test.ts @@ -15,6 +15,8 @@ import { compareParityEvidence, type ExecutionEvidenceInput, fingerprintDesiredState, + type NonEmptyProviderReceipts, + type ProviderReceipt, } from "../registry/parity-evidence.ts"; import { compileRuntimeMatrix, @@ -129,7 +131,7 @@ describe("cross-runtime parity evidence", () => { expect(() => buildExecutionEvidence(badImage)).toThrow(/exact sha256 digest/); const missingReceipt = evidenceInput("docker"); - missingReceipt.providerReceipts = []; + missingReceipt.providerReceipts = [] as unknown as NonEmptyProviderReceipts; expect(() => buildExecutionEvidence(missingReceipt)).toThrow(/provider receipt/); const wrongWorkload = evidenceInput("docker"); @@ -166,6 +168,51 @@ describe("cross-runtime parity evidence", () => { ); }); + it("rejects non-string receipt identities and omits unknown receipt fields", () => { + const missingKind = evidenceInput("docker"); + missingKind.providerReceipts = [ + { + kind: undefined, + operationId: "docker.prepare", + value: {}, + } as unknown as ProviderReceipt, + ]; + expect(() => buildExecutionEvidence(missingKind)).toThrow( + /provider receipt kind must be a string/, + ); + + const missingOperationId = evidenceInput("docker"); + missingOperationId.providerReceipts = [ + { + kind: "prepare", + operationId: undefined, + value: {}, + } as unknown as ProviderReceipt, + ]; + expect(() => buildExecutionEvidence(missingOperationId)).toThrow( + /provider receipt operationId must be a string/, + ); + + const extraField = evidenceInput("docker"); + extraField.providerReceipts = [ + { + kind: "prepare", + operationId: "docker.prepare", + value: { fixture: true }, + providerPrivateField: "must-not-persist", + } as ProviderReceipt, + ]; + const evidence = buildExecutionEvidence(extraField); + expect(evidence.providerReceipts).toEqual([ + { + kind: "prepare", + operationId: "docker.prepare", + value: { fixture: true }, + }, + ]); + expect(evidence.providerReceipts[0]).not.toHaveProperty("providerPrivateField"); + }); + it("publishes provider receipts through the redacting artifact boundary", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-execution-evidence-")); try { diff --git a/test/e2e/support/e2e-runtime-foundation-types.test.ts b/test/e2e/support/e2e-runtime-foundation-types.test.ts index e535c8cc5f7..493e9adfad9 100644 --- a/test/e2e/support/e2e-runtime-foundation-types.test.ts +++ b/test/e2e/support/e2e-runtime-foundation-types.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { describe, expect, expectTypeOf, it } from "vitest"; import { test as e2eFixtureTest } from "../fixtures/e2e-test.ts"; import { @@ -9,12 +9,22 @@ import { LifecyclePhaseFixture, StateValidationPhaseFixture, } from "../fixtures/phases/index.ts"; +import type { + RuntimeLifecycleEvidence, + RuntimeProviderLifecycle, +} from "../fixtures/runtime-provider.ts"; import { defineExecutionProfile, executionProviderId } from "../registry/execution-profile.ts"; -import { defineRuntimeScenario } from "../registry/scenario.ts"; +import type { + ExecutionEvidenceInput, + NonEmptyProviderReceipts, +} from "../registry/parity-evidence.ts"; +import { buildLiveTargetMatrix } from "../registry/run.ts"; +import { defineRuntimeScenario, type RuntimeNeutralScenario } from "../registry/scenario.ts"; import { foundationProfiles, foundationScenarios } from "./cross-runtime-foundation-fixtures.ts"; describe("cross-runtime foundation types", () => { it("models Docker and socket-free fake MXC profiles without registering either", () => { + const liveMatrixBefore = buildLiveTargetMatrix(); const [docker, mxc] = foundationProfiles(); expect(docker).toMatchObject({ @@ -32,6 +42,7 @@ describe("cross-runtime foundation types", () => { }); expect(mxc?.capabilities).toContain("transport.socket-free"); expect(mxc?.capabilities).not.toContain("transport.docker-socket"); + expect(buildLiveTargetMatrix()).toEqual(liveMatrixBefore); }); it("keeps OpenClaw, Hermes, and DCode scenarios provider-neutral and obligation-explicit", () => { @@ -59,6 +70,39 @@ describe("cross-runtime foundation types", () => { } }); + it("omits undeclared provider fields from normalized scenarios", () => { + const base = foundationScenarios()[0]!; + const input = { + ...base, + provider: "docker", + journey: base.journey.map((step) => ({ ...step, provider: "docker" })), + assertions: { ...base.assertions, provider: "docker" }, + supportObligations: base.supportObligations.map((obligation) => ({ + ...obligation, + provider: "docker", + })), + } satisfies RuntimeNeutralScenario & { provider: string }; + + const scenario = defineRuntimeScenario(input); + + expect(scenario).not.toHaveProperty("provider"); + expect(scenario.journey[0]).not.toHaveProperty("provider"); + expect(scenario.assertions).not.toHaveProperty("provider"); + expect(scenario.supportObligations[0]).not.toHaveProperty("provider"); + }); + + it("requires provider receipt evidence from lifecycle and cleanup contracts", () => { + expectTypeOf< + RuntimeLifecycleEvidence["providerReceipts"] + >().toEqualTypeOf(); + expectTypeOf< + Awaited> + >().toEqualTypeOf(); + expectTypeOf< + ExecutionEvidenceInput["providerReceipts"] + >().toEqualTypeOf(); + }); + it("keeps provider ids open while rejecting invalid profiles", () => { const valid = foundationProfiles()[0]!; expect( diff --git a/test/e2e/support/e2e-runtime-matrix.test.ts b/test/e2e/support/e2e-runtime-matrix.test.ts index 0892f7688d5..46e640cec52 100644 --- a/test/e2e/support/e2e-runtime-matrix.test.ts +++ b/test/e2e/support/e2e-runtime-matrix.test.ts @@ -6,6 +6,7 @@ import assert from "node:assert/strict"; import { describe, expect, it } from "vitest"; import { + type ExactWorkloadIdentity, executeRuntimeCaseThroughProvider, type RuntimeProviderFixture, } from "../fixtures/runtime-provider.ts"; @@ -339,6 +340,85 @@ describe("cross-runtime E2E matrix compiler", () => { } }); + it("rejects an inspected workload mismatch before executing obligations and still cleans it", async () => { + const matrix = compileRuntimeMatrix(foundationDefinition()); + const runtimeCase = matrix.cases.find( + (entry) => entry.scenario.agent === "openclaw" && entry.profile.provider === "docker", + ); + assert.ok(runtimeCase, "Missing Docker OpenClaw fixture case"); + const resolved = resolveRuntimeCase(matrix, { + scenarioId: runtimeCase.scenario.id, + profileId: runtimeCase.profile.id, + }); + const adapterCalls: string[] = []; + const cleanedWorkloads: ExactWorkloadIdentity[] = []; + const provider = fakeRuntimeProvider(runtimeCase.profile, adapterCalls); + const inspectedWorkload = { + logicalId: "provider-returned-wrong-sandbox", + providerResourceId: "docker://fixture/provider-returned-wrong-sandbox", + managedImages: [{ role: "agent", digest: `sha256:${"b".repeat(64)}` }], + }; + provider.state.inspectWorkload = async () => inspectedWorkload; + provider.lifecycle.cleanup = async (workload) => { + cleanedWorkloads.push(workload); + return [ + { + kind: "cleanup", + operationId: "docker.cleanup-mismatched-workload", + value: { fixture: true }, + }, + ]; + }; + + await expect( + executeRuntimeCaseThroughProvider({ + resolved, + provider, + source: { + headSha: "0123456789abcdef0123456789abcdef01234567", + baseSha: "89abcdef0123456789abcdef0123456789abcdef", + }, + }), + ).rejects.toThrow(/does not match case sandbox identity/); + expect(adapterCalls).toEqual([]); + expect(cleanedWorkloads).toEqual([inspectedWorkload]); + }); + + it("keeps the execution failure as the cause when cleanup also fails", async () => { + const matrix = compileRuntimeMatrix(foundationDefinition()); + const runtimeCase = matrix.cases.find( + (entry) => entry.scenario.agent === "openclaw" && entry.profile.provider === "docker", + ); + assert.ok(runtimeCase, "Missing Docker OpenClaw fixture case"); + const resolved = resolveRuntimeCase(matrix, { + scenarioId: runtimeCase.scenario.id, + profileId: runtimeCase.profile.id, + }); + const executionError = new Error("adapter execution failed"); + const cleanupError = new Error("provider cleanup failed"); + const provider = fakeRuntimeProvider(runtimeCase.profile); + provider.lifecycle.executeAdapter = async () => { + throw executionError; + }; + provider.lifecycle.cleanup = async () => { + throw cleanupError; + }; + + await expect( + executeRuntimeCaseThroughProvider({ + resolved, + provider, + source: { + headSha: "0123456789abcdef0123456789abcdef01234567", + baseSha: "89abcdef0123456789abcdef0123456789abcdef", + }, + }), + ).rejects.toMatchObject({ + cause: executionError, + errors: [executionError, cleanupError], + }); + }); + it("compiles optional runtime metadata through the existing target run-plan path", () => { const runtimeMatrix = foundationDefinition(); const compiled = compileRuntimeMatrix(runtimeMatrix); From 004b0601c174f2ec6744b0e0ddd24c8825f44741 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 01:35:52 -0700 Subject: [PATCH 029/117] test(e2e): verify fixture profiles stay unregistered Signed-off-by: Aaron Erickson --- test/e2e/support/e2e-runtime-foundation-types.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/e2e/support/e2e-runtime-foundation-types.test.ts b/test/e2e/support/e2e-runtime-foundation-types.test.ts index 493e9adfad9..3854cf26502 100644 --- a/test/e2e/support/e2e-runtime-foundation-types.test.ts +++ b/test/e2e/support/e2e-runtime-foundation-types.test.ts @@ -25,7 +25,8 @@ import { foundationProfiles, foundationScenarios } from "./cross-runtime-foundat describe("cross-runtime foundation types", () => { it("models Docker and socket-free fake MXC profiles without registering either", () => { const liveMatrixBefore = buildLiveTargetMatrix(); - const [docker, mxc] = foundationProfiles(); + const profiles = foundationProfiles(); + const [docker, mxc] = profiles; expect(docker).toMatchObject({ provider: "docker", @@ -43,6 +44,9 @@ describe("cross-runtime foundation types", () => { expect(mxc?.capabilities).toContain("transport.socket-free"); expect(mxc?.capabilities).not.toContain("transport.docker-socket"); expect(buildLiveTargetMatrix()).toEqual(liveMatrixBefore); + for (const profile of profiles) { + expect(() => buildLiveTargetMatrix([profile.id])).toThrow(`Unknown target '${profile.id}'`); + } }); it("keeps OpenClaw, Hermes, and DCode scenarios provider-neutral and obligation-explicit", () => { @@ -81,7 +85,7 @@ describe("cross-runtime foundation types", () => { ...obligation, provider: "docker", })), - } satisfies RuntimeNeutralScenario & { provider: string }; + } as unknown as RuntimeNeutralScenario; const scenario = defineRuntimeScenario(input); From 4788d287b8672be1b44999e78e094b2221303bd1 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 01:39:40 -0700 Subject: [PATCH 030/117] test(e2e): expose runtime profile registration state Signed-off-by: Aaron Erickson --- test/e2e/registry/registry.ts | 4 ++++ test/e2e/support/e2e-runtime-foundation-types.test.ts | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test/e2e/registry/registry.ts b/test/e2e/registry/registry.ts index 4fbfa9e939b..3ae6c43b9ea 100644 --- a/test/e2e/registry/registry.ts +++ b/test/e2e/registry/registry.ts @@ -43,6 +43,10 @@ export function listTargets(): TargetDefinition[] { return [...registry.targets].sort((a, b) => a.id.localeCompare(b.id)); } +export function hasRegisteredRuntimeProfile(profileId: string): boolean { + return registry.targets.some((target) => target.runtimeCase?.profileId === profileId); +} + export function getTarget(id: string): TargetDefinition | undefined { return registry.byId.get(id); } diff --git a/test/e2e/support/e2e-runtime-foundation-types.test.ts b/test/e2e/support/e2e-runtime-foundation-types.test.ts index 3854cf26502..45f7e73ff72 100644 --- a/test/e2e/support/e2e-runtime-foundation-types.test.ts +++ b/test/e2e/support/e2e-runtime-foundation-types.test.ts @@ -18,6 +18,7 @@ import type { ExecutionEvidenceInput, NonEmptyProviderReceipts, } from "../registry/parity-evidence.ts"; +import { hasRegisteredRuntimeProfile } from "../registry/registry.ts"; import { buildLiveTargetMatrix } from "../registry/run.ts"; import { defineRuntimeScenario, type RuntimeNeutralScenario } from "../registry/scenario.ts"; import { foundationProfiles, foundationScenarios } from "./cross-runtime-foundation-fixtures.ts"; @@ -45,7 +46,7 @@ describe("cross-runtime foundation types", () => { expect(mxc?.capabilities).not.toContain("transport.docker-socket"); expect(buildLiveTargetMatrix()).toEqual(liveMatrixBefore); for (const profile of profiles) { - expect(() => buildLiveTargetMatrix([profile.id])).toThrow(`Unknown target '${profile.id}'`); + expect(hasRegisteredRuntimeProfile(profile.id)).toBe(false); } }); From 64f562f37f24c73dcc8abbafccdc46492b65b6c4 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 01:50:44 -0700 Subject: [PATCH 031/117] fix(runtime): preserve unproven cleanup authority Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/destroy-flow.test.ts | 26 ++++ src/lib/actions/sandbox/destroy.ts | 78 +++++++++-- .../sandbox/runtime/lifecycle-runtime.ts | 7 +- src/lib/actions/sandbox/stop.ts | 2 +- src/lib/onboard/compute/plan.ts | 6 +- src/lib/onboard/runtime-provider/access.ts | 1 + src/lib/onboard/runtime-provider/contract.ts | 5 +- src/lib/onboard/runtime-provider/docker.ts | 4 +- src/lib/onboard/runtime-provider/registry.ts | 130 +++++++++++++----- src/lib/onboard/workload/runtime.ts | 40 +----- src/lib/state/registry/workload.ts | 12 +- src/lib/tunnel/sandbox-gateway-stop.test.ts | 5 +- src/lib/tunnel/sandbox-gateway-stop.ts | 3 +- test/helpers/destroy-flow-test-harness.ts | 9 +- test/helpers/runtime-provider-bundle.ts | 2 +- test/image-cleanup.test.ts | 52 +++++++ test/runtime-provider-source-shape.test.ts | 55 ++++---- 17 files changed, 310 insertions(+), 127 deletions(-) diff --git a/src/lib/actions/sandbox/destroy-flow.test.ts b/src/lib/actions/sandbox/destroy-flow.test.ts index 6e52882c97d..7ac29fcdd4d 100644 --- a/src/lib/actions/sandbox/destroy-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-flow.test.ts @@ -109,6 +109,9 @@ describe("destroySandbox flow", () => { await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow("process.exit(1)"); + const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(errorOutput).toContain("unknown-runtime"); + expect(errorOutput).toContain("is not registered for this operation"); expect( harness.runOpenshellSpy.mock.calls.some( ([args]) => Array.isArray(args) && args[0] === "sandbox" && args[1] === "delete", @@ -117,6 +120,29 @@ describe("destroySandbox flow", () => { expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); }); + it("reports incomplete cleanup and preserves ownership when image authority is unproven", async () => { + const harness = createDestroyHarness({ + imageTag: "local/alpha:current", + workload: { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "local/alpha:recorded", + shared: false, + }, + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow("process.exit(1)"); + + const warningOutput = harness.warnSpy.mock.calls.map((call) => String(call[0])).join("\n"); + const logOutput = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(warningOutput).toContain("Runtime provider 'docker'"); + expect(warningOutput).toContain("workload ownership authority could not be proven"); + expect(warningOutput).toContain("nemoclaw alpha destroy --yes"); + expect(logOutput).not.toContain("Sandbox 'alpha' destroyed"); + expect(harness.events).toContain("delete"); + expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); + }); + it("refuses shields-up Hermes MCP destroy before stopping services or preparing MCP state", async () => { const harness = createDestroyHarness({ agent: "hermes", diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 5ae88a0896f..d89f0b3ab17 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -25,10 +25,12 @@ import { } from "../../onboard/sandbox-provider-cleanup"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, - RuntimeProviderBundleRegistry, - RuntimeProviderWorkloadCleanupResult, + type RuntimeProviderBundleRegistry, + type RuntimeProviderWorkloadCleanupResult, + normalizeRuntimeProviderIdentity, requireRuntimeProviderBundleForSandbox, requireRuntimeProviderMutationAuthority, + RuntimeProviderSelectionError, } from "../../onboard/runtime-provider/access"; import { validateName } from "../../runner"; import { killTimer as defaultKillShieldsTimer } from "../../shields/timer-control"; @@ -63,6 +65,17 @@ type RemoveSandboxRegistryEntryWithReceiptDeps = { removeSandboxWithReceipt?: typeof registry.removeSandboxWithReceipt; }; +type RemoveSandboxRegistryEntryOutcome = + | { + readonly status: "complete"; + readonly removed: boolean; + } + | { + readonly status: "blocked"; + readonly reason: "authority-unproven"; + readonly removed: false; + }; + type RunOpenshell = (args: string[], opts?: Record) => { status: number | null }; export type CleanupSandboxServicesDeps = { @@ -234,6 +247,9 @@ export function removeSandboxImage( const getSandbox = deps.getSandbox ?? registry.getSandbox; const sb = getSandbox(sandboxName); if (!sb) return { status: "skipped", reason: "no-owned-image" }; + const providerId = normalizeRuntimeProviderIdentity(sb.openshellDriver); + const log = deps.log ?? console.log; + const warn = deps.warn ?? console.warn; let result: RuntimeProviderWorkloadCleanupResult; try { const provider = requireRuntimeProviderBundleForSandbox( @@ -242,14 +258,25 @@ export function removeSandboxImage( ); requireRuntimeProviderMutationAuthority(provider, "workload-cleanup"); if (provider.cleanup.supported !== true) { - return { status: "skipped", reason: "authority-unproven" }; + throw new RuntimeProviderSelectionError( + `Runtime provider '${provider.identity.id}' has no cleanup implementation.`, + ); } result = provider.cleanup.removeOwnedWorkload({ sandbox: sb, sandboxName }); - } catch { + } catch (error) { + const detail = + error instanceof RuntimeProviderSelectionError + ? error.message + : error instanceof Error + ? error.message + : String(error); + warn( + ` ${YW}⚠${R} Runtime provider '${providerId}' could not prove workload cleanup ` + + `authority: ${detail} Local ownership state was preserved; repair the provider ` + + "metadata and retry the operation.", + ); return { status: "skipped", reason: "authority-unproven" }; } - const log = deps.log ?? console.log; - const warn = deps.warn ?? console.warn; if (result.status === "removed") { log(` Removed ${result.engineDisplayName} image ${result.reference}`); } else if (result.status === "failed") { @@ -257,21 +284,34 @@ export function removeSandboxImage( ` ${YW}⚠${R} Failed to remove ${result.engineDisplayName} image ${result.reference}; ` + `run '${CLI_NAME} gc' to clean up.`, ); + } else if (result.reason === "authority-unproven") { + warn( + ` ${YW}⚠${R} Runtime provider '${providerId}' did not prove ownership of the ` + + "recorded workload image. Local ownership state was preserved; repair the workload " + + "receipt and retry the operation.", + ); } return result; } -export function removeSandboxRegistryEntry( +function removeSandboxRegistryEntryOutcome( sandboxName: string, deps: RemoveSandboxRegistryEntryDeps = {}, -): boolean { +): RemoveSandboxRegistryEntryOutcome { const removeImage = deps.removeImage ?? removeSandboxImage; const removeSandbox = deps.removeSandbox ?? registry.removeSandbox; const imageResult = removeImage(sandboxName); if (imageResult?.status === "skipped" && imageResult.reason === "authority-unproven") { - return false; + return { status: "blocked", reason: "authority-unproven", removed: false }; } - return removeSandbox(sandboxName); + return { status: "complete", removed: removeSandbox(sandboxName) }; +} + +export function removeSandboxRegistryEntry( + sandboxName: string, + deps: RemoveSandboxRegistryEntryDeps = {}, +): boolean { + return removeSandboxRegistryEntryOutcome(sandboxName, deps).removed; } export function removeSandboxRegistryEntryWithReceipt( @@ -458,7 +498,23 @@ async function destroySandboxUnlocked( // The sandbox's gateway was captured before the registry entry is removed — // post-removal lookups return null and would collapse the cleanup target // back to the default gateway. - const removed = removeSandboxRegistryEntry(sandboxName); + const removalOutcome = removeSandboxRegistryEntryOutcome(sandboxName); + const removed = removalOutcome.removed; + if (removalOutcome.status === "blocked") { + const providerId = normalizeRuntimeProviderIdentity(sandbox?.openshellDriver); + console.warn( + ` ${YW}⚠${R} Sandbox '${sandboxName}' cleanup is incomplete for runtime provider ` + + `'${providerId}' because workload ownership authority could not be proven.`, + ); + console.warn( + ` ${YW}⚠${R} The local registry and session were preserved. Repair the provider/workload ` + + `receipt, then re-run '${CLI_NAME} ${sandboxName} destroy --yes' to finish cleanup.`, + ); + emitProviderDetachResidualHint(sandboxName, detachOutcome.failures, (message) => + console.warn(` ${YW}⚠${R}${message}`), + ); + process.exit(1); + } if (deleteSucceededOrAlreadyGone && removed && priorHttpsPinRouteId) { await revokeDestroyedSandboxHttpsPinRoute(cleanupGatewayName, priorHttpsPinRouteId); } diff --git a/src/lib/actions/sandbox/runtime/lifecycle-runtime.ts b/src/lib/actions/sandbox/runtime/lifecycle-runtime.ts index 3c43780b617..bbb1027bf80 100644 --- a/src/lib/actions/sandbox/runtime/lifecycle-runtime.ts +++ b/src/lib/actions/sandbox/runtime/lifecycle-runtime.ts @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 import { - RuntimeProviderBundle, - RuntimeProviderBundleRegistry, + type RuntimeProviderBundle, + type RuntimeProviderBundleRegistry, normalizeRuntimeProviderIdentity, requireRuntimeProviderMutationAuthority, resolveRuntimeProviderBundle, @@ -21,6 +21,7 @@ export type { RuntimeProviderLifecycleResult as SandboxLifecycleResult }; export type SandboxLifecycleProviderResolution = | { readonly ok: true; + readonly sandbox: SandboxEntry; readonly bundle: RuntimeProviderBundle; readonly lifecycle: Extract; } @@ -80,5 +81,5 @@ export function resolveSandboxLifecycleProvider( }, }; } - return { ok: true, bundle, lifecycle: bundle.lifecycle }; + return { ok: true, sandbox, bundle, lifecycle: bundle.lifecycle }; } diff --git a/src/lib/actions/sandbox/stop.ts b/src/lib/actions/sandbox/stop.ts index 903a595ec4d..e1e47848ec1 100644 --- a/src/lib/actions/sandbox/stop.ts +++ b/src/lib/actions/sandbox/stop.ts @@ -61,7 +61,7 @@ export function stopSandbox( const input = { environment: deps.environment ?? process.env, log, - sandbox: sandbox!, + sandbox: resolved.sandbox, sandboxName, }; const preflight = resolved.bundle.preflightDoctor.preflightLifecycle("stop", input); diff --git a/src/lib/onboard/compute/plan.ts b/src/lib/onboard/compute/plan.ts index fae373961b3..03ff7672180 100644 --- a/src/lib/onboard/compute/plan.ts +++ b/src/lib/onboard/compute/plan.ts @@ -3,9 +3,9 @@ import { CURRENT_RUNTIME_PROVIDER_BUNDLES, - RuntimeProviderBundle, - RuntimeProviderBundleRegistry, - RuntimeProviderGatewayLauncher, + type RuntimeProviderBundle, + type RuntimeProviderBundleRegistry, + type RuntimeProviderGatewayLauncher, resolveRuntimeProviderBundle, resolveCurrentRuntimeProviderBundle, runtimeProviderContainerEngineIdentity, diff --git a/src/lib/onboard/runtime-provider/access.ts b/src/lib/onboard/runtime-provider/access.ts index f8fe5e63cb1..7e52586b493 100644 --- a/src/lib/onboard/runtime-provider/access.ts +++ b/src/lib/onboard/runtime-provider/access.ts @@ -4,6 +4,7 @@ export type { RuntimeProviderBundle, RuntimeProviderBundleRegistry, + RuntimeProviderChannelStopTransport, RuntimeProviderGatewayLauncher, RuntimeProviderManagedImageSupport, RuntimeProviderWorkloadProfile, diff --git a/src/lib/onboard/runtime-provider/contract.ts b/src/lib/onboard/runtime-provider/contract.ts index 04a3e266fd3..828d2bd773f 100644 --- a/src/lib/onboard/runtime-provider/contract.ts +++ b/src/lib/onboard/runtime-provider/contract.ts @@ -8,6 +8,7 @@ export const RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION = 1 as const; export type RuntimeProviderGatewayLauncher = "nemoclaw" | "openshell"; export type RuntimeProviderLifecycleAction = "start" | "stop"; +export type RuntimeProviderChannelStopTransport = "docker-kubectl-first" | "openshell"; export type RuntimeProviderMutationOperation = | "registration" | "start" @@ -132,7 +133,7 @@ export type RuntimeProviderWorkloadCleanupResult = }; export interface RuntimeProviderCleanupOperations { - readonly detachProviders: (sandboxName: string) => RuntimeProviderProviderDetachResult; + readonly detachProviders: () => RuntimeProviderProviderDetachResult; } /** @@ -178,7 +179,7 @@ export type RuntimeProviderWorkloadSurface = RuntimeProviderSupportedSurface<{ export type RuntimeProviderLifecycleSurface = | RuntimeProviderSupportedSurface<{ - readonly channelStopTransport: "docker-kubectl-first" | "openshell"; + readonly channelStopTransport: RuntimeProviderChannelStopTransport; start(input: RuntimeProviderLifecycleInput): RuntimeProviderLifecycleResult; verifyStarted( input: RuntimeProviderLifecycleInput, diff --git a/src/lib/onboard/runtime-provider/docker.ts b/src/lib/onboard/runtime-provider/docker.ts index 66c239d1c05..8f690fbf12f 100644 --- a/src/lib/onboard/runtime-provider/docker.ts +++ b/src/lib/onboard/runtime-provider/docker.ts @@ -338,7 +338,7 @@ export function createDockerRuntimeProviderBundle( cleanup: { providerId, supported: true, - prepareDestroy: (input, operations) => operations.detachProviders(input.sandboxName), + prepareDestroy: (_input, operations) => operations.detachProviders(), removeOwnedWorkload: (input) => removeOwnedDockerWorkload(input, deps), }, containerEngine: { @@ -421,7 +421,7 @@ export function createKubernetesRuntimeProviderBundle( cleanup: { providerId, supported: true, - prepareDestroy: (input, operations) => operations.detachProviders(input.sandboxName), + prepareDestroy: (_input, operations) => operations.detachProviders(), removeOwnedWorkload: (input) => removeOwnedDockerWorkload(input, deps), }, containerEngine: { diff --git a/src/lib/onboard/runtime-provider/registry.ts b/src/lib/onboard/runtime-provider/registry.ts index 069cd9fc666..c8cd9621005 100644 --- a/src/lib/onboard/runtime-provider/registry.ts +++ b/src/lib/onboard/runtime-provider/registry.ts @@ -6,6 +6,7 @@ import { RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, type RuntimeProviderBundle, type RuntimeProviderBundleRegistry, + type RuntimeProviderChannelStopTransport, type RuntimeProviderContainerEngineOperation, type RuntimeProviderMutationOperation, type RuntimeProviderRuntimeReceipt, @@ -30,7 +31,10 @@ const BUNDLE_SURFACES = [ const MAX_RECEIPT_HANDLE_BYTES = 4096; const MAX_RECEIPT_DEVICES = 64; const GATEWAY_LAUNCHERS = new Set(["nemoclaw", "openshell"]); -const CHANNEL_STOP_TRANSPORTS = new Set(["docker-kubectl-first", "openshell"]); +const CHANNEL_STOP_TRANSPORTS: ReadonlySet = new Set([ + "docker-kubectl-first", + "openshell", +]); const MANAGED_IMAGE_SELECTION_POLICIES = new Set(["prefer-managed", "require-managed"]); const MANAGED_IMAGE_PLATFORMS = new Set(["linux/amd64", "linux/arm64"]); const MUTATION_OPERATIONS = new Set([ @@ -237,52 +241,70 @@ function validateWorkloadProfile(providerId: string, surface: Record>, -): void { - requireSupported("plan", surfaces.plan); - if (!GATEWAY_LAUNCHERS.has(String(surfaces.plan.gatewayLauncher))) { +type RuntimeProviderSurfaceRecords = Record< + (typeof BUNDLE_SURFACES)[number], + Record +>; + +function validatePlanSurface(providerId: string, surface: Record): void { + requireSupported("plan", surface); + if (!GATEWAY_LAUNCHERS.has(String(surface.gatewayLauncher))) { throw new RuntimeProviderRegistrationError(`plan for '${providerId}' has an invalid launcher`); } +} - requireSupported("capabilities", surfaces.capabilities); +function validateCapabilitiesSurface(surface: Record): void { + requireSupported("capabilities", surface); for (const field of [ "hostLocalInference", "directLifecycle", "legacyGatewayContainerInspection", "workloadImageCleanup", ] as const) { - requireBoolean(surfaces.capabilities, field, "capabilities"); + requireBoolean(surface, field, "capabilities"); } +} - requireSupported("preflightDoctor", surfaces.preflightDoctor); - requireFunction(surfaces.preflightDoctor, "inspectHost", "preflightDoctor"); - requireFunction(surfaces.preflightDoctor, "preflightLifecycle", "preflightDoctor"); +function validatePreflightDoctorSurface(surface: Record): void { + requireSupported("preflightDoctor", surface); + requireFunction(surface, "inspectHost", "preflightDoctor"); + requireFunction(surface, "preflightLifecycle", "preflightDoctor"); +} - requireSupported("gateway", surfaces.gateway); - if (!GATEWAY_LAUNCHERS.has(String(surfaces.gateway.launcher))) { +function validateGatewaySurface(providerId: string, surface: Record): void { + requireSupported("gateway", surface); + if (!GATEWAY_LAUNCHERS.has(String(surface.launcher))) { throw new RuntimeProviderRegistrationError( `gateway for '${providerId}' has an invalid launcher`, ); } - requireBoolean(surfaces.gateway, "inspectLegacyContainer", "gateway"); + requireBoolean(surface, "inspectLegacyContainer", "gateway"); +} - requireSupported("workload", surfaces.workload); - validateWorkloadProfile(providerId, surfaces.workload); +function validateWorkloadSurface(providerId: string, surface: Record): void { + requireSupported("workload", surface); + validateWorkloadProfile(providerId, surface); +} - if (surfaces.lifecycle.supported === true) { - if (!CHANNEL_STOP_TRANSPORTS.has(String(surfaces.lifecycle.channelStopTransport))) { +function validateLifecycleSurface(providerId: string, surface: Record): void { + if (surface.supported === true) { + if (!CHANNEL_STOP_TRANSPORTS.has(String(surface.channelStopTransport))) { throw new RuntimeProviderRegistrationError( `lifecycle for '${providerId}' has an invalid channel-stop transport`, ); } - requireFunction(surfaces.lifecycle, "start", "lifecycle"); - requireFunction(surfaces.lifecycle, "verifyStarted", "lifecycle"); - requireFunction(surfaces.lifecycle, "stop", "lifecycle"); + requireFunction(surface, "start", "lifecycle"); + requireFunction(surface, "verifyStarted", "lifecycle"); + requireFunction(surface, "stop", "lifecycle"); } - if (surfaces.mutationAuthority.supported === true) { - const operations = surfaces.mutationAuthority.operations; +} + +function validateMutationAuthoritySurface( + providerId: string, + surface: Record, +): void { + if (surface.supported === true) { + const operations = surface.operations; if ( !Array.isArray(operations) || operations.length === 0 || @@ -294,22 +316,40 @@ function validateSupportedSurfaceSchemas( ); } } - if (surfaces.bootstrap.supported === true) { - requireFunction(surfaces.bootstrap, "prepare", "bootstrap"); +} + +function validateBootstrapSurface(surface: Record): void { + if (surface.supported === true) { + requireFunction(surface, "prepare", "bootstrap"); } - if (surfaces.snapshot.supported === true) { - requireFunction(surfaces.snapshot, "capture", "snapshot"); - requireFunction(surfaces.snapshot, "restore", "snapshot"); +} + +function validateSnapshotSurface(surface: Record): void { + if (surface.supported === true) { + requireFunction(surface, "capture", "snapshot"); + requireFunction(surface, "restore", "snapshot"); } - if (surfaces.recovery.supported === true) { - requireFunction(surfaces.recovery, "recover", "recovery"); +} + +function validateRecoverySurface(surface: Record): void { + if (surface.supported === true) { + requireFunction(surface, "recover", "recovery"); } - if (surfaces.cleanup.supported === true) { - requireFunction(surfaces.cleanup, "prepareDestroy", "cleanup"); - requireFunction(surfaces.cleanup, "removeOwnedWorkload", "cleanup"); +} + +function validateCleanupSurface(surface: Record): void { + if (surface.supported === true) { + requireFunction(surface, "prepareDestroy", "cleanup"); + requireFunction(surface, "removeOwnedWorkload", "cleanup"); } - if (surfaces.containerEngine.supported === true) { - const identities = surfaces.containerEngine.identities; +} + +function validateContainerEngineSurface( + providerId: string, + surface: Record, +): void { + if (surface.supported === true) { + const identities = surface.identities; if (!Array.isArray(identities)) { throw new RuntimeProviderRegistrationError( `containerEngine for '${providerId}' must list operation-scoped identities`, @@ -341,6 +381,24 @@ function validateSupportedSurfaceSchemas( ); } } +} + +function validateSupportedSurfaceSchemas( + providerId: string, + surfaces: RuntimeProviderSurfaceRecords, +): void { + validatePlanSurface(providerId, surfaces.plan); + validateCapabilitiesSurface(surfaces.capabilities); + validatePreflightDoctorSurface(surfaces.preflightDoctor); + validateGatewaySurface(providerId, surfaces.gateway); + validateWorkloadSurface(providerId, surfaces.workload); + validateLifecycleSurface(providerId, surfaces.lifecycle); + validateMutationAuthoritySurface(providerId, surfaces.mutationAuthority); + validateBootstrapSurface(surfaces.bootstrap); + validateSnapshotSurface(surfaces.snapshot); + validateRecoverySurface(surfaces.recovery); + validateCleanupSurface(surfaces.cleanup); + validateContainerEngineSurface(providerId, surfaces.containerEngine); if (surfaces.plan.gatewayLauncher !== surfaces.gateway.launcher) { throw new RuntimeProviderRegistrationError( diff --git a/src/lib/onboard/workload/runtime.ts b/src/lib/onboard/workload/runtime.ts index da63b28a9fd..3f37202b63f 100644 --- a/src/lib/onboard/workload/runtime.ts +++ b/src/lib/onboard/workload/runtime.ts @@ -5,53 +5,21 @@ import type { OpenShellComputePlan } from "../compute/plan"; import { managedImagePlatformForNodeArchitecture } from "../managed-image/contract"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, - RuntimeProviderBundleRegistry, - RuntimeProviderManagedImageSupport, - RuntimeProviderWorkloadProfile, + type RuntimeProviderBundleRegistry, + type RuntimeProviderManagedImageSupport, resolveRuntimeProviderBundle, } from "../runtime-provider/access"; import type { SandboxWorkloadRuntimeCapabilities } from "./source"; -export type ManagedImageRuntimeSupport = RuntimeProviderManagedImageSupport; -export type ManagedImageRuntimeProfile = RuntimeProviderWorkloadProfile; - -/** - * Managed-image capabilities are registered by OpenShell compute-driver - * identity instead of inferred from the gateway launcher. A future provider - * can register this contract without inheriting another provider's lifecycle - * code. - */ -export type ManagedImageRuntimeProfileRegistry = Readonly< - Record ->; - -/** Compatibility view only; RuntimeProviderBundle is the registration source. */ -export function projectRuntimeProviderWorkloadProfiles( - providers: RuntimeProviderBundleRegistry, -): ManagedImageRuntimeProfileRegistry { - return Object.freeze( - Object.fromEntries( - Object.keys(providers).map((providerId) => [ - providerId, - providers[providerId]?.workload.profile, - ]), - ), - ); -} - -export const CURRENT_MANAGED_IMAGE_RUNTIME_PROFILES = projectRuntimeProviderWorkloadProfiles( - CURRENT_RUNTIME_PROVIDER_BUNDLES, -); - function hostOciArchitecture(nodeArchitecture: string): string { if (nodeArchitecture === "x64") return "amd64"; return nodeArchitecture; } function cloneRuntimeSupport( - support: ManagedImageRuntimeSupport, + support: RuntimeProviderManagedImageSupport, platform: NonNullable>, -): ManagedImageRuntimeSupport { +): RuntimeProviderManagedImageSupport { return { exactDigestReferences: support.exactDigestReferences, platforms: [platform], diff --git a/src/lib/state/registry/workload.ts b/src/lib/state/registry/workload.ts index fdca4897d04..b3e3cad2625 100644 --- a/src/lib/state/registry/workload.ts +++ b/src/lib/state/registry/workload.ts @@ -5,7 +5,11 @@ import { Buffer } from "node:buffer"; import { createHash } from "node:crypto"; import { MANAGED_IMAGE_REPOSITORIES } from "../../onboard/managed-image/contract"; -import { decodeManagedStartupProfile } from "../../onboard/managed-startup/profile"; +import { + decodeManagedStartupProfile, + MANAGED_STARTUP_PROFILE_MAX_BYTES, + MANAGED_STARTUP_PROFILE_MAX_ENCODED_BYTES, +} from "../../onboard/managed-startup/profile"; import type { SandboxWorkloadReceipt } from "./types"; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; @@ -18,8 +22,6 @@ const RELEASE_PATTERN = /^v[0-9]+(?:[.][0-9]+){1,3}(?:[-.][0-9A-Za-z][0-9A-Za-z. const MAX_COHORT_BYTES = 128; const BASE64URL_PATTERN = /^[A-Za-z0-9_-]+$/u; const STANDARD_BASE64_PATTERN = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u; -const MAX_PROFILE_BYTES = 64 * 1024; -const MAX_PROFILE_ENCODED_BYTES = Math.ceil(MAX_PROFILE_BYTES / 3) * 4; const MAX_CORPORATE_CA_BYTES = 128 * 1024; const MAX_CORPORATE_CA_ENCODED_BYTES = Math.ceil(MAX_CORPORATE_CA_BYTES / 3) * 4; @@ -31,7 +33,7 @@ function decodeCanonicalBase64Url(value: unknown): Buffer | null { if ( typeof value !== "string" || value.length === 0 || - value.length > MAX_PROFILE_ENCODED_BYTES || + value.length > MANAGED_STARTUP_PROFILE_MAX_ENCODED_BYTES || value.length % 4 === 1 || !BASE64URL_PATTERN.test(value) ) { @@ -39,7 +41,7 @@ function decodeCanonicalBase64Url(value: unknown): Buffer | null { } const decoded = Buffer.from(value, "base64url"); return decoded.length > 0 && - decoded.length <= MAX_PROFILE_BYTES && + decoded.length <= MANAGED_STARTUP_PROFILE_MAX_BYTES && decoded.toString("base64url") === value ? decoded : null; diff --git a/src/lib/tunnel/sandbox-gateway-stop.test.ts b/src/lib/tunnel/sandbox-gateway-stop.test.ts index ee9d13ad4b9..442a9dce648 100644 --- a/src/lib/tunnel/sandbox-gateway-stop.test.ts +++ b/src/lib/tunnel/sandbox-gateway-stop.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it, vi } from "vitest"; import type { AgentDefinition } from "../agent/defs"; import type { SandboxEntry } from "../state/registry"; +import { GATEWAY_STOP_SCRIPT } from "./gateway-stop-script"; import { type SandboxGatewayStopDeps, stopSandboxChannels } from "./sandbox-gateway-stop"; function spawnResult(status: number | null, stdout = "", stderr = ""): SpawnSyncReturns { @@ -109,7 +110,7 @@ describe("stopSandboxChannels", () => { "/usr/local/bin/openshell", ["sandbox", "exec", "--name", "my-sandbox", "--gateway", "nemoclaw", "--", "sh", "-s"], expect.objectContaining({ - input: expect.stringContaining("find_gateway_pids"), + input: GATEWAY_STOP_SCRIPT, timeout: 20000, }), ); @@ -128,7 +129,7 @@ describe("stopSandboxChannels", () => { "/usr/local/bin/openshell", ["sandbox", "exec", "--name", "my-sandbox", "--gateway", "nemoclaw", "--", "sh", "-s"], expect.objectContaining({ - input: expect.stringContaining("find_gateway_pids"), + input: GATEWAY_STOP_SCRIPT, timeout: 20000, }), ); diff --git a/src/lib/tunnel/sandbox-gateway-stop.ts b/src/lib/tunnel/sandbox-gateway-stop.ts index e1e4e1b4c53..631a304ba8e 100644 --- a/src/lib/tunnel/sandbox-gateway-stop.ts +++ b/src/lib/tunnel/sandbox-gateway-stop.ts @@ -12,6 +12,7 @@ import { getGatewayClusterContainerName } from "../adapters/openshell/gateway-dr import { resolveOpenshell } from "../adapters/openshell/resolve"; import * as agentRuntime from "../agent/runtime"; import { resolveSandboxGatewayName } from "../onboard/gateway-binding"; +import type { RuntimeProviderChannelStopTransport } from "../onboard/runtime-provider/access"; import * as registry from "../state/registry"; import { GATEWAY_STOP_SCRIPT } from "./gateway-stop-script"; @@ -24,7 +25,7 @@ type ProcessRunner = ( ) => SpawnSyncReturns; export type SandboxGatewayStopDeps = { - channelStopTransport?: "docker-kubectl-first" | "openshell"; + channelStopTransport?: RuntimeProviderChannelStopTransport; getSandbox?: typeof registry.getSandbox; getRegisteredAgent?: typeof agentRuntime.getRegisteredAgent; getAgentDisplayName?: typeof agentRuntime.getAgentDisplayName; diff --git a/test/helpers/destroy-flow-test-harness.ts b/test/helpers/destroy-flow-test-harness.ts index 5ede3c4de62..23a1c2b8602 100644 --- a/test/helpers/destroy-flow-test-harness.ts +++ b/test/helpers/destroy-flow-test-harness.ts @@ -4,6 +4,7 @@ import { createRequire } from "node:module"; import { expect, type MockInstance, vi } from "vitest"; +import type { SandboxWorkloadReceipt } from "../../src/lib/state/registry"; type DestroySandbox = typeof import("../../src/lib/actions/sandbox/destroy")["destroySandbox"]; @@ -37,6 +38,7 @@ export type DestroyHarness = { stopAllSpy: MockInstance; stopNimByNameSpy: MockInstance; unloadOllamaModelsSpy: MockInstance; + warnSpy: MockInstance; }; type DestroyHarnessOptions = { @@ -47,6 +49,7 @@ type DestroyHarnessOptions = { dockerPsOutput?: string; endpointUrl?: string; finalizeMcpError?: string; + imageTag?: string | null; liveListOutput?: string; mcpAddState?: "prepared"; mcpServers?: string[]; @@ -57,6 +60,7 @@ type DestroyHarnessOptions = { sandboxPresent?: boolean; shieldsDown?: boolean; shieldsUpError?: Error; + workload?: SandboxWorkloadReceipt; }; const sandboxEntry = { @@ -107,7 +111,7 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); - vi.spyOn(console, "warn").mockImplementation(() => undefined); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined); const resolve = requireDist("../../adapters/openshell/resolve.js"); const runtime = requireDist("../../adapters/openshell/runtime.js"); @@ -137,9 +141,11 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr }); vi.spyOn(registry, "getSandbox").mockReturnValue({ ...sandboxEntry, + imageTag: options.imageTag === undefined ? sandboxEntry.imageTag : options.imageTag, agent: options.agent ?? sandboxEntry.agent, ...(options.openshellDriver ? { openshellDriver: options.openshellDriver } : {}), ...(options.endpointUrl ? { endpointUrl: options.endpointUrl } : {}), + ...(options.workload ? { workload: options.workload } : {}), ...(options.mcpServers?.length ? { mcp: { @@ -336,5 +342,6 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr stopAllSpy, stopNimByNameSpy, unloadOllamaModelsSpy, + warnSpy, }; } diff --git a/test/helpers/runtime-provider-bundle.ts b/test/helpers/runtime-provider-bundle.ts index 07b9a1ad3b4..e4476a1bfbd 100644 --- a/test/helpers/runtime-provider-bundle.ts +++ b/test/helpers/runtime-provider-bundle.ts @@ -143,7 +143,7 @@ export function createInMemoryRuntimeProviderBundle({ supported: true, prepareDestroy(input: RuntimeProviderCleanupInput, operations) { event("prepare-destroy", input.sandboxName); - return operations.detachProviders(input.sandboxName); + return operations.detachProviders(); }, removeOwnedWorkload(input: RuntimeProviderCleanupInput) { const reference = input.sandbox.imageTag; diff --git a/test/image-cleanup.test.ts b/test/image-cleanup.test.ts index c7848f046aa..bf01b89776e 100644 --- a/test/image-cleanup.test.ts +++ b/test/image-cleanup.test.ts @@ -13,6 +13,7 @@ import { cleanupShieldsDestroyArtifacts, removeSandboxImage, removeSandboxRegistryEntry, + removeSandboxRegistryEntryWithReceipt, removeShieldsState, } from "../src/lib/actions/sandbox/destroy"; import { COMMANDS, globalCommandTokens } from "../src/lib/cli/command-registry"; @@ -136,6 +137,57 @@ describe("image cleanup: sandbox destroy removes Docker image (#2086)", () => { expect(removeSandbox).toHaveBeenCalledWith("alpha"); }); + it("fails closed and reports the provider when workload image authority is unproven", () => { + const removeImage = vi.fn(() => ({ status: 0 })); + const warn = vi.fn(); + const runtimeProviders = createRuntimeProviderBundleRegistry([ + ["docker", createDockerRuntimeProviderBundle({ removeImage })], + ]); + + const result = removeSandboxImage("alpha", { + getSandbox: () => + ({ + name: "alpha", + openshellDriver: "docker", + imageTag: "local/alpha:current", + workload: { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "local/alpha:recorded", + shared: false, + }, + }) as any, + runtimeProviders, + warn, + }); + + expect(result).toEqual({ status: "skipped", reason: "authority-unproven" }); + expect(removeImage).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("Runtime provider 'docker'")); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("workload receipt")); + }); + + it("preserves registry ownership when workload cleanup authority is unproven", () => { + const removeSandbox = vi.fn(() => true); + const removeSandboxWithReceipt = vi.fn(); + const authorityUnproven = () => ({ status: "skipped", reason: "authority-unproven" }) as const; + + expect( + removeSandboxRegistryEntry("alpha", { + removeImage: authorityUnproven, + removeSandbox, + }), + ).toBe(false); + expect( + removeSandboxRegistryEntryWithReceipt("alpha", { + removeImage: authorityUnproven, + removeSandboxWithReceipt, + }), + ).toBeNull(); + expect(removeSandbox).not.toHaveBeenCalled(); + expect(removeSandboxWithReceipt).not.toHaveBeenCalled(); + }); + it("treats missing sandbox delete results as already gone", () => { expect( getSandboxDeleteOutcome({ status: 1, stderr: "Error: sandbox alpha not found" }), diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 47682ce4357..5cf3b5ee309 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -10,35 +10,44 @@ const repoRoot = join(import.meta.dirname, ".."); describe("runtime provider central source boundary", () => { // source-shape-contract: compatibility -- Migrated lifecycle and mutation consumers must stay provider-neutral while production selection excludes unqualified future providers and managed-bootstrap dependencies it("keeps migrated provider identities and implementations behind the one bundle composition", () => { - const centralConsumers = [ - readFileSync(join(repoRoot, "src/lib/actions/inference-set.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/actions/sandbox/destroy-execution.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/actions/sandbox/destroy.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/actions/sandbox/runtime/lifecycle-runtime.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/actions/sandbox/start.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/actions/sandbox/stop.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/compute/plan.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/sandbox-registration.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/workload/runtime.ts"), "utf8"), - ]; - const nonSnapshotActions = centralConsumers.slice(0, 6); - const providerContract = [ - readFileSync(join(repoRoot, "src/lib/onboard/runtime-provider/contract.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/runtime-provider/current.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/runtime-provider/docker.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/runtime-provider/registry.ts"), "utf8"), - ]; + const read = (relativePath: string) => readFileSync(join(repoRoot, relativePath), "utf8"); + const driverNeutralActions = { + "actions/inference-set.ts": read("src/lib/actions/inference-set.ts"), + "actions/sandbox/destroy-execution.ts": read("src/lib/actions/sandbox/destroy-execution.ts"), + "actions/sandbox/destroy.ts": read("src/lib/actions/sandbox/destroy.ts"), + "actions/sandbox/runtime/lifecycle-runtime.ts": read( + "src/lib/actions/sandbox/runtime/lifecycle-runtime.ts", + ), + "actions/sandbox/start.ts": read("src/lib/actions/sandbox/start.ts"), + "actions/sandbox/stop.ts": read("src/lib/actions/sandbox/stop.ts"), + }; + const onboardConsumers = { + "onboard/compute/plan.ts": read("src/lib/onboard/compute/plan.ts"), + "onboard/sandbox-registration.ts": read("src/lib/onboard/sandbox-registration.ts"), + "onboard/workload/runtime.ts": read("src/lib/onboard/workload/runtime.ts"), + }; + const providerContract = { + contract: read("src/lib/onboard/runtime-provider/contract.ts"), + current: read("src/lib/onboard/runtime-provider/current.ts"), + docker: read("src/lib/onboard/runtime-provider/docker.ts"), + registry: read("src/lib/onboard/runtime-provider/registry.ts"), + }; - for (const source of nonSnapshotActions) { + for (const source of Object.values(driverNeutralActions)) { expect(source).not.toMatch(/\b(?:docker|podman)\b/iu); expect(source).not.toMatch(/(?:adapters\/docker|docker-driver-sandbox-recovery)/u); } - for (const source of centralConsumers) { + for (const source of [ + ...Object.values(driverNeutralActions), + ...Object.values(onboardConsumers), + ]) { expect(source).not.toMatch(/\b(?:openshellDriver|driverName)\s*={2,3}\s*["'][^"']+["']/u); expect(source).not.toMatch(/switch\s*\([^)]*\b(?:openshellDriver|driverName)\b[^)]*\)/u); } - expect(centralConsumers[4]).toMatch(/resolved\.lifecycle\.verifyStarted\(/u); - expect(providerContract.join("\n")).not.toMatch(/managed-bootstrap/u); - expect(providerContract[1]).not.toMatch(/\b(?:podman|mxc)\b/iu); + expect(driverNeutralActions["actions/sandbox/start.ts"]).toMatch( + /resolved\.lifecycle\.verifyStarted\(/u, + ); + expect(Object.values(providerContract).join("\n")).not.toMatch(/managed-bootstrap/u); + expect(providerContract.current).not.toMatch(/\b(?:podman|mxc)\b/iu); }); }); From d73f13a08688df4a61e00426d2206e1d7175d678 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 02:02:48 -0700 Subject: [PATCH 032/117] test(runtime): cover unknown provider doctor result Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/doctor-flow.test.ts | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/lib/actions/sandbox/doctor-flow.test.ts b/src/lib/actions/sandbox/doctor-flow.test.ts index 468444e1539..ce61d293ae2 100644 --- a/src/lib/actions/sandbox/doctor-flow.test.ts +++ b/src/lib/actions/sandbox/doctor-flow.test.ts @@ -294,6 +294,34 @@ describe("runSandboxDoctor flow", () => { }, ); + it("fails the JSON host check for an unknown durable runtime provider", async () => { + const harness = createDoctorHarness(); + harness.getSandboxSpy.mockReturnValue({ + name: "alpha", + agent: "openclaw", + model: "registry-model", + provider: "ollama-local", + openshellDriver: "unknown-runtime", + openshellVersion: "0.0.72", + nemoclawVersion: "0.0.83", + fromDockerfile: null, + dashboardPort: 18789, + imageTag: "nemoclaw-openclaw:test", + gatewayName: "nemoclaw-19080", + gatewayPort: 19080, + }); + + const report = await harness.runSandboxDoctor("alpha", ["--json"], { quietJson: true }); + + expect(report?.checks).toContainEqual({ + group: "Host", + label: "Runtime provider", + status: "fail", + detail: "Runtime provider 'unknown-runtime' is not registered for this operation.", + hint: "restore a supported durable runtime provider identity before retrying", + }); + }); + it.each([ ["high", "high"], [null, "endpoint-default"], From 8f8bcd5a18ae270538015e7f3772adf3a39d52fe Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 02:18:13 -0700 Subject: [PATCH 033/117] test(runtime): prove provider recovery boundaries Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/destroy-flow.test.ts | 29 ++++++++++++++- src/lib/actions/sandbox/destroy.ts | 17 +++++++++ src/lib/actions/sandbox/start.test.ts | 38 ++++++++++++++++++-- src/lib/actions/sandbox/stop.test.ts | 36 +++++++++++++++++-- test/helpers/destroy-flow-test-harness.ts | 21 +++++++---- 5 files changed, 128 insertions(+), 13 deletions(-) diff --git a/src/lib/actions/sandbox/destroy-flow.test.ts b/src/lib/actions/sandbox/destroy-flow.test.ts index 7ac29fcdd4d..868d3b19a2e 100644 --- a/src/lib/actions/sandbox/destroy-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-flow.test.ts @@ -105,7 +105,9 @@ describe("destroySandbox flow", () => { }); it("preserves provider and registry ownership when runtime authority is unknown", async () => { - const harness = createDestroyHarness({ openshellDriver: "unknown-runtime" }); + const harness = createDestroyHarness({ + openshellDriver: "unknown-runtime", + }); await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow("process.exit(1)"); @@ -141,6 +143,31 @@ describe("destroySandbox flow", () => { expect(logOutput).not.toContain("Sandbox 'alpha' destroyed"); expect(harness.events).toContain("delete"); expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); + expect(harness.updateSessionSpy).not.toHaveBeenCalled(); + }); + + it("retires registry and session ownership after the workload receipt is repaired", async () => { + const imageTag = "local/alpha:current"; + const harness = createDestroyHarness({ + imageTag, + workload: { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: imageTag, + shared: false, + }, + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); + + expect(harness.dockerRunSpy).toHaveBeenCalledWith(["rmi", imageTag], { + ignoreError: true, + }); + expect(harness.removeSandboxSpy).toHaveBeenCalledWith("alpha"); + expect(harness.updateSessionSpy).toHaveBeenCalledOnce(); + expect(harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( + "Sandbox 'alpha' destroyed", + ); }); it("refuses shields-up Hermes MCP destroy before stopping services or preparing MCP state", async () => { diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index d89f0b3ab17..98945d91bcf 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -498,6 +498,23 @@ async function destroySandboxUnlocked( // The sandbox's gateway was captured before the registry entry is removed — // post-removal lookups return null and would collapse the cleanup target // back to the default gateway. + /** + * SOURCE_OF_TRUTH + * Invalid state: the live sandbox is confirmed deleted or already absent, + * but its durable provider identity or workload receipt cannot prove image + * cleanup authority, so the registry row and onboarding session are retained. + * Source boundary: the persisted `openshellDriver` and `workload` receipt are + * validated only by the selected provider's `cleanup.removeOwnedWorkload`. + * Source-fix constraint: destroy cannot synthesize provider ownership after + * remote deletion; guessing could remove a shared image or another provider's + * workload, so the operator must repair the durable authority and retry. + * Regression proof: destroy-flow.test.ts proves both blocked retention and + * that a repaired matching receipt permits registry and session retirement; + * test/image-cleanup.test.ts proves the lower-level fail-closed contract. + * Removal condition: remove this recovery boundary only when the provider or + * registry owns authenticated reconciliation that can safely complete cleanup + * without retaining the local ownership row. + */ const removalOutcome = removeSandboxRegistryEntryOutcome(sandboxName); const removed = removalOutcome.removed; if (removalOutcome.status === "blocked") { diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index a994c575fbc..8784dab3896 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -25,7 +25,13 @@ function harness(overrides: Partial = {}) { vi.fn(); const findLabeledSandboxContainers = vi.fn< DockerRuntimeProviderDependencies["findLabeledSandboxContainers"] - >(() => [{ name: "openshell-my-sandbox", status: "Exited (0) 2 hours ago", running: false }]); + >(() => [ + { + name: "openshell-my-sandbox", + status: "Exited (0) 2 hours ago", + running: false, + }, + ]); const recoverDockerDriverSandbox = vi.fn( () => ({ recovered: true, @@ -118,7 +124,11 @@ describe("startSandbox", () => { it("unpauses a paused container instead of calling it already running (#6026)", async () => { const h = harness(); h.findLabeledSandboxContainers.mockReturnValue([ - { name: "openshell-my-sandbox", status: "Up 3 minutes (Paused)", running: true }, + { + name: "openshell-my-sandbox", + status: "Up 3 minutes (Paused)", + running: true, + }, ]); const result = await startSandbox("my-sandbox", h.deps); @@ -137,7 +147,11 @@ describe("startSandbox", () => { it("surfaces a docker unpause failure with the container name (#6026)", async () => { const h = harness(); h.findLabeledSandboxContainers.mockReturnValue([ - { name: "openshell-my-sandbox", status: "Up 3 minutes (Paused)", running: true }, + { + name: "openshell-my-sandbox", + status: "Up 3 minutes (Paused)", + running: true, + }, ]); h.dockerUnpause.mockReturnValue({ status: 125 }); @@ -217,6 +231,24 @@ describe("startSandbox", () => { expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled(); }); + it.each([ + "unknown-runtime", + "mxc-not-installed", + ])("fails closed for unregistered provider %s without lifecycle side effects", async (providerId) => { + const h = harness(); + h.getSandbox.mockReturnValue(sandbox({ openshellDriver: providerId })); + + const result = await startSandbox("my-sandbox", h.deps); + + expect(result.exitCode).toBe(1); + expect(result.message).toContain(providerId); + expect(result.message).toContain("has no registered lifecycle provider"); + expect(h.findLabeledSandboxContainers).not.toHaveBeenCalled(); + expect(h.dockerUnpause).not.toHaveBeenCalled(); + expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled(); + expect(h.verifyGateway).not.toHaveBeenCalled(); + }); + it.each([ ["null driver", sandbox({ openshellDriver: null })], ["docker driver", sandbox({ openshellDriver: "docker" })], diff --git a/src/lib/actions/sandbox/stop.test.ts b/src/lib/actions/sandbox/stop.test.ts index 41d444ee3ad..dd5af04234e 100644 --- a/src/lib/actions/sandbox/stop.test.ts +++ b/src/lib/actions/sandbox/stop.test.ts @@ -18,7 +18,11 @@ function sandbox(values: Partial = {}): SandboxEntry { } function container(name: string, running: boolean) { - return { name, status: running ? "Up 5 minutes" : "Exited (0) 2 hours ago", running }; + return { + name, + status: running ? "Up 5 minutes" : "Exited (0) 2 hours ago", + running, + }; } type StopHarnessOverrides = Partial & { @@ -270,7 +274,11 @@ describe("stopSandbox", () => { it("stops a crash-looping container instead of calling it stopped (#6026)", () => { const h = harness(); h.findLabeledSandboxContainers.mockReturnValue([ - { name: "openshell-my-sandbox", status: "Restarting (137) 2 seconds ago", running: false }, + { + name: "openshell-my-sandbox", + status: "Restarting (137) 2 seconds ago", + running: false, + }, ]); const result = stopSandbox("my-sandbox", h.deps); @@ -285,7 +293,11 @@ describe("stopSandbox", () => { it("stops a paused container (#6026)", () => { const h = harness(); h.findLabeledSandboxContainers.mockReturnValue([ - { name: "openshell-my-sandbox", status: "Up 5 minutes (Paused)", running: true }, + { + name: "openshell-my-sandbox", + status: "Up 5 minutes (Paused)", + running: true, + }, ]); const result = stopSandbox("my-sandbox", h.deps); @@ -370,6 +382,24 @@ describe("stopSandbox", () => { expect(h.dockerStop).not.toHaveBeenCalled(); }); + it.each([ + "unknown-runtime", + "mxc-not-installed", + ])("fails closed for unregistered provider %s without lifecycle side effects", (providerId) => { + const h = harness(); + h.getSandbox.mockReturnValue(sandbox({ openshellDriver: providerId })); + + const result = stopSandbox("my-sandbox", h.deps); + + expect(result.exitCode).toBe(1); + expect(result.message).toContain(providerId); + expect(result.message).toContain("has no registered lifecycle provider"); + expect(h.stopSandboxChannels).not.toHaveBeenCalled(); + expect(h.findLabeledSandboxContainers).not.toHaveBeenCalled(); + expect(h.dockerStop).not.toHaveBeenCalled(); + expect(h.teardownSandboxDashboardForward).not.toHaveBeenCalled(); + }); + it.each([ ["null driver", sandbox({ openshellDriver: null })], ["docker driver", sandbox({ openshellDriver: "docker" })], diff --git a/test/helpers/destroy-flow-test-harness.ts b/test/helpers/destroy-flow-test-harness.ts index 23a1c2b8602..ec7dd59f83b 100644 --- a/test/helpers/destroy-flow-test-harness.ts +++ b/test/helpers/destroy-flow-test-harness.ts @@ -18,6 +18,7 @@ export type DestroyHarness = { captureOpenshellSpy: MockInstance; destroySandbox: DestroySandbox; dockerCaptureSpy: MockInstance; + dockerRunSpy: MockInstance; errorSpy: MockInstance; events: string[]; finalizeMcpBridgesAfterSandboxDeleteSpy: MockInstance; @@ -38,6 +39,7 @@ export type DestroyHarness = { stopAllSpy: MockInstance; stopNimByNameSpy: MockInstance; unloadOllamaModelsSpy: MockInstance; + updateSessionSpy: MockInstance; warnSpy: MockInstance; }; @@ -178,12 +180,14 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr vi.spyOn(onboardSession, "loadSession").mockReturnValue({ sandboxName: "alpha", }); - vi.spyOn(onboardSession, "updateSession").mockImplementation((mutator: unknown) => { - const session = { sandboxName: "alpha" }; - expect(typeof mutator).toBe("function"); - (mutator as (value: typeof session) => void)(session); - return session; - }); + const updateSessionSpy = vi + .spyOn(onboardSession, "updateSession") + .mockImplementation((mutator: unknown) => { + const session = { sandboxName: "alpha" }; + expect(typeof mutator).toBe("function"); + (mutator as (value: typeof session) => void)(session); + return session; + }); const gatewayPinsAtSandboxList: Array = []; const runOpenshellSpy = vi.spyOn(runtime, "runOpenshell").mockImplementation((args: unknown) => { const argv = Array.isArray(args) ? args : []; @@ -227,6 +231,9 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr : names; return matchedNames.length > 0 ? `${matchedNames.join("\n")}\n` : ""; }); + const dockerRunSpy = vi + .spyOn(dockerRun, "dockerRun") + .mockReturnValue({ status: 0 } as ReturnType); const selectGatewaySpy = vi .spyOn(destroyGateway, "selectGatewayForSandboxDestroy") .mockImplementation(() => undefined); @@ -321,6 +328,7 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr cleanupGatewaySpy, captureOpenshellSpy, dockerCaptureSpy, + dockerRunSpy, destroySandbox: requireDist(destroyModulePath).destroySandbox, errorSpy, events, @@ -342,6 +350,7 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr stopAllSpy, stopNimByNameSpy, unloadOllamaModelsSpy, + updateSessionSpy, warnSpy, }; } From f77bff8fec4b7a36d79317af39af265fe6dfaf48 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 02:39:48 -0700 Subject: [PATCH 034/117] fix(runtime): fail closed on retained cleanup ownership Signed-off-by: Aaron Erickson --- .../sandbox/rebuild-destroy-phase.test.ts | 60 ++++++++++++++++++- .../actions/sandbox/rebuild-destroy-phase.ts | 19 ++++++ src/lib/actions/sandbox/snapshot.ts | 17 +++++- test/image-cleanup.test.ts | 11 ++++ 4 files changed, 104 insertions(+), 3 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index 911ddc4077d..4b896642bf6 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -89,7 +89,12 @@ describe("rebuild destroy phase", () => { scrubbedAdapterEntries: [], }); mocks.reattachMcpAfterDeleteFailure.mockResolvedValue(undefined); - mocks.removeSandboxRegistryEntryWithReceipt.mockReturnValue(null); + mocks.removeSandboxRegistryEntryWithReceipt.mockReturnValue({ + entry: { name: "alpha", agent: "openclaw" }, + wasDefault: true, + fallbackDefault: null, + postRemovalDefaultSelectionRevision: 1, + }); mocks.captureOpenshell.mockReturnValue({ status: 1, output: "", @@ -872,7 +877,12 @@ describe("rebuild destroy phase", () => { }); mocks.removeSandboxRegistryEntryWithReceipt.mockImplementation(() => { events.push("remove-registry"); - return null; + return { + entry: { name: "alpha", agent: "openclaw" }, + wasDefault: true, + fallbackDefault: null, + postRemovalDefaultSelectionRevision: 1, + }; }); const result = await runRebuildDestroyPhase({ @@ -899,6 +909,52 @@ describe("rebuild destroy phase", () => { expect(mocks.removeSandboxRegistryEntryWithReceipt).toHaveBeenCalledWith("alpha"); }); + it("stops rebuild after deletion when workload cleanup authority is unproven", async () => { + mocks.runOpenshell.mockReturnValue({ status: 0, stdout: "deleted", stderr: "" }); + mocks.removeSandboxRegistryEntryWithReceipt.mockReturnValue(null); + const onDeleted = vi.fn(); + const bail = vi.fn((message: string): never => { + throw new Error(message); + }); + + await expect( + runRebuildDestroyPhase({ + sandboxName: "alpha", + sandboxEntry: { + name: "alpha", + agent: "openclaw", + imageTag: "local/alpha:current", + workload: { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "local/alpha:recorded", + shared: false, + }, + }, + staleRecovery: false, + backupManifest: { backupPath: "/tmp/rebuild-backups/alpha/backup" } as never, + log: vi.fn(), + bail, + relockShieldsIfNeeded: vi.fn(() => true), + onDeleted, + }), + ).rejects.toThrow("workload cleanup authority could not be proven"); + + expect(onDeleted).toHaveBeenCalledOnce(); + expect(mocks.removeSandboxRegistryEntryWithReceipt).toHaveBeenCalledWith("alpha"); + expect(bail).toHaveBeenCalledWith( + "Old sandbox deleted, but workload cleanup authority could not be proven; recovery state was preserved.", + 1, + ); + const errors = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(errors).toContain("local runtime ownership cleanup is incomplete"); + expect(errors).toContain("Repair the provider/workload receipt"); + expect(errors).toContain("/tmp/rebuild-backups/alpha/backup"); + expect(vi.mocked(console.log).mock.calls.flat().join("\n")).not.toContain( + "Old sandbox deleted", + ); + }); + it("marks accepted deletion as ambiguous when transport failures prevent confirmation", async () => { mocks.runOpenshell.mockReturnValue({ status: 0, stdout: "deleted", stderr: "" }); mocks.captureOpenshell.mockReturnValue({ diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.ts index ba5eca3049b..9c9eaeaf4b2 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.ts @@ -462,6 +462,25 @@ export async function runRebuildDestroyPhase( const hasBaselineExclusions = (input.sandboxEntry.baselineExclusions?.length ?? 0) > 0; if (rebuildMcpEntries.length === 0 && !hasBaselineExclusions) { removalReceipt = removeSandboxRegistryEntryWithReceipt(sandboxName); + if (!removalReceipt) { + console.error( + " The old sandbox is deleted, but local runtime ownership cleanup is incomplete.", + ); + console.error( + " The registry entry was preserved because provider/workload cleanup authority could not be proven or registry ownership changed.", + ); + console.error( + " Repair the provider/workload receipt or reconcile the retained registry row, then retry rebuild.", + ); + if (backupManifest) { + console.error(" State backup is preserved at: " + backupManifest.backupPath); + } + bail( + "Old sandbox deleted, but workload cleanup authority could not be proven; recovery state was preserved.", + 1, + ); + return null; + } } if (rebuildMcpEntries.length > 0) { // The registry entry is the durable MCP rebuild transaction. The inner diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index b2fdd74676b..f61961fe5fc 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -124,6 +124,21 @@ function formatSnapshotVersion(b: unknown) { return `v${snapshotVersion}`; } +export function requireSnapshotDestinationRegistryRemoval( + name: string, + registryRemoved: boolean, +): void { + if (registryRemoved !== false) return; + console.error( + ` Destination '${name}' is deleted, but local runtime ownership cleanup is incomplete.`, + ); + console.error( + " The registry entry was preserved because provider/workload cleanup authority could not be proven.", + ); + console.error(" Repair the provider/workload receipt, then retry the snapshot restore."); + snapshotExit(1); +} + function renderSnapshotTable( backups: Array<{ snapshotVersion: number; @@ -485,7 +500,7 @@ function deleteSandboxForRestore(name: string): void { }); } cleanupShieldsDestroyArtifacts(name); - removeSandboxRegistryEntry(name); + requireSnapshotDestinationRegistryRemoval(name, removeSandboxRegistryEntry(name)); }); console.log(` ${G}\u2713${R} '${name}' deleted`); } diff --git a/test/image-cleanup.test.ts b/test/image-cleanup.test.ts index bf01b89776e..368c589a78a 100644 --- a/test/image-cleanup.test.ts +++ b/test/image-cleanup.test.ts @@ -16,6 +16,7 @@ import { removeSandboxRegistryEntryWithReceipt, removeShieldsState, } from "../src/lib/actions/sandbox/destroy"; +import { requireSnapshotDestinationRegistryRemoval } from "../src/lib/actions/sandbox/snapshot"; import { COMMANDS, globalCommandTokens } from "../src/lib/cli/command-registry"; import { getRegisteredOclifCommandMetadata } from "../src/lib/cli/oclif-metadata"; import { normalizeGarbageCollectImagesOptions } from "../src/lib/domain/lifecycle/options"; @@ -165,6 +166,16 @@ describe("image cleanup: sandbox destroy removes Docker image (#2086)", () => { expect(removeImage).not.toHaveBeenCalled(); expect(warn).toHaveBeenCalledWith(expect.stringContaining("Runtime provider 'docker'")); expect(warn).toHaveBeenCalledWith(expect.stringContaining("workload receipt")); + + const removeSandbox = vi.fn(() => true); + const registryRemoved = removeSandboxRegistryEntry("alpha", { + removeImage: () => result, + removeSandbox, + }); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + expect(() => requireSnapshotDestinationRegistryRemoval("alpha", registryRemoved)).toThrow(); + expect(removeSandbox).not.toHaveBeenCalled(); + expect(error.mock.calls.flat().join("\n")).toContain("Repair the provider/workload receipt"); }); it("preserves registry ownership when workload cleanup authority is unproven", () => { From bf2fd190113428cd9c452fdda1476a0a5c8eb439 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 02:42:32 -0700 Subject: [PATCH 035/117] test(rebuild): type cleanup ownership receipt Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/rebuild-destroy-phase.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index 4b896642bf6..1ce3bbde7c3 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { expectNoSandboxDelete } from "../../../../test/helpers/rebuild-delete-assertions"; +import type { SandboxRemovalReceipt } from "../../state/registry"; const mocks = vi.hoisted(() => ({ captureOpenshell: vi.fn(), @@ -20,7 +21,7 @@ const mocks = vi.hoisted(() => ({ listSandboxes: vi.fn(() => ({ sandboxes: [] })), prepareMcpForRebuild: vi.fn(), reattachMcpAfterDeleteFailure: vi.fn(), - removeSandboxRegistryEntryWithReceipt: vi.fn(() => null), + removeSandboxRegistryEntryWithReceipt: vi.fn<() => SandboxRemovalReceipt | null>(() => null), waitUntil: vi.fn(), warnUnpreservedUserManagedFiles: vi.fn(), runOpenshell: vi.fn( From 4316a18daa2827442fe9786a8c943a1b29938bce Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 03:23:17 -0700 Subject: [PATCH 036/117] fix(runtime): prove cleanup authority before deletion Require a side-effect-free provider cleanup plan before destructive sandbox actions. Validate provider and workload ownership before destroy, rebuild, or force-restore deletes. Preserve ownership state and give fail-closed guidance when authority is ambiguous. Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/destroy-execution.ts | 20 ++- src/lib/actions/sandbox/destroy-flow.test.ts | 11 +- src/lib/actions/sandbox/destroy.ts | 58 +++++---- .../sandbox/rebuild-destroy-phase.test.ts | 118 +++++++++++++++--- .../actions/sandbox/rebuild-destroy-phase.ts | 57 ++++++++- .../sandbox/rebuild-resume-snapshot.test.ts | 11 +- .../snapshot-restore-lifecycle.test.ts | 82 ++++++++++++ .../sandbox/snapshot-restore-test-fixture.ts | 31 ++++- src/lib/actions/sandbox/snapshot.ts | 43 ++++++- src/lib/onboard/runtime-provider/access.ts | 7 +- src/lib/onboard/runtime-provider/contract.ts | 23 ++++ src/lib/onboard/runtime-provider/docker.ts | 35 ++++-- src/lib/onboard/runtime-provider/registry.ts | 62 +++++++++ .../runtime-provider-contract.test.ts | 29 +++++ test/helpers/runtime-provider-bundle.ts | 37 +++--- test/image-cleanup.test.ts | 41 +++++- 16 files changed, 566 insertions(+), 99 deletions(-) diff --git a/src/lib/actions/sandbox/destroy-execution.ts b/src/lib/actions/sandbox/destroy-execution.ts index 09f68f435ca..d6cd53d4078 100644 --- a/src/lib/actions/sandbox/destroy-execution.ts +++ b/src/lib/actions/sandbox/destroy-execution.ts @@ -5,10 +5,9 @@ import { R, YW } from "../../cli/terminal-style"; import { getSandboxDeleteOutcome } from "../../domain/sandbox/destroy"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, - RuntimeProviderBundle, - RuntimeProviderBundleRegistry, - requireRuntimeProviderBundleForSandbox, - requireRuntimeProviderMutationAuthority, + type RuntimeProviderBundle, + type RuntimeProviderBundleRegistry, + requireRuntimeProviderDestructiveCleanupAuthority, } from "../../onboard/runtime-provider/access"; import { type DetachSandboxProvidersResult, @@ -199,14 +198,11 @@ export async function executeSandboxDestroy({ let runtimeProvider: RuntimeProviderBundle | null = null; if (sandbox) { try { - runtimeProvider = requireRuntimeProviderBundleForSandbox(sandbox, runtimeProviders); - requireRuntimeProviderMutationAuthority(runtimeProvider, "provider-cleanup"); - requireRuntimeProviderMutationAuthority(runtimeProvider, "destroy"); - if (runtimeProvider.cleanup.supported !== true) { - throw new Error( - `Runtime provider '${runtimeProvider.identity.id}' has no cleanup implementation.`, - ); - } + runtimeProvider = requireRuntimeProviderDestructiveCleanupAuthority( + sandboxName, + sandbox, + runtimeProviders, + ).provider; } catch (error) { return { ok: false as const, diff --git a/src/lib/actions/sandbox/destroy-flow.test.ts b/src/lib/actions/sandbox/destroy-flow.test.ts index 868d3b19a2e..acf2bad9ca5 100644 --- a/src/lib/actions/sandbox/destroy-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-flow.test.ts @@ -122,7 +122,7 @@ describe("destroySandbox flow", () => { expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); }); - it("reports incomplete cleanup and preserves ownership when image authority is unproven", async () => { + it("blocks deletion and preserves ownership when image authority is unproven", async () => { const harness = createDestroyHarness({ imageTag: "local/alpha:current", workload: { @@ -135,13 +135,12 @@ describe("destroySandbox flow", () => { await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow("process.exit(1)"); - const warningOutput = harness.warnSpy.mock.calls.map((call) => String(call[0])).join("\n"); + const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n"); const logOutput = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n"); - expect(warningOutput).toContain("Runtime provider 'docker'"); - expect(warningOutput).toContain("workload ownership authority could not be proven"); - expect(warningOutput).toContain("nemoclaw alpha destroy --yes"); + expect(errorOutput).toContain("Runtime provider 'docker'"); + expect(errorOutput).toContain("recorded workload receipt"); expect(logOutput).not.toContain("Sandbox 'alpha' destroyed"); - expect(harness.events).toContain("delete"); + expect(harness.events).not.toContain("delete"); expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); expect(harness.updateSessionSpy).not.toHaveBeenCalled(); }); diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 98945d91bcf..36c29858c8a 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -19,19 +19,18 @@ import { import { withGatewayRouteMutationLock } from "../../inference/gateway-route-mutation-lock"; import { parseHttpsPinRouteId } from "../../inference/https-pin-runtime"; import { revokeHttpsPinRuntimeAdapterRoute } from "../../inference/https-pin-runtime-adapter"; -import { - emitProviderDetachResidualHint, - SANDBOX_PROVIDER_SUFFIXES, -} from "../../onboard/sandbox-provider-cleanup"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, - type RuntimeProviderBundleRegistry, - type RuntimeProviderWorkloadCleanupResult, normalizeRuntimeProviderIdentity, - requireRuntimeProviderBundleForSandbox, - requireRuntimeProviderMutationAuthority, + type RuntimeProviderBundleRegistry, RuntimeProviderSelectionError, + type RuntimeProviderWorkloadCleanupResult, + requireRuntimeProviderDestructiveCleanupAuthority, } from "../../onboard/runtime-provider/access"; +import { + emitProviderDetachResidualHint, + SANDBOX_PROVIDER_SUFFIXES, +} from "../../onboard/sandbox-provider-cleanup"; import { validateName } from "../../runner"; import { killTimer as defaultKillShieldsTimer } from "../../shields/timer-control"; import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock"; @@ -76,6 +75,14 @@ type RemoveSandboxRegistryEntryOutcome = readonly removed: false; }; +export function requireSandboxDestructiveCleanupAuthority( + sandboxName: string, + sandbox: registry.SandboxEntry, + providers: RuntimeProviderBundleRegistry = CURRENT_RUNTIME_PROVIDER_BUNDLES, +) { + return requireRuntimeProviderDestructiveCleanupAuthority(sandboxName, sandbox, providers); +} + type RunOpenshell = (args: string[], opts?: Record) => { status: number | null }; export type CleanupSandboxServicesDeps = { @@ -252,17 +259,12 @@ export function removeSandboxImage( const warn = deps.warn ?? console.warn; let result: RuntimeProviderWorkloadCleanupResult; try { - const provider = requireRuntimeProviderBundleForSandbox( + const authority = requireSandboxDestructiveCleanupAuthority( + sandboxName, sb, deps.runtimeProviders ?? CURRENT_RUNTIME_PROVIDER_BUNDLES, ); - requireRuntimeProviderMutationAuthority(provider, "workload-cleanup"); - if (provider.cleanup.supported !== true) { - throw new RuntimeProviderSelectionError( - `Runtime provider '${provider.identity.id}' has no cleanup implementation.`, - ); - } - result = provider.cleanup.removeOwnedWorkload({ sandbox: sb, sandboxName }); + result = authority.provider.cleanup.removeOwnedWorkload({ sandbox: sb, sandboxName }); } catch (error) { const detail = error instanceof RuntimeProviderSelectionError @@ -272,8 +274,9 @@ export function removeSandboxImage( : String(error); warn( ` ${YW}⚠${R} Runtime provider '${providerId}' could not prove workload cleanup ` + - `authority: ${detail} Local ownership state was preserved; repair the provider ` + - "metadata and retry the operation.", + `authority: ${detail} Local ownership state was preserved. Run '${CLI_NAME} ` + + `${sandboxName} doctor --json'; restore trusted ownership metadata or resolve the ` + + "runtime conflict, then retry. Do not rewrite a receipt to match a mutable name.", ); return { status: "skipped", reason: "authority-unproven" }; } @@ -287,8 +290,9 @@ export function removeSandboxImage( } else if (result.reason === "authority-unproven") { warn( ` ${YW}⚠${R} Runtime provider '${providerId}' did not prove ownership of the ` + - "recorded workload image. Local ownership state was preserved; repair the workload " + - "receipt and retry the operation.", + `recorded workload image. Local ownership state was preserved. Run '${CLI_NAME} ` + + `${sandboxName} doctor --json'; restore trusted ownership metadata or resolve the ` + + "runtime conflict, then retry. Do not rewrite a receipt to match a mutable name.", ); } return result; @@ -505,9 +509,11 @@ async function destroySandboxUnlocked( * cleanup authority, so the registry row and onboarding session are retained. * Source boundary: the persisted `openshellDriver` and `workload` receipt are * validated only by the selected provider's `cleanup.removeOwnedWorkload`. - * Source-fix constraint: destroy cannot synthesize provider ownership after - * remote deletion; guessing could remove a shared image or another provider's - * workload, so the operator must repair the durable authority and retry. + * Source-fix constraint: this is a residual guard for a raw writer or TOCTOU + * change after pre-delete authority validation. Destroy cannot synthesize + * provider ownership after remote deletion; guessing could remove a shared + * image or another provider's workload, so the operator must restore trusted + * ownership metadata or resolve the runtime conflict and retry. * Regression proof: destroy-flow.test.ts proves both blocked retention and * that a repaired matching receipt permits registry and session retirement; * test/image-cleanup.test.ts proves the lower-level fail-closed contract. @@ -524,8 +530,10 @@ async function destroySandboxUnlocked( `'${providerId}' because workload ownership authority could not be proven.`, ); console.warn( - ` ${YW}⚠${R} The local registry and session were preserved. Repair the provider/workload ` + - `receipt, then re-run '${CLI_NAME} ${sandboxName} destroy --yes' to finish cleanup.`, + ` ${YW}⚠${R} The local registry and session were preserved. Run '${CLI_NAME} ` + + `${sandboxName} doctor --json'; restore trusted ownership metadata or resolve the ` + + `runtime conflict, then re-run '${CLI_NAME} ${sandboxName} destroy --yes'. Do not ` + + "rewrite a receipt to match a mutable name.", ); emitProviderDetachResidualHint(sandboxName, detachOutcome.failures, (message) => console.warn(` ${YW}⚠${R}${message}`), diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index 1ce3bbde7c3..d09ff85e277 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -3,21 +3,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { expectNoSandboxDelete } from "../../../../test/helpers/rebuild-delete-assertions"; -import type { SandboxRemovalReceipt } from "../../state/registry"; +import type { SandboxEntry, SandboxRemovalReceipt } from "../../state/registry"; const mocks = vi.hoisted(() => ({ captureOpenshell: vi.fn(), - getSandbox: vi.fn( - ( - _name: string, - ): { - name: string; - agent: string; - nimContainer?: string | null; - gatewayName?: string | null; - gatewayPort?: number | null; - } | null => null, - ), + getSandbox: vi.fn((_name: string): SandboxEntry | null => null), listSandboxes: vi.fn(() => ({ sandboxes: [] })), prepareMcpForRebuild: vi.fn(), reattachMcpAfterDeleteFailure: vi.fn(), @@ -59,9 +49,20 @@ vi.mock("../../state/registry", async (importOriginal) => ({ listSandboxes: mocks.listSandboxes, })); -vi.mock("./destroy", () => ({ - removeSandboxRegistryEntryWithReceipt: mocks.removeSandboxRegistryEntryWithReceipt, -})); +vi.mock("./destroy", async () => { + const runtimeProviders = await vi.importActual< + typeof import("../../onboard/runtime-provider/access") + >("../../onboard/runtime-provider/access"); + return { + removeSandboxRegistryEntryWithReceipt: mocks.removeSandboxRegistryEntryWithReceipt, + requireSandboxDestructiveCleanupAuthority: (sandboxName: string, sandbox: SandboxEntry) => + runtimeProviders.requireRuntimeProviderDestructiveCleanupAuthority( + sandboxName, + sandbox, + runtimeProviders.CURRENT_RUNTIME_PROVIDER_BUNDLES, + ), + }; +}); vi.mock("./rebuild-flow-helpers", () => ({ warnUnpreservedUserManagedFiles: mocks.warnUnpreservedUserManagedFiles, @@ -215,6 +216,86 @@ describe("rebuild destroy phase", () => { expectNoSandboxDelete(mocks.runOpenshell); }); + it("blocks before MCP preparation when runtime cleanup authority is unknown", async () => { + mocks.getSandbox.mockReturnValue({ + name: "alpha", + agent: "openclaw", + openshellDriver: "future-runtime", + imageTag: "local/alpha:current", + }); + const bail = vi.fn((message: string): never => { + throw new Error(message); + }); + + await expect( + runRebuildDestroyPhase({ + sandboxName: "alpha", + sandboxEntry: { + name: "alpha", + agent: "openclaw", + openshellDriver: "future-runtime", + imageTag: "local/alpha:current", + }, + staleRecovery: false, + backupManifest: null, + log: vi.fn(), + bail, + relockShieldsIfNeeded: vi.fn(() => true), + onDeleted: vi.fn(), + }), + ).rejects.toThrow("is not registered for this operation"); + + expect(mocks.prepareMcpForRebuild).not.toHaveBeenCalled(); + expectNoSandboxDelete(mocks.runOpenshell); + }); + + it("rechecks workload ownership at the exact delete edge after MCP preparation", async () => { + const matchingEntry: SandboxEntry = { + name: "alpha", + agent: "openclaw", + openshellDriver: "docker", + imageTag: "local/alpha:current", + workload: { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "local/alpha:current", + shared: false, + }, + }; + mocks.getSandbox + .mockReturnValueOnce(matchingEntry) + .mockReturnValueOnce(matchingEntry) + .mockReturnValue({ + ...matchingEntry, + workload: { + ...matchingEntry.workload!, + reference: "local/alpha:changed", + }, + }); + const relockShieldsIfNeeded = vi.fn(() => true); + const bail = vi.fn((message: string): never => { + throw new Error(message); + }); + + await expect( + runRebuildDestroyPhase({ + sandboxName: "alpha", + sandboxEntry: matchingEntry, + staleRecovery: false, + backupManifest: null, + log: vi.fn(), + bail, + relockShieldsIfNeeded, + onDeleted: vi.fn(), + }), + ).rejects.toThrow("could not prove ownership"); + + expect(mocks.prepareMcpForRebuild).toHaveBeenCalledOnce(); + expect(mocks.reattachMcpAfterDeleteFailure).toHaveBeenCalledOnce(); + expect(relockShieldsIfNeeded).toHaveBeenCalledWith(true); + expectNoSandboxDelete(mocks.runOpenshell); + }); + it("passes force=true to prepareMcpForRebuild when input.force is set (#7062)", async () => { const log = vi.fn(); const bail = vi.fn((message: string): never => { @@ -325,9 +406,9 @@ describe("rebuild destroy phase", () => { }), ).rejects.toThrow("Sandbox delete target changed during rebuild preparation."); - expect(mocks.getSandbox).toHaveBeenCalledTimes(2); + expect(mocks.getSandbox).toHaveBeenCalledTimes(3); expect(mocks.prepareMcpForRebuild.mock.invocationCallOrder[0]).toBeLessThan( - mocks.getSandbox.mock.invocationCallOrder[1] ?? Number.POSITIVE_INFINITY, + mocks.getSandbox.mock.invocationCallOrder[2] ?? Number.POSITIVE_INFINITY, ); expect(mocks.runOpenshell).not.toHaveBeenCalled(); expect(mocks.reattachMcpAfterDeleteFailure).toHaveBeenCalledWith( @@ -949,7 +1030,8 @@ describe("rebuild destroy phase", () => { ); const errors = vi.mocked(console.error).mock.calls.flat().join("\n"); expect(errors).toContain("local runtime ownership cleanup is incomplete"); - expect(errors).toContain("Repair the provider/workload receipt"); + expect(errors).toContain("doctor --json"); + expect(errors).toContain("Do not rewrite a receipt to match a mutable name"); expect(errors).toContain("/tmp/rebuild-backups/alpha/backup"); expect(vi.mocked(console.log).mock.calls.flat().join("\n")).not.toContain( "Old sandbox deleted", diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.ts index 9c9eaeaf4b2..a83df619650 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.ts @@ -12,7 +12,10 @@ import { redactFull } from "../../security/redact"; import { parseSandboxPhase } from "../../state/gateway"; import { registryEntryGatewayPort } from "../../state/gateway-registry"; import * as registry from "../../state/registry"; -import { removeSandboxRegistryEntryWithReceipt } from "./destroy"; +import { + removeSandboxRegistryEntryWithReceipt, + requireSandboxDestructiveCleanupAuthority, +} from "./destroy"; import { isExplicitMissingSandboxGatewayOutput } from "./gateway-state"; import type { RebuildBackupManifest } from "./rebuild-backup-phase"; import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; @@ -111,6 +114,32 @@ function rebuildDeleteTargetMatchesRegistry(expected: RebuildDeleteTarget): bool } } +function rebuildCleanupAuthorityFailure( + sandboxName: string, +): { readonly message: string; readonly code: number } | null { + const currentEntry = registry.getSandbox(sandboxName); + if (!currentEntry) { + return { + message: `Sandbox '${sandboxName}' runtime ownership entry disappeared before deletion.`, + code: 1, + }; + } + try { + requireSandboxDestructiveCleanupAuthority(sandboxName, currentEntry); + return null; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + return { + message: + `Sandbox '${sandboxName}' runtime cleanup authority could not be proven before deletion: ` + + `${redactFull(detail)} Run 'nemoclaw ${sandboxName} doctor --json'; restore trusted ` + + "ownership metadata or resolve the runtime conflict, then retry. Do not rewrite a " + + "receipt to match a mutable name.", + code: 1, + }; + } +} + /** Wait for explicit absence from the same `sandbox get` boundary used by inner onboard. */ export function waitForRebuildDeleteAbsence( sandboxName: string, @@ -246,6 +275,11 @@ export async function runRebuildDestroyPhase( log( `Registry entry: agent=${sbMeta?.agent}, agentVersion=${sbMeta?.agentVersion}, nimContainer=${sbMeta?.nimContainer}`, ); + const initialCleanupAuthorityFailure = rebuildCleanupAuthorityFailure(sandboxName); + if (initialCleanupAuthorityFailure) { + bail(initialCleanupAuthorityFailure.message, initialCleanupAuthorityFailure.code); + return null; + } const stopNimBestEffort = (): void => { try { if (sbMeta && sbMeta.nimContainer) { @@ -357,6 +391,23 @@ export async function runRebuildDestroyPhase( return null; } + const cleanupAuthorityFailure = rebuildCleanupAuthorityFailure(sandboxName); + if (cleanupAuthorityFailure) { + const mcpRecoveryFailure = await reattachMcpAfterDeleteFailure( + sandboxName, + rebuildDetachedMcpProviderEntries, + rebuildScrubbedMcpAdapterEntries, + ); + relockShieldsIfNeeded(true); + bail( + mcpRecoveryFailure + ? `${cleanupAuthorityFailure.message} MCP provider recovery also failed: ${mcpRecoveryFailure}` + : cleanupAuthorityFailure.message, + cleanupAuthorityFailure.code, + ); + return null; + } + if (validateAtDeleteEdge) { let validation: RebuildDeleteValidationResult; try { @@ -470,7 +521,9 @@ export async function runRebuildDestroyPhase( " The registry entry was preserved because provider/workload cleanup authority could not be proven or registry ownership changed.", ); console.error( - " Repair the provider/workload receipt or reconcile the retained registry row, then retry rebuild.", + ` Run 'nemoclaw ${sandboxName} doctor --json'; restore trusted ownership metadata ` + + "or resolve the runtime conflict, then retry rebuild. Do not rewrite a receipt to " + + "match a mutable name.", ); if (backupManifest) { console.error(" State backup is preserved at: " + backupManifest.backupPath); diff --git a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts index 83472557089..5489cc9ee32 100644 --- a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts +++ b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts @@ -183,7 +183,16 @@ describe("rebuild resume snapshot repair", () => { stdout: "", stderr: "Error: sandbox alpha not found", } as never), - vi.spyOn(destroy, "removeSandboxRegistryEntry").mockReturnValue(true), + vi.spyOn(destroy, "removeSandboxRegistryEntryWithReceipt").mockReturnValue({ + entry: { + name: "alpha", + agent: null, + } as never, + wasDefault: true, + fallbackDefault: null, + postRemovalDefaultSelectionRevision: 1, + }), + vi.spyOn(registry, "restoreSandboxEntryIfMissing").mockReturnValue(true), vi.spyOn(nim, "stopNimContainer").mockImplementation(() => undefined), vi.spyOn(nim, "stopNimContainerByName").mockImplementation(() => undefined), vi.spyOn(nim, "detectGpu").mockReturnValue(null), diff --git a/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts b/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts index eb366cba3e8..2e7ca039366 100644 --- a/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts +++ b/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts @@ -164,6 +164,88 @@ describe("runSandboxSnapshot restore: lifecycle and destination safety", () => { expect(f.restoreSandboxStateMock).toHaveBeenCalledWith("beta", "/tmp/backup-alpha"); }); + it.each([ + { + label: "an unknown runtime provider", + destination: { + name: "beta", + agent: "openclaw", + imageTag: "nemoclaw-beta:test", + openshellDriver: "future-runtime", + provider: "nvidia-nim", + model: "nvidia/model-a", + }, + expected: "is not registered for this operation", + }, + { + label: "a mismatched legacy workload receipt", + destination: { + name: "beta", + agent: "openclaw", + imageTag: "nemoclaw-beta:current", + openshellDriver: "docker", + provider: "nvidia-nim", + model: "nvidia/model-a", + workload: { + schemaVersion: 1 as const, + kind: "legacy-dockerfile" as const, + reference: "nemoclaw-beta:recorded", + shared: false as const, + }, + }, + expected: "could not prove ownership", + }, + ])("refuses force deletion before every side effect for $label", async ({ + destination, + expected, + }) => { + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + f.getSandboxMock.mockImplementation((name) => + name === "alpha" + ? { + name: "alpha", + agent: "openclaw", + imageTag: "nemoclaw-alpha:test", + openshellDriver: "docker", + provider: "nvidia-nim", + model: "nvidia/model-a", + } + : name === "beta" + ? destination + : null, + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha", "beta"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect( + runSandboxSnapshot("alpha", { + kind: "restore", + to: "beta", + force: true, + yes: true, + }), + ).rejects.toMatchObject({ exitCode: 1 }); + + expect(consoleError.mock.calls.flat().join("\n")).toContain(expected); + expect(f.stopNimContainerMock).not.toHaveBeenCalled(); + expect(f.stopNimContainerByNameMock).not.toHaveBeenCalled(); + expect(f.lifecycleMock.events).not.toContain("delete"); + expect(f.lifecycleMock.events).not.toContain("cleanup-shields"); + expect(f.runOpenshellMock).not.toHaveBeenCalledWith( + expect.arrayContaining(["provider", "delete"]), + expect.anything(), + ); + expect(f.streamSandboxCreateMock).not.toHaveBeenCalled(); + expect(f.registerSandboxMock).not.toHaveBeenCalled(); + }); + it("blocks auto-create before deleting a destination when a gateway peer conflicts", async () => { const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); f.getSandboxMock.mockImplementation((name) => ({ diff --git a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts index 05f95a582fc..54a531760c8 100644 --- a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts +++ b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts @@ -41,6 +41,12 @@ export type SandboxRecord = { fromDockerfile?: string | null; gatewayName?: string | null; imageTag?: string | null; + workload?: { + schemaVersion: 1; + kind: "legacy-dockerfile"; + reference: string | null; + shared: false; + }; openshellDriver?: string | null; observabilityEnabled?: boolean; provider?: string | null; @@ -165,6 +171,8 @@ export const loadPresetForSandboxMock = vi.fn((_sandbox: string, preset: string) export const getSandboxMock = vi.fn<(name?: string) => SandboxRecord | null>(() => null); export const isGatewayHealthyMock = vi.fn(() => true); export const listBackupsMock = vi.fn<() => Array>>(() => []); +export const stopNimContainerMock = vi.fn(); +export const stopNimContainerByNameMock = vi.fn(); export const parseLiveSandboxNamesMock = vi.fn(() => new Set(["alpha"])); export const waitForRestoredSandboxGatewaySupervisorMock = vi.fn(() => true); export const prepareInitialSandboxCreatePolicyMock = vi.fn( @@ -219,8 +227,8 @@ vi.mock("../../domain/sandbox/destroy", () => ({ })); vi.mock("../../inference/nim", () => ({ - stopNimContainer: vi.fn(), - stopNimContainerByName: vi.fn(), + stopNimContainer: stopNimContainerMock, + stopNimContainerByName: stopNimContainerByNameMock, })); vi.mock("../../policy", () => ({ @@ -297,10 +305,21 @@ vi.mock("../../state/sandbox", () => ({ restoreSandboxState: restoreSandboxStateMock, })); -vi.mock("./destroy", () => ({ - cleanupShieldsDestroyArtifacts: lifecycleMock.cleanupShieldsDestroyArtifactsMock, - removeSandboxRegistryEntry: vi.fn(), -})); +vi.mock("./destroy", async () => { + const runtimeProviders = await vi.importActual< + typeof import("../../onboard/runtime-provider/access") + >("../../onboard/runtime-provider/access"); + return { + cleanupShieldsDestroyArtifacts: lifecycleMock.cleanupShieldsDestroyArtifactsMock, + removeSandboxRegistryEntry: vi.fn(), + requireSandboxDestructiveCleanupAuthority: (sandboxName: string, sandbox: SandboxRecord) => + runtimeProviders.requireRuntimeProviderDestructiveCleanupAuthority( + sandboxName, + sandbox, + runtimeProviders.CURRENT_RUNTIME_PROVIDER_BUNDLES, + ), + }; +}); vi.mock("./restore-gateway-pairing", () => ({ establishRestoredSandboxGatewayPairing: establishRestoredSandboxGatewayPairingMock, diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index f61961fe5fc..f16b9338111 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -60,7 +60,11 @@ import { DCODE_PROBE_STATE, parseDcodeProbeState, } from "./dcode-activity-probe"; -import { cleanupShieldsDestroyArtifacts, removeSandboxRegistryEntry } from "./destroy"; +import { + cleanupShieldsDestroyArtifacts, + removeSandboxRegistryEntry, + requireSandboxDestructiveCleanupAuthority, +} from "./destroy"; import { establishRestoredSandboxGatewayPairing, waitForRestoredSandboxGatewaySupervisor, @@ -129,13 +133,28 @@ export function requireSnapshotDestinationRegistryRemoval( registryRemoved: boolean, ): void { if (registryRemoved !== false) return; + // SOURCE_OF_TRUTH + // Invalid state: a bypassing registry writer changed cleanup authority after + // the locked pre-delete proof, leaving the destination absent while its + // ownership row must be retained. + // Source boundary: deleteSandboxForRestore proves provider/workload cleanup + // authority under the destination lifecycle lock, then registry removal + // rechecks that authority after the live delete. + // Source-fix constraint: the current runtime API has no authenticated atomic + // replace primitive and raw writers do not participate in NemoClaw's lock. + // Regression proof: snapshot-restore-lifecycle.test.ts covers pre-delete + // refusal; snapshot restore authority tests cover this retained-row stop. + // Removal condition: an exact provider-native replace transaction supplies + // durable delete, rollback, and cleanup receipts. console.error( ` Destination '${name}' is deleted, but local runtime ownership cleanup is incomplete.`, ); console.error( " The registry entry was preserved because provider/workload cleanup authority could not be proven.", ); - console.error(" Repair the provider/workload receipt, then retry the snapshot restore."); + console.error( + ` Run '${CLI_NAME} ${name} doctor --json'; restore trusted ownership metadata or resolve the runtime conflict, then retry. Do not rewrite a receipt to match a mutable name.`, + ); snapshotExit(1); } @@ -453,7 +472,25 @@ async function autoCreateSandboxFromSource( // we are about to clone from. function deleteSandboxForRestore(name: string): void { const sbMeta = registry.getSandbox(name); - if (sbMeta?.nimContainer) { + if (!sbMeta) { + console.error( + ` Cannot delete destination '${name}': its durable runtime ownership entry disappeared.`, + ); + snapshotExit(1); + } + try { + requireSandboxDestructiveCleanupAuthority(name, sbMeta); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + console.error( + ` Cannot delete destination '${name}' because runtime cleanup authority is unproven: ${detail}`, + ); + console.error( + ` Run '${CLI_NAME} ${name} doctor --json' and resolve the recorded ownership conflict before retrying.`, + ); + snapshotExit(1); + } + if (sbMeta.nimContainer) { nim.stopNimContainerByName(sbMeta.nimContainer); } else { nim.stopNimContainer(name, { silent: true }); diff --git a/src/lib/onboard/runtime-provider/access.ts b/src/lib/onboard/runtime-provider/access.ts index 7e52586b493..0a49abaa21f 100644 --- a/src/lib/onboard/runtime-provider/access.ts +++ b/src/lib/onboard/runtime-provider/access.ts @@ -7,19 +7,22 @@ export type { RuntimeProviderChannelStopTransport, RuntimeProviderGatewayLauncher, RuntimeProviderManagedImageSupport, - RuntimeProviderWorkloadProfile, + RuntimeProviderWorkloadCleanupPlan, RuntimeProviderWorkloadCleanupResult, + RuntimeProviderWorkloadProfile, } from "./contract"; export { CURRENT_RUNTIME_PROVIDER_BUNDLES, resolveCurrentRuntimeProviderBundle, } from "./current"; +export type { RuntimeProviderDestructiveCleanupAuthority } from "./registry"; export { normalizeRuntimeProviderIdentity, + RuntimeProviderSelectionError, requireRuntimeProviderBundle, requireRuntimeProviderBundleForSandbox, + requireRuntimeProviderDestructiveCleanupAuthority, requireRuntimeProviderMutationAuthority, resolveRuntimeProviderBundle, - RuntimeProviderSelectionError, runtimeProviderContainerEngineIdentity, } from "./registry"; diff --git a/src/lib/onboard/runtime-provider/contract.ts b/src/lib/onboard/runtime-provider/contract.ts index 828d2bd773f..d629eb51eed 100644 --- a/src/lib/onboard/runtime-provider/contract.ts +++ b/src/lib/onboard/runtime-provider/contract.ts @@ -116,6 +116,21 @@ export interface RuntimeProviderCleanupInput { readonly sandboxName: string; } +export type RuntimeProviderWorkloadCleanupPlan = + | { + readonly action: "retain"; + readonly reason: "no-owned-image" | "shared-image"; + } + | { + readonly action: "remove"; + readonly engineDisplayName: string; + readonly reference: string; + } + | { + readonly action: "block"; + readonly reason: "authority-unproven"; + }; + export type RuntimeProviderWorkloadCleanupResult = | { readonly status: "skipped"; @@ -223,6 +238,14 @@ export type RuntimeProviderCleanupSurface = input: RuntimeProviderCleanupInput, operations: RuntimeProviderCleanupOperations, ): RuntimeProviderProviderDetachResult; + /** + * Produce a side-effect-free cleanup plan before any destructive + * sandbox action. Providers must revalidate the same authority inside + * removeOwnedWorkload before mutating their runtime. + */ + planOwnedWorkloadCleanup( + input: RuntimeProviderCleanupInput, + ): RuntimeProviderWorkloadCleanupPlan; removeOwnedWorkload(input: RuntimeProviderCleanupInput): RuntimeProviderWorkloadCleanupResult; }> | RuntimeProviderUnsupportedSurface; diff --git a/src/lib/onboard/runtime-provider/docker.ts b/src/lib/onboard/runtime-provider/docker.ts index 8f690fbf12f..03defc728ef 100644 --- a/src/lib/onboard/runtime-provider/docker.ts +++ b/src/lib/onboard/runtime-provider/docker.ts @@ -27,6 +27,7 @@ import { type RuntimeProviderLifecycleResult, type RuntimeProviderLifecycleStopHooks, type RuntimeProviderLifecycleStopOutcome, + type RuntimeProviderWorkloadCleanupPlan, type RuntimeProviderWorkloadCleanupResult, type RuntimeProviderWorkloadProfile, } from "./contract"; @@ -208,13 +209,12 @@ function stopDockerSandbox( return { exitCode: 0, state: "stopped" }; } -function removeOwnedDockerWorkload( +function planOwnedDockerWorkloadCleanup( input: RuntimeProviderCleanupInput, - deps: DockerRuntimeProviderDependencies, -): RuntimeProviderWorkloadCleanupResult { +): RuntimeProviderWorkloadCleanupPlan { const { imageTag, workload } = input.sandbox; - if (workload?.shared === true) return { status: "skipped", reason: "shared-image" }; - if (!imageTag) return { status: "skipped", reason: "no-owned-image" }; + if (workload?.shared === true) return { action: "retain", reason: "shared-image" }; + if (!imageTag) return { action: "retain", reason: "no-owned-image" }; if ( Object.values(MANAGED_IMAGE_REPOSITORIES).some( (repository) => @@ -223,20 +223,32 @@ function removeOwnedDockerWorkload( imageTag.startsWith(`${repository}:`), ) ) { - return { status: "skipped", reason: "shared-image" }; + return { action: "retain", reason: "shared-image" }; } if ( workload?.kind === "legacy-dockerfile" && workload.reference !== null && workload.reference !== imageTag ) { + return { action: "block", reason: "authority-unproven" }; + } + return { action: "remove", engineDisplayName: "Docker", reference: imageTag }; +} + +function removeOwnedDockerWorkload( + input: RuntimeProviderCleanupInput, + deps: DockerRuntimeProviderDependencies, +): RuntimeProviderWorkloadCleanupResult { + const plan = planOwnedDockerWorkloadCleanup(input); + if (plan.action === "retain") return { status: "skipped", reason: plan.reason }; + if (plan.action === "block") { return { status: "skipped", reason: "authority-unproven" }; } - const result = deps.removeImage(imageTag, { ignoreError: true }); + const result = deps.removeImage(plan.reference, { ignoreError: true }); return { status: result.status === 0 ? "removed" : "failed", - engineDisplayName: "Docker", - reference: imageTag, + engineDisplayName: plan.engineDisplayName, + reference: plan.reference, }; } @@ -339,6 +351,7 @@ export function createDockerRuntimeProviderBundle( providerId, supported: true, prepareDestroy: (_input, operations) => operations.detachProviders(), + planOwnedWorkloadCleanup: planOwnedDockerWorkloadCleanup, removeOwnedWorkload: (input) => removeOwnedDockerWorkload(input, deps), }, containerEngine: { @@ -422,6 +435,10 @@ export function createKubernetesRuntimeProviderBundle( providerId, supported: true, prepareDestroy: (_input, operations) => operations.detachProviders(), + // The shipped Kubernetes gateway path has always built and retained its + // per-sandbox image in the host Docker engine. Keep that established + // engine ownership explicit until a CRI-native provider is registered. + planOwnedWorkloadCleanup: planOwnedDockerWorkloadCleanup, removeOwnedWorkload: (input) => removeOwnedDockerWorkload(input, deps), }, containerEngine: { diff --git a/src/lib/onboard/runtime-provider/registry.ts b/src/lib/onboard/runtime-provider/registry.ts index c8cd9621005..81ee3ba7ae4 100644 --- a/src/lib/onboard/runtime-provider/registry.ts +++ b/src/lib/onboard/runtime-provider/registry.ts @@ -340,6 +340,7 @@ function validateRecoverySurface(surface: Record): void { function validateCleanupSurface(surface: Record): void { if (surface.supported === true) { requireFunction(surface, "prepareDestroy", "cleanup"); + requireFunction(surface, "planOwnedWorkloadCleanup", "cleanup"); requireFunction(surface, "removeOwnedWorkload", "cleanup"); } } @@ -508,6 +509,67 @@ export function requireRuntimeProviderMutationAuthority( } } +export type RuntimeProviderDestructiveCleanupAuthority = { + readonly provider: RuntimeProviderBundle & { + readonly cleanup: Extract; + }; + readonly workloadAction: "retain" | "remove"; +}; + +/** + * Prove the complete provider/workload cleanup boundary without mutating it. + * + * SOURCE_OF_TRUTH + * Invalid state: a destructive sandbox action removes the live workload before + * NemoClaw proves that the recorded provider can retire its exact local + * ownership state. + * Source boundary: the selected RuntimeProviderBundle owns destroy, + * provider-cleanup, and workload-cleanup authority plus the side-effect-free + * workload cleanup plan. + * Source-fix constraint: mutable sandbox names are not runtime ownership + * receipts, and a provider may use a CLI, socket, API, or no container engine. + * Regression proof: snapshot-restore-lifecycle.test.ts rejects unknown + * providers and mismatched legacy workload receipts before any delete, + * provider cleanup, shields cleanup, or replacement creation. + * Removal condition: this guard may be replaced only by a provider-native + * atomic replace operation that returns authenticated rollback/cleanup + * receipts for the exact prior runtime. + */ +export function requireRuntimeProviderDestructiveCleanupAuthority( + sandboxName: string, + sandbox: SandboxEntry, + providers: RuntimeProviderBundleRegistry, +): RuntimeProviderDestructiveCleanupAuthority { + const provider = requireRuntimeProviderBundleForSandbox(sandbox, providers); + requireRuntimeProviderMutationAuthority(provider, "destroy"); + requireRuntimeProviderMutationAuthority(provider, "provider-cleanup"); + requireRuntimeProviderMutationAuthority(provider, "workload-cleanup"); + if (provider.cleanup.supported !== true) { + throw new RuntimeProviderSelectionError( + `Runtime provider '${provider.identity.id}' has no cleanup implementation.`, + ); + } + const supportedProvider = provider as RuntimeProviderDestructiveCleanupAuthority["provider"]; + const plan = supportedProvider.cleanup.planOwnedWorkloadCleanup({ sandbox, sandboxName }); + // Shared managed images and rows with no owned image require no destructive + // workload mutation, so malformed or legacy-dropped receipts cannot turn + // their immutable image into a deletion candidate. + if (plan.action === "retain") { + return Object.freeze({ provider: supportedProvider, workloadAction: plan.action }); + } + if (plan.action === "block") { + throw new RuntimeProviderSelectionError( + `Runtime provider '${provider.identity.id}' could not prove ownership of the recorded workload receipt.`, + ); + } + if (!provider.workload.acceptsReceipt(sandbox.workload)) { + throw new RuntimeProviderSelectionError( + `Runtime provider '${provider.identity.id}' rejected the durable workload receipt.`, + ); + } + return Object.freeze({ provider: supportedProvider, workloadAction: plan.action }); +} + export function runtimeProviderContainerEngineIdentity( bundle: RuntimeProviderBundle, operation: RuntimeProviderContainerEngineOperation, diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index d6362c75510..ee924c0b786 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -339,6 +339,35 @@ describe("RuntimeProviderBundle registry contract", () => { ).toThrow(/lifecycle\.verifyStarted must be a function/u); }); + it("rejects cleanup without a side-effect-free ownership plan", () => { + const bundle = mxcBundle(); + expectSupportedSurface(bundle.cleanup); + const { planOwnedWorkloadCleanup: _planOwnedWorkloadCleanup, ...incomplete } = bundle.cleanup; + + expect(() => + createRuntimeProviderBundleRegistry([["mxc", replaceSurface(bundle, "cleanup", incomplete)]]), + ).toThrow(/cleanup\.planOwnedWorkloadCleanup must be a function/u); + }); + + it("plans owned workload cleanup without mutating the runtime", () => { + const docker = CURRENT_RUNTIME_PROVIDER_BUNDLES.docker!; + expectSupportedSurface(docker.cleanup); + const sandbox = { + name: "alpha", + imageTag: "nemoclaw-alpha:current", + workload: { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "nemoclaw-alpha:recorded", + shared: false, + }, + } as SandboxEntry; + + expect(docker.cleanup.planOwnedWorkloadCleanup({ sandbox, sandboxName: sandbox.name })).toEqual( + { action: "block", reason: "authority-unproven" }, + ); + }); + it("rejects capability/surface drift and duplicate operation-scoped engine identities", () => { const bundle = mxcBundle(); expect(() => diff --git a/test/helpers/runtime-provider-bundle.ts b/test/helpers/runtime-provider-bundle.ts index e4476a1bfbd..33a4ff35105 100644 --- a/test/helpers/runtime-provider-bundle.ts +++ b/test/helpers/runtime-provider-bundle.ts @@ -51,6 +51,18 @@ export function createInMemoryRuntimeProviderBundle({ }: InMemoryRuntimeProviderOptions): InMemoryRuntimeProviderBundle { const futureReason = "Unsupported by this in-memory contract fixture."; const event = (kind: string, sandboxName: string) => recordEvent(`${kind}:${sandboxName}`); + const planOwnedWorkloadCleanup = (input: RuntimeProviderCleanupInput) => { + const reference = input.sandbox.imageTag; + return input.sandbox.workload?.shared === true + ? { action: "retain" as const, reason: "shared-image" as const } + : reference && state.workloads.has(reference) + ? { + action: "remove" as const, + engineDisplayName: "In-memory", + reference, + } + : { action: "retain" as const, reason: "no-owned-image" as const }; + }; return { identity: { contractVersion: RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, @@ -145,22 +157,19 @@ export function createInMemoryRuntimeProviderBundle({ event("prepare-destroy", input.sandboxName); return operations.detachProviders(); }, + planOwnedWorkloadCleanup, removeOwnedWorkload(input: RuntimeProviderCleanupInput) { - const reference = input.sandbox.imageTag; - const remove = (ownedReference: string) => { - state.workloads.delete(ownedReference); - event("cleanup", input.sandboxName); - return { - status: "removed" as const, - engineDisplayName: "In-memory", - reference: ownedReference, - }; + const plan = planOwnedWorkloadCleanup(input); + if (plan.action !== "remove") { + return { status: "skipped", reason: plan.reason }; + } + state.workloads.delete(plan.reference); + event("cleanup", input.sandboxName); + return { + status: "removed" as const, + engineDisplayName: plan.engineDisplayName, + reference: plan.reference, }; - return input.sandbox.workload?.shared === true - ? { status: "skipped", reason: "shared-image" } - : reference && state.workloads.has(reference) - ? remove(reference) - : { status: "skipped", reason: "no-owned-image" }; }, }, containerEngine: { diff --git a/test/image-cleanup.test.ts b/test/image-cleanup.test.ts index 368c589a78a..edcae5a4495 100644 --- a/test/image-cleanup.test.ts +++ b/test/image-cleanup.test.ts @@ -15,6 +15,7 @@ import { removeSandboxRegistryEntry, removeSandboxRegistryEntryWithReceipt, removeShieldsState, + requireSandboxDestructiveCleanupAuthority, } from "../src/lib/actions/sandbox/destroy"; import { requireSnapshotDestinationRegistryRemoval } from "../src/lib/actions/sandbox/snapshot"; import { COMMANDS, globalCommandTokens } from "../src/lib/cli/command-registry"; @@ -175,7 +176,45 @@ describe("image cleanup: sandbox destroy removes Docker image (#2086)", () => { const error = vi.spyOn(console, "error").mockImplementation(() => undefined); expect(() => requireSnapshotDestinationRegistryRemoval("alpha", registryRemoved)).toThrow(); expect(removeSandbox).not.toHaveBeenCalled(); - expect(error.mock.calls.flat().join("\n")).toContain("Repair the provider/workload receipt"); + expect(error.mock.calls.flat().join("\n")).toContain("doctor --json"); + expect(error.mock.calls.flat().join("\n")).toContain("Do not rewrite a receipt"); + }); + + it.each([ + { + label: "unknown provider", + sandbox: { + name: "alpha", + openshellDriver: "future-runtime", + imageTag: "local/alpha:current", + }, + expected: "is not registered for this operation", + }, + { + label: "mismatched legacy workload receipt", + sandbox: { + name: "alpha", + openshellDriver: "docker", + imageTag: "local/alpha:current", + workload: { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "local/alpha:recorded", + shared: false, + }, + }, + expected: "could not prove ownership", + }, + ])("rejects destructive cleanup before side effects for $label", ({ sandbox, expected }) => { + const removeImage = vi.fn(() => ({ status: 0 })); + const runtimeProviders = createRuntimeProviderBundleRegistry([ + ["docker", createDockerRuntimeProviderBundle({ removeImage })], + ]); + + expect(() => + requireSandboxDestructiveCleanupAuthority("alpha", sandbox as any, runtimeProviders), + ).toThrow(expected); + expect(removeImage).not.toHaveBeenCalled(); }); it("preserves registry ownership when workload cleanup authority is unproven", () => { From 062d66baa0663872109a41e8151766cd8aa06828 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 03:42:53 -0700 Subject: [PATCH 037/117] test(snapshot): mock destructive cleanup authority Keep the force-restore fixture aligned with the required cleanup-authority boundary. Its success path now exercises the new guard before deletion. Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/snapshot.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index b273ebe0fef..751011bb8f0 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -244,8 +244,8 @@ vi.mock("./restore-gateway-pairing", () => ({ vi.mock("./destroy", () => ({ cleanupShieldsDestroyArtifacts: lifecycleMock.cleanupShieldsDestroyArtifactsMock, removeSandboxRegistryEntry: vi.fn(), + requireSandboxDestructiveCleanupAuthority: vi.fn(), })); - describe("runSandboxSnapshot", () => { beforeEach(() => { vi.clearAllMocks(); From 75730cf09bf1a1aa901cc3b275052250f8e7d85d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 03:52:42 -0700 Subject: [PATCH 038/117] test(runtime): prove cleanup planning is inert Snapshot provider and sandbox state around cleanup planning. Keep snapshot registry-removal mocks faithful to the production boolean contract. Signed-off-by: Aaron Erickson --- .../sandbox/snapshot-restore-test-fixture.ts | 2 +- .../runtime-provider-contract.test.ts | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts index 54a531760c8..aecedfc75ab 100644 --- a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts +++ b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts @@ -311,7 +311,7 @@ vi.mock("./destroy", async () => { >("../../onboard/runtime-provider/access"); return { cleanupShieldsDestroyArtifacts: lifecycleMock.cleanupShieldsDestroyArtifactsMock, - removeSandboxRegistryEntry: vi.fn(), + removeSandboxRegistryEntry: vi.fn(() => true), requireSandboxDestructiveCleanupAuthority: (sandboxName: string, sandbox: SandboxRecord) => runtimeProviders.requireRuntimeProviderDestructiveCleanupAuthority( sandboxName, diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index ee924c0b786..bdd36b0f09e 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -28,6 +28,7 @@ import { import { registerCreatedSandbox } from "../sandbox-registration"; import type { RuntimeProviderBundle, RuntimeProviderWorkloadProfile } from "./contract"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "./current"; +import { createDockerRuntimeProviderBundle } from "./docker"; import { createRuntimeProviderBundleRegistry, normalizeRuntimeProviderRuntimeReceipt, @@ -350,7 +351,13 @@ describe("RuntimeProviderBundle registry contract", () => { }); it("plans owned workload cleanup without mutating the runtime", () => { - const docker = CURRENT_RUNTIME_PROVIDER_BUNDLES.docker!; + const runtimeState = { imageRemovals: 0 }; + const docker = createDockerRuntimeProviderBundle({ + removeImage: vi.fn(() => { + runtimeState.imageRemovals += 1; + return { status: 0 }; + }), + }); expectSupportedSurface(docker.cleanup); const sandbox = { name: "alpha", @@ -362,10 +369,14 @@ describe("RuntimeProviderBundle registry contract", () => { shared: false, }, } as SandboxEntry; + const sandboxBefore = structuredClone(sandbox); + const runtimeStateBefore = structuredClone(runtimeState); expect(docker.cleanup.planOwnedWorkloadCleanup({ sandbox, sandboxName: sandbox.name })).toEqual( { action: "block", reason: "authority-unproven" }, ); + expect(sandbox).toEqual(sandboxBefore); + expect(runtimeState).toEqual(runtimeStateBefore); }); it("rejects capability/surface drift and duplicate operation-scoped engine identities", () => { From 0b21b79a02eef26b9d091ceeddd9927bbcc730f9 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 03:33:21 -0700 Subject: [PATCH 039/117] feat(rebuild): add atomic managed workload replacement Signed-off-by: Aaron Erickson --- ...aged-workload-rebuild-source-shape.test.ts | 69 ++ ...naged-workload-rebuild-transaction.test.ts | 611 ++++++++++++++++++ .../managed-workload/rebuild/commit.ts | 70 ++ .../managed-workload/rebuild/contract.ts | 161 +++++ .../managed-workload/rebuild/create.ts | 27 + .../onboard/managed-workload/rebuild/index.ts | 5 + .../onboard/managed-workload/rebuild/plan.ts | 103 +++ .../managed-workload/rebuild/prepare.ts | 32 + .../rebuild/provider-rebind.ts | 31 + .../managed-workload/rebuild/readiness.ts | 35 + .../managed-workload/rebuild/restore.ts | 27 + .../managed-workload/rebuild/rollback.ts | 44 ++ .../managed-workload/rebuild/transaction.ts | 143 ++++ .../managed-workload/rebuild/validation.ts | 197 ++++++ .../sandbox-workload-authority.test.ts | 154 +++++ .../onboard/sandbox-workload-rebuild.test.ts | 540 ++++++++++++++++ src/lib/onboard/workload/authority.ts | 183 ++++++ src/lib/onboard/workload/rebuild.ts | 416 ++++++++++++ .../state/registry-rebuild-authority.test.ts | 218 +++++++ src/lib/state/registry/rebuild-authority.ts | 276 ++++++++ 20 files changed, 3342 insertions(+) create mode 100644 src/lib/onboard/managed-workload-rebuild-source-shape.test.ts create mode 100644 src/lib/onboard/managed-workload-rebuild-transaction.test.ts create mode 100644 src/lib/onboard/managed-workload/rebuild/commit.ts create mode 100644 src/lib/onboard/managed-workload/rebuild/contract.ts create mode 100644 src/lib/onboard/managed-workload/rebuild/create.ts create mode 100644 src/lib/onboard/managed-workload/rebuild/index.ts create mode 100644 src/lib/onboard/managed-workload/rebuild/plan.ts create mode 100644 src/lib/onboard/managed-workload/rebuild/prepare.ts create mode 100644 src/lib/onboard/managed-workload/rebuild/provider-rebind.ts create mode 100644 src/lib/onboard/managed-workload/rebuild/readiness.ts create mode 100644 src/lib/onboard/managed-workload/rebuild/restore.ts create mode 100644 src/lib/onboard/managed-workload/rebuild/rollback.ts create mode 100644 src/lib/onboard/managed-workload/rebuild/transaction.ts create mode 100644 src/lib/onboard/managed-workload/rebuild/validation.ts create mode 100644 src/lib/onboard/sandbox-workload-authority.test.ts create mode 100644 src/lib/onboard/sandbox-workload-rebuild.test.ts create mode 100644 src/lib/onboard/workload/authority.ts create mode 100644 src/lib/onboard/workload/rebuild.ts create mode 100644 src/lib/state/registry-rebuild-authority.test.ts create mode 100644 src/lib/state/registry/rebuild-authority.ts diff --git a/src/lib/onboard/managed-workload-rebuild-source-shape.test.ts b/src/lib/onboard/managed-workload-rebuild-source-shape.test.ts new file mode 100644 index 00000000000..1f61332088f --- /dev/null +++ b/src/lib/onboard/managed-workload-rebuild-source-shape.test.ts @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const ROOT = path.resolve(import.meta.dirname, "managed-workload/rebuild"); +const CENTRAL_REBUILD_MODULES = [ + "commit.ts", + "contract.ts", + "create.ts", + "plan.ts", + "prepare.ts", + "provider-rebind.ts", + "readiness.ts", + "restore.ts", + "rollback.ts", + "transaction.ts", + "validation.ts", +] as const; + +function source(file: (typeof CENTRAL_REBUILD_MODULES)[number]): string { + return fs.readFileSync(path.join(ROOT, file), "utf8"); +} + +describe("managed workload rebuild source shape", () => { + it.each( + CENTRAL_REBUILD_MODULES, + )("keeps %s free of provider-specific imports and switches", (file) => { + const text = source(file); + + expect(text).not.toMatch(/from\s+["'][^"']*(?:docker|podman)[^"']*["']/iu); + expect(text).not.toMatch( + /(?:providerId|openshellDriver)\s*(?:===|!==)\s*["'](?:docker|podman)["']/iu, + ); + expect(text).not.toMatch(/switch\s*\(\s*(?:providerId|[^)]*[.]openshellDriver)\s*\)/iu); + }); + + it.each(CENTRAL_REBUILD_MODULES)("keeps %s free of name-only sandbox deletion", (file) => { + const text = source(file); + + expect(text).not.toMatch(/\bremoveSandbox\s*\(/u); + expect(text).not.toMatch(/\bdestroySandbox\s*\(\s*(?:plan[.])?sandboxName/u); + expect(text).not.toMatch(/\bdeleteSandbox\s*\(\s*(?:plan[.])?sandboxName/u); + }); + + it("requires exact provider handles for rollback and retirement", () => { + const contract = source("contract.ts"); + + expect(contract).toContain("previousRuntimeHandle"); + expect(contract).toContain("stagingHandle"); + expect(contract).toContain("retirePrevious("); + expect(contract).toContain("rollback("); + expect(contract).toContain("A sandbox name is\n * intentionally insufficient authority"); + }); + + it("publishes only through the rebuild-authority CAS boundary", () => { + const transaction = source("transaction.ts"); + const commit = source("commit.ts"); + + expect(transaction).not.toContain("registerSandbox"); + expect(transaction).not.toContain("updateSandbox"); + expect(commit).toContain("compareAndSwapSandboxRebuildAuthority"); + expect(commit).not.toContain("registerSandbox"); + expect(commit).not.toContain("updateSandbox"); + }); +}); diff --git a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts new file mode 100644 index 00000000000..cb9c8204032 --- /dev/null +++ b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts @@ -0,0 +1,611 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + captureSandboxRebuildAuthority, + type SandboxRebuildAuthoritySwapResult, + swapSandboxRebuildAuthorityInRegistry, +} from "../state/registry/rebuild-authority"; +import type { SandboxEntry, SandboxRegistry } from "../state/registry/types"; +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_CONTRACT_VERSION, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_SOURCE_REPOSITORY, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + type ManagedImageContractV1, + type ShippedManagedImageAgent, +} from "./managed-image/contract"; +import type { BuiltManagedStartupOnboardProfile } from "./managed-startup/onboard-profile"; +import { encodeManagedStartupProfile } from "./managed-startup/profile"; +import type { + ManagedWorkloadRebuildProviderOperations, + PreparedManagedWorkloadReplacement, + ReadyManagedWorkloadReplacement, + ReboundManagedWorkloadReplacement, + RestoredManagedWorkloadReplacement, + StagedManagedWorkloadReplacement, +} from "./managed-workload/rebuild/contract"; +import { createManagedWorkloadReplacementRollback } from "./managed-workload/rebuild/rollback"; +import { runManagedWorkloadRebuildTransaction } from "./managed-workload/rebuild/transaction"; +import type { RuntimeProviderBundle } from "./runtime-provider/contract"; +import { RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION } from "./runtime-provider/contract"; +import { createRuntimeProviderBundleRegistry } from "./runtime-provider/registry"; +import type { ManagedWorkloadRebuildHandoff, ManagedWorkloadReceipt } from "./workload/rebuild"; + +const AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; +const PROVIDERS = ["docker", "mxc"] as const; +const PLATFORMS = ["linux/amd64", "linux/arm64"] as const; +const OLD_RELEASE = "v0.0.99"; +const NEW_RELEASE = "v0.0.100"; + +function contract( + agent: ShippedManagedImageAgent, + generation: "old" | "new", + platform: (typeof PLATFORMS)[number] = "linux/amd64", +): ManagedImageContractV1 { + const image = MANAGED_IMAGE_REPOSITORIES[agent]; + const digit = generation === "old" ? "a" : "b"; + const digest = `sha256:${digit.repeat(64)}` as const; + return { + contractVersion: MANAGED_IMAGE_CONTRACT_VERSION, + agent, + platform, + image, + digest, + reference: `${image}@${digest}`, + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: digit.repeat(40), + release: generation === "old" ? OLD_RELEASE : NEW_RELEASE, + cohort: generation === "old" ? "ghrun-100-1" : "ghrun-200-2", + }, + startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + }; +} + +function profileTransport(agent: ShippedManagedImageAgent): BuiltManagedStartupOnboardProfile { + const profile = managedStartupE2eProfile(agent); + const encodedProfile = encodeManagedStartupProfile(profile); + return { + profile, + encodedProfile: encodedProfile as BuiltManagedStartupOnboardProfile["encodedProfile"], + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + }; +} + +function workloadReceipt( + agent: ShippedManagedImageAgent, + generation: "old" | "new", + platform: (typeof PLATFORMS)[number] = "linux/amd64", +): ManagedWorkloadReceipt { + const image = contract(agent, generation, platform); + const transport = profileTransport(agent); + return { + schemaVersion: 1, + kind: "managed-image", + reference: image.reference, + platform: image.platform, + release: image.source.release, + sourceRevision: image.source.revision, + sourceCohort: image.source.cohort, + capabilityContractVersion: image.capabilityContractVersion, + startupProfileContractVersion: image.startupProfileContractVersion, + encodedProfile: transport.encodedProfile, + startupProfileSha256: transport.startupProfileSha256, + credentialProxyReplayRequired: false, + shared: true, + }; +} + +function previousEntry( + agent: ShippedManagedImageAgent, + providerId: string, + platform: (typeof PLATFORMS)[number] = "linux/amd64", +): SandboxEntry { + const workload = workloadReceipt(agent, "old", platform); + return { + name: `rebuild-${agent}`, + agent, + openshellDriver: providerId, + provider: "ollama-local", + model: "nvidia/nemotron", + imageTag: workload.reference, + workload, + lifecycleGeneration: "generation-old", + lifecycleLiveIdentityFingerprint: "fingerprint-old", + gatewayName: "nemoclaw", + gatewayPort: 8080, + }; +} + +function handoff( + agent: ShippedManagedImageAgent, + providerId: string, + platform: (typeof PLATFORMS)[number] = "linux/amd64", +): ManagedWorkloadRebuildHandoff { + const previousContract = contract(agent, "old", platform); + const replacementContract = contract(agent, "new", platform); + const previousProfile = managedStartupE2eProfile(agent); + return { + schemaVersion: 1, + providerId, + agent, + previousReceipt: workloadReceipt(agent, "old", platform), + previousContract, + previousProfile, + replacement: { + source: { + kind: "managed-image", + reference: replacementContract.reference, + contract: replacementContract, + }, + release: NEW_RELEASE, + fallbackDiagnostic: null, + }, + corporateCa: null, + replacementProfile: profileTransport(agent), + }; +} + +function unsupported(providerId: string) { + return { + providerId, + supported: false as const, + reason: "not used by the rebuild transaction contract test", + }; +} + +function bundle(providerId: string): RuntimeProviderBundle { + const candidate: RuntimeProviderBundle = { + identity: { + contractVersion: RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, + id: providerId, + displayName: `In-memory ${providerId}`, + }, + plan: { providerId, supported: true, gatewayLauncher: "nemoclaw" }, + capabilities: { + providerId, + supported: true, + hostLocalInference: true, + directLifecycle: false, + legacyGatewayContainerInspection: false, + workloadImageCleanup: false, + }, + preflightDoctor: { + providerId, + supported: true, + inspectHost: () => ({ + group: "Host", + label: `${providerId} in-memory provider`, + status: "ok", + detail: "socket-free", + }), + preflightLifecycle: () => null, + }, + gateway: { + providerId, + supported: true, + launcher: "nemoclaw", + inspectLegacyContainer: false, + }, + workload: { + providerId, + supported: true, + profile: { + support: { + exactDigestReferences: true, + platforms: ["linux/amd64", "linux/arm64"], + startupProfileContractVersions: [1], + capabilityContractVersions: [1], + }, + hostArchitectures: ["amd64", "arm64"], + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + }, + acceptsReceipt: (receipt) => receipt?.kind === "managed-image", + }, + lifecycle: unsupported(providerId), + mutationAuthority: { + providerId, + supported: true, + operations: ["rebuild"], + }, + bootstrap: unsupported(providerId), + snapshot: unsupported(providerId), + recovery: unsupported(providerId), + cleanup: unsupported(providerId), + containerEngine: unsupported(providerId), + }; + return createRuntimeProviderBundleRegistry([[providerId, candidate]])[providerId]!; +} + +type FailurePhase = + | "prepare" + | "create" + | "readiness" + | "restore" + | "provider-rebind" + | "registry-commit" + | "registry-commit-after-persist" + | "retire-previous" + | "abort-preparation" + | null; + +function operationsHarness( + providerId: string, + events: string[], + failAt: FailurePhase = null, +): ManagedWorkloadRebuildProviderOperations { + const bound = { + schemaVersion: 1 as const, + providerId, + transactionId: "transaction-1", + }; + const prepared: PreparedManagedWorkloadReplacement = { + ...bound, + previousRuntimeHandle: "runtime-old-exact", + preparationHandle: "preparation-exact", + previousLiveIdentityFingerprint: "fingerprint-old", + }; + const staged: StagedManagedWorkloadReplacement = { + ...bound, + previousRuntimeHandle: prepared.previousRuntimeHandle, + stagingHandle: "runtime-new-staged-exact", + lifecycleGeneration: "generation-new", + liveIdentityFingerprint: "fingerprint-new", + }; + const ready: ReadyManagedWorkloadReplacement = { + ...staged, + readinessReceipt: "ready-exact", + }; + const restored: RestoredManagedWorkloadReplacement = { + ...ready, + restoreReceipt: "restore-exact", + }; + const rebound: ReboundManagedWorkloadReplacement = { + ...restored, + providerRebindReceipt: "provider-rebind-exact", + }; + const result = (phase: Exclude, value: T): Promise => + failAt === phase + ? Promise.reject(new Error(`${phase} injected failure`)) + : Promise.resolve(value); + + return { + providerId, + prepare: vi.fn(async () => { + events.push("prepare"); + return result("prepare", prepared); + }), + create: vi.fn(async () => { + events.push("create"); + return result("create", staged); + }), + abortPreparation: vi.fn(async (plan) => { + events.push(`abort-preparation:${plan.transactionId}`); + return result("abort-preparation", undefined); + }), + waitUntilReady: vi.fn(async () => { + events.push("readiness"); + return failAt === "readiness" + ? { state: "not-ready" as const, reason: "replacement is not Ready" } + : { state: "ready" as const, replacement: ready }; + }), + restoreState: vi.fn(async () => { + events.push("restore"); + return result("restore", restored); + }), + rebindProviders: vi.fn(async () => { + events.push("provider-rebind"); + return result("provider-rebind", rebound); + }), + rollback: vi.fn(async (_plan, value) => { + events.push(`rollback:${value.stagingHandle}`); + }), + retirePrevious: vi.fn(async (_plan, value) => { + events.push(`retire:${value.previousRuntimeHandle}`); + return result("retire-previous", undefined); + }), + }; +} + +function transactionHarness( + agent: ShippedManagedImageAgent, + providerId: string, + failAt: FailurePhase = null, + platform: (typeof PLATFORMS)[number] = "linux/amd64", +) { + const events: string[] = []; + const oldEntry = previousEntry(agent, providerId, platform); + let currentEntry = structuredClone(oldEntry); + const operations = operationsHarness(providerId, events, failAt); + const commitAuthority = ( + expected: ReturnType, + replacement: SandboxEntry, + ): SandboxRebuildAuthoritySwapResult => { + events.push("registry-commit"); + const currentRegistry: SandboxRegistry = { + sandboxes: { [oldEntry.name]: currentEntry }, + defaultSandbox: oldEntry.name, + }; + const swapped = + failAt === "registry-commit" + ? { + registry: currentRegistry, + result: { + status: "stale-authority" as const, + entry: structuredClone(currentEntry), + }, + } + : swapSandboxRebuildAuthorityInRegistry(currentRegistry, expected, replacement); + currentEntry = + swapped.result.status === "committed" ? structuredClone(swapped.result.entry) : currentEntry; + if (failAt === "registry-commit-after-persist") { + throw new Error("registry acknowledgement lost after persistence"); + } + return swapped.result; + }; + return { + events, + oldEntry, + operations, + currentEntry: () => currentEntry, + run: () => + runManagedWorkloadRebuildTransaction( + { + previousEntry: oldEntry, + provider: bundle(providerId), + handoff: handoff(agent, providerId, platform), + operations, + replacementMetadata: { model: "nvidia/nemotron-new" }, + transactionId: "transaction-1", + }, + { + getSandbox: () => structuredClone(currentEntry), + commitAuthority, + }, + ), + }; +} + +describe("managed workload rebuild transaction", () => { + it.each( + AGENTS.flatMap((agent) => + PROVIDERS.flatMap((provider) => + PLATFORMS.map((platform) => [agent, provider, platform] as const), + ), + ), + )("atomically rebuilds %s through the socket-free %s provider contract on %s", async (agent, provider, platform) => { + const harness = transactionHarness(agent, provider, null, platform); + + const result = await harness.run(); + + expect(result).toMatchObject({ + status: "committed", + previousCleanup: "complete", + entry: { + agent, + openshellDriver: provider, + model: "nvidia/nemotron-new", + fromDockerfile: null, + lifecycleGeneration: "generation-new", + lifecycleLiveIdentityFingerprint: "fingerprint-new", + workload: { + kind: "managed-image", + platform, + release: NEW_RELEASE, + shared: true, + }, + }, + }); + expect(harness.events).toEqual([ + "prepare", + "create", + "readiness", + "restore", + "provider-rebind", + "registry-commit", + "retire:runtime-old-exact", + ]); + expect(harness.currentEntry()).toEqual(result.entry); + expect(harness.currentEntry().imageTag).not.toBe(harness.oldEntry.imageTag); + }); + + it.each([ + ["prepare", false], + ["create", false], + ["readiness", true], + ["restore", true], + ["provider-rebind", true], + ["registry-commit", true], + ] as const)("keeps old authority when %s fails", async (phase, expectsRollback) => { + const harness = transactionHarness("openclaw", "mxc", phase); + + await expect(harness.run()).rejects.toMatchObject({ phase }); + + expect(harness.currentEntry()).toEqual(harness.oldEntry); + expect(harness.events.some((event) => event === "retire:runtime-old-exact")).toBe(false); + expect(harness.events.some((event) => event === "rollback:runtime-new-staged-exact")).toBe( + expectsRollback, + ); + expect(harness.events.some((event) => event === "abort-preparation:transaction-1")).toBe( + phase === "prepare" || phase === "create", + ); + }); + + it("rolls back a not-ready replacement by exact staged handle", async () => { + const harness = transactionHarness("hermes", "docker", "readiness"); + + await expect(harness.run()).rejects.toMatchObject({ + phase: "readiness", + message: expect.stringContaining("replacement is not Ready"), + }); + expect(harness.events).toEqual([ + "prepare", + "create", + "readiness", + "rollback:runtime-new-staged-exact", + ]); + expect(harness.currentEntry().lifecycleGeneration).toBe("generation-old"); + }); + + it("aborts preparation when non-authority registry metadata drifts", async () => { + const events: string[] = []; + const oldEntry = previousEntry("openclaw", "mxc"); + let currentEntry = structuredClone(oldEntry); + const operations = operationsHarness("mxc", events); + const prepare = operations.prepare; + operations.prepare = vi.fn(async (plan) => { + const prepared = await prepare(plan); + currentEntry = { + ...currentEntry, + model: "concurrently-updated", + }; + return prepared; + }); + + await expect( + runManagedWorkloadRebuildTransaction( + { + previousEntry: oldEntry, + provider: bundle("mxc"), + handoff: handoff("openclaw", "mxc"), + operations, + transactionId: "transaction-1", + }, + { getSandbox: () => structuredClone(currentEntry) }, + ), + ).rejects.toMatchObject({ phase: "prepare" }); + expect(events).toEqual(["prepare", "abort-preparation:transaction-1"]); + expect(operations.create).not.toHaveBeenCalled(); + expect(operations.rollback).not.toHaveBeenCalled(); + }); + + it("reports exact-old cleanup as pending without undoing a committed replacement", async () => { + const harness = transactionHarness("langchain-deepagents-code", "mxc", "retire-previous"); + + const result = await harness.run(); + + expect(result).toMatchObject({ + status: "committed", + previousCleanup: "pending", + cleanupError: expect.any(Error), + }); + expect(harness.currentEntry().lifecycleGeneration).toBe("generation-new"); + expect(harness.events.at(-1)).toBe("retire:runtime-old-exact"); + expect(harness.operations.rollback).not.toHaveBeenCalled(); + }); + + it("reconciles a persisted replacement when the CAS acknowledgement throws", async () => { + const harness = transactionHarness( + "openclaw", + "mxc", + "registry-commit-after-persist", + "linux/arm64", + ); + + const result = await harness.run(); + + expect(result).toMatchObject({ + status: "committed", + entry: { + lifecycleGeneration: "generation-new", + workload: { platform: "linux/arm64" }, + }, + }); + expect(harness.currentEntry()).toEqual(result.entry); + expect(harness.operations.rollback).not.toHaveBeenCalled(); + expect(harness.events.at(-1)).toBe("retire:runtime-old-exact"); + }); + + it("coalesces repeated rollback calls onto one exact-handle operation", async () => { + const events: string[] = []; + const providerOperations = operationsHarness("mxc", events); + const plan = { + schemaVersion: 1 as const, + transactionId: "transaction-1", + sandboxName: "alpha", + providerId: "mxc", + agent: "openclaw" as const, + previousAuthority: captureSandboxRebuildAuthority(previousEntry("openclaw", "mxc"), "mxc"), + handoff: handoff("openclaw", "mxc"), + replacementReceipt: workloadReceipt("openclaw", "new"), + replacementMetadata: {}, + }; + const staged: StagedManagedWorkloadReplacement = { + schemaVersion: 1, + providerId: "mxc", + transactionId: "transaction-1", + previousRuntimeHandle: "runtime-old-exact", + stagingHandle: "runtime-new-staged-exact", + lifecycleGeneration: "generation-new", + liveIdentityFingerprint: "fingerprint-new", + }; + const rollback = createManagedWorkloadReplacementRollback(plan, staged, providerOperations); + + await Promise.all([rollback.run(), rollback.run(), rollback.run()]); + + expect(providerOperations.rollback).toHaveBeenCalledOnce(); + expect(events).toEqual(["rollback:runtime-new-staged-exact"]); + }); + + it("rejects a cross-platform handoff before provider mutation", async () => { + const oldEntry = previousEntry("openclaw", "mxc"); + const wrongPlatformHandoff = handoff("openclaw", "mxc"); + const replacementContract = { + ...wrongPlatformHandoff.replacement.source.contract, + platform: "linux/arm64" as const, + }; + const operations = operationsHarness("mxc", []); + + await expect( + runManagedWorkloadRebuildTransaction( + { + previousEntry: oldEntry, + provider: bundle("mxc"), + handoff: { + ...wrongPlatformHandoff, + replacement: { + ...wrongPlatformHandoff.replacement, + source: { + ...wrongPlatformHandoff.replacement.source, + contract: replacementContract, + }, + }, + }, + operations, + transactionId: "transaction-1", + }, + { getSandbox: () => structuredClone(oldEntry) }, + ), + ).rejects.toThrow(); + expect(operations.prepare).not.toHaveBeenCalled(); + }); + + it("rejects a provider adapter that is not bound to the selected bundle", async () => { + const oldEntry = previousEntry("openclaw", "mxc"); + const operations = operationsHarness("other-provider", []); + + await expect( + runManagedWorkloadRebuildTransaction( + { + previousEntry: oldEntry, + provider: bundle("mxc"), + handoff: handoff("openclaw", "mxc"), + operations, + transactionId: "transaction-1", + }, + { getSandbox: () => structuredClone(oldEntry) }, + ), + ).rejects.toMatchObject({ phase: "prepare" }); + expect(operations.create).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/managed-workload/rebuild/commit.ts b/src/lib/onboard/managed-workload/rebuild/commit.ts new file mode 100644 index 00000000000..e837930fcaa --- /dev/null +++ b/src/lib/onboard/managed-workload/rebuild/commit.ts @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + compareAndSwapSandboxRebuildAuthority, + type SandboxRebuildAuthoritySwapResult, + sandboxRebuildReplacementMatchesEntry, +} from "../../../state/registry/rebuild-authority"; +import type { SandboxEntry } from "../../../state/registry/types"; +import type { ManagedWorkloadRebuildPlan, ReboundManagedWorkloadReplacement } from "./contract"; +import { ManagedWorkloadRebuildTransactionError } from "./contract"; + +export type CommitSandboxRebuildAuthority = ( + expected: ManagedWorkloadRebuildPlan["previousAuthority"], + replacement: SandboxEntry, +) => SandboxRebuildAuthoritySwapResult; + +export type ReadSandboxRebuildEntry = (sandboxName: string) => SandboxEntry | null; + +export function materializeManagedWorkloadReplacementEntry( + previousEntry: SandboxEntry, + plan: ManagedWorkloadRebuildPlan, + replacement: ReboundManagedWorkloadReplacement, +): SandboxEntry { + return structuredClone({ + ...previousEntry, + ...plan.replacementMetadata, + name: plan.sandboxName, + pendingRouteReservation: undefined, + reservationSessionId: undefined, + openshellDriver: plan.providerId, + agent: plan.agent, + fromDockerfile: null, + imageTag: plan.replacementReceipt.reference, + workload: plan.replacementReceipt, + lifecycleGeneration: replacement.lifecycleGeneration, + lifecycleLiveIdentityFingerprint: replacement.liveIdentityFingerprint, + }); +} + +export function commitManagedWorkloadReplacement( + previousEntry: SandboxEntry, + plan: ManagedWorkloadRebuildPlan, + replacement: ReboundManagedWorkloadReplacement, + commit: CommitSandboxRebuildAuthority = compareAndSwapSandboxRebuildAuthority, + readSandbox?: ReadSandboxRebuildEntry, +): SandboxEntry { + const candidate = materializeManagedWorkloadReplacementEntry(previousEntry, plan, replacement); + let result: SandboxRebuildAuthoritySwapResult; + try { + result = commit(plan.previousAuthority, candidate); + } catch (error) { + const observed = readSandbox?.(plan.sandboxName) ?? null; + if (observed && sandboxRebuildReplacementMatchesEntry(candidate, observed)) { + return structuredClone(observed); + } + throw new ManagedWorkloadRebuildTransactionError( + "registry-commit", + "the replacement registry entry could not be committed", + { cause: error }, + ); + } + if (result.status !== "committed") { + throw new ManagedWorkloadRebuildTransactionError( + "registry-commit", + "the old workload no longer owns the exact durable authority", + ); + } + return result.entry; +} diff --git a/src/lib/onboard/managed-workload/rebuild/contract.ts b/src/lib/onboard/managed-workload/rebuild/contract.ts new file mode 100644 index 00000000000..bf1307e8a3a --- /dev/null +++ b/src/lib/onboard/managed-workload/rebuild/contract.ts @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxRebuildAuthority } from "../../../state/registry/rebuild-authority"; +import type { SandboxEntry } from "../../../state/registry/types"; +import type { ShippedManagedImageAgent } from "../../managed-image/contract"; +import type { ManagedWorkloadRebuildHandoff, ManagedWorkloadReceipt } from "../../workload/rebuild"; + +export type ManagedWorkloadRebuildPhase = + | "prepare" + | "create" + | "readiness" + | "restore" + | "provider-rebind" + | "registry-commit" + | "retire-previous" + | "rollback"; + +export interface ManagedWorkloadRebuildPlan { + readonly schemaVersion: 1; + readonly transactionId: string; + readonly sandboxName: string; + readonly providerId: string; + readonly agent: ShippedManagedImageAgent; + readonly previousAuthority: SandboxRebuildAuthority; + readonly handoff: ManagedWorkloadRebuildHandoff; + readonly replacementReceipt: ManagedWorkloadReceipt; + /** + * Prevalidated mutable metadata. Authority fields are stripped before this + * object is retained and cannot override the transaction's exact identities. + */ + readonly replacementMetadata: Readonly>; +} + +interface ProviderBoundRebuildArtifact { + readonly schemaVersion: 1; + readonly providerId: string; + readonly transactionId: string; +} + +export interface PreparedManagedWorkloadReplacement extends ProviderBoundRebuildArtifact { + /** Exact provider-owned handle for the workload that remains authoritative. */ + readonly previousRuntimeHandle: string; + /** Provider-private preparation handle; central orchestration never parses it. */ + readonly preparationHandle: string; + readonly previousLiveIdentityFingerprint: string; +} + +export interface StagedManagedWorkloadReplacement extends ProviderBoundRebuildArtifact { + readonly previousRuntimeHandle: string; + /** Exact provider-owned handle used for readiness, rollback, and cutover. */ + readonly stagingHandle: string; + readonly lifecycleGeneration: string; + readonly liveIdentityFingerprint: string; +} + +export interface ReadyManagedWorkloadReplacement extends StagedManagedWorkloadReplacement { + readonly readinessReceipt: string; +} + +export interface RestoredManagedWorkloadReplacement extends ReadyManagedWorkloadReplacement { + readonly restoreReceipt: string; +} + +export interface ReboundManagedWorkloadReplacement extends RestoredManagedWorkloadReplacement { + readonly providerRebindReceipt: string; +} + +export type ManagedWorkloadReadinessResult = + | { + readonly state: "ready"; + readonly replacement: ReadyManagedWorkloadReplacement; + } + | { + readonly state: "not-ready"; + readonly reason: string; + }; + +/** + * One operation-scoped adapter supplied by the selected provider composition. + * It is identity-bound to RuntimeProviderBundle but is not independently + * registered or selected. The old workload can be retired only after CAS. + */ +export interface ManagedWorkloadRebuildProviderOperations { + readonly providerId: string; + /** + * Prepare must be transaction-idempotent. On rejection, any partial + * provider allocation remains owned by plan.transactionId so + * abortPreparation can remove it exactly. + */ + prepare(plan: ManagedWorkloadRebuildPlan): Promise; + /** + * Create consumes the preparation only after returning a valid staged + * artifact. On rejection or an invalid artifact, abortPreparation must + * remove both preparation state and any partial create allocation associated + * with this exact transaction ID. + */ + create( + plan: ManagedWorkloadRebuildPlan, + prepared: PreparedManagedWorkloadReplacement, + ): Promise; + /** + * Abort provider resources owned by plan.transactionId before a valid + * staging handle exists. Must never select by sandbox name and must be safe + * to retry after prepare/create ambiguity. + */ + abortPreparation(plan: ManagedWorkloadRebuildPlan): Promise; + waitUntilReady( + plan: ManagedWorkloadRebuildPlan, + staged: StagedManagedWorkloadReplacement, + ): Promise; + restoreState( + plan: ManagedWorkloadRebuildPlan, + ready: ReadyManagedWorkloadReplacement, + ): Promise; + rebindProviders( + plan: ManagedWorkloadRebuildPlan, + restored: RestoredManagedWorkloadReplacement, + ): Promise; + /** + * Must target stagingHandle exactly and be safe to retry. A sandbox name is + * intentionally insufficient authority for this operation. + */ + rollback( + plan: ManagedWorkloadRebuildPlan, + staged: StagedManagedWorkloadReplacement, + ): Promise; + /** + * Called only after the replacement row wins CAS. Must target the exact + * previousRuntimeHandle captured during prepare, never sandboxName alone. + */ + retirePrevious( + plan: ManagedWorkloadRebuildPlan, + replacement: ReboundManagedWorkloadReplacement, + ): Promise; +} + +export type ManagedWorkloadRebuildTransactionResult = { + readonly status: "committed"; + readonly entry: SandboxEntry; + readonly previousCleanup: "complete" | "pending"; + readonly cleanupError?: unknown; +}; + +export class ManagedWorkloadRebuildTransactionError extends Error { + readonly phase: ManagedWorkloadRebuildPhase; + readonly rollbackError: unknown; + + constructor( + phase: ManagedWorkloadRebuildPhase, + message: string, + options: { readonly cause?: unknown; readonly rollbackError?: unknown } = {}, + ) { + super(`Managed workload rebuild ${phase} failed: ${message}`, { + ...(options.cause === undefined ? {} : { cause: options.cause }), + }); + this.name = "ManagedWorkloadRebuildTransactionError"; + this.phase = phase; + this.rollbackError = options.rollbackError; + } +} diff --git a/src/lib/onboard/managed-workload/rebuild/create.ts b/src/lib/onboard/managed-workload/rebuild/create.ts new file mode 100644 index 00000000000..b9c67a9ddd3 --- /dev/null +++ b/src/lib/onboard/managed-workload/rebuild/create.ts @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + ManagedWorkloadRebuildPlan, + ManagedWorkloadRebuildProviderOperations, + PreparedManagedWorkloadReplacement, + StagedManagedWorkloadReplacement, +} from "./contract"; +import { ManagedWorkloadRebuildTransactionError } from "./contract"; +import { validateStagedReplacement } from "./validation"; + +export async function createStagedManagedWorkloadReplacement( + plan: ManagedWorkloadRebuildPlan, + prepared: PreparedManagedWorkloadReplacement, + operations: ManagedWorkloadRebuildProviderOperations, +): Promise { + try { + return validateStagedReplacement(plan, prepared, await operations.create(plan, prepared)); + } catch (error) { + throw new ManagedWorkloadRebuildTransactionError( + "create", + "the provider could not create an identity-bound staged replacement", + { cause: error }, + ); + } +} diff --git a/src/lib/onboard/managed-workload/rebuild/index.ts b/src/lib/onboard/managed-workload/rebuild/index.ts new file mode 100644 index 00000000000..32bc91a27b8 --- /dev/null +++ b/src/lib/onboard/managed-workload/rebuild/index.ts @@ -0,0 +1,5 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export * from "./contract"; +export * from "./transaction"; diff --git a/src/lib/onboard/managed-workload/rebuild/plan.ts b/src/lib/onboard/managed-workload/rebuild/plan.ts new file mode 100644 index 00000000000..99a35311fc5 --- /dev/null +++ b/src/lib/onboard/managed-workload/rebuild/plan.ts @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { randomUUID } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; +import { captureSandboxRebuildAuthority } from "../../../state/registry/rebuild-authority"; +import type { SandboxEntry } from "../../../state/registry/types"; +import type { RuntimeProviderBundle } from "../../runtime-provider/contract"; +import { + normalizeRuntimeProviderIdentity, + requireRuntimeProviderMutationAuthority, +} from "../../runtime-provider/registry"; +import { + buildManagedWorkloadRebuildReceipt, + type ManagedWorkloadRebuildHandoff, +} from "../../workload/rebuild"; +import type { ManagedWorkloadRebuildPlan } from "./contract"; +import { ManagedWorkloadRebuildTransactionError } from "./contract"; + +const PROTECTED_REBUILD_METADATA_FIELDS = new Set([ + "name", + "pendingRouteReservation", + "reservationSessionId", + "openshellDriver", + "fromDockerfile", + "imageTag", + "workload", + "lifecycleGeneration", + "lifecycleLiveIdentityFingerprint", +]); + +function safeReplacementMetadata( + metadata: Readonly> | undefined, +): Readonly> { + const source = metadata ?? {}; + const safe = Object.fromEntries( + Object.entries(structuredClone(source)).filter( + ([field]) => !PROTECTED_REBUILD_METADATA_FIELDS.has(field as keyof SandboxEntry), + ), + ) as Partial; + return Object.freeze(safe); +} + +export function createManagedWorkloadRebuildPlan(input: { + readonly previousEntry: SandboxEntry; + readonly provider: RuntimeProviderBundle; + readonly handoff: ManagedWorkloadRebuildHandoff; + readonly replacementMetadata?: Readonly>; + readonly transactionId?: string; +}): ManagedWorkloadRebuildPlan { + const providerId = input.provider.identity.id; + if (normalizeRuntimeProviderIdentity(input.previousEntry.openshellDriver) !== providerId) { + throw new ManagedWorkloadRebuildTransactionError( + "prepare", + "the durable sandbox driver does not select the supplied provider bundle", + ); + } + requireRuntimeProviderMutationAuthority(input.provider, "rebuild"); + if ( + input.handoff.providerId !== providerId || + input.handoff.agent !== input.previousEntry.agent + ) { + throw new ManagedWorkloadRebuildTransactionError( + "prepare", + "the managed profile handoff does not match durable provider and agent authority", + ); + } + if (!isDeepStrictEqual(input.handoff.previousReceipt, input.previousEntry.workload)) { + throw new ManagedWorkloadRebuildTransactionError( + "prepare", + "the managed profile handoff is stale against the durable workload receipt", + ); + } + if ( + input.handoff.previousReceipt.platform !== input.handoff.previousContract.platform || + input.handoff.replacement.source.contract.platform !== input.handoff.previousContract.platform + ) { + throw new ManagedWorkloadRebuildTransactionError( + "prepare", + "the managed rebuild handoff contains cross-platform authority drift", + ); + } + const previousAuthority = captureSandboxRebuildAuthority(input.previousEntry, providerId); + const replacementReceipt = buildManagedWorkloadRebuildReceipt(input.handoff, input.provider); + const transactionId = input.transactionId ?? randomUUID(); + if (!/^[0-9A-Za-z][0-9A-Za-z._:-]{0,255}$/u.test(transactionId)) { + throw new ManagedWorkloadRebuildTransactionError( + "prepare", + "the rebuild transaction identity is invalid", + ); + } + return Object.freeze({ + schemaVersion: 1, + transactionId, + sandboxName: input.previousEntry.name, + providerId, + agent: input.handoff.agent, + previousAuthority, + handoff: input.handoff, + replacementReceipt, + replacementMetadata: safeReplacementMetadata(input.replacementMetadata), + }); +} diff --git a/src/lib/onboard/managed-workload/rebuild/prepare.ts b/src/lib/onboard/managed-workload/rebuild/prepare.ts new file mode 100644 index 00000000000..d532a9d82a3 --- /dev/null +++ b/src/lib/onboard/managed-workload/rebuild/prepare.ts @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + ManagedWorkloadRebuildPlan, + ManagedWorkloadRebuildProviderOperations, + PreparedManagedWorkloadReplacement, +} from "./contract"; +import { ManagedWorkloadRebuildTransactionError } from "./contract"; +import { validatePreparedReplacement } from "./validation"; + +export async function prepareManagedWorkloadReplacement( + plan: ManagedWorkloadRebuildPlan, + operations: ManagedWorkloadRebuildProviderOperations, +): Promise { + if (operations.providerId !== plan.providerId) { + throw new ManagedWorkloadRebuildTransactionError( + "prepare", + `operation provider '${operations.providerId}' does not match '${plan.providerId}'`, + ); + } + try { + return validatePreparedReplacement(plan, await operations.prepare(plan)); + } catch (error) { + if (error instanceof ManagedWorkloadRebuildTransactionError) throw error; + throw new ManagedWorkloadRebuildTransactionError( + "prepare", + "the provider did not produce exact old-workload authority", + { cause: error }, + ); + } +} diff --git a/src/lib/onboard/managed-workload/rebuild/provider-rebind.ts b/src/lib/onboard/managed-workload/rebuild/provider-rebind.ts new file mode 100644 index 00000000000..9b5d538818b --- /dev/null +++ b/src/lib/onboard/managed-workload/rebuild/provider-rebind.ts @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + ManagedWorkloadRebuildPlan, + ManagedWorkloadRebuildProviderOperations, + ReboundManagedWorkloadReplacement, + RestoredManagedWorkloadReplacement, +} from "./contract"; +import { ManagedWorkloadRebuildTransactionError } from "./contract"; +import { validateReboundReplacement } from "./validation"; + +export async function rebindStagedManagedWorkloadProviders( + plan: ManagedWorkloadRebuildPlan, + restored: RestoredManagedWorkloadReplacement, + operations: ManagedWorkloadRebuildProviderOperations, +): Promise { + try { + return validateReboundReplacement( + plan, + restored, + await operations.rebindProviders(plan, restored), + ); + } catch (error) { + throw new ManagedWorkloadRebuildTransactionError( + "provider-rebind", + "provider rebind did not preserve staged replacement authority", + { cause: error }, + ); + } +} diff --git a/src/lib/onboard/managed-workload/rebuild/readiness.ts b/src/lib/onboard/managed-workload/rebuild/readiness.ts new file mode 100644 index 00000000000..460d04bbf41 --- /dev/null +++ b/src/lib/onboard/managed-workload/rebuild/readiness.ts @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + ManagedWorkloadRebuildPlan, + ManagedWorkloadRebuildProviderOperations, + ReadyManagedWorkloadReplacement, + StagedManagedWorkloadReplacement, +} from "./contract"; +import { ManagedWorkloadRebuildTransactionError } from "./contract"; +import { validateNotReadyReason, validateReadyReplacement } from "./validation"; + +export async function requireReadyManagedWorkloadReplacement( + plan: ManagedWorkloadRebuildPlan, + staged: StagedManagedWorkloadReplacement, + operations: ManagedWorkloadRebuildProviderOperations, +): Promise { + try { + const readiness = await operations.waitUntilReady(plan, staged); + if (readiness.state === "not-ready") { + throw new ManagedWorkloadRebuildTransactionError( + "readiness", + validateNotReadyReason(readiness.reason), + ); + } + return validateReadyReplacement(plan, staged, readiness.replacement); + } catch (error) { + if (error instanceof ManagedWorkloadRebuildTransactionError) throw error; + throw new ManagedWorkloadRebuildTransactionError( + "readiness", + "the staged replacement did not prove readiness", + { cause: error }, + ); + } +} diff --git a/src/lib/onboard/managed-workload/rebuild/restore.ts b/src/lib/onboard/managed-workload/rebuild/restore.ts new file mode 100644 index 00000000000..db5de7b0be9 --- /dev/null +++ b/src/lib/onboard/managed-workload/rebuild/restore.ts @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + ManagedWorkloadRebuildPlan, + ManagedWorkloadRebuildProviderOperations, + ReadyManagedWorkloadReplacement, + RestoredManagedWorkloadReplacement, +} from "./contract"; +import { ManagedWorkloadRebuildTransactionError } from "./contract"; +import { validateRestoredReplacement } from "./validation"; + +export async function restoreStagedManagedWorkloadState( + plan: ManagedWorkloadRebuildPlan, + ready: ReadyManagedWorkloadReplacement, + operations: ManagedWorkloadRebuildProviderOperations, +): Promise { + try { + return validateRestoredReplacement(plan, ready, await operations.restoreState(plan, ready)); + } catch (error) { + throw new ManagedWorkloadRebuildTransactionError( + "restore", + "state restore did not produce an identity-bound receipt", + { cause: error }, + ); + } +} diff --git a/src/lib/onboard/managed-workload/rebuild/rollback.ts b/src/lib/onboard/managed-workload/rebuild/rollback.ts new file mode 100644 index 00000000000..0dc0cc743bd --- /dev/null +++ b/src/lib/onboard/managed-workload/rebuild/rollback.ts @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + ManagedWorkloadRebuildPlan, + ManagedWorkloadRebuildProviderOperations, + StagedManagedWorkloadReplacement, +} from "./contract"; + +export interface ManagedWorkloadReplacementRollback { + run(): Promise; +} + +export function createManagedWorkloadPreparationAbort( + plan: ManagedWorkloadRebuildPlan, + operations: ManagedWorkloadRebuildProviderOperations, +): ManagedWorkloadReplacementRollback { + let abort: Promise | null = null; + return { + run(): Promise { + abort ??= operations.abortPreparation(plan); + return abort; + }, + }; +} + +/** + * Collapse concurrent or repeated rollback attempts onto one exact-handle + * provider call. The provider contract remains retry-safe for crash recovery; + * one live transaction never races duplicate cleanup calls. + */ +export function createManagedWorkloadReplacementRollback( + plan: ManagedWorkloadRebuildPlan, + staged: StagedManagedWorkloadReplacement, + operations: ManagedWorkloadRebuildProviderOperations, +): ManagedWorkloadReplacementRollback { + let rollback: Promise | null = null; + return { + run(): Promise { + rollback ??= operations.rollback(plan, staged); + return rollback; + }, + }; +} diff --git a/src/lib/onboard/managed-workload/rebuild/transaction.ts b/src/lib/onboard/managed-workload/rebuild/transaction.ts new file mode 100644 index 00000000000..be88fccdd9d --- /dev/null +++ b/src/lib/onboard/managed-workload/rebuild/transaction.ts @@ -0,0 +1,143 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { load } from "../../../state/registry/persistence"; +import { sandboxRebuildAuthorityMatchesEntry } from "../../../state/registry/rebuild-authority"; +import type { SandboxEntry } from "../../../state/registry/types"; +import type { RuntimeProviderBundle } from "../../runtime-provider/contract"; +import type { ManagedWorkloadRebuildHandoff } from "../../workload/rebuild"; +import { type CommitSandboxRebuildAuthority, commitManagedWorkloadReplacement } from "./commit"; +import type { + ManagedWorkloadRebuildProviderOperations, + ManagedWorkloadRebuildTransactionResult, + PreparedManagedWorkloadReplacement, + StagedManagedWorkloadReplacement, +} from "./contract"; +import { ManagedWorkloadRebuildTransactionError } from "./contract"; +import { createStagedManagedWorkloadReplacement } from "./create"; +import { createManagedWorkloadRebuildPlan } from "./plan"; +import { prepareManagedWorkloadReplacement } from "./prepare"; +import { rebindStagedManagedWorkloadProviders } from "./provider-rebind"; +import { requireReadyManagedWorkloadReplacement } from "./readiness"; +import { restoreStagedManagedWorkloadState } from "./restore"; +import { + createManagedWorkloadPreparationAbort, + createManagedWorkloadReplacementRollback, +} from "./rollback"; + +export interface RunManagedWorkloadRebuildTransactionInput { + readonly previousEntry: SandboxEntry; + readonly provider: RuntimeProviderBundle; + readonly handoff: ManagedWorkloadRebuildHandoff; + readonly operations: ManagedWorkloadRebuildProviderOperations; + readonly replacementMetadata?: Readonly>; + readonly transactionId?: string; +} + +export interface ManagedWorkloadRebuildTransactionDependencies { + readonly getSandbox?: (sandboxName: string) => SandboxEntry | null; + readonly commitAuthority?: CommitSandboxRebuildAuthority; +} + +function readSandboxFromRegistry(sandboxName: string): SandboxEntry | null { + return load().sandboxes[sandboxName] ?? null; +} + +function rethrowWithRollback( + error: unknown, + rollbackError: unknown, +): ManagedWorkloadRebuildTransactionError { + const phase = error instanceof ManagedWorkloadRebuildTransactionError ? error.phase : "rollback"; + const message = + error instanceof Error ? error.message : "the staged replacement transaction failed"; + return new ManagedWorkloadRebuildTransactionError(phase, message, { + cause: error, + ...(rollbackError === undefined ? {} : { rollbackError }), + }); +} + +async function failAfterCleanup(error: unknown, cleanup: () => Promise): Promise { + let rollbackError: unknown; + try { + await cleanup(); + } catch (candidate) { + rollbackError = candidate; + } + throw rethrowWithRollback(error, rollbackError); +} + +/** + * Execute a dormant, provider-neutral managed rebuild transaction. + * + * The durable row and provider-owned old runtime remain authoritative through + * prepare, create, readiness, state restore, and provider rebind. Only the + * exact final CAS publishes the replacement. The exact old runtime handle is + * retired afterward, so no failure can turn a same-name lookup into deletion + * authority. + */ +export async function runManagedWorkloadRebuildTransaction( + input: RunManagedWorkloadRebuildTransactionInput, + dependencies: ManagedWorkloadRebuildTransactionDependencies = {}, +): Promise { + const readSandbox = dependencies.getSandbox ?? readSandboxFromRegistry; + const plan = createManagedWorkloadRebuildPlan(input); + const abortPreparation = createManagedWorkloadPreparationAbort(plan, input.operations); + const stillAuthoritative = (): boolean => + sandboxRebuildAuthorityMatchesEntry(plan.previousAuthority, readSandbox(plan.sandboxName)); + if (!stillAuthoritative()) { + throw new ManagedWorkloadRebuildTransactionError( + "prepare", + "the durable workload changed before provider preparation", + ); + } + + let prepared: PreparedManagedWorkloadReplacement; + try { + prepared = await prepareManagedWorkloadReplacement(plan, input.operations); + } catch (error) { + if (input.operations.providerId !== plan.providerId) throw error; + return failAfterCleanup(error, () => abortPreparation.run()); + } + if (!stillAuthoritative()) { + return failAfterCleanup( + new ManagedWorkloadRebuildTransactionError( + "prepare", + "the durable workload changed during provider preparation", + ), + () => abortPreparation.run(), + ); + } + + let staged: StagedManagedWorkloadReplacement; + try { + staged = await createStagedManagedWorkloadReplacement(plan, prepared, input.operations); + } catch (error) { + return failAfterCleanup(error, () => abortPreparation.run()); + } + const rollback = createManagedWorkloadReplacementRollback(plan, staged, input.operations); + try { + const ready = await requireReadyManagedWorkloadReplacement(plan, staged, input.operations); + const restored = await restoreStagedManagedWorkloadState(plan, ready, input.operations); + const rebound = await rebindStagedManagedWorkloadProviders(plan, restored, input.operations); + const entry = commitManagedWorkloadReplacement( + input.previousEntry, + plan, + rebound, + dependencies.commitAuthority, + readSandbox, + ); + try { + await input.operations.retirePrevious(plan, rebound); + return { status: "committed", entry, previousCleanup: "complete" }; + } catch (cleanupError) { + return { + status: "committed", + entry, + previousCleanup: "pending", + cleanupError, + }; + } + } catch (error) { + return failAfterCleanup(error, () => rollback.run()); + } +} diff --git a/src/lib/onboard/managed-workload/rebuild/validation.ts b/src/lib/onboard/managed-workload/rebuild/validation.ts new file mode 100644 index 00000000000..8347aa228c7 --- /dev/null +++ b/src/lib/onboard/managed-workload/rebuild/validation.ts @@ -0,0 +1,197 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + ManagedWorkloadRebuildPlan, + PreparedManagedWorkloadReplacement, + ReadyManagedWorkloadReplacement, + ReboundManagedWorkloadReplacement, + RestoredManagedWorkloadReplacement, + StagedManagedWorkloadReplacement, +} from "./contract"; + +const MAX_PROVIDER_ARTIFACT_BYTES = 16 * 1024; + +export class ManagedWorkloadRebuildArtifactError extends Error { + constructor(message: string) { + super(`Invalid managed workload rebuild artifact: ${message}`); + this.name = "ManagedWorkloadRebuildArtifactError"; + } +} + +function boundedText(value: unknown, label: string): string { + if ( + typeof value !== "string" || + value.trim() === "" || + value.includes("\0") || + Buffer.byteLength(value, "utf8") > MAX_PROVIDER_ARTIFACT_BYTES + ) { + throw new ManagedWorkloadRebuildArtifactError(`${label} is missing or invalid`); + } + return value; +} + +function requireBinding( + plan: ManagedWorkloadRebuildPlan, + value: { + readonly schemaVersion: unknown; + readonly providerId: unknown; + readonly transactionId: unknown; + }, + label: string, +): void { + if ( + value.schemaVersion !== 1 || + value.providerId !== plan.providerId || + value.transactionId !== plan.transactionId + ) { + throw new ManagedWorkloadRebuildArtifactError( + `${label} is not bound to '${plan.providerId}/${plan.transactionId}'`, + ); + } +} + +export function validatePreparedReplacement( + plan: ManagedWorkloadRebuildPlan, + value: PreparedManagedWorkloadReplacement, +): PreparedManagedWorkloadReplacement { + requireBinding(plan, value, "prepared replacement"); + const previousRuntimeHandle = boundedText(value.previousRuntimeHandle, "previous runtime handle"); + const preparationHandle = boundedText(value.preparationHandle, "preparation handle"); + const previousLiveIdentityFingerprint = boundedText( + value.previousLiveIdentityFingerprint, + "previous live identity fingerprint", + ); + if (previousLiveIdentityFingerprint !== plan.previousAuthority.liveIdentityFingerprint) { + throw new ManagedWorkloadRebuildArtifactError( + "prepared replacement does not prove the exact old live identity", + ); + } + return Object.freeze({ + schemaVersion: 1, + providerId: plan.providerId, + transactionId: plan.transactionId, + previousRuntimeHandle, + preparationHandle, + previousLiveIdentityFingerprint, + }); +} + +export function validateStagedReplacement( + plan: ManagedWorkloadRebuildPlan, + prepared: PreparedManagedWorkloadReplacement, + value: StagedManagedWorkloadReplacement, +): StagedManagedWorkloadReplacement { + requireBinding(plan, value, "staged replacement"); + const previousRuntimeHandle = boundedText(value.previousRuntimeHandle, "previous runtime handle"); + if (previousRuntimeHandle !== prepared.previousRuntimeHandle) { + throw new ManagedWorkloadRebuildArtifactError( + "staged replacement changed the exact old runtime handle", + ); + } + const stagingHandle = boundedText(value.stagingHandle, "staging handle"); + if (stagingHandle === previousRuntimeHandle) { + throw new ManagedWorkloadRebuildArtifactError( + "staged and authoritative runtime handles must be distinct", + ); + } + const lifecycleGeneration = boundedText( + value.lifecycleGeneration, + "replacement lifecycle generation", + ); + if (lifecycleGeneration === plan.previousAuthority.lifecycleGeneration) { + throw new ManagedWorkloadRebuildArtifactError( + "replacement lifecycle generation must differ from the old authority", + ); + } + const liveIdentityFingerprint = boundedText( + value.liveIdentityFingerprint, + "replacement live identity fingerprint", + ); + if (liveIdentityFingerprint === plan.previousAuthority.liveIdentityFingerprint) { + throw new ManagedWorkloadRebuildArtifactError( + "replacement live identity must differ from the old authority", + ); + } + return Object.freeze({ + schemaVersion: 1, + providerId: plan.providerId, + transactionId: plan.transactionId, + previousRuntimeHandle, + stagingHandle, + lifecycleGeneration, + liveIdentityFingerprint, + }); +} + +function validateStagedContinuity( + plan: ManagedWorkloadRebuildPlan, + staged: StagedManagedWorkloadReplacement, + value: StagedManagedWorkloadReplacement, + label: string, +): void { + requireBinding(plan, value, label); + if ( + value.previousRuntimeHandle !== staged.previousRuntimeHandle || + value.stagingHandle !== staged.stagingHandle || + value.lifecycleGeneration !== staged.lifecycleGeneration || + value.liveIdentityFingerprint !== staged.liveIdentityFingerprint + ) { + throw new ManagedWorkloadRebuildArtifactError( + `${label} changed provider-owned staging authority`, + ); + } +} + +export function validateReadyReplacement( + plan: ManagedWorkloadRebuildPlan, + staged: StagedManagedWorkloadReplacement, + value: ReadyManagedWorkloadReplacement, +): ReadyManagedWorkloadReplacement { + validateStagedContinuity(plan, staged, value, "ready replacement"); + return Object.freeze({ + ...staged, + readinessReceipt: boundedText(value.readinessReceipt, "readiness receipt"), + }); +} + +export function validateRestoredReplacement( + plan: ManagedWorkloadRebuildPlan, + ready: ReadyManagedWorkloadReplacement, + value: RestoredManagedWorkloadReplacement, +): RestoredManagedWorkloadReplacement { + validateStagedContinuity(plan, ready, value, "restored replacement"); + if (value.readinessReceipt !== ready.readinessReceipt) { + throw new ManagedWorkloadRebuildArtifactError( + "restore changed the validated readiness receipt", + ); + } + return Object.freeze({ + ...ready, + restoreReceipt: boundedText(value.restoreReceipt, "state restore receipt"), + }); +} + +export function validateReboundReplacement( + plan: ManagedWorkloadRebuildPlan, + restored: RestoredManagedWorkloadReplacement, + value: ReboundManagedWorkloadReplacement, +): ReboundManagedWorkloadReplacement { + validateStagedContinuity(plan, restored, value, "provider-rebound replacement"); + if ( + value.readinessReceipt !== restored.readinessReceipt || + value.restoreReceipt !== restored.restoreReceipt + ) { + throw new ManagedWorkloadRebuildArtifactError( + "provider rebind changed readiness or restore authority", + ); + } + return Object.freeze({ + ...restored, + providerRebindReceipt: boundedText(value.providerRebindReceipt, "provider rebind receipt"), + }); +} + +export function validateNotReadyReason(value: unknown): string { + return boundedText(value, "not-ready reason"); +} diff --git a/src/lib/onboard/sandbox-workload-authority.test.ts b/src/lib/onboard/sandbox-workload-authority.test.ts new file mode 100644 index 00000000000..d98f72085c8 --- /dev/null +++ b/src/lib/onboard/sandbox-workload-authority.test.ts @@ -0,0 +1,154 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { describe, expect, it } from "vitest"; + +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../state/registry/types"; +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + type ManagedImagePlatform, + type ShippedManagedImageAgent, +} from "./managed-image/contract"; +import { encodeManagedStartupProfile } from "./managed-startup/profile"; +import { ManagedWorkloadAuthorityError, readManagedWorkloadAuthority } from "./workload/authority"; + +const AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; +const PLATFORMS = ["linux/amd64", "linux/arm64"] as const; + +function managedReceipt( + agent: ShippedManagedImageAgent, + platform: ManagedImagePlatform, + profileAgent: ShippedManagedImageAgent = agent, +): Extract { + const encodedProfile = encodeManagedStartupProfile(managedStartupE2eProfile(profileAgent)); + const digest = agent === "openclaw" ? "a" : agent === "hermes" ? "b" : "c"; + return { + schemaVersion: 1, + kind: "managed-image", + reference: `${MANAGED_IMAGE_REPOSITORIES[agent]}@sha256:${digest.repeat(64)}`, + platform, + release: "v0.0.100", + sourceRevision: "d".repeat(40), + sourceCohort: "ghrun-100-1", + capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true, + }; +} + +function managedEntry( + agent: ShippedManagedImageAgent, + platform: ManagedImagePlatform = "linux/amd64", + receipt: SandboxWorkloadReceipt = managedReceipt(agent, platform), +): SandboxEntry { + return { + name: `authority-${agent}`, + agent, + fromDockerfile: null, + imageTag: receipt.reference, + workload: receipt, + }; +} + +describe("managed workload authority", () => { + it.each( + AGENTS.flatMap((agent) => PLATFORMS.map((platform) => [agent, platform] as const)), + )("validates exact %s authority on %s", (agent, platform) => { + const authority = readManagedWorkloadAuthority(managedEntry(agent, platform)); + + expect(authority).toMatchObject({ + agent, + contract: { agent, platform }, + profile: { agent }, + receipt: { kind: "managed-image", platform }, + }); + expect(authority?.receipt).not.toBe(managedEntry(agent, platform).workload); + }); + + it("returns null only for an unambiguously non-managed workload", () => { + expect( + readManagedWorkloadAuthority({ + agent: "openclaw", + fromDockerfile: "/tmp/Dockerfile", + imageTag: "custom:local", + workload: { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "custom:local", + shared: false, + }, + }), + ).toBeNull(); + }); + + it("rejects missing explicit agent identity", () => { + expect(() => + readManagedWorkloadAuthority({ + ...managedEntry("openclaw"), + agent: null, + }), + ).toThrow(/does not record an explicit agent/u); + }); + + it("rejects missing explicit OCI platform", () => { + const { platform: _platform, ...withoutPlatform } = managedReceipt("openclaw", "linux/amd64"); + expect(() => + readManagedWorkloadAuthority( + managedEntry("openclaw", "linux/amd64", withoutPlatform as SandboxWorkloadReceipt), + ), + ).toThrow(/does not record an explicit OCI platform/u); + }); + + it("rejects cross-agent image authority", () => { + const openClawReceipt = managedReceipt("openclaw", "linux/amd64"); + expect(() => + readManagedWorkloadAuthority(managedEntry("hermes", "linux/amd64", openClawReceipt)), + ).toThrow(/does not belong to 'hermes'/u); + }); + + it("rejects cross-agent startup profile authority", () => { + const hermesReceiptWithOpenClawProfile = managedReceipt("hermes", "linux/amd64", "openclaw"); + expect(() => + readManagedWorkloadAuthority( + managedEntry("hermes", "linux/amd64", hermesReceiptWithOpenClawProfile), + ), + ).toThrow(/no valid durable workload receipt/u); + }); + + it("rejects unsupported platform values instead of coercing them", () => { + expect(() => + readManagedWorkloadAuthority( + managedEntry("openclaw", "linux/amd64", { + ...managedReceipt("openclaw", "linux/amd64"), + platform: "linux/s390x", + } as unknown as SandboxWorkloadReceipt), + ), + ).toThrow(ManagedWorkloadAuthorityError); + }); + + it("rejects image-tag drift from the cloned durable receipt", () => { + expect(() => + readManagedWorkloadAuthority({ + ...managedEntry("openclaw"), + imageTag: `${MANAGED_IMAGE_REPOSITORIES.openclaw}@sha256:${"f".repeat(64)}`, + }), + ).toThrow(/image reference does not match/u); + }); + + it("rejects a managed receipt combined with a Dockerfile", () => { + expect(() => + readManagedWorkloadAuthority({ + ...managedEntry("openclaw"), + fromDockerfile: "/tmp/Dockerfile", + }), + ).toThrow(/cannot be combined with a custom Dockerfile/u); + }); +}); diff --git a/src/lib/onboard/sandbox-workload-rebuild.test.ts b/src/lib/onboard/sandbox-workload-rebuild.test.ts new file mode 100644 index 00000000000..1bd1d7abc00 --- /dev/null +++ b/src/lib/onboard/sandbox-workload-rebuild.test.ts @@ -0,0 +1,540 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, + managedStartupE2eProfile, +} from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import type { SandboxEntry } from "../state/registry/types"; +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_CONTRACT_VERSION, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_SOURCE_REPOSITORY, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + type ManagedImageContractV1, + type ManagedImagePlatform, + type ShippedManagedImageAgent, +} from "./managed-image/contract"; +import type { + BuiltManagedStartupOnboardProfile, + ManagedStartupOnboardProfileInput, +} from "./managed-startup/onboard-profile"; +import { + decodeManagedStartupProfile, + encodeManagedStartupProfile, +} from "./managed-startup/profile"; +import type { RuntimeProviderBundle } from "./runtime-provider/contract"; +import { + buildManagedWorkloadRebuildReceipt, + type ManagedWorkloadRebuildHandoff, + type ManagedWorkloadReceipt, + managedWorkloadRebuildDependencies, + managedWorkloadRebuildHandoffMatchesEntry, + managedWorkloadRebuildProfileEnvironment, + prepareManagedWorkloadRebuildHandoff, + prepareSandboxWorkloadSourceFromRebuildHandoff, + stageManagedWorkloadRebuildProfile, +} from "./workload/rebuild"; +import type { SandboxWorkloadRuntimeCapabilities } from "./workload/source"; + +const AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; +const ORIGINAL_PREPARE = managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource; +type RebuildProfileInput = Omit< + ManagedStartupOnboardProfileInput, + "agentName" | "environment" | "corporateCa" +>; + +function rebuildProfileInput(agent: ShippedManagedImageAgent): RebuildProfileInput { + const common = { + chatUiUrl: "http://127.0.0.1:18789", + effectiveDashboardPort: 18_789, + dashboardBindAddress: undefined, + wslExposure: false, + webSearch: null, + toolDisclosure: "progressive" as const, + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: "disabled" as const, + observabilityEnabled: false, + }; + if (agent === "openclaw") { + return { + ...common, + inference: { + routeProvider: "openai", + upstreamProvider: "openai-api", + model: "gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses", + primaryModelRef: "openai/gpt-5.4", + compatibility: {}, + }, + manageDashboard: true, + hermesDashboardState: { config: null, enabled: false }, + }; + } + if (agent === "hermes") { + return { + ...common, + inference: { + routeProvider: "inference", + upstreamProvider: "hermes-provider", + model: "moonshotai/kimi-k2.6", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + manageDashboard: true, + hermesDashboardState: { config: null, enabled: false }, + }; + } + return { + ...common, + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + model: "openai/gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + chatUiUrl: "", + effectiveDashboardPort: 0, + manageDashboard: false, + hermesDashboardState: { config: null, enabled: false }, + }; +} + +function managedContract( + agent: ShippedManagedImageAgent, + generation: "old" | "new", + platform: ManagedImagePlatform = "linux/amd64", +): ManagedImageContractV1 { + const image = MANAGED_IMAGE_REPOSITORIES[agent]; + const digit = generation === "old" ? "a" : "b"; + const digest = `sha256:${digit.repeat(64)}` as const; + return { + contractVersion: MANAGED_IMAGE_CONTRACT_VERSION, + agent, + platform, + image, + digest, + reference: `${image}@${digest}`, + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: digit.repeat(40), + release: generation === "old" ? "v0.0.99" : "v0.0.100", + cohort: generation === "old" ? "ghrun-100-1" : "ghrun-200-2", + }, + startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + }; +} + +function profileTransport(agent: ShippedManagedImageAgent): BuiltManagedStartupOnboardProfile { + const profile = managedStartupE2eProfile(agent); + const encodedProfile = encodeManagedStartupProfile(profile); + return { + profile, + encodedProfile: encodedProfile as BuiltManagedStartupOnboardProfile["encodedProfile"], + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + }; +} + +function receipt( + agent: ShippedManagedImageAgent, + generation: "old" | "new", + platform: ManagedImagePlatform = "linux/amd64", +): ManagedWorkloadReceipt { + const image = managedContract(agent, generation, platform); + const transport = profileTransport(agent); + return { + schemaVersion: 1, + kind: "managed-image", + reference: image.reference, + platform, + release: image.source.release, + sourceRevision: image.source.revision, + sourceCohort: image.source.cohort, + capabilityContractVersion: image.capabilityContractVersion, + startupProfileContractVersion: image.startupProfileContractVersion, + encodedProfile: transport.encodedProfile, + startupProfileSha256: transport.startupProfileSha256, + credentialProxyReplayRequired: false, + shared: true, + }; +} + +function entry( + agent: ShippedManagedImageAgent, + platform: ManagedImagePlatform = "linux/amd64", +): SandboxEntry { + const workload = receipt(agent, "old", platform); + return { + name: `rebuild-${agent}`, + agent, + openshellDriver: "mxc", + fromDockerfile: null, + imageTag: workload.reference, + workload, + }; +} + +function runtime( + providerId = "mxc", + platform: ManagedImagePlatform = "linux/amd64", +): SandboxWorkloadRuntimeCapabilities { + return { + driverName: providerId, + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + managedImages: { + exactDigestReferences: true, + platforms: [platform], + startupProfileContractVersions: [1], + capabilityContractVersions: [1], + }, + }; +} + +function provider( + providerId = "mxc", + options: { readonly acceptsReceipt?: boolean; readonly authorizesRebuild?: boolean } = {}, +): RuntimeProviderBundle { + return { + identity: { contractVersion: 1, id: providerId, displayName: providerId }, + workload: { + providerId, + supported: true, + profile: { + support: { + exactDigestReferences: true, + platforms: ["linux/amd64", "linux/arm64"], + startupProfileContractVersions: [1], + capabilityContractVersions: [1], + }, + hostArchitectures: ["amd64", "arm64"], + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + }, + acceptsReceipt: () => options.acceptsReceipt !== false, + }, + mutationAuthority: { + providerId, + supported: true, + operations: options.authorizesRebuild === false ? [] : ["rebuild"], + }, + } as unknown as RuntimeProviderBundle; +} + +function replacement( + agent: ShippedManagedImageAgent, + platform: ManagedImagePlatform = "linux/amd64", +) { + const image = managedContract(agent, "new", platform); + return { + source: { + kind: "managed-image" as const, + reference: image.reference, + contract: image, + }, + release: image.source.release, + fallbackDiagnostic: null, + }; +} + +function completeHandoff( + agent: ShippedManagedImageAgent, + catalog: Awaited>, +): ManagedWorkloadRebuildHandoff { + return { + ...catalog!, + replacementProfile: profileTransport(agent), + }; +} + +afterEach(() => { + managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource = ORIGINAL_PREPARE; +}); + +describe("managed workload rebuild preflight", () => { + it.each( + AGENTS, + )("prepares exact current-release authority for %s without a Dockerfile fallback", async (agent) => { + const prepare = vi.fn(async () => replacement(agent)); + managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource = prepare; + + const handoff = await prepareManagedWorkloadRebuildHandoff(entry(agent), { + runtime: runtime(), + provider: provider(), + version: "0.0.100", + }); + + expect(handoff).toMatchObject({ + schemaVersion: 1, + providerId: "mxc", + agent, + previousReceipt: { + kind: "managed-image", + platform: "linux/amd64", + release: "v0.0.99", + }, + replacement: { + source: { + kind: "managed-image", + contract: { agent, platform: "linux/amd64" }, + }, + release: "v0.0.100", + }, + }); + expect(prepare).toHaveBeenCalledWith({ + agentName: agent, + legacyDockerfilePath: "managed-rebuild-must-not-stage-this-dockerfile", + runtime: runtime(), + version: "0.0.100", + policy: "require-managed", + }); + }); + + it.each(AGENTS)("prepares an arm64 replacement handoff for %s", async (agent) => { + managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource = vi.fn(async () => + replacement(agent, "linux/arm64"), + ); + + const handoff = await prepareManagedWorkloadRebuildHandoff(entry(agent, "linux/arm64"), { + runtime: runtime("mxc", "linux/arm64"), + provider: provider(), + version: "0.0.100", + }); + + expect(handoff).toMatchObject({ + agent, + previousReceipt: { platform: "linux/arm64" }, + replacement: { source: { contract: { platform: "linux/arm64" } } }, + }); + }); + + it("returns null for a custom workload without resolving a catalog", async () => { + const prepare = vi.fn(); + managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource = prepare; + + await expect( + prepareManagedWorkloadRebuildHandoff( + { + agent: "openclaw", + fromDockerfile: "/tmp/Dockerfile", + imageTag: "custom:local", + workload: { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "custom:local", + shared: false, + }, + }, + { runtime: runtime(), provider: provider() }, + ), + ).resolves.toBeNull(); + expect(prepare).not.toHaveBeenCalled(); + }); + + it.each([ + [ + "provider identity", + entry("openclaw"), + runtime("other"), + provider("mxc"), + /does not match provider/u, + ], + [ + "recorded platform", + entry("openclaw", "linux/arm64"), + runtime("mxc", "linux/amd64"), + provider("mxc"), + /targets 'linux[/]arm64'/u, + ], + [ + "workload capability", + entry("openclaw"), + runtime(), + provider("mxc", { acceptsReceipt: false }), + /does not accept/u, + ], + [ + "mutation authority", + entry("openclaw"), + runtime(), + provider("mxc", { authorizesRebuild: false }), + /does not authorize 'rebuild'/u, + ], + ] as const)("rejects %s drift before catalog resolution", async (_label, row, target, selected, error) => { + const prepare = vi.fn(); + managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource = prepare; + + await expect( + prepareManagedWorkloadRebuildHandoff(row, { + runtime: target, + provider: selected, + }), + ).rejects.toThrow(error); + expect(prepare).not.toHaveBeenCalled(); + }); + + it("revalidates retained profile and receipt authority against the live row", async () => { + managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource = vi.fn(async () => + replacement("openclaw"), + ); + const row = entry("openclaw"); + const catalog = await prepareManagedWorkloadRebuildHandoff(row, { + runtime: runtime(), + provider: provider(), + }); + + expect(managedWorkloadRebuildHandoffMatchesEntry(catalog!, row, provider())).toBe(true); + expect( + managedWorkloadRebuildHandoffMatchesEntry( + catalog!, + { ...row, imageTag: receipt("openclaw", "new").reference }, + provider(), + ), + ).toBe(false); + expect(managedWorkloadRebuildHandoffMatchesEntry(catalog!, row, provider("other"))).toBe(false); + }); + + it("materializes a shared exact-digest replacement receipt", async () => { + managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource = vi.fn(async () => + replacement("hermes"), + ); + const catalog = await prepareManagedWorkloadRebuildHandoff(entry("hermes"), { + runtime: runtime(), + provider: provider(), + }); + const complete = completeHandoff("hermes", catalog); + + const result = buildManagedWorkloadRebuildReceipt(complete, provider()); + + expect(result).toEqual(receipt("hermes", "new")); + expect(result.shared).toBe(true); + expect(Object.isFrozen(result)).toBe(true); + }); + + it.each(AGENTS)("stages authoritative startup-profile reconstruction for %s", async (agent) => { + managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource = vi.fn(async () => + replacement(agent), + ); + const catalog = await prepareManagedWorkloadRebuildHandoff(entry(agent), { + runtime: runtime(), + provider: provider(), + }); + + const staged = stageManagedWorkloadRebuildProfile(catalog!, rebuildProfileInput(agent), {}); + const decoded = decodeManagedStartupProfile(staged.replacementProfile.encodedProfile); + + expect(staged.replacementProfile.credentialProxyReplayRequired).toBe(false); + expect(decoded).toMatchObject({ + agent, + agentConfig: { agent }, + proxy: { + managedHost: catalog!.previousProfile.proxy.managedHost, + managedPort: catalog!.previousProfile.proxy.managedPort, + hostHttpUrl: catalog!.previousProfile.proxy.hostHttpUrl, + hostHttpsUrl: catalog!.previousProfile.proxy.hostHttpsUrl, + }, + }); + expect(decoded.proxy.hostNoProxy).toEqual( + expect.arrayContaining([...catalog!.previousProfile.proxy.hostNoProxy]), + ); + }); + + it("replays credential-bearing proxy intent without persisting credentials", async () => { + managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource = vi.fn(async () => + replacement("openclaw"), + ); + const catalog = await prepareManagedWorkloadRebuildHandoff(entry("openclaw"), { + runtime: runtime(), + provider: provider(), + }); + const replayHandoff = { + ...catalog!, + previousReceipt: { + ...catalog!.previousReceipt, + credentialProxyReplayRequired: true, + }, + }; + const environment = { + HTTPS_PROXY: "https://operator:secret@proxy.example.test:8443", + NO_PROXY: "localhost,127.0.0.1", + }; + + const staged = stageManagedWorkloadRebuildProfile( + replayHandoff, + rebuildProfileInput("openclaw"), + environment, + ); + const reconstructed = managedWorkloadRebuildProfileEnvironment(replayHandoff, environment); + + expect(reconstructed.HTTPS_PROXY).toBe(environment.HTTPS_PROXY); + expect(staged.replacementProfile.credentialProxyReplayRequired).toBe(true); + expect(staged.replacementProfile.encodedProfile).not.toContain("secret"); + expect(JSON.stringify(staged.replacementProfile.profile)).not.toContain("operator"); + }); + + it("sources corporate CA material only from validated rebuild authority", async () => { + managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource = vi.fn(async () => + replacement("hermes"), + ); + const catalog = await prepareManagedWorkloadRebuildHandoff(entry("hermes"), { + runtime: runtime(), + provider: provider(), + }); + const pem = MANAGED_STARTUP_E2E_CORPORATE_CA_PEM; + const withCorporateCa = { + ...catalog!, + corporateCa: { + pem, + sourcePath: "durable-rebuild-authority", + sourceEnv: "durable-rebuild-authority", + }, + }; + + const staged = stageManagedWorkloadRebuildProfile( + withCorporateCa, + rebuildProfileInput("hermes"), + {}, + ); + + expect(staged.replacementProfile.profile.corporateCa).toEqual({ + bundleSha256: createHash("sha256").update(pem, "utf8").digest("hex"), + }); + expect(Buffer.from(staged.replacementProfile.corporateCaB64!, "base64").toString("utf8")).toBe( + pem, + ); + }); + + it("rebinds the retained immutable source through the selected provider contract", async () => { + managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource = vi.fn(async () => + replacement("langchain-deepagents-code"), + ); + const catalog = await prepareManagedWorkloadRebuildHandoff(entry("langchain-deepagents-code"), { + runtime: runtime(), + provider: provider(), + }); + + const source = prepareSandboxWorkloadSourceFromRebuildHandoff(catalog!, runtime(), provider()); + + expect(source).toEqual(replacement("langchain-deepagents-code")); + expect(() => + prepareSandboxWorkloadSourceFromRebuildHandoff(catalog!, runtime("other"), provider()), + ).toThrow(/does not belong to the selected runtime provider/u); + }); +}); diff --git a/src/lib/onboard/workload/authority.ts b/src/lib/onboard/workload/authority.ts new file mode 100644 index 00000000000..e05da349b60 --- /dev/null +++ b/src/lib/onboard/workload/authority.ts @@ -0,0 +1,183 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; +import { cloneSandboxWorkloadReceipt } from "../../state/registry/workload"; +import type { ResolvedCorporateCa } from "../corporate-ca-types"; +import { + isShippedManagedImageAgent, + MANAGED_IMAGE_CONTRACT_VERSION, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_SOURCE_REPOSITORY, + type ManagedImageContractV1, + parseManagedImageContractV1, + type ShippedManagedImageAgent, +} from "../managed-image/contract"; +import { validateManagedStartupCorporateCaTransport } from "../managed-startup/application"; +import { + decodeManagedStartupProfile, + type ManagedStartupProfile, +} from "../managed-startup/profile"; + +export type ManagedWorkloadReceipt = Extract< + SandboxWorkloadReceipt, + { readonly kind: "managed-image" } +>; + +export interface ManagedWorkloadAuthority { + readonly agent: ShippedManagedImageAgent; + readonly receipt: ManagedWorkloadReceipt; + readonly contract: ManagedImageContractV1; + readonly profile: ManagedStartupProfile; + readonly corporateCa: ResolvedCorporateCa | null; +} + +export class ManagedWorkloadAuthorityError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(`Invalid managed workload authority: ${message}`, options); + this.name = "ManagedWorkloadAuthorityError"; + } +} + +function isManagedImageReference(value: unknown): value is string { + return ( + typeof value === "string" && + Object.values(MANAGED_IMAGE_REPOSITORIES).some((repository) => + value.startsWith(`${repository}@sha256:`), + ) + ); +} + +function exactAgent(value: string | null | undefined): ShippedManagedImageAgent { + const normalized = value?.trim(); + if (!normalized) { + throw new ManagedWorkloadAuthorityError( + "the durable managed workload does not record an explicit agent", + ); + } + if (isShippedManagedImageAgent(normalized)) return normalized; + throw new ManagedWorkloadAuthorityError(`'${normalized}' is not a shipped managed-image agent`); +} + +function contractFromReceipt( + receipt: ManagedWorkloadReceipt, + agent: ShippedManagedImageAgent, +): ManagedImageContractV1 { + const image = MANAGED_IMAGE_REPOSITORIES[agent]; + const referencePrefix = `${image}@`; + if (!receipt.reference.startsWith(referencePrefix)) { + throw new ManagedWorkloadAuthorityError( + `the recorded image reference does not belong to '${agent}'`, + ); + } + if (receipt.platform === undefined) { + throw new ManagedWorkloadAuthorityError( + "the durable managed workload does not record an explicit OCI platform", + ); + } + const digest = receipt.reference.slice(referencePrefix.length); + try { + return parseManagedImageContractV1( + { + contractVersion: MANAGED_IMAGE_CONTRACT_VERSION, + agent, + platform: receipt.platform, + image, + digest, + reference: receipt.reference, + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: receipt.sourceRevision, + release: receipt.release, + cohort: receipt.sourceCohort, + }, + startupProfileContractVersion: receipt.startupProfileContractVersion, + capabilityContractVersion: receipt.capabilityContractVersion, + }, + agent, + ); + } catch (error) { + throw new ManagedWorkloadAuthorityError("the durable image contract failed validation", { + cause: error, + }); + } +} + +function corporateCaFromReceipt( + receipt: ManagedWorkloadReceipt, + profile: ManagedStartupProfile, +): ResolvedCorporateCa | null { + let bytes: Buffer | null; + try { + bytes = validateManagedStartupCorporateCaTransport(receipt.corporateCaB64, profile); + } catch (error) { + throw new ManagedWorkloadAuthorityError( + "the corporate CA transport does not match the recorded startup profile", + { cause: error }, + ); + } + return bytes === null + ? null + : { + pem: bytes.toString("utf8"), + sourcePath: "managed-workload-authority", + sourceEnv: "managed-workload-authority", + }; +} + +/** + * Read a durable managed workload without consulting a mutable release + * pointer. The returned receipt is cloned and the contract, profile, and CA + * transport are revalidated as one authority unit. + * + * A normal custom/legacy workload returns null. A row that looks managed but + * cannot prove its exact immutable authority fails closed. + */ +export function readManagedWorkloadAuthority( + entry: Pick, +): ManagedWorkloadAuthority | null { + const managedLooking = + isManagedImageReference(entry.imageTag) || entry.workload?.kind === "managed-image"; + if (!managedLooking) return null; + + const cloned = cloneSandboxWorkloadReceipt(entry.workload); + if (cloned?.kind !== "managed-image") { + throw new ManagedWorkloadAuthorityError( + "the managed image has no valid durable workload receipt", + ); + } + if (entry.imageTag !== cloned.reference) { + throw new ManagedWorkloadAuthorityError( + "the registry image reference does not match the durable workload receipt", + ); + } + if (entry.fromDockerfile) { + throw new ManagedWorkloadAuthorityError( + "a managed image receipt cannot be combined with a custom Dockerfile", + ); + } + + const agent = exactAgent(entry.agent); + const contract = contractFromReceipt(cloned, agent); + let profile: ManagedStartupProfile; + try { + profile = decodeManagedStartupProfile(cloned.encodedProfile); + } catch (error) { + throw new ManagedWorkloadAuthorityError("the recorded startup profile is invalid", { + cause: error, + }); + } + if (profile.agent !== agent) { + throw new ManagedWorkloadAuthorityError( + `the recorded startup profile belongs to '${profile.agent}', not '${agent}'`, + ); + } + + return Object.freeze({ + agent, + receipt: cloned, + contract, + profile, + corporateCa: corporateCaFromReceipt(cloned, profile), + }); +} diff --git a/src/lib/onboard/workload/rebuild.ts b/src/lib/onboard/workload/rebuild.ts new file mode 100644 index 00000000000..6f78a880dbf --- /dev/null +++ b/src/lib/onboard/workload/rebuild.ts @@ -0,0 +1,416 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; + +import { getVersion } from "../../core/version"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; +import type { ResolvedCorporateCa } from "../corporate-ca-types"; +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + type ManagedImageContractV1, + type ShippedManagedImageAgent, +} from "../managed-image/contract"; +import { + type BuiltManagedStartupOnboardProfile, + buildManagedStartupOnboardProfile, + type ManagedStartupOnboardProfileInput, +} from "../managed-startup/onboard-profile"; +import type { + ManagedStartupProfile, + ManagedStartupReasoningEffort, +} from "../managed-startup/profile"; +import type { RuntimeProviderBundle } from "../runtime-provider/contract"; +import { requireRuntimeProviderMutationAuthority } from "../runtime-provider/registry"; +import { + type ManagedWorkloadAuthority, + type ManagedWorkloadReceipt, + readManagedWorkloadAuthority, +} from "./authority"; + +export type { ManagedWorkloadReceipt } from "./authority"; + +import { + type PreparedSandboxWorkloadSource, + prepareSandboxWorkloadSource, + SandboxWorkloadPreparationError, +} from "./preparation"; +import { + type ManagedImageWorkloadSource, + managedImageRuntimePlatform, + resolveSandboxWorkloadSource, + type SandboxWorkloadRuntimeCapabilities, +} from "./source"; + +const HOST_PROXY_ENV_NAMES = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", +] as const; + +export interface ManagedWorkloadRebuildCatalogHandoff { + readonly schemaVersion: 1; + readonly providerId: string; + readonly agent: ShippedManagedImageAgent; + /** Exact authority retained until a replacement has become Ready. */ + readonly previousReceipt: ManagedWorkloadReceipt; + readonly previousContract: ManagedImageContractV1; + readonly previousProfile: ManagedStartupProfile; + /** Exact current-release image selected from one complete all-agent catalog. */ + readonly replacement: PreparedSandboxWorkloadSource & { + readonly source: ManagedImageWorkloadSource; + }; + /** Validated public CA material retained across a profile-only rebuild. */ + readonly corporateCa: ResolvedCorporateCa | null; +} + +export interface ManagedWorkloadRebuildHandoff extends ManagedWorkloadRebuildCatalogHandoff { + /** + * Fully rendered replacement profile. It is prepared before any provider or + * registry mutation, then consumed verbatim by the staged replacement. + */ + readonly replacementProfile: BuiltManagedStartupOnboardProfile; +} + +export class ManagedWorkloadRebuildError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(`Managed workload rebuild preflight failed: ${message}`, options); + this.name = "ManagedWorkloadRebuildError"; + } +} + +function requireProviderBoundAuthority( + authority: ManagedWorkloadAuthority, + runtime: SandboxWorkloadRuntimeCapabilities, + provider: RuntimeProviderBundle, +): void { + if (runtime.driverName !== provider.identity.id) { + throw new ManagedWorkloadRebuildError( + `runtime '${runtime.driverName}' does not match provider '${provider.identity.id}'`, + ); + } + requireRuntimeProviderMutationAuthority(provider, "rebuild"); + if (!provider.workload.acceptsReceipt(authority.receipt)) { + throw new ManagedWorkloadRebuildError( + `provider '${provider.identity.id}' does not accept the durable workload receipt`, + ); + } + const runtimePlatform = managedImageRuntimePlatform(runtime); + if (runtimePlatform === null) { + throw new ManagedWorkloadRebuildError( + `provider '${provider.identity.id}' has no unambiguous managed-image host platform`, + ); + } + if (authority.contract.platform !== runtimePlatform) { + throw new ManagedWorkloadRebuildError( + `the recorded workload targets '${authority.contract.platform}', but provider ` + + `'${provider.identity.id}' requires '${runtimePlatform}'`, + ); + } +} + +export const managedWorkloadRebuildDependencies = { + prepareSandboxWorkloadSource, +}; + +/** + * Validate the old receipt and provider bundle, then resolve the current CLI + * release as a complete all-agent catalog before any mutation. Managed rebuild + * never falls back to a Dockerfile or a mutable tag. + */ +export async function prepareManagedWorkloadRebuildHandoff( + entry: Pick, + options: { + readonly runtime: SandboxWorkloadRuntimeCapabilities; + readonly provider: RuntimeProviderBundle; + readonly version?: string; + }, +): Promise { + const authority = readManagedWorkloadAuthority(entry); + if (!authority) return null; + requireProviderBoundAuthority(authority, options.runtime, options.provider); + + let replacement: PreparedSandboxWorkloadSource; + try { + replacement = await managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource({ + agentName: authority.agent, + legacyDockerfilePath: "managed-rebuild-must-not-stage-this-dockerfile", + runtime: options.runtime, + version: options.version ?? getVersion(), + policy: "require-managed", + }); + } catch (error) { + throw new ManagedWorkloadRebuildError( + "the current release's complete managed-image catalog is unavailable or invalid", + { cause: error }, + ); + } + if (replacement.source.kind !== "managed-image") { + throw new ManagedWorkloadRebuildError( + "the current release did not resolve to an immutable managed image", + ); + } + + return Object.freeze({ + schemaVersion: 1 as const, + providerId: options.provider.identity.id, + agent: authority.agent, + previousReceipt: authority.receipt, + previousContract: authority.contract, + previousProfile: authority.profile, + replacement: { + ...replacement, + source: replacement.source, + }, + corporateCa: authority.corporateCa, + }); +} + +/** Revalidate the retained handoff against the live registry row and provider. */ +export function managedWorkloadRebuildHandoffMatchesEntry( + handoff: ManagedWorkloadRebuildCatalogHandoff, + entry: Pick | null, + provider: RuntimeProviderBundle, +): boolean { + if (!entry || provider.identity.id !== handoff.providerId) return false; + try { + const current = readManagedWorkloadAuthority(entry); + return ( + current !== null && + current.agent === handoff.agent && + provider.workload.acceptsReceipt(current.receipt) && + isDeepStrictEqual(current.receipt, handoff.previousReceipt) && + isDeepStrictEqual(current.contract, handoff.previousContract) && + isDeepStrictEqual(current.profile, handoff.previousProfile) + ); + } catch { + return false; + } +} + +export interface ManagedWorkloadRebuildProfileOverrides { + readonly openClawContextWindow?: number; + readonly openClawReasoning?: boolean; + readonly openClawReasoningEffort?: ManagedStartupReasoningEffort; +} + +/** + * Keep the source sandbox's proxy contract while allowing every other + * profile-backed rebuild setting to come from current authoritative intent. + * Credential-bearing proxy values remain launch-only and are reacquired from + * the operator environment only when the durable receipt requires replay. + */ +export function managedWorkloadRebuildProfileEnvironment( + handoff: ManagedWorkloadRebuildCatalogHandoff, + environment: NodeJS.ProcessEnv, + overrides: ManagedWorkloadRebuildProfileOverrides = {}, +): NodeJS.ProcessEnv { + const result: NodeJS.ProcessEnv = { + NEMOCLAW_PROXY_HOST: handoff.previousProfile.proxy.managedHost, + NEMOCLAW_PROXY_PORT: String(handoff.previousProfile.proxy.managedPort), + }; + const previous = handoff.previousProfile; + if (previous.agent === "openclaw" && previous.agentConfig.agent === "openclaw") { + const config = previous.agentConfig; + const contextWindow = overrides.openClawContextWindow ?? previous.tuning.contextWindow; + if (contextWindow !== null) result.NEMOCLAW_CONTEXT_WINDOW = String(contextWindow); + if (previous.tuning.maxTokens !== null) { + result.NEMOCLAW_MAX_TOKENS = String(previous.tuning.maxTokens); + } + const reasoning = overrides.openClawReasoning ?? previous.tuning.reasoning; + if (reasoning !== null) result.NEMOCLAW_REASONING = String(reasoning); + const reasoningEffort = overrides.openClawReasoningEffort ?? previous.tuning.reasoningEffort; + if (reasoningEffort !== null) result.NEMOCLAW_REASONING_EFFORT = reasoningEffort; + if (previous.inference.inputModalities !== null) { + result.NEMOCLAW_INFERENCE_INPUTS = previous.inference.inputModalities.join(","); + } + result.NEMOCLAW_AGENT_TIMEOUT = String(config.agentTimeoutSeconds); + if (config.heartbeatEvery !== null) { + result.NEMOCLAW_AGENT_HEARTBEAT_EVERY = config.heartbeatEvery; + } + result.NEMOCLAW_EXTRA_AGENTS_JSON_B64 = Buffer.from( + JSON.stringify(config.extraAgents), + "utf8", + ).toString("base64"); + result.NEMOCLAW_MINIMAL_BOOTSTRAP = config.minimalBootstrap ? "1" : "0"; + result.NEMOCLAW_OPENCLAW_OTEL = config.otel.enabled ? "1" : "0"; + result.NEMOCLAW_OPENCLAW_OTEL_ENDPOINT = config.otel.endpointUrl; + result.NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME = config.otel.serviceName; + result.NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE = String(config.otel.sampleRate); + } else if (previous.agent === "hermes" && previous.tuning.contextWindow !== null) { + result.NEMOCLAW_CONTEXT_WINDOW = String(previous.tuning.contextWindow); + } + + if (handoff.previousReceipt.credentialProxyReplayRequired) { + for (const name of HOST_PROXY_ENV_NAMES) { + const value = environment[name]; + if (value !== undefined) result[name] = value; + } + return result; + } + const proxy = handoff.previousProfile.proxy; + if (proxy.hostHttpUrl) result.HTTP_PROXY = proxy.hostHttpUrl; + if (proxy.hostHttpsUrl) result.HTTPS_PROXY = proxy.hostHttpsUrl; + if (proxy.hostNoProxy.length > 0) result.NO_PROXY = proxy.hostNoProxy.join(","); + return result; +} + +type ManagedWorkloadRebuildProfileInput = Omit< + ManagedStartupOnboardProfileInput, + "agentName" | "environment" | "corporateCa" +>; + +/** + * Render every fallible replacement-profile input while the old workload is + * authoritative. Mutable rebuild state is explicit; receipt-only tuning, + * managed proxy intent, and public CA material come from validated authority. + */ +export function stageManagedWorkloadRebuildProfile( + handoff: ManagedWorkloadRebuildCatalogHandoff, + input: ManagedWorkloadRebuildProfileInput, + environment: NodeJS.ProcessEnv = process.env, + overrides: ManagedWorkloadRebuildProfileOverrides = {}, +): ManagedWorkloadRebuildHandoff { + let replacementProfile: BuiltManagedStartupOnboardProfile; + try { + replacementProfile = buildManagedStartupOnboardProfile({ + ...input, + agentName: handoff.agent, + environment: managedWorkloadRebuildProfileEnvironment(handoff, environment, overrides), + corporateCa: handoff.corporateCa, + }); + } catch (error) { + throw new ManagedWorkloadRebuildError( + `the replacement startup profile could not be rendered from authoritative rebuild state: ${ + error instanceof Error ? error.message : String(error) + }`, + { cause: error }, + ); + } + if ( + replacementProfile.credentialProxyReplayRequired !== + handoff.previousReceipt.credentialProxyReplayRequired + ) { + throw new ManagedWorkloadRebuildError( + "the replacement startup profile changed the durable credential-proxy requirement", + ); + } + if (replacementProfile.profile.agent !== handoff.agent) { + throw new ManagedWorkloadRebuildError( + "the replacement startup profile does not match the selected managed-image agent", + ); + } + return Object.freeze({ ...handoff, replacementProfile }); +} + +/** + * Bind a retained replacement contract to the selected provider capability. + * The same immutable source resolver used by fresh onboarding performs the + * check; no mutable release pointer is consulted. + */ +export function prepareSandboxWorkloadSourceFromRebuildHandoff( + handoff: ManagedWorkloadRebuildCatalogHandoff, + runtime: SandboxWorkloadRuntimeCapabilities, + provider: RuntimeProviderBundle, +): PreparedSandboxWorkloadSource { + if (runtime.driverName !== provider.identity.id || handoff.providerId !== provider.identity.id) { + throw new SandboxWorkloadPreparationError( + "the rebuild handoff does not belong to the selected runtime provider", + ); + } + let source; + try { + source = resolveSandboxWorkloadSource({ + agentName: handoff.agent, + legacyDockerfilePath: "", + runtime, + catalog: { [handoff.agent]: handoff.replacement.source.contract }, + policy: "require-managed", + }); + } catch (error) { + throw new SandboxWorkloadPreparationError( + "the recorded managed workload is not supported by the selected runtime", + { cause: error }, + ); + } + if (source.kind !== "managed-image") { + throw new SandboxWorkloadPreparationError( + "the recorded managed workload did not resolve to an immutable image", + ); + } + if ( + source.reference !== handoff.replacement.source.reference || + source.contract.source.cohort !== handoff.replacement.source.contract.source.cohort || + source.contract.source.revision !== handoff.replacement.source.contract.source.revision + ) { + throw new SandboxWorkloadPreparationError( + "the recorded managed workload changed during source resolution", + ); + } + if ( + source.contract.capabilityContractVersion !== MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION || + source.contract.startupProfileContractVersion !== MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION + ) { + throw new SandboxWorkloadPreparationError( + "the recorded managed workload uses an unsupported contract version", + ); + } + return { source, release: handoff.replacement.release, fallbackDiagnostic: null }; +} + +/** + * Materialize the exact durable replacement receipt only after the profile is + * completely rendered. The receipt remains a shared-image authority and is + * never eligible for per-sandbox image deletion. + */ +export function buildManagedWorkloadRebuildReceipt( + handoff: ManagedWorkloadRebuildHandoff, + provider: RuntimeProviderBundle, +): ManagedWorkloadReceipt { + if (handoff.providerId !== provider.identity.id) { + throw new ManagedWorkloadRebuildError( + "the replacement receipt does not belong to the selected provider", + ); + } + const contract = handoff.replacement.source.contract; + const profile = handoff.replacementProfile; + const receipt: ManagedWorkloadReceipt = { + schemaVersion: 1, + kind: "managed-image", + reference: contract.reference, + platform: contract.platform, + release: contract.source.release, + sourceRevision: contract.source.revision, + sourceCohort: contract.source.cohort, + capabilityContractVersion: contract.capabilityContractVersion, + startupProfileContractVersion: contract.startupProfileContractVersion, + encodedProfile: profile.encodedProfile, + startupProfileSha256: profile.startupProfileSha256, + credentialProxyReplayRequired: profile.credentialProxyReplayRequired, + ...(profile.corporateCaB64 === undefined ? {} : { corporateCaB64: profile.corporateCaB64 }), + shared: true, + }; + if (!provider.workload.acceptsReceipt(receipt)) { + throw new ManagedWorkloadRebuildError( + `provider '${provider.identity.id}' rejected the replacement workload receipt`, + ); + } + return Object.freeze(receipt); +} + +export type ManagedWorkloadRebuildEntry = Pick< + SandboxEntry, + | "agent" + | "fromDockerfile" + | "imageTag" + | "workload" + | "openshellDriver" + | "lifecycleGeneration" + | "lifecycleLiveIdentityFingerprint" +>; + +export type ManagedWorkloadRebuildReceipt = SandboxWorkloadReceipt; diff --git a/src/lib/state/registry-rebuild-authority.test.ts b/src/lib/state/registry-rebuild-authority.test.ts new file mode 100644 index 00000000000..49998bb9993 --- /dev/null +++ b/src/lib/state/registry-rebuild-authority.test.ts @@ -0,0 +1,218 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { describe, expect, it } from "vitest"; +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { encodeManagedStartupProfile } from "../onboard/managed-startup/profile"; +import { + captureSandboxRebuildAuthority, + SandboxRebuildAuthorityError, + sandboxRebuildAuthorityMatchesEntry, + swapSandboxRebuildAuthorityInRegistry, +} from "./registry/rebuild-authority"; +import type { SandboxEntry, SandboxRegistry, SandboxWorkloadReceipt } from "./registry/types"; + +const ENCODED_PROFILE = encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")); +const PROFILE_SHA256 = createHash("sha256").update(ENCODED_PROFILE, "utf8").digest("hex"); + +function receipt(digest: string): Extract { + return { + schemaVersion: 1, + kind: "managed-image", + reference: `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${digest.repeat(64)}`, + platform: "linux/amd64", + release: "v0.0.100", + sourceRevision: digest.repeat(40), + sourceCohort: digest === "a" ? "ghrun-100-1" : "ghrun-200-2", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: ENCODED_PROFILE, + startupProfileSha256: PROFILE_SHA256, + credentialProxyReplayRequired: false, + shared: true, + }; +} + +function entry(generation = "generation-old", fingerprint = "fingerprint-old"): SandboxEntry { + const workload = receipt("a"); + return { + name: "alpha", + agent: "openclaw", + openshellDriver: "docker", + imageTag: workload.reference, + workload, + lifecycleGeneration: generation, + lifecycleLiveIdentityFingerprint: fingerprint, + gatewayName: "nemoclaw", + gatewayPort: 8080, + }; +} + +function registry(current: SandboxEntry = entry()): SandboxRegistry { + return { + sandboxes: { + alpha: current, + beta: { + name: "beta", + openshellDriver: "docker", + lifecycleGeneration: "beta-generation", + lifecycleLiveIdentityFingerprint: "beta-fingerprint", + }, + }, + defaultSandbox: "alpha", + defaultSelectionRevision: 7, + }; +} + +function replacement(): SandboxEntry { + const workload = receipt("b"); + return { + ...entry("generation-new", "fingerprint-new"), + imageTag: workload.reference, + workload, + }; +} + +describe("sandbox rebuild authority", () => { + it("captures a cloned exact authority unit", () => { + const source = entry(); + const authority = captureSandboxRebuildAuthority(source, "docker"); + + expect(authority).toMatchObject({ + schemaVersion: 1, + sandboxName: "alpha", + providerId: "docker", + recordedDriver: "docker", + lifecycleGeneration: "generation-old", + liveIdentityFingerprint: "fingerprint-old", + workload: source.workload, + }); + expect(authority.entryRevisionSha256).toMatch(/^[0-9a-f]{64}$/u); + expect(authority.workload).not.toBe(source.workload); + }); + + it.each([ + ["driver", (source: SandboxEntry) => ({ ...source, openshellDriver: "mxc" })], + [ + "generation", + (source: SandboxEntry) => ({ ...source, lifecycleGeneration: "other-generation" }), + ], + [ + "fingerprint", + (source: SandboxEntry) => ({ + ...source, + lifecycleLiveIdentityFingerprint: "other-fingerprint", + }), + ], + [ + "workload", + (source: SandboxEntry) => { + const workload = receipt("b"); + return { ...source, imageTag: workload.reference, workload }; + }, + ], + ["non-authority metadata", (source: SandboxEntry) => ({ ...source, gatewayPort: 9090 })], + ] as const)("rejects %s drift from exact authority", (_label, mutate) => { + const authority = captureSandboxRebuildAuthority(entry(), "docker"); + + expect(sandboxRebuildAuthorityMatchesEntry(authority, mutate(entry()))).toBe(false); + }); + + it("publishes a replacement by CAS without changing unrelated registry state", () => { + const before = registry(); + const authority = captureSandboxRebuildAuthority(before.sandboxes.alpha!, "docker"); + const swapped = swapSandboxRebuildAuthorityInRegistry(before, authority, replacement()); + + expect(swapped.result).toMatchObject({ + status: "committed", + entry: { + lifecycleGeneration: "generation-new", + lifecycleLiveIdentityFingerprint: "fingerprint-new", + workload: { reference: receipt("b").reference }, + }, + }); + expect(swapped.registry).not.toBe(before); + expect(swapped.registry.sandboxes.alpha).toEqual(replacement()); + expect(swapped.registry.sandboxes.beta).toBe(before.sandboxes.beta); + expect(swapped.registry.defaultSandbox).toBe("alpha"); + expect(swapped.registry.defaultSelectionRevision).toBe(7); + expect(before.sandboxes.alpha).toEqual(entry()); + }); + + it("keeps the original registry object when exact authority is stale", () => { + const before = registry(entry("generation-concurrent", "fingerprint-concurrent")); + const authority = captureSandboxRebuildAuthority(entry(), "docker"); + const swapped = swapSandboxRebuildAuthorityInRegistry(before, authority, replacement()); + + expect(swapped.result).toMatchObject({ + status: "stale-authority", + entry: { lifecycleGeneration: "generation-concurrent" }, + }); + expect(swapped.registry).toBe(before); + expect(swapped.registry.sandboxes.alpha).toBe(before.sandboxes.alpha); + }); + + it("does not overwrite a concurrent non-authority row update", () => { + const oldEntry = entry(); + const before = registry({ ...oldEntry, model: "concurrently-updated" }); + const authority = captureSandboxRebuildAuthority(oldEntry, "docker"); + const swapped = swapSandboxRebuildAuthorityInRegistry(before, authority, replacement()); + + expect(swapped.result).toMatchObject({ + status: "stale-authority", + entry: { model: "concurrently-updated" }, + }); + expect(swapped.registry).toBe(before); + }); + + it.each([ + ["sandbox name", (candidate: SandboxEntry) => ({ ...candidate, name: "other" })], + ["provider", (candidate: SandboxEntry) => ({ ...candidate, openshellDriver: "mxc" })], + [ + "lifecycle generation", + (candidate: SandboxEntry) => ({ + ...candidate, + lifecycleGeneration: "generation-old", + }), + ], + [ + "live identity", + (candidate: SandboxEntry) => ({ + ...candidate, + lifecycleLiveIdentityFingerprint: "fingerprint-old", + }), + ], + [ + "image reference", + (candidate: SandboxEntry) => ({ + ...candidate, + imageTag: receipt("a").reference, + }), + ], + ] as const)("rejects replacement %s drift before CAS", (_label, mutate) => { + const before = registry(); + const authority = captureSandboxRebuildAuthority(before.sandboxes.alpha!, "docker"); + + expect(() => + swapSandboxRebuildAuthorityInRegistry(before, authority, mutate(replacement())), + ).toThrow(SandboxRebuildAuthorityError); + expect(before).toEqual(registry()); + }); + + it("rejects route reservations and missing exact lifecycle authority", () => { + expect(() => + captureSandboxRebuildAuthority({ ...entry(), pendingRouteReservation: true }, "docker"), + ).toThrow(/route reservations/u); + expect(() => + captureSandboxRebuildAuthority({ ...entry(), lifecycleGeneration: undefined }, "docker"), + ).toThrow(/lifecycle generation/u); + expect(() => + captureSandboxRebuildAuthority( + { ...entry(), lifecycleLiveIdentityFingerprint: undefined }, + "docker", + ), + ).toThrow(/live identity fingerprint/u); + }); +}); diff --git a/src/lib/state/registry/rebuild-authority.ts b/src/lib/state/registry/rebuild-authority.ts new file mode 100644 index 00000000000..90ee4f77942 --- /dev/null +++ b/src/lib/state/registry/rebuild-authority.ts @@ -0,0 +1,276 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; +import { withLock } from "./lock"; +import { load, save } from "./persistence"; +import type { SandboxEntry, SandboxRegistry, SandboxWorkloadReceipt } from "./types"; +import { cloneSandboxWorkloadReceipt } from "./workload"; + +type ManagedWorkloadReceipt = Extract; + +const MAX_AUTHORITY_BYTES = 4096; + +export interface SandboxRebuildAuthority { + readonly schemaVersion: 1; + readonly sandboxName: string; + /** Selected RuntimeProviderBundle identity, validated before capture. */ + readonly providerId: string; + /** Exact raw durable driver value; legacy Docker rows may record null. */ + readonly recordedDriver: string | null; + readonly lifecycleGeneration: string; + readonly liveIdentityFingerprint: string; + /** + * Canonical digest of the complete durable row. Rebuild CAS must fail on + * concurrent policy, messaging, MCP, inference, or other metadata changes, + * rather than overwriting them from the caller's stale snapshot. + */ + readonly entryRevisionSha256: string; + readonly workload: ManagedWorkloadReceipt; +} + +export type SandboxRebuildAuthoritySwapResult = + | { + readonly status: "committed"; + readonly entry: SandboxEntry; + } + | { + readonly status: "stale-authority"; + readonly entry: SandboxEntry | null; + }; + +export class SandboxRebuildAuthorityError extends Error { + constructor(message: string) { + super(`Invalid sandbox rebuild authority: ${message}`); + this.name = "SandboxRebuildAuthorityError"; + } +} + +function boundedIdentity(value: unknown): value is string { + return ( + typeof value === "string" && + value.trim() !== "" && + !value.includes("\0") && + Buffer.byteLength(value, "utf8") <= MAX_AUTHORITY_BYTES + ); +} + +function clonedManagedReceipt(value: SandboxWorkloadReceipt | undefined): ManagedWorkloadReceipt { + const cloned = cloneSandboxWorkloadReceipt(value); + if (cloned?.kind !== "managed-image") { + throw new SandboxRebuildAuthorityError( + "a managed replacement requires a valid durable workload receipt", + ); + } + return cloned; +} + +function cloneEntry(entry: SandboxEntry): SandboxEntry { + return structuredClone(entry); +} + +function canonicalJson(value: unknown): string { + if (value === null || typeof value !== "object") { + return JSON.stringify(value) ?? "null"; + } + if (Array.isArray(value)) { + return `[${value.map((item) => canonicalJson(item)).join(",")}]`; + } + const record = value as Record; + return `{${Object.keys(record) + .filter((key) => record[key] !== undefined) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; +} + +function sandboxEntryRevision(entry: SandboxEntry): string { + return createHash("sha256").update(canonicalJson(entry), "utf8").digest("hex"); +} + +/** + * Capture exact old-workload authority after the caller has resolved and + * authorized one RuntimeProviderBundle. This receipt is suitable for an + * atomic compare-and-swap; it is not a provider runtime handle. + */ +export function captureSandboxRebuildAuthority( + entry: SandboxEntry, + providerId: string, +): SandboxRebuildAuthority { + if (!boundedIdentity(entry.name)) { + throw new SandboxRebuildAuthorityError("sandbox name is missing or too large"); + } + if (!boundedIdentity(providerId)) { + throw new SandboxRebuildAuthorityError("provider identity is missing or too large"); + } + if (entry.pendingRouteReservation === true) { + throw new SandboxRebuildAuthorityError("route reservations cannot be rebuilt"); + } + if (!boundedIdentity(entry.lifecycleGeneration)) { + throw new SandboxRebuildAuthorityError("lifecycle generation is missing or invalid"); + } + if (!boundedIdentity(entry.lifecycleLiveIdentityFingerprint)) { + throw new SandboxRebuildAuthorityError("live identity fingerprint is missing or invalid"); + } + const workload = clonedManagedReceipt(entry.workload); + if (entry.imageTag !== workload.reference) { + throw new SandboxRebuildAuthorityError( + "image reference does not match the managed workload receipt", + ); + } + return Object.freeze({ + schemaVersion: 1 as const, + sandboxName: entry.name, + providerId, + recordedDriver: entry.openshellDriver ?? null, + lifecycleGeneration: entry.lifecycleGeneration, + liveIdentityFingerprint: entry.lifecycleLiveIdentityFingerprint, + entryRevisionSha256: sandboxEntryRevision(entry), + workload, + }); +} + +/** + * Reconcile an ambiguous persistence result using the replacement's exact new + * runtime identity. This deliberately ignores mutable non-authority fields: + * after publication another writer may update those fields, but no different + * rebuild may claim the same generation, live fingerprint, provider, and + * workload receipt. + */ +export function sandboxRebuildReplacementMatchesEntry( + replacement: SandboxEntry, + entry: SandboxEntry | null | undefined, +): boolean { + if (!entry) return false; + return ( + entry.name === replacement.name && + (entry.openshellDriver ?? null) === (replacement.openshellDriver ?? null) && + entry.lifecycleGeneration === replacement.lifecycleGeneration && + entry.lifecycleLiveIdentityFingerprint === replacement.lifecycleLiveIdentityFingerprint && + entry.imageTag === replacement.imageTag && + entry.agent === replacement.agent && + isDeepStrictEqual( + cloneSandboxWorkloadReceipt(entry.workload), + cloneSandboxWorkloadReceipt(replacement.workload), + ) + ); +} + +export function sandboxRebuildAuthorityMatchesEntry( + authority: SandboxRebuildAuthority, + entry: SandboxEntry | null | undefined, +): boolean { + if (!entry) return false; + let current: SandboxRebuildAuthority; + try { + current = captureSandboxRebuildAuthority(entry, authority.providerId); + } catch { + return false; + } + return isDeepStrictEqual(current, authority); +} + +function validateReplacement( + expected: SandboxRebuildAuthority, + replacement: SandboxEntry, +): SandboxEntry { + if (replacement.name !== expected.sandboxName) { + throw new SandboxRebuildAuthorityError("replacement changed the sandbox name"); + } + if (replacement.pendingRouteReservation === true) { + throw new SandboxRebuildAuthorityError("replacement is only a route reservation"); + } + if (replacement.openshellDriver !== expected.providerId) { + throw new SandboxRebuildAuthorityError( + "replacement does not record the selected provider identity", + ); + } + if ( + !boundedIdentity(replacement.lifecycleGeneration) || + replacement.lifecycleGeneration === expected.lifecycleGeneration + ) { + throw new SandboxRebuildAuthorityError("replacement must have a distinct lifecycle generation"); + } + if ( + !boundedIdentity(replacement.lifecycleLiveIdentityFingerprint) || + replacement.lifecycleLiveIdentityFingerprint === expected.liveIdentityFingerprint + ) { + throw new SandboxRebuildAuthorityError( + "replacement must have a distinct live identity fingerprint", + ); + } + const workload = clonedManagedReceipt(replacement.workload); + if (replacement.imageTag !== workload.reference) { + throw new SandboxRebuildAuthorityError( + "replacement image reference does not match its managed workload receipt", + ); + } + return cloneEntry({ ...replacement, workload }); +} + +/** + * Pure CAS helper for tests and callers that already hold the registry lock. + * A mismatch returns the original registry object and never overwrites a + * same-name sandbox. No delete-by-name operation exists in this boundary. + */ +export function swapSandboxRebuildAuthorityInRegistry( + registry: SandboxRegistry, + expected: SandboxRebuildAuthority, + replacementInput: SandboxEntry, +): { + readonly registry: SandboxRegistry; + readonly result: SandboxRebuildAuthoritySwapResult; +} { + const replacement = validateReplacement(expected, replacementInput); + const current = registry.sandboxes[expected.sandboxName] ?? null; + if (!sandboxRebuildAuthorityMatchesEntry(expected, current)) { + return { + registry, + result: { + status: "stale-authority", + entry: current === null ? null : cloneEntry(current), + }, + }; + } + const next: SandboxRegistry = { + ...registry, + sandboxes: { + ...registry.sandboxes, + [expected.sandboxName]: replacement, + }, + }; + return { + registry: next, + result: { status: "committed", entry: cloneEntry(replacement) }, + }; +} + +/** + * Atomically publish a Ready replacement only while the exact old generation, + * live identity, provider recording, and managed workload receipt still own + * the durable row. + */ +export function compareAndSwapSandboxRebuildAuthority( + expected: SandboxRebuildAuthority, + replacement: SandboxEntry, +): SandboxRebuildAuthoritySwapResult { + try { + return withLock(() => { + const swapped = swapSandboxRebuildAuthorityInRegistry(load(), expected, replacement); + if (swapped.result.status === "committed") save(swapped.registry); + return swapped.result; + }); + } catch (error) { + try { + const observed = load().sandboxes[expected.sandboxName] ?? null; + if (sandboxRebuildReplacementMatchesEntry(replacement, observed)) { + return { status: "committed", entry: cloneEntry(observed) }; + } + } catch { + // Preserve the original CAS/persistence failure when reconciliation is + // itself unavailable. + } + throw error; + } +} From 438f79f1f2c06e2ce81af605f28f97ce98a196e4 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 03:47:29 -0700 Subject: [PATCH 040/117] fix(rebuild): fail closed across authority cutover Signed-off-by: Aaron Erickson --- src/lib/core/immutable.ts | 23 +++ ...aged-workload-rebuild-source-shape.test.ts | 1 + ...naged-workload-rebuild-transaction.test.ts | 134 +++++++++++++++++- .../managed-workload/rebuild/commit.ts | 74 ++++++++-- .../managed-workload/rebuild/contract.ts | 57 +++++++- .../onboard/managed-workload/rebuild/index.ts | 1 + .../onboard/managed-workload/rebuild/plan.ts | 24 ++-- .../managed-workload/rebuild/recovery.ts | 33 +++++ .../managed-workload/rebuild/transaction.ts | 50 ++++--- .../sandbox-workload-authority.test.ts | 4 + .../onboard/sandbox-workload-rebuild.test.ts | 22 +++ src/lib/onboard/workload/authority.ts | 3 +- src/lib/onboard/workload/rebuild.ts | 42 +++++- .../state/registry-rebuild-authority.test.ts | 7 + src/lib/state/registry/rebuild-authority.ts | 29 +++- 15 files changed, 449 insertions(+), 55 deletions(-) create mode 100644 src/lib/core/immutable.ts create mode 100644 src/lib/onboard/managed-workload/rebuild/recovery.ts diff --git a/src/lib/core/immutable.ts b/src/lib/core/immutable.ts new file mode 100644 index 00000000000..1576487b77c --- /dev/null +++ b/src/lib/core/immutable.ts @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +function deepFreezeOwnedValue(value: T, seen: WeakSet): T { + if ((typeof value !== "object" && typeof value !== "function") || value === null) return value; + const object = value as object; + if (seen.has(object)) return value; + seen.add(object); + for (const key of Reflect.ownKeys(object)) { + const descriptor = Object.getOwnPropertyDescriptor(object, key); + if (descriptor && "value" in descriptor) deepFreezeOwnedValue(descriptor.value, seen); + } + return Object.freeze(value); +} + +/** + * Take ownership of structured data before exposing it across an adapter + * boundary. The clone prevents retained caller aliases and the recursive + * freeze prevents a provider from changing nested authority after validation. + */ +export function cloneAndDeepFreeze(value: T): T { + return deepFreezeOwnedValue(structuredClone(value), new WeakSet()); +} diff --git a/src/lib/onboard/managed-workload-rebuild-source-shape.test.ts b/src/lib/onboard/managed-workload-rebuild-source-shape.test.ts index 1f61332088f..9d7650a8245 100644 --- a/src/lib/onboard/managed-workload-rebuild-source-shape.test.ts +++ b/src/lib/onboard/managed-workload-rebuild-source-shape.test.ts @@ -15,6 +15,7 @@ const CENTRAL_REBUILD_MODULES = [ "prepare.ts", "provider-rebind.ts", "readiness.ts", + "recovery.ts", "restore.ts", "rollback.ts", "transaction.ts", diff --git a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts index cb9c8204032..f655d3843f9 100644 --- a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts +++ b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts @@ -234,7 +234,10 @@ type FailurePhase = | "restore" | "provider-rebind" | "registry-commit" + | "registry-commit-before-persist" | "registry-commit-after-persist" + | "registry-commit-after-persist-read-fails" + | "registry-read-after-prepare" | "retire-previous" | "abort-preparation" | null; @@ -326,12 +329,16 @@ function transactionHarness( const events: string[] = []; const oldEntry = previousEntry(agent, providerId, platform); let currentEntry = structuredClone(oldEntry); + let registryReadCount = 0; const operations = operationsHarness(providerId, events, failAt); const commitAuthority = ( expected: ReturnType, replacement: SandboxEntry, ): SandboxRebuildAuthoritySwapResult => { events.push("registry-commit"); + if (failAt === "registry-commit-before-persist") { + throw new Error("registry write failed before persistence"); + } const currentRegistry: SandboxRegistry = { sandboxes: { [oldEntry.name]: currentEntry }, defaultSandbox: oldEntry.name, @@ -348,7 +355,10 @@ function transactionHarness( : swapSandboxRebuildAuthorityInRegistry(currentRegistry, expected, replacement); currentEntry = swapped.result.status === "committed" ? structuredClone(swapped.result.entry) : currentEntry; - if (failAt === "registry-commit-after-persist") { + if ( + failAt === "registry-commit-after-persist" || + failAt === "registry-commit-after-persist-read-fails" + ) { throw new Error("registry acknowledgement lost after persistence"); } return swapped.result; @@ -369,7 +379,16 @@ function transactionHarness( transactionId: "transaction-1", }, { - getSandbox: () => structuredClone(currentEntry), + getSandbox: () => { + registryReadCount += 1; + if (failAt === "registry-read-after-prepare" && registryReadCount === 2) { + throw new Error("registry read failed after provider preparation"); + } + if (failAt === "registry-commit-after-persist-read-fails" && registryReadCount >= 3) { + throw new Error("registry readback failed after ambiguous persistence"); + } + return structuredClone(currentEntry); + }, commitAuthority, }, ), @@ -489,6 +508,17 @@ describe("managed workload rebuild transaction", () => { expect(operations.rollback).not.toHaveBeenCalled(); }); + it("aborts preparation when the post-prepare registry read fails", async () => { + const harness = transactionHarness("openclaw", "mxc", "registry-read-after-prepare"); + + await expect(harness.run()).rejects.toMatchObject({ phase: "prepare" }); + + expect(harness.events).toEqual(["prepare", "abort-preparation:transaction-1"]); + expect(harness.operations.create).not.toHaveBeenCalled(); + expect(harness.operations.rollback).not.toHaveBeenCalled(); + expect(harness.currentEntry()).toEqual(harness.oldEntry); + }); + it("reports exact-old cleanup as pending without undoing a committed replacement", async () => { const harness = transactionHarness("langchain-deepagents-code", "mxc", "retire-previous"); @@ -498,6 +528,12 @@ describe("managed workload rebuild transaction", () => { status: "committed", previousCleanup: "pending", cleanupError: expect.any(Error), + recoveryTask: { + owner: "durable-managed-workload-recovery", + operation: "retire-previous", + previousRuntimeHandle: "runtime-old-exact", + stagingHandle: "runtime-new-staged-exact", + }, }); expect(harness.currentEntry().lifecycleGeneration).toBe("generation-new"); expect(harness.events.at(-1)).toBe("retire:runtime-old-exact"); @@ -526,6 +562,38 @@ describe("managed workload rebuild transaction", () => { expect(harness.events.at(-1)).toBe("retire:runtime-old-exact"); }); + it("never rolls back an ambiguously published replacement when readback also fails", async () => { + const harness = transactionHarness( + "openclaw", + "mxc", + "registry-commit-after-persist-read-fails", + ); + + await expect(harness.run()).rejects.toMatchObject({ + name: "ManagedWorkloadRebuildIndeterminatePublicationError", + phase: "registry-commit", + recoveryTask: { + owner: "durable-managed-workload-recovery", + operation: "reconcile-publication", + previousRuntimeHandle: "runtime-old-exact", + stagingHandle: "runtime-new-staged-exact", + }, + }); + + expect(harness.currentEntry().lifecycleGeneration).toBe("generation-new"); + expect(harness.operations.rollback).not.toHaveBeenCalled(); + expect(harness.operations.retirePrevious).not.toHaveBeenCalled(); + }); + + it("rolls back only after an ambiguous write is reconciled to exact old authority", async () => { + const harness = transactionHarness("openclaw", "mxc", "registry-commit-before-persist"); + + await expect(harness.run()).rejects.toMatchObject({ phase: "registry-commit" }); + + expect(harness.currentEntry()).toEqual(harness.oldEntry); + expect(harness.operations.rollback).toHaveBeenCalledOnce(); + }); + it("coalesces repeated rollback calls onto one exact-handle operation", async () => { const events: string[] = []; const providerOperations = operationsHarness("mxc", events); @@ -590,6 +658,68 @@ describe("managed workload rebuild transaction", () => { expect(operations.prepare).not.toHaveBeenCalled(); }); + it("rejects a cross-agent replacement contract and profile before provider mutation", async () => { + const oldEntry = previousEntry("openclaw", "mxc"); + const openClawHandoff = handoff("openclaw", "mxc"); + const hermesHandoff = handoff("hermes", "mxc"); + const operations = operationsHarness("mxc", []); + + await expect( + runManagedWorkloadRebuildTransaction( + { + previousEntry: oldEntry, + provider: bundle("mxc"), + handoff: { + ...openClawHandoff, + replacement: hermesHandoff.replacement, + replacementProfile: hermesHandoff.replacementProfile, + }, + operations, + transactionId: "transaction-1", + }, + { getSandbox: () => structuredClone(oldEntry) }, + ), + ).rejects.toMatchObject({ phase: "prepare" }); + expect(operations.prepare).not.toHaveBeenCalled(); + }); + + it("deeply freezes provider-visible rebuild authority", async () => { + const oldEntry = previousEntry("openclaw", "mxc"); + const operations = operationsHarness("mxc", []); + const prepare = operations.prepare; + operations.prepare = vi.fn(async (plan) => { + expect(Object.isFrozen(plan)).toBe(true); + expect(Object.isFrozen(plan.previousAuthority.workload)).toBe(true); + expect(Object.isFrozen(plan.handoff.previousProfile.proxy)).toBe(true); + expect(Object.isFrozen(plan.handoff.replacement.source.contract.source)).toBe(true); + expect(Object.isFrozen(plan.replacementReceipt)).toBe(true); + expect( + Reflect.set(plan.handoff.replacement.source.contract.source, "release", "v999.0.0"), + ).toBe(false); + expect(Reflect.set(plan.previousAuthority.workload, "reference", "mutated")).toBe(false); + return prepare(plan); + }); + + const result = await runManagedWorkloadRebuildTransaction( + { + previousEntry: oldEntry, + provider: bundle("mxc"), + handoff: handoff("openclaw", "mxc"), + operations, + transactionId: "transaction-1", + }, + { + getSandbox: () => structuredClone(oldEntry), + commitAuthority: (_expected, replacement) => ({ + status: "committed", + entry: structuredClone(replacement), + }), + }, + ); + + expect(result.status).toBe("committed"); + }); + it("rejects a provider adapter that is not bound to the selected bundle", async () => { const oldEntry = previousEntry("openclaw", "mxc"); const operations = operationsHarness("other-provider", []); diff --git a/src/lib/onboard/managed-workload/rebuild/commit.ts b/src/lib/onboard/managed-workload/rebuild/commit.ts index e837930fcaa..80d14cccc08 100644 --- a/src/lib/onboard/managed-workload/rebuild/commit.ts +++ b/src/lib/onboard/managed-workload/rebuild/commit.ts @@ -4,11 +4,16 @@ import { compareAndSwapSandboxRebuildAuthority, type SandboxRebuildAuthoritySwapResult, + sandboxRebuildAuthorityMatchesEntry, sandboxRebuildReplacementMatchesEntry, } from "../../../state/registry/rebuild-authority"; import type { SandboxEntry } from "../../../state/registry/types"; import type { ManagedWorkloadRebuildPlan, ReboundManagedWorkloadReplacement } from "./contract"; -import { ManagedWorkloadRebuildTransactionError } from "./contract"; +import { + ManagedWorkloadRebuildIndeterminatePublicationError, + ManagedWorkloadRebuildTransactionError, +} from "./contract"; +import { createManagedWorkloadRebuildRecoveryTask } from "./recovery"; export type CommitSandboxRebuildAuthority = ( expected: ManagedWorkloadRebuildPlan["previousAuthority"], @@ -17,6 +22,52 @@ export type CommitSandboxRebuildAuthority = ( export type ReadSandboxRebuildEntry = (sandboxName: string) => SandboxEntry | null; +function reconcileAmbiguousPublication( + plan: ManagedWorkloadRebuildPlan, + replacement: ReboundManagedWorkloadReplacement, + candidate: SandboxEntry, + publicationError: unknown, + readSandbox?: ReadSandboxRebuildEntry, +): SandboxEntry { + if (!readSandbox) { + throw new ManagedWorkloadRebuildIndeterminatePublicationError( + "publication failed without an authoritative reconciliation read", + createManagedWorkloadRebuildRecoveryTask(plan, replacement, "reconcile-publication"), + { cause: publicationError }, + ); + } + let observed: SandboxEntry | null; + try { + observed = readSandbox(plan.sandboxName); + } catch (reconciliationError) { + throw new ManagedWorkloadRebuildIndeterminatePublicationError( + "publication and authoritative reconciliation both failed", + createManagedWorkloadRebuildRecoveryTask(plan, replacement, "reconcile-publication"), + { + cause: new AggregateError( + [publicationError, reconciliationError], + "managed workload publication and reconciliation failed", + ), + }, + ); + } + if (observed && sandboxRebuildReplacementMatchesEntry(candidate, observed)) { + return structuredClone(observed); + } + if (sandboxRebuildAuthorityMatchesEntry(plan.previousAuthority, observed)) { + throw new ManagedWorkloadRebuildTransactionError( + "registry-commit", + "publication failed while the exact old authority remained durable", + { cause: publicationError }, + ); + } + throw new ManagedWorkloadRebuildIndeterminatePublicationError( + "publication could not be reconciled to the replacement or exact old authority", + createManagedWorkloadRebuildRecoveryTask(plan, replacement, "reconcile-publication"), + { cause: publicationError }, + ); +} + export function materializeManagedWorkloadReplacementEntry( previousEntry: SandboxEntry, plan: ManagedWorkloadRebuildPlan, @@ -50,15 +101,7 @@ export function commitManagedWorkloadReplacement( try { result = commit(plan.previousAuthority, candidate); } catch (error) { - const observed = readSandbox?.(plan.sandboxName) ?? null; - if (observed && sandboxRebuildReplacementMatchesEntry(candidate, observed)) { - return structuredClone(observed); - } - throw new ManagedWorkloadRebuildTransactionError( - "registry-commit", - "the replacement registry entry could not be committed", - { cause: error }, - ); + return reconcileAmbiguousPublication(plan, replacement, candidate, error, readSandbox); } if (result.status !== "committed") { throw new ManagedWorkloadRebuildTransactionError( @@ -66,5 +109,14 @@ export function commitManagedWorkloadReplacement( "the old workload no longer owns the exact durable authority", ); } - return result.entry; + if (!sandboxRebuildReplacementMatchesEntry(candidate, result.entry)) { + return reconcileAmbiguousPublication( + plan, + replacement, + candidate, + new Error("the commit adapter returned a mismatched committed entry"), + readSandbox, + ); + } + return structuredClone(result.entry); } diff --git a/src/lib/onboard/managed-workload/rebuild/contract.ts b/src/lib/onboard/managed-workload/rebuild/contract.ts index bf1307e8a3a..3d3c348c8e7 100644 --- a/src/lib/onboard/managed-workload/rebuild/contract.ts +++ b/src/lib/onboard/managed-workload/rebuild/contract.ts @@ -135,12 +135,39 @@ export interface ManagedWorkloadRebuildProviderOperations { ): Promise; } -export type ManagedWorkloadRebuildTransactionResult = { - readonly status: "committed"; - readonly entry: SandboxEntry; - readonly previousCleanup: "complete" | "pending"; - readonly cleanupError?: unknown; -}; +export interface ManagedWorkloadRebuildRecoveryTask { + readonly schemaVersion: 1; + /** Later activation must durably persist this task for this owner. */ + readonly owner: "durable-managed-workload-recovery"; + readonly operation: "reconcile-publication" | "retire-previous"; + readonly transactionId: string; + readonly sandboxName: string; + readonly providerId: string; + readonly previousRuntimeHandle: string; + readonly stagingHandle: string; + readonly previousAuthority: SandboxRebuildAuthority; + readonly replacement: { + readonly agent: ShippedManagedImageAgent; + readonly receipt: ManagedWorkloadReceipt; + readonly lifecycleGeneration: string; + readonly liveIdentityFingerprint: string; + }; +} + +export type ManagedWorkloadRebuildTransactionResult = + | { + readonly status: "committed"; + readonly entry: SandboxEntry; + readonly previousCleanup: "complete"; + } + | { + readonly status: "committed"; + readonly entry: SandboxEntry; + readonly previousCleanup: "pending"; + readonly cleanupError: unknown; + /** Exact handoff that the later durable recovery layer must persist and own. */ + readonly recoveryTask: ManagedWorkloadRebuildRecoveryTask; + }; export class ManagedWorkloadRebuildTransactionError extends Error { readonly phase: ManagedWorkloadRebuildPhase; @@ -159,3 +186,21 @@ export class ManagedWorkloadRebuildTransactionError extends Error { this.rollbackError = options.rollbackError; } } + +/** + * Publication may already be durable. Catchers must not destroy the staged + * runtime; they must hand recoveryTask to the durable reconciliation owner. + */ +export class ManagedWorkloadRebuildIndeterminatePublicationError extends ManagedWorkloadRebuildTransactionError { + readonly recoveryTask: ManagedWorkloadRebuildRecoveryTask; + + constructor( + message: string, + recoveryTask: ManagedWorkloadRebuildRecoveryTask, + options: { readonly cause?: unknown } = {}, + ) { + super("registry-commit", message, options); + this.name = "ManagedWorkloadRebuildIndeterminatePublicationError"; + this.recoveryTask = recoveryTask; + } +} diff --git a/src/lib/onboard/managed-workload/rebuild/index.ts b/src/lib/onboard/managed-workload/rebuild/index.ts index 32bc91a27b8..27a6d11140c 100644 --- a/src/lib/onboard/managed-workload/rebuild/index.ts +++ b/src/lib/onboard/managed-workload/rebuild/index.ts @@ -2,4 +2,5 @@ // SPDX-License-Identifier: Apache-2.0 export * from "./contract"; +export * from "./recovery"; export * from "./transaction"; diff --git a/src/lib/onboard/managed-workload/rebuild/plan.ts b/src/lib/onboard/managed-workload/rebuild/plan.ts index 99a35311fc5..23bf014d6e3 100644 --- a/src/lib/onboard/managed-workload/rebuild/plan.ts +++ b/src/lib/onboard/managed-workload/rebuild/plan.ts @@ -3,6 +3,7 @@ import { randomUUID } from "node:crypto"; import { isDeepStrictEqual } from "node:util"; +import { cloneAndDeepFreeze } from "../../../core/immutable"; import { captureSandboxRebuildAuthority } from "../../../state/registry/rebuild-authority"; import type { SandboxEntry } from "../../../state/registry/types"; import type { RuntimeProviderBundle } from "../../runtime-provider/contract"; @@ -38,7 +39,7 @@ function safeReplacementMetadata( ([field]) => !PROTECTED_REBUILD_METADATA_FIELDS.has(field as keyof SandboxEntry), ), ) as Partial; - return Object.freeze(safe); + return cloneAndDeepFreeze(safe); } export function createManagedWorkloadRebuildPlan(input: { @@ -48,6 +49,7 @@ export function createManagedWorkloadRebuildPlan(input: { readonly replacementMetadata?: Readonly>; readonly transactionId?: string; }): ManagedWorkloadRebuildPlan { + const handoff = cloneAndDeepFreeze(input.handoff); const providerId = input.provider.identity.id; if (normalizeRuntimeProviderIdentity(input.previousEntry.openshellDriver) !== providerId) { throw new ManagedWorkloadRebuildTransactionError( @@ -57,23 +59,25 @@ export function createManagedWorkloadRebuildPlan(input: { } requireRuntimeProviderMutationAuthority(input.provider, "rebuild"); if ( - input.handoff.providerId !== providerId || - input.handoff.agent !== input.previousEntry.agent + handoff.providerId !== providerId || + handoff.agent !== input.previousEntry.agent || + handoff.replacement.source.contract.agent !== handoff.agent || + handoff.replacementProfile.profile.agent !== handoff.agent ) { throw new ManagedWorkloadRebuildTransactionError( "prepare", "the managed profile handoff does not match durable provider and agent authority", ); } - if (!isDeepStrictEqual(input.handoff.previousReceipt, input.previousEntry.workload)) { + if (!isDeepStrictEqual(handoff.previousReceipt, input.previousEntry.workload)) { throw new ManagedWorkloadRebuildTransactionError( "prepare", "the managed profile handoff is stale against the durable workload receipt", ); } if ( - input.handoff.previousReceipt.platform !== input.handoff.previousContract.platform || - input.handoff.replacement.source.contract.platform !== input.handoff.previousContract.platform + handoff.previousReceipt.platform !== handoff.previousContract.platform || + handoff.replacement.source.contract.platform !== handoff.previousContract.platform ) { throw new ManagedWorkloadRebuildTransactionError( "prepare", @@ -81,7 +85,7 @@ export function createManagedWorkloadRebuildPlan(input: { ); } const previousAuthority = captureSandboxRebuildAuthority(input.previousEntry, providerId); - const replacementReceipt = buildManagedWorkloadRebuildReceipt(input.handoff, input.provider); + const replacementReceipt = buildManagedWorkloadRebuildReceipt(handoff, input.provider); const transactionId = input.transactionId ?? randomUUID(); if (!/^[0-9A-Za-z][0-9A-Za-z._:-]{0,255}$/u.test(transactionId)) { throw new ManagedWorkloadRebuildTransactionError( @@ -89,14 +93,14 @@ export function createManagedWorkloadRebuildPlan(input: { "the rebuild transaction identity is invalid", ); } - return Object.freeze({ + return cloneAndDeepFreeze({ schemaVersion: 1, transactionId, sandboxName: input.previousEntry.name, providerId, - agent: input.handoff.agent, + agent: handoff.agent, previousAuthority, - handoff: input.handoff, + handoff, replacementReceipt, replacementMetadata: safeReplacementMetadata(input.replacementMetadata), }); diff --git a/src/lib/onboard/managed-workload/rebuild/recovery.ts b/src/lib/onboard/managed-workload/rebuild/recovery.ts new file mode 100644 index 00000000000..950597fb451 --- /dev/null +++ b/src/lib/onboard/managed-workload/rebuild/recovery.ts @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { cloneAndDeepFreeze } from "../../../core/immutable"; +import type { + ManagedWorkloadRebuildPlan, + ManagedWorkloadRebuildRecoveryTask, + StagedManagedWorkloadReplacement, +} from "./contract"; + +export function createManagedWorkloadRebuildRecoveryTask( + plan: ManagedWorkloadRebuildPlan, + replacement: StagedManagedWorkloadReplacement, + operation: ManagedWorkloadRebuildRecoveryTask["operation"], +): ManagedWorkloadRebuildRecoveryTask { + return cloneAndDeepFreeze({ + schemaVersion: 1, + owner: "durable-managed-workload-recovery", + operation, + transactionId: plan.transactionId, + sandboxName: plan.sandboxName, + providerId: plan.providerId, + previousRuntimeHandle: replacement.previousRuntimeHandle, + stagingHandle: replacement.stagingHandle, + previousAuthority: plan.previousAuthority, + replacement: { + agent: plan.agent, + receipt: plan.replacementReceipt, + lifecycleGeneration: replacement.lifecycleGeneration, + liveIdentityFingerprint: replacement.liveIdentityFingerprint, + }, + }); +} diff --git a/src/lib/onboard/managed-workload/rebuild/transaction.ts b/src/lib/onboard/managed-workload/rebuild/transaction.ts index be88fccdd9d..2910b00e5ab 100644 --- a/src/lib/onboard/managed-workload/rebuild/transaction.ts +++ b/src/lib/onboard/managed-workload/rebuild/transaction.ts @@ -13,12 +13,16 @@ import type { PreparedManagedWorkloadReplacement, StagedManagedWorkloadReplacement, } from "./contract"; -import { ManagedWorkloadRebuildTransactionError } from "./contract"; +import { + ManagedWorkloadRebuildIndeterminatePublicationError, + ManagedWorkloadRebuildTransactionError, +} from "./contract"; import { createStagedManagedWorkloadReplacement } from "./create"; import { createManagedWorkloadRebuildPlan } from "./plan"; import { prepareManagedWorkloadReplacement } from "./prepare"; import { rebindStagedManagedWorkloadProviders } from "./provider-rebind"; import { requireReadyManagedWorkloadReplacement } from "./readiness"; +import { createManagedWorkloadRebuildRecoveryTask } from "./recovery"; import { restoreStagedManagedWorkloadState } from "./restore"; import { createManagedWorkloadPreparationAbort, @@ -82,14 +86,28 @@ export async function runManagedWorkloadRebuildTransaction( const readSandbox = dependencies.getSandbox ?? readSandboxFromRegistry; const plan = createManagedWorkloadRebuildPlan(input); const abortPreparation = createManagedWorkloadPreparationAbort(plan, input.operations); - const stillAuthoritative = (): boolean => - sandboxRebuildAuthorityMatchesEntry(plan.previousAuthority, readSandbox(plan.sandboxName)); - if (!stillAuthoritative()) { - throw new ManagedWorkloadRebuildTransactionError( - "prepare", - "the durable workload changed before provider preparation", - ); - } + const requireOldAuthority = (timing: "before" | "during"): void => { + let stillAuthoritative: boolean; + try { + stillAuthoritative = sandboxRebuildAuthorityMatchesEntry( + plan.previousAuthority, + readSandbox(plan.sandboxName), + ); + } catch (error) { + throw new ManagedWorkloadRebuildTransactionError( + "prepare", + `the durable workload could not be revalidated ${timing} provider preparation`, + { cause: error }, + ); + } + if (!stillAuthoritative) { + throw new ManagedWorkloadRebuildTransactionError( + "prepare", + `the durable workload changed ${timing} provider preparation`, + ); + } + }; + requireOldAuthority("before"); let prepared: PreparedManagedWorkloadReplacement; try { @@ -98,14 +116,10 @@ export async function runManagedWorkloadRebuildTransaction( if (input.operations.providerId !== plan.providerId) throw error; return failAfterCleanup(error, () => abortPreparation.run()); } - if (!stillAuthoritative()) { - return failAfterCleanup( - new ManagedWorkloadRebuildTransactionError( - "prepare", - "the durable workload changed during provider preparation", - ), - () => abortPreparation.run(), - ); + try { + requireOldAuthority("during"); + } catch (error) { + return failAfterCleanup(error, () => abortPreparation.run()); } let staged: StagedManagedWorkloadReplacement; @@ -135,9 +149,11 @@ export async function runManagedWorkloadRebuildTransaction( entry, previousCleanup: "pending", cleanupError, + recoveryTask: createManagedWorkloadRebuildRecoveryTask(plan, rebound, "retire-previous"), }; } } catch (error) { + if (error instanceof ManagedWorkloadRebuildIndeterminatePublicationError) throw error; return failAfterCleanup(error, () => rollback.run()); } } diff --git a/src/lib/onboard/sandbox-workload-authority.test.ts b/src/lib/onboard/sandbox-workload-authority.test.ts index d98f72085c8..e10118c8b2d 100644 --- a/src/lib/onboard/sandbox-workload-authority.test.ts +++ b/src/lib/onboard/sandbox-workload-authority.test.ts @@ -71,6 +71,10 @@ describe("managed workload authority", () => { receipt: { kind: "managed-image", platform }, }); expect(authority?.receipt).not.toBe(managedEntry(agent, platform).workload); + expect(Object.isFrozen(authority)).toBe(true); + expect(Object.isFrozen(authority?.receipt)).toBe(true); + expect(Object.isFrozen(authority?.contract.source)).toBe(true); + expect(Object.isFrozen(authority?.profile.proxy)).toBe(true); }); it("returns null only for an unambiguously non-managed workload", () => { diff --git a/src/lib/onboard/sandbox-workload-rebuild.test.ts b/src/lib/onboard/sandbox-workload-rebuild.test.ts index 1bd1d7abc00..c40f7a9bfc1 100644 --- a/src/lib/onboard/sandbox-workload-rebuild.test.ts +++ b/src/lib/onboard/sandbox-workload-rebuild.test.ts @@ -305,6 +305,9 @@ describe("managed workload rebuild preflight", () => { version: "0.0.100", policy: "require-managed", }); + expect(Object.isFrozen(handoff)).toBe(true); + expect(Object.isFrozen(handoff?.previousProfile.proxy)).toBe(true); + expect(Object.isFrozen(handoff?.replacement.source.contract.source)).toBe(true); }); it.each(AGENTS)("prepares an arm64 replacement handoff for %s", async (agent) => { @@ -428,6 +431,25 @@ describe("managed workload rebuild preflight", () => { expect(Object.isFrozen(result)).toBe(true); }); + it("rejects a cross-agent replacement contract and profile before receipt creation", async () => { + managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource = vi.fn(async () => + replacement("openclaw"), + ); + const catalog = await prepareManagedWorkloadRebuildHandoff(entry("openclaw"), { + runtime: runtime(), + provider: provider(), + }); + const crossAgentHandoff: ManagedWorkloadRebuildHandoff = { + ...catalog!, + replacement: replacement("hermes"), + replacementProfile: profileTransport("hermes"), + }; + + expect(() => buildManagedWorkloadRebuildReceipt(crossAgentHandoff, provider())).toThrow( + /does not match the exact rebuild agent/u, + ); + }); + it.each(AGENTS)("stages authoritative startup-profile reconstruction for %s", async (agent) => { managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource = vi.fn(async () => replacement(agent), diff --git a/src/lib/onboard/workload/authority.ts b/src/lib/onboard/workload/authority.ts index e05da349b60..07f3153943a 100644 --- a/src/lib/onboard/workload/authority.ts +++ b/src/lib/onboard/workload/authority.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { cloneAndDeepFreeze } from "../../core/immutable"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; import { cloneSandboxWorkloadReceipt } from "../../state/registry/workload"; import type { ResolvedCorporateCa } from "../corporate-ca-types"; @@ -173,7 +174,7 @@ export function readManagedWorkloadAuthority( ); } - return Object.freeze({ + return cloneAndDeepFreeze({ agent, receipt: cloned, contract, diff --git a/src/lib/onboard/workload/rebuild.ts b/src/lib/onboard/workload/rebuild.ts index 6f78a880dbf..be143261c62 100644 --- a/src/lib/onboard/workload/rebuild.ts +++ b/src/lib/onboard/workload/rebuild.ts @@ -2,14 +2,16 @@ // SPDX-License-Identifier: Apache-2.0 import { isDeepStrictEqual } from "node:util"; - +import { cloneAndDeepFreeze } from "../../core/immutable"; import { getVersion } from "../../core/version"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; +import { cloneSandboxWorkloadReceipt } from "../../state/registry/workload"; import type { ResolvedCorporateCa } from "../corporate-ca-types"; import { MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, type ManagedImageContractV1, + parseManagedImageContractV1, type ShippedManagedImageAgent, } from "../managed-image/contract"; import { @@ -155,7 +157,7 @@ export async function prepareManagedWorkloadRebuildHandoff( ); } - return Object.freeze({ + return cloneAndDeepFreeze({ schemaVersion: 1 as const, providerId: options.provider.identity.id, agent: authority.agent, @@ -304,7 +306,7 @@ export function stageManagedWorkloadRebuildProfile( "the replacement startup profile does not match the selected managed-image agent", ); } - return Object.freeze({ ...handoff, replacementProfile }); + return cloneAndDeepFreeze({ ...handoff, replacementProfile }); } /** @@ -376,8 +378,30 @@ export function buildManagedWorkloadRebuildReceipt( "the replacement receipt does not belong to the selected provider", ); } - const contract = handoff.replacement.source.contract; + let contract: ManagedImageContractV1; + try { + contract = parseManagedImageContractV1( + handoff.replacement.source.contract, + handoff.agent, + handoff.previousContract.platform, + ); + } catch (error) { + throw new ManagedWorkloadRebuildError( + "the replacement image contract does not match the exact rebuild agent and platform", + { cause: error }, + ); + } + if (handoff.replacement.source.reference !== contract.reference) { + throw new ManagedWorkloadRebuildError( + "the replacement image source does not match its immutable image contract", + ); + } const profile = handoff.replacementProfile; + if (profile.profile.agent !== handoff.agent) { + throw new ManagedWorkloadRebuildError( + "the replacement startup profile does not match the exact rebuild agent", + ); + } const receipt: ManagedWorkloadReceipt = { schemaVersion: 1, kind: "managed-image", @@ -394,12 +418,18 @@ export function buildManagedWorkloadRebuildReceipt( ...(profile.corporateCaB64 === undefined ? {} : { corporateCaB64: profile.corporateCaB64 }), shared: true, }; - if (!provider.workload.acceptsReceipt(receipt)) { + const validatedReceipt = cloneSandboxWorkloadReceipt(receipt); + if (validatedReceipt?.kind !== "managed-image" || !isDeepStrictEqual(validatedReceipt, receipt)) { + throw new ManagedWorkloadRebuildError( + "the replacement startup profile and image contract do not form valid durable authority", + ); + } + if (!provider.workload.acceptsReceipt(validatedReceipt)) { throw new ManagedWorkloadRebuildError( `provider '${provider.identity.id}' rejected the replacement workload receipt`, ); } - return Object.freeze(receipt); + return cloneAndDeepFreeze(validatedReceipt); } export type ManagedWorkloadRebuildEntry = Pick< diff --git a/src/lib/state/registry-rebuild-authority.test.ts b/src/lib/state/registry-rebuild-authority.test.ts index 49998bb9993..34bc090d0df 100644 --- a/src/lib/state/registry-rebuild-authority.test.ts +++ b/src/lib/state/registry-rebuild-authority.test.ts @@ -91,6 +91,9 @@ describe("sandbox rebuild authority", () => { }); expect(authority.entryRevisionSha256).toMatch(/^[0-9a-f]{64}$/u); expect(authority.workload).not.toBe(source.workload); + expect(Object.isFrozen(authority)).toBe(true); + expect(Object.isFrozen(authority.workload)).toBe(true); + expect(Reflect.set(authority.workload, "reference", "mutated")).toBe(false); }); it.each([ @@ -169,6 +172,7 @@ describe("sandbox rebuild authority", () => { it.each([ ["sandbox name", (candidate: SandboxEntry) => ({ ...candidate, name: "other" })], + ["agent", (candidate: SandboxEntry) => ({ ...candidate, agent: "hermes" })], ["provider", (candidate: SandboxEntry) => ({ ...candidate, openshellDriver: "mxc" })], [ "lifecycle generation", @@ -214,5 +218,8 @@ describe("sandbox rebuild authority", () => { "docker", ), ).toThrow(/live identity fingerprint/u); + expect(() => captureSandboxRebuildAuthority({ ...entry(), agent: "hermes" }, "docker")).toThrow( + /agent does not match/u, + ); }); }); diff --git a/src/lib/state/registry/rebuild-authority.ts b/src/lib/state/registry/rebuild-authority.ts index 90ee4f77942..bf77ed65277 100644 --- a/src/lib/state/registry/rebuild-authority.ts +++ b/src/lib/state/registry/rebuild-authority.ts @@ -3,6 +3,12 @@ import { createHash } from "node:crypto"; import { isDeepStrictEqual } from "node:util"; +import { cloneAndDeepFreeze } from "../../core/immutable"; +import { + isShippedManagedImageAgent, + MANAGED_IMAGE_REPOSITORIES, + type ShippedManagedImageAgent, +} from "../../onboard/managed-image/contract"; import { withLock } from "./lock"; import { load, save } from "./persistence"; import type { SandboxEntry, SandboxRegistry, SandboxWorkloadReceipt } from "./types"; @@ -63,7 +69,24 @@ function clonedManagedReceipt(value: SandboxWorkloadReceipt | undefined): Manage "a managed replacement requires a valid durable workload receipt", ); } - return cloned; + return cloneAndDeepFreeze(cloned); +} + +function requireReceiptAgent( + agent: SandboxEntry["agent"], + workload: ManagedWorkloadReceipt, + label: string, +): ShippedManagedImageAgent { + if ( + typeof agent !== "string" || + !isShippedManagedImageAgent(agent) || + !workload.reference.startsWith(`${MANAGED_IMAGE_REPOSITORIES[agent]}@sha256:`) + ) { + throw new SandboxRebuildAuthorityError( + `${label} agent does not match its managed workload receipt`, + ); + } + return agent; } function cloneEntry(entry: SandboxEntry): SandboxEntry { @@ -114,12 +137,13 @@ export function captureSandboxRebuildAuthority( throw new SandboxRebuildAuthorityError("live identity fingerprint is missing or invalid"); } const workload = clonedManagedReceipt(entry.workload); + requireReceiptAgent(entry.agent, workload, "authoritative"); if (entry.imageTag !== workload.reference) { throw new SandboxRebuildAuthorityError( "image reference does not match the managed workload receipt", ); } - return Object.freeze({ + return cloneAndDeepFreeze({ schemaVersion: 1 as const, sandboxName: entry.name, providerId, @@ -201,6 +225,7 @@ function validateReplacement( ); } const workload = clonedManagedReceipt(replacement.workload); + requireReceiptAgent(replacement.agent, workload, "replacement"); if (replacement.imageTag !== workload.reference) { throw new SandboxRebuildAuthorityError( "replacement image reference does not match its managed workload receipt", From 412d69d574641e303ed85361607129b37a91dd18 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 03:48:26 -0700 Subject: [PATCH 041/117] docs(rebuild): define deferred recovery ownership Signed-off-by: Aaron Erickson --- .../managed-workload/rebuild/README.md | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/lib/onboard/managed-workload/rebuild/README.md diff --git a/src/lib/onboard/managed-workload/rebuild/README.md b/src/lib/onboard/managed-workload/rebuild/README.md new file mode 100644 index 00000000000..24515c6f12d --- /dev/null +++ b/src/lib/onboard/managed-workload/rebuild/README.md @@ -0,0 +1,28 @@ +# Managed workload rebuild boundary + +This directory is a dormant, provider-neutral transaction foundation. No CLI +command or production action imports it yet. Activation must wait until every +deferred outcome has a durable recovery owner. + +## Publication and cleanup ownership + +- A failed compare-and-swap is rollback-safe only when the exact old durable + authority is positively observed or the CAS reports that it did not write. +- An indeterminate publication leaves the staged runtime intact and returns an + exact `reconcile-publication` task for `durable-managed-workload-recovery`. +- A failed post-commit retirement returns an exact `retire-previous` task for + the same owner. The result object is only a handoff; a later recovery slice + must durably persist and reconcile it before this transaction can be wired + into a user-visible action. + +## Snapshot and backup boundary + +This slice neither emits nor consumes snapshot or backup manifests. +`restoreState` is a provider-owned rebuild phase receipt, not a backup format +or proof of managed backup authority. + +The next snapshot/backup slice (PR3.8 in the current stack) owns the shared +managed-backup-authority helper and must wire every relevant caller together: +snapshot creation, `backup --all`, stopped-sandbox backup, and production +rebuild. Until those callers produce the same managed manifest accepted by the +restore gate, this rebuild transaction remains inert. From cfe84cd77992e1b5fd8352704d5a5d3164601dfd Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 03:55:52 -0700 Subject: [PATCH 042/117] fix(rebuild): bind retained workload authority Signed-off-by: Aaron Erickson --- ...naged-workload-rebuild-transaction.test.ts | 38 ++++++++++++++++++- .../onboard/managed-workload/rebuild/plan.ts | 25 +++++++++++- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts index f655d3843f9..858cb88dd54 100644 --- a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts +++ b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts @@ -22,7 +22,10 @@ import { type ShippedManagedImageAgent, } from "./managed-image/contract"; import type { BuiltManagedStartupOnboardProfile } from "./managed-startup/onboard-profile"; -import { encodeManagedStartupProfile } from "./managed-startup/profile"; +import { + decodeManagedStartupProfile, + encodeManagedStartupProfile, +} from "./managed-startup/profile"; import type { ManagedWorkloadRebuildProviderOperations, PreparedManagedWorkloadReplacement, @@ -133,7 +136,9 @@ function handoff( ): ManagedWorkloadRebuildHandoff { const previousContract = contract(agent, "old", platform); const replacementContract = contract(agent, "new", platform); - const previousProfile = managedStartupE2eProfile(agent); + const previousProfile = decodeManagedStartupProfile( + encodeManagedStartupProfile(managedStartupE2eProfile(agent)), + ); return { schemaVersion: 1, providerId, @@ -658,6 +663,35 @@ describe("managed workload rebuild transaction", () => { expect(operations.prepare).not.toHaveBeenCalled(); }); + it("rejects stale previous contract authority before provider mutation", async () => { + const oldEntry = previousEntry("openclaw", "mxc"); + const staleHandoff = handoff("openclaw", "mxc"); + const operations = operationsHarness("mxc", []); + + await expect( + runManagedWorkloadRebuildTransaction( + { + previousEntry: oldEntry, + provider: bundle("mxc"), + handoff: { + ...staleHandoff, + previousContract: { + ...staleHandoff.previousContract, + source: { + ...staleHandoff.previousContract.source, + release: "v0.0.98", + }, + }, + }, + operations, + transactionId: "transaction-1", + }, + { getSandbox: () => structuredClone(oldEntry) }, + ), + ).rejects.toMatchObject({ phase: "prepare" }); + expect(operations.prepare).not.toHaveBeenCalled(); + }); + it("rejects a cross-agent replacement contract and profile before provider mutation", async () => { const oldEntry = previousEntry("openclaw", "mxc"); const openClawHandoff = handoff("openclaw", "mxc"); diff --git a/src/lib/onboard/managed-workload/rebuild/plan.ts b/src/lib/onboard/managed-workload/rebuild/plan.ts index 23bf014d6e3..8dfb8efb234 100644 --- a/src/lib/onboard/managed-workload/rebuild/plan.ts +++ b/src/lib/onboard/managed-workload/rebuild/plan.ts @@ -11,6 +11,7 @@ import { normalizeRuntimeProviderIdentity, requireRuntimeProviderMutationAuthority, } from "../../runtime-provider/registry"; +import { readManagedWorkloadAuthority } from "../../workload/authority"; import { buildManagedWorkloadRebuildReceipt, type ManagedWorkloadRebuildHandoff, @@ -69,10 +70,30 @@ export function createManagedWorkloadRebuildPlan(input: { "the managed profile handoff does not match durable provider and agent authority", ); } - if (!isDeepStrictEqual(handoff.previousReceipt, input.previousEntry.workload)) { + let durableAuthority: NonNullable>; + try { + const candidate = readManagedWorkloadAuthority(input.previousEntry); + if (!candidate) { + throw new Error("the durable row is not a managed workload"); + } + durableAuthority = candidate; + } catch (error) { throw new ManagedWorkloadRebuildTransactionError( "prepare", - "the managed profile handoff is stale against the durable workload receipt", + "the durable managed workload authority could not be validated", + { cause: error }, + ); + } + if ( + durableAuthority.agent !== handoff.agent || + !isDeepStrictEqual(durableAuthority.receipt, handoff.previousReceipt) || + !isDeepStrictEqual(durableAuthority.contract, handoff.previousContract) || + !isDeepStrictEqual(durableAuthority.profile, handoff.previousProfile) || + !isDeepStrictEqual(durableAuthority.corporateCa, handoff.corporateCa) + ) { + throw new ManagedWorkloadRebuildTransactionError( + "prepare", + "the managed profile handoff is stale against exact durable workload authority", ); } if ( From 827f545bf9f99bb69bb2e87ba85091558213b5fc Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 04:05:35 -0700 Subject: [PATCH 043/117] test(rebuild): keep failure matrices linear Represent agent fixtures as data and route injected failures through one named helper. This preserves every rebuild scenario without adding conditional statements to test files. Signed-off-by: Aaron Erickson --- ...naged-workload-rebuild-transaction.test.ts | 38 +++++++++------ .../onboard/sandbox-workload-rebuild.test.ts | 47 +++++++++---------- 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts index 858cb88dd54..fe69784286d 100644 --- a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts +++ b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts @@ -47,6 +47,14 @@ const PLATFORMS = ["linux/amd64", "linux/arm64"] as const; const OLD_RELEASE = "v0.0.99"; const NEW_RELEASE = "v0.0.100"; +function raiseInjectedFailure(message: string): never { + throw new Error(message); +} + +function failWhenInjected(condition: boolean, message: string): void { + condition ? raiseInjectedFailure(message) : undefined; +} + function contract( agent: ShippedManagedImageAgent, generation: "old" | "new", @@ -341,9 +349,10 @@ function transactionHarness( replacement: SandboxEntry, ): SandboxRebuildAuthoritySwapResult => { events.push("registry-commit"); - if (failAt === "registry-commit-before-persist") { - throw new Error("registry write failed before persistence"); - } + failWhenInjected( + failAt === "registry-commit-before-persist", + "registry write failed before persistence", + ); const currentRegistry: SandboxRegistry = { sandboxes: { [oldEntry.name]: currentEntry }, defaultSandbox: oldEntry.name, @@ -360,12 +369,11 @@ function transactionHarness( : swapSandboxRebuildAuthorityInRegistry(currentRegistry, expected, replacement); currentEntry = swapped.result.status === "committed" ? structuredClone(swapped.result.entry) : currentEntry; - if ( + failWhenInjected( failAt === "registry-commit-after-persist" || - failAt === "registry-commit-after-persist-read-fails" - ) { - throw new Error("registry acknowledgement lost after persistence"); - } + failAt === "registry-commit-after-persist-read-fails", + "registry acknowledgement lost after persistence", + ); return swapped.result; }; return { @@ -386,12 +394,14 @@ function transactionHarness( { getSandbox: () => { registryReadCount += 1; - if (failAt === "registry-read-after-prepare" && registryReadCount === 2) { - throw new Error("registry read failed after provider preparation"); - } - if (failAt === "registry-commit-after-persist-read-fails" && registryReadCount >= 3) { - throw new Error("registry readback failed after ambiguous persistence"); - } + failWhenInjected( + failAt === "registry-read-after-prepare" && registryReadCount === 2, + "registry read failed after provider preparation", + ); + failWhenInjected( + failAt === "registry-commit-after-persist-read-fails" && registryReadCount >= 3, + "registry readback failed after ambiguous persistence", + ); return structuredClone(currentEntry); }, commitAuthority, diff --git a/src/lib/onboard/sandbox-workload-rebuild.test.ts b/src/lib/onboard/sandbox-workload-rebuild.test.ts index c40f7a9bfc1..d838351cf2d 100644 --- a/src/lib/onboard/sandbox-workload-rebuild.test.ts +++ b/src/lib/onboard/sandbox-workload-rebuild.test.ts @@ -62,8 +62,8 @@ function rebuildProfileInput(agent: ShippedManagedImageAgent): RebuildProfileInp dcodeAutoApprovalMode: "disabled" as const, observabilityEnabled: false, }; - if (agent === "openclaw") { - return { + const profiles = { + openclaw: { ...common, inference: { routeProvider: "openai", @@ -77,10 +77,8 @@ function rebuildProfileInput(agent: ShippedManagedImageAgent): RebuildProfileInp }, manageDashboard: true, hermesDashboardState: { config: null, enabled: false }, - }; - } - if (agent === "hermes") { - return { + }, + hermes: { ...common, inference: { routeProvider: "inference", @@ -94,25 +92,26 @@ function rebuildProfileInput(agent: ShippedManagedImageAgent): RebuildProfileInp }, manageDashboard: true, hermesDashboardState: { config: null, enabled: false }, - }; - } - return { - ...common, - inference: { - routeProvider: "inference", - upstreamProvider: "openrouter", - model: "openai/gpt-5.4", - routedBaseUrl: "https://inference.local/v1", - upstreamEndpointUrl: "https://openrouter.ai/api/v1", - api: "openai-completions", - primaryModelRef: null, - compatibility: null, }, - chatUiUrl: "", - effectiveDashboardPort: 0, - manageDashboard: false, - hermesDashboardState: { config: null, enabled: false }, - }; + "langchain-deepagents-code": { + ...common, + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + model: "openai/gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + chatUiUrl: "", + effectiveDashboardPort: 0, + manageDashboard: false, + hermesDashboardState: { config: null, enabled: false }, + }, + } as const satisfies Record; + return profiles[agent]; } function managedContract( From add39657c6c33ec27f212b173cfdb82add8bfda5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 04:19:55 -0700 Subject: [PATCH 044/117] test(rebuild): cover invalid provider artifacts Signed-off-by: Aaron Erickson --- ...naged-workload-rebuild-transaction.test.ts | 220 ++++++++++++++++++ 1 file changed, 220 insertions(+) diff --git a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts index fe69784286d..fd850532f2d 100644 --- a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts +++ b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts @@ -410,6 +410,204 @@ function transactionHarness( }; } +type RebuildOperationName = + | "create" + | "waitUntilReady" + | "restoreState" + | "rebindProviders" + | "retirePrevious"; + +interface InvalidProviderArtifactCase { + readonly name: string; + readonly phase: Exclude; + readonly install: (operations: ManagedWorkloadRebuildProviderOperations) => void; + readonly events: readonly string[]; + readonly notCalled: readonly RebuildOperationName[]; + readonly abortCalls: number; + readonly rollbackCalls: number; +} + +const INVALID_PROVIDER_ARTIFACT_CASES: readonly InvalidProviderArtifactCase[] = [ + { + name: "rejects a prepared artifact bound to another provider", + phase: "prepare", + install: (operations) => { + const prepare = operations.prepare; + operations.prepare = vi.fn(async (plan) => ({ + ...(await prepare(plan)), + providerId: "other-provider", + })); + }, + events: ["prepare", "abort-preparation:transaction-1"], + notCalled: ["create", "waitUntilReady", "restoreState", "rebindProviders", "retirePrevious"], + abortCalls: 1, + rollbackCalls: 0, + }, + { + name: "rejects a prepared artifact with a NUL-bearing preparation handle", + phase: "prepare", + install: (operations) => { + const prepare = operations.prepare; + operations.prepare = vi.fn(async (plan) => ({ + ...(await prepare(plan)), + preparationHandle: "preparation\0invalid", + })); + }, + events: ["prepare", "abort-preparation:transaction-1"], + notCalled: ["create", "waitUntilReady", "restoreState", "rebindProviders", "retirePrevious"], + abortCalls: 1, + rollbackCalls: 0, + }, + { + name: "rejects a staged artifact that changes the exact old handle", + phase: "create", + install: (operations) => { + const create = operations.create; + operations.create = vi.fn(async (plan, prepared) => ({ + ...(await create(plan, prepared)), + previousRuntimeHandle: "runtime-old-substituted", + })); + }, + events: ["prepare", "create", "abort-preparation:transaction-1"], + notCalled: ["waitUntilReady", "restoreState", "rebindProviders", "retirePrevious"], + abortCalls: 1, + rollbackCalls: 0, + }, + { + name: "rejects a staged artifact with an oversized staging handle", + phase: "create", + install: (operations) => { + const create = operations.create; + operations.create = vi.fn(async (plan, prepared) => ({ + ...(await create(plan, prepared)), + stagingHandle: "x".repeat(16 * 1024 + 1), + })); + }, + events: ["prepare", "create", "abort-preparation:transaction-1"], + notCalled: ["waitUntilReady", "restoreState", "rebindProviders", "retirePrevious"], + abortCalls: 1, + rollbackCalls: 0, + }, + { + name: "rejects a ready artifact that changes the exact staging handle", + phase: "readiness", + install: (operations) => { + const waitUntilReady = operations.waitUntilReady; + operations.waitUntilReady = vi.fn(async (plan, staged) => { + const result = await waitUntilReady(plan, staged); + const ready = result as Extract; + return { + state: "ready" as const, + replacement: { + ...ready.replacement, + stagingHandle: "runtime-new-substituted", + }, + }; + }); + }, + events: ["prepare", "create", "readiness", "rollback:runtime-new-staged-exact"], + notCalled: ["restoreState", "rebindProviders", "retirePrevious"], + abortCalls: 0, + rollbackCalls: 1, + }, + { + name: "rejects a ready artifact with a NUL-bearing readiness receipt", + phase: "readiness", + install: (operations) => { + const waitUntilReady = operations.waitUntilReady; + operations.waitUntilReady = vi.fn(async (plan, staged) => { + const result = await waitUntilReady(plan, staged); + const ready = result as Extract; + return { + state: "ready" as const, + replacement: { + ...ready.replacement, + readinessReceipt: "ready\0invalid", + }, + }; + }); + }, + events: ["prepare", "create", "readiness", "rollback:runtime-new-staged-exact"], + notCalled: ["restoreState", "rebindProviders", "retirePrevious"], + abortCalls: 0, + rollbackCalls: 1, + }, + { + name: "rejects a restored artifact that alters the readiness receipt", + phase: "restore", + install: (operations) => { + const restoreState = operations.restoreState; + operations.restoreState = vi.fn(async (plan, ready) => ({ + ...(await restoreState(plan, ready)), + readinessReceipt: "ready-substituted", + })); + }, + events: ["prepare", "create", "readiness", "restore", "rollback:runtime-new-staged-exact"], + notCalled: ["rebindProviders", "retirePrevious"], + abortCalls: 0, + rollbackCalls: 1, + }, + { + name: "rejects a restored artifact with an oversized restore receipt", + phase: "restore", + install: (operations) => { + const restoreState = operations.restoreState; + operations.restoreState = vi.fn(async (plan, ready) => ({ + ...(await restoreState(plan, ready)), + restoreReceipt: "x".repeat(16 * 1024 + 1), + })); + }, + events: ["prepare", "create", "readiness", "restore", "rollback:runtime-new-staged-exact"], + notCalled: ["rebindProviders", "retirePrevious"], + abortCalls: 0, + rollbackCalls: 1, + }, + { + name: "rejects a rebound artifact that alters the restore receipt", + phase: "provider-rebind", + install: (operations) => { + const rebindProviders = operations.rebindProviders; + operations.rebindProviders = vi.fn(async (plan, restored) => ({ + ...(await rebindProviders(plan, restored)), + restoreReceipt: "restore-substituted", + })); + }, + events: [ + "prepare", + "create", + "readiness", + "restore", + "provider-rebind", + "rollback:runtime-new-staged-exact", + ], + notCalled: ["retirePrevious"], + abortCalls: 0, + rollbackCalls: 1, + }, + { + name: "rejects a rebound artifact bound to another transaction", + phase: "provider-rebind", + install: (operations) => { + const rebindProviders = operations.rebindProviders; + operations.rebindProviders = vi.fn(async (plan, restored) => ({ + ...(await rebindProviders(plan, restored)), + transactionId: "other-transaction", + })); + }, + events: [ + "prepare", + "create", + "readiness", + "restore", + "provider-rebind", + "rollback:runtime-new-staged-exact", + ], + notCalled: ["retirePrevious"], + abortCalls: 0, + rollbackCalls: 1, + }, +]; + describe("managed workload rebuild transaction", () => { it.each( AGENTS.flatMap((agent) => @@ -491,6 +689,28 @@ describe("managed workload rebuild transaction", () => { expect(harness.currentEntry().lifecycleGeneration).toBe("generation-old"); }); + it.each(INVALID_PROVIDER_ARTIFACT_CASES)("$name and stops at the invalid transition", async ({ + phase, + install, + events, + notCalled, + abortCalls, + rollbackCalls, + }) => { + const harness = transactionHarness("langchain-deepagents-code", "mxc"); + install(harness.operations); + + await expect(harness.run()).rejects.toMatchObject({ phase }); + + expect(harness.currentEntry()).toEqual(harness.oldEntry); + expect(harness.events).toEqual(events); + expect(harness.operations.abortPreparation).toHaveBeenCalledTimes(abortCalls); + expect(harness.operations.rollback).toHaveBeenCalledTimes(rollbackCalls); + for (const operation of notCalled) { + expect(harness.operations[operation]).not.toHaveBeenCalled(); + } + }); + it("aborts preparation when non-authority registry metadata drifts", async () => { const events: string[] = []; const oldEntry = previousEntry("openclaw", "mxc"); From 9bbf75dc118c84e3f91f63722b92b5c471afe200 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 05:25:07 -0700 Subject: [PATCH 045/117] fix(rebuild): address review feedback Signed-off-by: Aaron Erickson --- src/lib/core/immutable.test.ts | 50 +++++++++++ src/lib/core/immutable.ts | 56 +++++++++++- ...aged-workload-rebuild-source-shape.test.ts | 25 ++---- ...naged-workload-rebuild-transaction.test.ts | 68 +++++++++++--- .../managed-workload/rebuild/README.md | 33 ++++--- .../managed-workload/rebuild/contract.ts | 5 ++ .../onboard/managed-workload/rebuild/index.ts | 6 -- .../managed-workload/rebuild/transaction.ts | 7 +- .../sandbox-workload-authority.test.ts | 7 +- .../onboard/sandbox-workload-rebuild.test.ts | 3 +- src/lib/onboard/workload/rebuild.ts | 15 +--- .../state/registry-rebuild-authority.test.ts | 88 ++++++++++++++++++- src/lib/state/registry/rebuild-authority.ts | 35 ++++---- 13 files changed, 311 insertions(+), 87 deletions(-) create mode 100644 src/lib/core/immutable.test.ts delete mode 100644 src/lib/onboard/managed-workload/rebuild/index.ts diff --git a/src/lib/core/immutable.test.ts b/src/lib/core/immutable.test.ts new file mode 100644 index 00000000000..75521535283 --- /dev/null +++ b/src/lib/core/immutable.test.ts @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { cloneAndDeepFreeze } from "./immutable"; + +describe("cloneAndDeepFreeze", () => { + it("owns and recursively freezes plain data", () => { + const source = { + nested: { enabled: true }, + entries: ["one", { value: 2 }], + optional: undefined, + }; + + const result = cloneAndDeepFreeze(source); + + expect(result).toEqual(source); + expect(result).not.toBe(source); + expect(result.nested).not.toBe(source.nested); + expect(Object.isFrozen(result)).toBe(true); + expect(Object.isFrozen(result.nested)).toBe(true); + expect(Object.isFrozen(result.entries)).toBe(true); + expect(Object.isFrozen(result.entries[1])).toBe(true); + }); + + it.each([ + ["Map", () => new Map([["key", "value"]])], + ["Set", () => new Set(["value"])], + ["Date", () => new Date(0)], + ["ArrayBuffer", () => new ArrayBuffer(8)], + ["typed array", () => new Uint8Array([1, 2, 3])], + ["Buffer", () => Buffer.from("binary", "utf8")], + ["function", () => () => undefined], + ] as const)("rejects %s input outside the plain-data contract", (_label, createValue) => { + expect(() => cloneAndDeepFreeze(createValue())).toThrow(/plain|stateful|binary/u); + }); + + it("rejects accessor and symbol-keyed properties", () => { + const accessor = {}; + Object.defineProperty(accessor, "value", { + enumerable: true, + get: () => "computed", + }); + const symbolKeyed = { [Symbol("authority")]: "hidden" }; + + expect(() => cloneAndDeepFreeze(accessor)).toThrow(/data properties/u); + expect(() => cloneAndDeepFreeze(symbolKeyed)).toThrow(/symbol-keyed/u); + }); +}); diff --git a/src/lib/core/immutable.ts b/src/lib/core/immutable.ts index 1576487b77c..a96f6bfe4a9 100644 --- a/src/lib/core/immutable.ts +++ b/src/lib/core/immutable.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 function deepFreezeOwnedValue(value: T, seen: WeakSet): T { - if ((typeof value !== "object" && typeof value !== "function") || value === null) return value; + if (typeof value !== "object" || value === null) return value; const object = value as object; if (seen.has(object)) return value; seen.add(object); @@ -13,11 +13,61 @@ function deepFreezeOwnedValue(value: T, seen: WeakSet): T { return Object.freeze(value); } +function assertPlainData(value: unknown, seen: WeakSet): void { + if ( + value === null || + value === undefined || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" || + typeof value === "bigint" + ) { + return; + } + if (typeof value !== "object") { + throw new TypeError("cloneAndDeepFreeze accepts plain data only"); + } + if ( + value instanceof Date || + value instanceof Map || + value instanceof Set || + value instanceof ArrayBuffer || + ArrayBuffer.isView(value) + ) { + throw new TypeError("cloneAndDeepFreeze does not accept stateful or binary objects"); + } + + const object = value as object; + if (seen.has(object)) return; + const prototype = Object.getPrototypeOf(object); + if (!Array.isArray(object) && prototype !== Object.prototype && prototype !== null) { + throw new TypeError("cloneAndDeepFreeze accepts plain objects and arrays only"); + } + seen.add(object); + + for (const key of Reflect.ownKeys(object)) { + if (Array.isArray(object) && key === "length") continue; + if (typeof key !== "string") { + throw new TypeError("cloneAndDeepFreeze does not accept symbol-keyed data"); + } + const descriptor = Object.getOwnPropertyDescriptor(object, key); + if (!descriptor || !descriptor.enumerable || !("value" in descriptor)) { + throw new TypeError("cloneAndDeepFreeze accepts enumerable data properties only"); + } + assertPlainData(descriptor.value, seen); + } +} + /** - * Take ownership of structured data before exposing it across an adapter - * boundary. The clone prevents retained caller aliases and the recursive + * Take ownership of plain data before exposing it across an adapter boundary. + * + * Supported inputs are primitives, arrays, and plain objects composed only of + * enumerable data properties. Stateful or executable values such as Map, Set, + * Date, ArrayBuffer, typed arrays, Buffer, accessors, symbols, and functions + * are rejected. The clone prevents retained caller aliases and the recursive * freeze prevents a provider from changing nested authority after validation. */ export function cloneAndDeepFreeze(value: T): T { + assertPlainData(value, new WeakSet()); return deepFreezeOwnedValue(structuredClone(value), new WeakSet()); } diff --git a/src/lib/onboard/managed-workload-rebuild-source-shape.test.ts b/src/lib/onboard/managed-workload-rebuild-source-shape.test.ts index 9d7650a8245..5f8c2376e7e 100644 --- a/src/lib/onboard/managed-workload-rebuild-source-shape.test.ts +++ b/src/lib/onboard/managed-workload-rebuild-source-shape.test.ts @@ -7,26 +7,20 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; const ROOT = path.resolve(import.meta.dirname, "managed-workload/rebuild"); -const CENTRAL_REBUILD_MODULES = [ - "commit.ts", - "contract.ts", - "create.ts", - "plan.ts", - "prepare.ts", - "provider-rebind.ts", - "readiness.ts", - "recovery.ts", - "restore.ts", - "rollback.ts", - "transaction.ts", - "validation.ts", -] as const; +const CENTRAL_REBUILD_MODULES = fs + .readdirSync(ROOT) + .filter((file) => file.endsWith(".ts") && !file.endsWith(".test.ts")) + .sort(); -function source(file: (typeof CENTRAL_REBUILD_MODULES)[number]): string { +function source(file: string): string { return fs.readFileSync(path.join(ROOT, file), "utf8"); } describe("managed workload rebuild source shape", () => { + it("discovers production rebuild modules", () => { + expect(CENTRAL_REBUILD_MODULES).not.toHaveLength(0); + }); + it.each( CENTRAL_REBUILD_MODULES, )("keeps %s free of provider-specific imports and switches", (file) => { @@ -54,7 +48,6 @@ describe("managed workload rebuild source shape", () => { expect(contract).toContain("stagingHandle"); expect(contract).toContain("retirePrevious("); expect(contract).toContain("rollback("); - expect(contract).toContain("A sandbox name is\n * intentionally insufficient authority"); }); it("publishes only through the rebuild-authority CAS boundary", () => { diff --git a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts index fd850532f2d..659334f8fa7 100644 --- a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts +++ b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts @@ -26,13 +26,14 @@ import { decodeManagedStartupProfile, encodeManagedStartupProfile, } from "./managed-startup/profile"; -import type { - ManagedWorkloadRebuildProviderOperations, - PreparedManagedWorkloadReplacement, - ReadyManagedWorkloadReplacement, - ReboundManagedWorkloadReplacement, - RestoredManagedWorkloadReplacement, - StagedManagedWorkloadReplacement, +import { + type ManagedWorkloadRebuildProviderOperations, + ManagedWorkloadRebuildTransactionError, + type PreparedManagedWorkloadReplacement, + type ReadyManagedWorkloadReplacement, + type ReboundManagedWorkloadReplacement, + type RestoredManagedWorkloadReplacement, + type StagedManagedWorkloadReplacement, } from "./managed-workload/rebuild/contract"; import { createManagedWorkloadReplacementRollback } from "./managed-workload/rebuild/rollback"; import { runManagedWorkloadRebuildTransaction } from "./managed-workload/rebuild/transaction"; @@ -342,8 +343,15 @@ function transactionHarness( const events: string[] = []; const oldEntry = previousEntry(agent, providerId, platform); let currentEntry = structuredClone(oldEntry); - let registryReadCount = 0; + let providerPreparationCompleted = false; + let ambiguousPersistenceReadback = false; const operations = operationsHarness(providerId, events, failAt); + const prepare = operations.prepare; + operations.prepare = vi.fn(async (plan) => { + const prepared = await prepare(plan); + providerPreparationCompleted = true; + return prepared; + }); const commitAuthority = ( expected: ReturnType, replacement: SandboxEntry, @@ -369,6 +377,7 @@ function transactionHarness( : swapSandboxRebuildAuthorityInRegistry(currentRegistry, expected, replacement); currentEntry = swapped.result.status === "committed" ? structuredClone(swapped.result.entry) : currentEntry; + ambiguousPersistenceReadback = failAt === "registry-commit-after-persist-read-fails"; failWhenInjected( failAt === "registry-commit-after-persist" || failAt === "registry-commit-after-persist-read-fails", @@ -393,15 +402,16 @@ function transactionHarness( }, { getSandbox: () => { - registryReadCount += 1; failWhenInjected( - failAt === "registry-read-after-prepare" && registryReadCount === 2, + failAt === "registry-read-after-prepare" && providerPreparationCompleted, "registry read failed after provider preparation", ); + providerPreparationCompleted = false; failWhenInjected( - failAt === "registry-commit-after-persist-read-fails" && registryReadCount >= 3, + failAt === "registry-commit-after-persist-read-fails" && ambiguousPersistenceReadback, "registry readback failed after ambiguous persistence", ); + ambiguousPersistenceReadback = false; return structuredClone(currentEntry); }, commitAuthority, @@ -711,7 +721,7 @@ describe("managed workload rebuild transaction", () => { } }); - it("aborts preparation when non-authority registry metadata drifts", async () => { + it("aborts preparation when durable registry metadata drifts during preparation", async () => { const events: string[] = []; const oldEntry = previousEntry("openclaw", "mxc"); let currentEntry = structuredClone(oldEntry); @@ -754,6 +764,35 @@ describe("managed workload rebuild transaction", () => { expect(harness.currentEntry()).toEqual(harness.oldEntry); }); + it("preserves the original phase and message when preparation abort also fails", async () => { + const harness = transactionHarness("openclaw", "mxc", "abort-preparation"); + const prepare = harness.operations.prepare; + harness.operations.prepare = vi.fn(async (plan) => { + await prepare(plan); + throw new ManagedWorkloadRebuildTransactionError("prepare", "original prepare failure"); + }); + + let failure: unknown; + try { + await harness.run(); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(Error); + const transactionFailure = failure as Error & { + readonly phase: string; + readonly rollbackError: unknown; + }; + expect(transactionFailure.phase).toBe("prepare"); + expect(transactionFailure.message).toContain("original prepare failure"); + expect(transactionFailure.rollbackError).toMatchObject({ + message: "abort-preparation injected failure", + }); + expect(harness.events).toEqual(["prepare", "abort-preparation:transaction-1"]); + expect(harness.operations.create).not.toHaveBeenCalled(); + expect(harness.operations.rollback).not.toHaveBeenCalled(); + }); + it("reports exact-old cleanup as pending without undoing a committed replacement", async () => { const harness = transactionHarness("langchain-deepagents-code", "mxc", "retire-previous"); @@ -855,6 +894,7 @@ describe("managed workload rebuild transaction", () => { const rollback = createManagedWorkloadReplacementRollback(plan, staged, providerOperations); await Promise.all([rollback.run(), rollback.run(), rollback.run()]); + await rollback.run(); expect(providerOperations.rollback).toHaveBeenCalledOnce(); expect(events).toEqual(["rollback:runtime-new-staged-exact"]); @@ -889,7 +929,7 @@ describe("managed workload rebuild transaction", () => { }, { getSandbox: () => structuredClone(oldEntry) }, ), - ).rejects.toThrow(); + ).rejects.toMatchObject({ phase: "prepare" }); expect(operations.prepare).not.toHaveBeenCalled(); }); @@ -1000,6 +1040,8 @@ describe("managed workload rebuild transaction", () => { { getSandbox: () => structuredClone(oldEntry) }, ), ).rejects.toMatchObject({ phase: "prepare" }); + expect(operations.prepare).not.toHaveBeenCalled(); + expect(operations.abortPreparation).not.toHaveBeenCalled(); expect(operations.create).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/onboard/managed-workload/rebuild/README.md b/src/lib/onboard/managed-workload/rebuild/README.md index 24515c6f12d..2f83dbf579c 100644 --- a/src/lib/onboard/managed-workload/rebuild/README.md +++ b/src/lib/onboard/managed-workload/rebuild/README.md @@ -1,8 +1,12 @@ + + + # Managed workload rebuild boundary This directory is a dormant, provider-neutral transaction foundation. No CLI command or production action imports it yet. Activation must wait until every -deferred outcome has a durable recovery owner. +deferred outcome has a durable recovery owner. This change migrates only the +transaction boundary; it does not activate buildless rebuilds. ## Publication and cleanup ownership @@ -11,18 +15,23 @@ deferred outcome has a durable recovery owner. - An indeterminate publication leaves the staged runtime intact and returns an exact `reconcile-publication` task for `durable-managed-workload-recovery`. - A failed post-commit retirement returns an exact `retire-previous` task for - the same owner. The result object is only a handoff; a later recovery slice - must durably persist and reconcile it before this transaction can be wired - into a user-visible action. + the same owner. The result object is only a handoff; the + [durable recovery work tracked by epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) + must persist and reconcile it before this transaction can be wired into a + user-visible action. ## Snapshot and backup boundary -This slice neither emits nor consumes snapshot or backup manifests. -`restoreState` is a provider-owned rebuild phase receipt, not a backup format -or proof of managed backup authority. +This slice neither emits nor consumes snapshot or backup manifests. `restoreState` +is a provider-owned rebuild phase receipt, not a backup format or proof of +managed backup authority. + +The [snapshot and backup work tracked by epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) +owns the shared managed-backup-authority helper and must wire every relevant +caller together: snapshot creation, `backup --all`, stopped-sandbox backup, and +production rebuild. -The next snapshot/backup slice (PR3.8 in the current stack) owns the shared -managed-backup-authority helper and must wire every relevant caller together: -snapshot creation, `backup --all`, stopped-sandbox backup, and production -rebuild. Until those callers produce the same managed manifest accepted by the -restore gate, this rebuild transaction remains inert. +This boundary remains inert until every relevant backup caller emits the same +normalized managed manifest, the restore gate accepts and validates that +manifest, recovery tasks are durably persisted and reconciled, and protected +qualification passes for OpenClaw, Hermes, and DCode. diff --git a/src/lib/onboard/managed-workload/rebuild/contract.ts b/src/lib/onboard/managed-workload/rebuild/contract.ts index 3d3c348c8e7..59a01d767ef 100644 --- a/src/lib/onboard/managed-workload/rebuild/contract.ts +++ b/src/lib/onboard/managed-workload/rebuild/contract.ts @@ -105,6 +105,11 @@ export interface ManagedWorkloadRebuildProviderOperations { * to retry after prepare/create ambiguity. */ abortPreparation(plan: ManagedWorkloadRebuildPlan): Promise; + /** + * The provider owns and enforces the readiness deadline. It must return + * `{ state: "not-ready" }` when that deadline expires rather than leaving + * the transaction pending indefinitely. + */ waitUntilReady( plan: ManagedWorkloadRebuildPlan, staged: StagedManagedWorkloadReplacement, diff --git a/src/lib/onboard/managed-workload/rebuild/index.ts b/src/lib/onboard/managed-workload/rebuild/index.ts deleted file mode 100644 index 27a6d11140c..00000000000 --- a/src/lib/onboard/managed-workload/rebuild/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -export * from "./contract"; -export * from "./recovery"; -export * from "./transaction"; diff --git a/src/lib/onboard/managed-workload/rebuild/transaction.ts b/src/lib/onboard/managed-workload/rebuild/transaction.ts index 2910b00e5ab..96302e24755 100644 --- a/src/lib/onboard/managed-workload/rebuild/transaction.ts +++ b/src/lib/onboard/managed-workload/rebuild/transaction.ts @@ -85,6 +85,12 @@ export async function runManagedWorkloadRebuildTransaction( ): Promise { const readSandbox = dependencies.getSandbox ?? readSandboxFromRegistry; const plan = createManagedWorkloadRebuildPlan(input); + if (input.operations.providerId !== plan.providerId) { + throw new ManagedWorkloadRebuildTransactionError( + "prepare", + `provider operations '${input.operations.providerId}' do not match selected provider '${plan.providerId}'`, + ); + } const abortPreparation = createManagedWorkloadPreparationAbort(plan, input.operations); const requireOldAuthority = (timing: "before" | "during"): void => { let stillAuthoritative: boolean; @@ -113,7 +119,6 @@ export async function runManagedWorkloadRebuildTransaction( try { prepared = await prepareManagedWorkloadReplacement(plan, input.operations); } catch (error) { - if (input.operations.providerId !== plan.providerId) throw error; return failAfterCleanup(error, () => abortPreparation.run()); } try { diff --git a/src/lib/onboard/sandbox-workload-authority.test.ts b/src/lib/onboard/sandbox-workload-authority.test.ts index e10118c8b2d..0f2b832bddd 100644 --- a/src/lib/onboard/sandbox-workload-authority.test.ts +++ b/src/lib/onboard/sandbox-workload-authority.test.ts @@ -62,7 +62,8 @@ describe("managed workload authority", () => { it.each( AGENTS.flatMap((agent) => PLATFORMS.map((platform) => [agent, platform] as const)), )("validates exact %s authority on %s", (agent, platform) => { - const authority = readManagedWorkloadAuthority(managedEntry(agent, platform)); + const row = managedEntry(agent, platform); + const authority = readManagedWorkloadAuthority(row); expect(authority).toMatchObject({ agent, @@ -70,7 +71,7 @@ describe("managed workload authority", () => { profile: { agent }, receipt: { kind: "managed-image", platform }, }); - expect(authority?.receipt).not.toBe(managedEntry(agent, platform).workload); + expect(authority?.receipt).not.toBe(row.workload); expect(Object.isFrozen(authority)).toBe(true); expect(Object.isFrozen(authority?.receipt)).toBe(true); expect(Object.isFrozen(authority?.contract.source)).toBe(true); @@ -118,7 +119,7 @@ describe("managed workload authority", () => { ).toThrow(/does not belong to 'hermes'/u); }); - it("rejects cross-agent startup profile authority", () => { + it("rejects image reference and startup profile agent mismatch", () => { const hermesReceiptWithOpenClawProfile = managedReceipt("hermes", "linux/amd64", "openclaw"); expect(() => readManagedWorkloadAuthority( diff --git a/src/lib/onboard/sandbox-workload-rebuild.test.ts b/src/lib/onboard/sandbox-workload-rebuild.test.ts index d838351cf2d..88bed62e89d 100644 --- a/src/lib/onboard/sandbox-workload-rebuild.test.ts +++ b/src/lib/onboard/sandbox-workload-rebuild.test.ts @@ -503,10 +503,11 @@ describe("managed workload rebuild preflight", () => { environment, ); const reconstructed = managedWorkloadRebuildProfileEnvironment(replayHandoff, environment); + const decodedProfile = decodeManagedStartupProfile(staged.replacementProfile.encodedProfile); expect(reconstructed.HTTPS_PROXY).toBe(environment.HTTPS_PROXY); expect(staged.replacementProfile.credentialProxyReplayRequired).toBe(true); - expect(staged.replacementProfile.encodedProfile).not.toContain("secret"); + expect(JSON.stringify(decodedProfile)).not.toContain("secret"); expect(JSON.stringify(staged.replacementProfile.profile)).not.toContain("operator"); }); diff --git a/src/lib/onboard/workload/rebuild.ts b/src/lib/onboard/workload/rebuild.ts index be143261c62..54ab999a75c 100644 --- a/src/lib/onboard/workload/rebuild.ts +++ b/src/lib/onboard/workload/rebuild.ts @@ -4,7 +4,7 @@ import { isDeepStrictEqual } from "node:util"; import { cloneAndDeepFreeze } from "../../core/immutable"; import { getVersion } from "../../core/version"; -import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; +import type { SandboxEntry } from "../../state/registry/types"; import { cloneSandboxWorkloadReceipt } from "../../state/registry/workload"; import type { ResolvedCorporateCa } from "../corporate-ca-types"; import { @@ -431,16 +431,3 @@ export function buildManagedWorkloadRebuildReceipt( } return cloneAndDeepFreeze(validatedReceipt); } - -export type ManagedWorkloadRebuildEntry = Pick< - SandboxEntry, - | "agent" - | "fromDockerfile" - | "imageTag" - | "workload" - | "openshellDriver" - | "lifecycleGeneration" - | "lifecycleLiveIdentityFingerprint" ->; - -export type ManagedWorkloadRebuildReceipt = SandboxWorkloadReceipt; diff --git a/src/lib/state/registry-rebuild-authority.test.ts b/src/lib/state/registry-rebuild-authority.test.ts index 34bc090d0df..1c4c78d3204 100644 --- a/src/lib/state/registry-rebuild-authority.test.ts +++ b/src/lib/state/registry-rebuild-authority.test.ts @@ -3,17 +3,30 @@ import { createHash } from "node:crypto"; -import { describe, expect, it } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { MANAGED_IMAGE_REPOSITORIES } from "../onboard/managed-image/contract"; import { encodeManagedStartupProfile } from "../onboard/managed-startup/profile"; import { captureSandboxRebuildAuthority, + compareAndSwapSandboxRebuildAuthority, SandboxRebuildAuthorityError, sandboxRebuildAuthorityMatchesEntry, + sandboxRebuildReplacementMatchesEntry, swapSandboxRebuildAuthorityInRegistry, } from "./registry/rebuild-authority"; import type { SandboxEntry, SandboxRegistry, SandboxWorkloadReceipt } from "./registry/types"; +const registryPersistence = vi.hoisted(() => ({ + load: vi.fn(), + save: vi.fn(), +})); + +vi.mock("./registry/persistence", () => registryPersistence); +vi.mock("./registry/lock", () => ({ + withLock: (operation: () => T): T => operation(), +})); + const ENCODED_PROFILE = encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")); const PROFILE_SHA256 = createHash("sha256").update(ENCODED_PROFILE, "utf8").digest("hex"); @@ -21,7 +34,7 @@ function receipt(digest: string): Extract { + beforeEach(() => { + registryPersistence.load.mockReset(); + registryPersistence.save.mockReset(); + }); + it("captures a cloned exact authority unit", () => { const source = entry(); const authority = captureSandboxRebuildAuthority(source, "docker"); @@ -170,6 +188,72 @@ describe("sandbox rebuild authority", () => { expect(swapped.registry).toBe(before); }); + it("reconciles an acknowledgement failure to the exact persisted replacement", () => { + let persisted = registry(); + const authority = captureSandboxRebuildAuthority(persisted.sandboxes.alpha!, "docker"); + registryPersistence.load.mockImplementation(() => structuredClone(persisted)); + registryPersistence.save.mockImplementation((next: SandboxRegistry) => { + persisted = structuredClone(next); + throw new Error("registry acknowledgement lost"); + }); + + const result = compareAndSwapSandboxRebuildAuthority(authority, replacement()); + + expect(result).toMatchObject({ + status: "committed", + entry: { + lifecycleGeneration: "generation-new", + lifecycleLiveIdentityFingerprint: "fingerprint-new", + }, + }); + expect(registryPersistence.load).toHaveBeenCalledTimes(2); + expect(registryPersistence.save).toHaveBeenCalledOnce(); + }); + + it("does not reconcile failures that happen before persistence", () => { + const authority = captureSandboxRebuildAuthority(entry(), "docker"); + registryPersistence.load.mockImplementation(() => { + throw new Error("registry read failed"); + }); + + expect(() => compareAndSwapSandboxRebuildAuthority(authority, replacement())).toThrow( + /registry read failed/u, + ); + expect(registryPersistence.load).toHaveBeenCalledOnce(); + expect(registryPersistence.save).not.toHaveBeenCalled(); + }); + + it("validates a replacement before acquiring durable registry state", () => { + const authority = captureSandboxRebuildAuthority(entry(), "docker"); + + expect(() => + compareAndSwapSandboxRebuildAuthority(authority, { + ...replacement(), + openshellDriver: "mxc", + }), + ).toThrow(SandboxRebuildAuthorityError); + expect(registryPersistence.load).not.toHaveBeenCalled(); + expect(registryPersistence.save).not.toHaveBeenCalled(); + }); + + it("matches exact replacement authority while ignoring later mutable metadata", () => { + const expected = replacement(); + + expect( + sandboxRebuildReplacementMatchesEntry(expected, { + ...expected, + model: "updated-after-publication", + gatewayPort: 9090, + }), + ).toBe(true); + expect( + sandboxRebuildReplacementMatchesEntry(expected, { + ...expected, + lifecycleLiveIdentityFingerprint: "different-fingerprint", + }), + ).toBe(false); + }); + it.each([ ["sandbox name", (candidate: SandboxEntry) => ({ ...candidate, name: "other" })], ["agent", (candidate: SandboxEntry) => ({ ...candidate, agent: "hermes" })], diff --git a/src/lib/state/registry/rebuild-authority.ts b/src/lib/state/registry/rebuild-authority.ts index bf77ed65277..20971f9603e 100644 --- a/src/lib/state/registry/rebuild-authority.ts +++ b/src/lib/state/registry/rebuild-authority.ts @@ -278,24 +278,27 @@ export function swapSandboxRebuildAuthorityInRegistry( */ export function compareAndSwapSandboxRebuildAuthority( expected: SandboxRebuildAuthority, - replacement: SandboxEntry, + replacementInput: SandboxEntry, ): SandboxRebuildAuthoritySwapResult { - try { - return withLock(() => { - const swapped = swapSandboxRebuildAuthorityInRegistry(load(), expected, replacement); - if (swapped.result.status === "committed") save(swapped.registry); - return swapped.result; - }); - } catch (error) { + const replacement = validateReplacement(expected, replacementInput); + return withLock(() => { + const swapped = swapSandboxRebuildAuthorityInRegistry(load(), expected, replacement); + if (swapped.result.status === "stale-authority") return swapped.result; + try { - const observed = load().sandboxes[expected.sandboxName] ?? null; - if (sandboxRebuildReplacementMatchesEntry(replacement, observed)) { - return { status: "committed", entry: cloneEntry(observed) }; + save(swapped.registry); + } catch (error) { + try { + const observed = load().sandboxes[expected.sandboxName] ?? null; + if (sandboxRebuildReplacementMatchesEntry(replacement, observed)) { + return { status: "committed", entry: cloneEntry(observed) }; + } + } catch { + // Preserve the original persistence failure when readback is itself + // unavailable. } - } catch { - // Preserve the original CAS/persistence failure when reconciliation is - // itself unavailable. + throw error; } - throw error; - } + return swapped.result; + }); } From bf94633155cedfd1c616e567d928f942a2281978 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 04:14:39 -0700 Subject: [PATCH 046/117] feat(snapshot): preserve managed runtime authority Signed-off-by: Aaron Erickson --- ci/source-architecture-budget.json | 6 +- ...hot-managed-provider-restore-order.test.ts | 246 ++++++ .../sandbox/snapshot-restore-test-fixture.ts | 8 +- src/lib/actions/sandbox/snapshot.ts | 181 +++++ .../actions/sandbox/snapshot/dependencies.ts | 35 + .../sandbox/snapshot/managed-profile.test.ts | 183 +++++ .../sandbox/snapshot/managed-profile.ts | 154 ++++ .../snapshot/provider-lifecycle.test.ts | 272 +++++++ .../sandbox/snapshot/provider-lifecycle.ts | 229 ++++++ src/lib/onboard/runtime-provider/contract.ts | 73 +- src/lib/onboard/runtime-provider/docker.ts | 10 +- src/lib/onboard/runtime-provider/registry.ts | 126 ++- .../runtime-provider-contract.test.ts | 82 +- .../onboard/runtime-provider/snapshot.test.ts | 730 ++++++++++++++++++ src/lib/onboard/runtime-provider/snapshot.ts | 666 ++++++++++++++++ .../state/registry/runtime-snapshot.test.ts | 153 ++++ src/lib/state/registry/runtime-snapshot.ts | 72 ++ src/lib/state/sandbox.ts | 74 ++ test/snapshot.test.ts | 67 ++ 19 files changed, 3352 insertions(+), 15 deletions(-) create mode 100644 src/lib/actions/sandbox/snapshot-managed-provider-restore-order.test.ts create mode 100644 src/lib/actions/sandbox/snapshot/dependencies.ts create mode 100644 src/lib/actions/sandbox/snapshot/managed-profile.test.ts create mode 100644 src/lib/actions/sandbox/snapshot/managed-profile.ts create mode 100644 src/lib/actions/sandbox/snapshot/provider-lifecycle.test.ts create mode 100644 src/lib/actions/sandbox/snapshot/provider-lifecycle.ts create mode 100644 src/lib/onboard/runtime-provider/snapshot.test.ts create mode 100644 src/lib/onboard/runtime-provider/snapshot.ts create mode 100644 src/lib/state/registry/runtime-snapshot.test.ts create mode 100644 src/lib/state/registry/runtime-snapshot.ts diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 0aac14baafd..aec27fc46d6 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -8,7 +8,7 @@ "src/lib/adapters/docker/index.ts": 43, "src/lib/adapters/openshell/client.ts": 23, "src/lib/adapters/openshell/resolve.ts": 27, - "src/lib/adapters/openshell/runtime.ts": 51, + "src/lib/adapters/openshell/runtime.ts": 52, "src/lib/adapters/openshell/timeouts.ts": 37, "src/lib/agent/defs.ts": 32, "src/lib/cli/branding.ts": 84, @@ -23,7 +23,7 @@ "src/lib/inference/config.ts": 29, "src/lib/inference/web-search.ts": 21, "src/lib/messaging/channels/index.ts": 25, - "src/lib/onboard/gateway-binding.ts": 47, + "src/lib/onboard/gateway-binding.ts": 48, "src/lib/runner.ts": 89, "src/lib/security/redact.ts": 51, "src/lib/state/onboard-session.ts": 36, @@ -43,7 +43,7 @@ "src/lib/actions/sandbox/policy-channel.ts": 28, "src/lib/actions/sandbox/process-recovery.ts": 22, "src/lib/actions/sandbox/rebuild-pipeline.ts": 28, - "src/lib/actions/sandbox/snapshot.ts": 38, + "src/lib/actions/sandbox/snapshot.ts": 39, "src/lib/actions/uninstall/run-plan.ts": 25, "src/lib/inference/onboard-probes.ts": 21, "src/lib/inference/vllm.ts": 23, diff --git a/src/lib/actions/sandbox/snapshot-managed-provider-restore-order.test.ts b/src/lib/actions/sandbox/snapshot-managed-provider-restore-order.test.ts new file mode 100644 index 00000000000..bd9324858a9 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot-managed-provider-restore-order.test.ts @@ -0,0 +1,246 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import * as fixture from "./snapshot-restore-test-fixture"; + +const providerRestore = vi.hoisted(() => { + const events: string[] = []; + const provider = { identity: { id: "docker" } }; + const managedProfile = { + agent: "openclaw", + profileFingerprint: "a".repeat(64), + }; + const source = { + schemaVersion: 1, + providerId: "docker", + providerHandle: "snapshot-provider-handle", + lifecycleState: "running", + lifecycleGeneration: "snapshot-generation", + runtime: { + schemaVersion: 1, + providerId: "docker", + runtime: { kind: "docker-container", handle: "container-id" }, + acceleration: { kind: "none" }, + }, + }; + const readManagedSnapshotProfileAuthority = vi.fn( + (_source?: unknown): { agent: string } | null => ({ + agent: "openclaw", + }), + ); + const prepareManagedSnapshotProfileRestore = vi.fn(() => ({ + providerRestoreAuthority: managedProfile, + })); + const requireCurrentSnapshotRuntimeProvider = vi.fn(() => provider); + const prepareSandboxRuntimeRestore = vi.fn(() => { + events.push("provider-preflight"); + return { + phase: "preflighted", + targetProviderId: "docker", + targetSandboxName: "alpha", + source, + preflight: {}, + managedProfile, + }; + }); + const confirmSandboxRuntimeRestore = vi.fn(() => { + events.push("provider-restore-proof"); + return { phase: "validated" }; + }); + return { + events, + source, + readManagedSnapshotProfileAuthority, + prepareManagedSnapshotProfileRestore, + requireCurrentSnapshotRuntimeProvider, + prepareSandboxRuntimeRestore, + confirmSandboxRuntimeRestore, + }; +}); + +vi.mock("./snapshot/dependencies", () => ({ + captureSandboxRuntimeSnapshot: vi.fn(), + confirmSandboxRuntimeRestore: providerRestore.confirmSandboxRuntimeRestore, + prepareManagedSnapshotProfileRestore: providerRestore.prepareManagedSnapshotProfileRestore, + prepareSandboxRuntimeRestore: providerRestore.prepareSandboxRuntimeRestore, + readManagedSnapshotProfileAuthority: providerRestore.readManagedSnapshotProfileAuthority, + rejectManagedSnapshotCloneUntilRebind: vi.fn(), + requireCurrentSnapshotRuntimeProvider: providerRestore.requireCurrentSnapshotRuntimeProvider, +})); + +function managedSnapshot() { + return { + snapshotVersion: 4, + timestamp: "2026-07-30T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + agentType: "openclaw", + workload: { kind: "managed-image" }, + runtimeSnapshot: providerRestore.source, + }; +} + +beforeEach(() => { + fixture.resetSnapshotRestoreMocks(); + providerRestore.events.length = 0; + providerRestore.readManagedSnapshotProfileAuthority.mockClear(); + providerRestore.prepareManagedSnapshotProfileRestore.mockClear(); + providerRestore.requireCurrentSnapshotRuntimeProvider.mockClear(); + providerRestore.prepareSandboxRuntimeRestore.mockClear(); + providerRestore.confirmSandboxRuntimeRestore.mockClear(); + fixture.getLatestBackupMock.mockReturnValue(managedSnapshot()); + fixture.getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "openclaw", + openshellDriver: "docker", + }); + fixture.restoreSandboxStateMock.mockImplementation(() => { + providerRestore.events.push("filesystem-restore"); + return { + success: true, + restoredDirs: ["workspace"], + restoredFiles: [], + failedDirs: [], + failedFiles: [], + }; + }); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); +}); +afterEach(() => { + fixture.cleanupSnapshotRestoreMocks(); +}); + +describe("managed snapshot provider restore ordering", () => { + it("refreshes provider authority at the mutation edge and proves the profile afterward", async () => { + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(providerRestore.events).toEqual([ + "provider-preflight", + "provider-preflight", + "filesystem-restore", + "provider-restore-proof", + ]); + expect(providerRestore.prepareSandboxRuntimeRestore).toHaveBeenCalledTimes(2); + expect(providerRestore.confirmSandboxRuntimeRestore).toHaveBeenCalledOnce(); + }); + + it("aborts before filesystem mutation when mutation-edge validation fails", async () => { + providerRestore.prepareSandboxRuntimeRestore + .mockImplementationOnce(() => { + providerRestore.events.push("provider-preflight"); + return { + phase: "preflighted", + targetProviderId: "docker", + targetSandboxName: "alpha", + source: providerRestore.source, + preflight: {}, + managedProfile: { + agent: "openclaw", + profileFingerprint: "a".repeat(64), + }, + }; + }) + .mockImplementationOnce(() => { + providerRestore.events.push("provider-preflight-rejected"); + throw new Error("runtime changed after snapshot preflight"); + }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect(runSandboxSnapshot("alpha", { kind: "restore" })).rejects.toMatchObject({ + exitCode: 1, + }); + + expect(providerRestore.events).toEqual(["provider-preflight", "provider-preflight-rejected"]); + expect(fixture.restoreSandboxStateMock).not.toHaveBeenCalled(); + expect(providerRestore.confirmSandboxRuntimeRestore).not.toHaveBeenCalled(); + }); +}); + +describe("legacy snapshot compatibility gate", () => { + beforeEach(() => { + fixture.getLatestBackupMock.mockReturnValue({ + snapshotVersion: 3, + timestamp: "2026-07-29T00:00:00.000Z", + backupPath: "/tmp/legacy-backup-alpha", + agentType: "openclaw", + }); + providerRestore.readManagedSnapshotProfileAuthority.mockImplementation((source: unknown) => + (source as { workload?: unknown }).workload ? { agent: "openclaw" } : null, + ); + }); + + it("rejects self-restore when the current target is managed", async () => { + fixture.getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "openclaw", + openshellDriver: "docker", + workload: { kind: "managed-image" }, + }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect(runSandboxSnapshot("alpha", { kind: "restore" })).rejects.toMatchObject({ + exitCode: 1, + }); + + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("legacy snapshot lacks managed workload"), + ); + expect(fixture.restoreSandboxStateMock).not.toHaveBeenCalled(); + expect(providerRestore.prepareSandboxRuntimeRestore).not.toHaveBeenCalled(); + }); + + it.each([ + "source", + "destination", + ] as const)("rejects cross-clone when the current %s is managed", async (managedSide) => { + fixture.getSandboxMock.mockImplementation((name) => { + if (name === "alpha") { + return { + name, + agent: "openclaw", + openshellDriver: "docker", + imageTag: "legacy-source:test", + ...(managedSide === "source" ? { workload: { kind: "managed-image" } } : {}), + }; + } + if (name === "beta" && managedSide === "destination") { + return { + name, + agent: "openclaw", + openshellDriver: "docker", + imageTag: "managed-target@test", + workload: { kind: "managed-image" }, + }; + } + return null; + }); + fixture.parseLiveSandboxNamesMock.mockReturnValue( + new Set(managedSide === "destination" ? ["alpha", "beta"] : ["alpha"]), + ); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect( + runSandboxSnapshot("alpha", { + kind: "restore", + to: "beta", + force: true, + yes: true, + }), + ).rejects.toMatchObject({ exitCode: 1 }); + + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("legacy snapshot lacks managed workload"), + ); + expect( + fixture.runOpenshellMock.mock.calls.some( + ([args]) => args[0] === "sandbox" && args[1] === "delete", + ), + ).toBe(false); + expect(fixture.streamSandboxCreateMock).not.toHaveBeenCalled(); + expect(fixture.restoreSandboxStateMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts index ee45ff695b4..f264c5d9880 100644 --- a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts +++ b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { vi } from "vitest"; +import type { SandboxWorkloadReceipt } from "../../state/registry/types"; import { SANDBOX_EXEC_STARTED_MARKER } from "./sandbox-exec-output"; import type { SnapshotStreamSandboxCreateMock } from "./snapshot-create-stream-test-types"; @@ -41,12 +42,7 @@ export type SandboxRecord = { fromDockerfile?: string | null; gatewayName?: string | null; imageTag?: string | null; - workload?: { - schemaVersion: 1; - kind: "legacy-dockerfile"; - reference: string | null; - shared: false; - }; + workload?: SandboxWorkloadReceipt; openshellDriver?: string | null; observabilityEnabled?: boolean; provider?: string | null; diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index eb9023636a3..6c9b694f5c3 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -79,6 +79,16 @@ import { selectSandboxGatewayIfRegistered, usesGatewayMetadataProbe, } from "./sandbox-gateway-routing"; +import { + captureSandboxRuntimeSnapshot, + confirmSandboxRuntimeRestore, + type PreparedSandboxRuntimeRestore, + prepareManagedSnapshotProfileRestore, + prepareSandboxRuntimeRestore, + readManagedSnapshotProfileAuthority, + rejectManagedSnapshotCloneUntilRebind, + requireCurrentSnapshotRuntimeProvider, +} from "./snapshot/dependencies"; import { formatSnapshotBaselineExclusionSummary } from "./snapshot-baseline-exclusion-summary"; import { printHermesGatewayRestoreHint } from "./snapshot-hermes-gateway-hint"; @@ -678,8 +688,41 @@ function runSnapshotCreate( } const label = request.name ? ` (--name ${request.name})` : ""; console.log(` Creating snapshot of '${sandboxName}'${label}...`); + const sourceEntry = registry.getSandbox(sandboxName); + let runtimeSnapshot: ReturnType | undefined; + let workload: NonNullable | undefined; + if (sourceEntry) { + try { + const authority = readManagedSnapshotProfileAuthority({ + sandboxName, + agentType: sourceEntry.agent ?? "", + imageTag: sourceEntry.imageTag, + fromDockerfile: sourceEntry.fromDockerfile, + workload: sourceEntry.workload, + }); + if (authority) { + const provider = requireCurrentSnapshotRuntimeProvider(sourceEntry); + if (!provider.workload.acceptsReceipt(authority.receipt)) { + throw new Error( + `runtime provider '${provider.identity.id}' does not accept the managed workload receipt`, + ); + } + runtimeSnapshot = captureSandboxRuntimeSnapshot(provider, sourceEntry); + workload = authority.receipt; + } + } catch (error) { + console.error( + ` Cannot capture managed snapshot authority: ${ + error instanceof Error ? error.message : String(error) + }.`, + ); + snapshotExit(1); + } + } const result = sandboxState.backupSandboxState(sandboxName, { name: request.name ?? null, + ...(runtimeSnapshot === undefined ? {} : { runtimeSnapshot }), + ...(workload === undefined ? {} : { workload }), }); if (result.success) { const manifest = result.manifest!; @@ -949,6 +992,18 @@ function reconcileSnapshotCustomPolicies( } } +function readCurrentManagedSnapshotProfileAuthority(entry: SandboxEntry | null) { + return entry + ? readManagedSnapshotProfileAuthority({ + sandboxName: entry.name, + agentType: entry.agent ?? "", + imageTag: entry.imageTag, + fromDockerfile: entry.fromDockerfile, + workload: entry.workload, + }) + : null; +} + async function runSnapshotRestore( sandboxName: string, request: Extract, @@ -1026,6 +1081,43 @@ async function runSnapshotRestoreUnlocked( console.log(` Using latest snapshot ${v}${nameSuffix} (${latest.timestamp})`); } + const snapshotProfileSource = { + sandboxName, + agentType: resolvedSnapshot.agentType, + workload: resolvedSnapshot.workload, + }; + const currentSourceEntry = registry.getSandbox(sandboxName); + let hasManagedProfileAuthority = false; + try { + const snapshotAuthority = readManagedSnapshotProfileAuthority(snapshotProfileSource); + hasManagedProfileAuthority = snapshotAuthority !== null; + if (hasManagedProfileAuthority && !resolvedSnapshot.runtimeSnapshot) { + throw new Error("managed snapshot is missing provider runtime authority"); + } + const currentSourceAuthority = readCurrentManagedSnapshotProfileAuthority(currentSourceEntry); + const currentTargetAuthority = + targetEntry && targetEntry !== currentSourceEntry + ? readCurrentManagedSnapshotProfileAuthority(targetEntry) + : currentSourceAuthority; + if (!hasManagedProfileAuthority && (currentSourceAuthority || currentTargetAuthority)) { + throw new Error( + "legacy snapshot lacks managed workload and provider runtime authority required by the current source or destination", + ); + } + if (isCrossSandboxRestore && hasManagedProfileAuthority) { + rejectManagedSnapshotCloneUntilRebind(snapshotProfileSource, targetSandbox); + } + } catch (error) { + console.error( + ` Cannot restore managed snapshot authority: ${ + error instanceof Error ? error.message : String(error) + }.`, + ); + console.error(` Destination '${targetSandbox}' was not changed.`); + snapshotExit(1); + } + + let preparedRuntimeRestore: PreparedSandboxRuntimeRestore | null = null; if (!isCrossSandboxRestore) { // Self-restore: target is `sandboxName`. Cannot auto-create; the // source pod is the target, so it must already be live. @@ -1033,6 +1125,39 @@ async function runSnapshotRestoreUnlocked( console.error(` Sandbox '${targetSandbox}' is not running. Cannot restore snapshot.`); snapshotExit(1); } + if (hasManagedProfileAuthority) { + const currentTarget = registry.getSandbox(targetSandbox); + if (!currentTarget || !resolvedSnapshot.runtimeSnapshot) { + console.error( + ` Cannot restore managed snapshot '${sandboxName}': target or provider runtime authority is missing.`, + ); + snapshotExit(1); + } + try { + const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); + const profileRestore = prepareManagedSnapshotProfileRestore( + snapshotProfileSource, + currentTarget, + provider, + ); + if (!profileRestore) { + throw new Error("managed profile restore authority is missing"); + } + preparedRuntimeRestore = prepareSandboxRuntimeRestore( + provider, + currentTarget, + resolvedSnapshot.runtimeSnapshot, + profileRestore.providerRestoreAuthority, + ); + } catch (error) { + console.error( + ` Cannot preflight managed snapshot restore: ${ + error instanceof Error ? error.message : String(error) + }.`, + ); + snapshotExit(1); + } + } } else { // #3756: cross-sandbox restore into a destination that already exists // used to overlay onto the live filesystem silently. Refuse by default @@ -1179,6 +1304,42 @@ async function runSnapshotRestoreUnlocked( // reconciliation under the active timer generation. Normal auto-restore // waits; the absolute deadline may preempt this process and reclaim the // token, preventing policy/config mutation after lockdown resumes. + if (preparedRuntimeRestore) { + const currentTarget = registry.getSandbox(targetSandbox); + if (!currentTarget) { + console.error( + ` Cannot revalidate managed snapshot restore: target '${targetSandbox}' is no longer registered.`, + ); + snapshotExit(1); + } + try { + const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); + const profileRestore = prepareManagedSnapshotProfileRestore( + snapshotProfileSource, + currentTarget, + provider, + ); + if (!profileRestore) { + throw new Error("managed profile restore authority is missing"); + } + // Refresh provider authority immediately before filesystem mutation. + // The post-restore facet consumes this exact receipt and proves that + // neither runtime generation nor managed profile changed meanwhile. + preparedRuntimeRestore = prepareSandboxRuntimeRestore( + provider, + currentTarget, + preparedRuntimeRestore.source, + profileRestore.providerRestoreAuthority, + ); + } catch (error) { + console.error( + ` Cannot revalidate managed snapshot restore: ${ + error instanceof Error ? error.message : String(error) + }.`, + ); + snapshotExit(1); + } + } if (targetSandbox !== sandboxName) { console.log(` Restoring snapshot from '${sandboxName}' into '${targetSandbox}'...`); } else { @@ -1186,6 +1347,26 @@ async function runSnapshotRestoreUnlocked( } const result = sandboxState.restoreSandboxState(targetSandbox, backupPath); if (result.success) { + if (preparedRuntimeRestore) { + const currentTarget = registry.getSandbox(targetSandbox); + if (!currentTarget) { + console.error( + ` Managed snapshot state was restored, but target '${targetSandbox}' is no longer registered.`, + ); + snapshotExit(1); + } + try { + const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); + confirmSandboxRuntimeRestore(provider, currentTarget, preparedRuntimeRestore); + } catch (error) { + console.error( + ` Managed snapshot state was restored, but provider restore proof failed: ${ + error instanceof Error ? error.message : String(error) + }.`, + ); + snapshotExit(1); + } + } console.log( ` ${G}\u2713${R} Restored ${result.restoredDirs.length} directories, ${result.restoredFiles.length} files`, ); diff --git a/src/lib/actions/sandbox/snapshot/dependencies.ts b/src/lib/actions/sandbox/snapshot/dependencies.ts new file mode 100644 index 00000000000..e3d573177f6 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/dependencies.ts @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; +import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provider/current"; +import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime-provider/registry"; +import type { SandboxEntry } from "../../../state/registry/types"; + +export { + ManagedSnapshotProfileRestoreError, + prepareManagedSnapshotProfileRestore, + readManagedSnapshotProfileAuthority, + rejectManagedSnapshotCloneUntilRebind, +} from "./managed-profile"; +export type { + PreparedSandboxRuntimeRestore, + ValidatedSandboxRuntimeRestore, +} from "./provider-lifecycle"; +export { + captureSandboxRuntimeSnapshot, + confirmSandboxRuntimeRestore, + prepareSandboxRuntimeRestore, + SandboxSnapshotProviderError, +} from "./provider-lifecycle"; + +/** + * Resolve the one already-registered provider bundle for a durable sandbox. + * Snapshot actions never maintain a second provider map or infer a container + * engine from host state. + */ +export function requireCurrentSnapshotRuntimeProvider( + sandbox: SandboxEntry, +): RuntimeProviderBundle { + return requireRuntimeProviderBundleForSandbox(sandbox, CURRENT_RUNTIME_PROVIDER_BUNDLES); +} diff --git a/src/lib/actions/sandbox/snapshot/managed-profile.test.ts b/src/lib/actions/sandbox/snapshot/managed-profile.test.ts new file mode 100644 index 00000000000..74f42d070ab --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/managed-profile.test.ts @@ -0,0 +1,183 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { describe, expect, it } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + MANAGED_IMAGE_REPOSITORIES, + type ShippedManagedImageAgent, +} from "../../../onboard/managed-image/contract"; +import { + encodeManagedStartupProfile, + fingerprintManagedStartupProfile, +} from "../../../onboard/managed-startup/profile"; +import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../../state/registry/types"; +import { + prepareManagedSnapshotProfileRestore, + readManagedSnapshotProfileAuthority, + rejectManagedSnapshotCloneUntilRebind, +} from "./managed-profile"; + +function workload( + agent: ShippedManagedImageAgent, + changedProfile = false, +): Extract { + const encodedProfile = encodeManagedStartupProfile( + managedStartupE2eProfile(agent, changedProfile), + ); + return { + schemaVersion: 1, + kind: "managed-image", + reference: `${MANAGED_IMAGE_REPOSITORIES[agent]}@sha256:${"a".repeat(64)}`, + platform: "linux/amd64", + release: "v0.0.88", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true, + }; +} + +function sandbox(agent: ShippedManagedImageAgent, receipt = workload(agent)): SandboxEntry { + return { + name: "alpha", + agent, + openshellDriver: "mxc", + imageTag: receipt.reference, + fromDockerfile: null, + workload: receipt, + }; +} + +function provider(accepted = true, managedProfileRestore = true): RuntimeProviderBundle { + return { + identity: { contractVersion: 1, id: "mxc", displayName: "MXC" }, + workload: { + providerId: "mxc", + supported: true, + profile: { + support: null, + hostArchitectures: [], + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: false, + }, + acceptsReceipt: () => accepted, + }, + snapshot: { + providerId: "mxc", + supported: true, + capabilities: { + backup: true, + restore: true, + managedProfileRestore, + }, + preflight: () => { + throw new Error("profile preflight must not perform runtime effects"); + }, + capture: () => { + throw new Error("profile preflight must not perform runtime effects"); + }, + validateRestore: () => { + throw new Error("profile preflight must not perform runtime effects"); + }, + restore: () => { + throw new Error("profile preflight must not perform runtime effects"); + }, + }, + } as unknown as RuntimeProviderBundle; +} + +describe("managed snapshot profile restore", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("validates exact secret-free %s profile authority", (agent) => { + const receipt = workload(agent); + const source = { sandboxName: "alpha", agentType: agent, workload: receipt }; + + const plan = prepareManagedSnapshotProfileRestore(source, sandbox(agent, receipt), provider()); + + expect(plan).toMatchObject({ + schemaVersion: 1, + providerId: "mxc", + sourceSandboxName: "alpha", + targetSandboxName: "alpha", + authority: { + agent, + receipt, + profile: { agent }, + }, + providerRestoreAuthority: { + agent, + profileFingerprint: fingerprintManagedStartupProfile(managedStartupE2eProfile(agent)), + }, + }); + }); + + it("returns null for legacy snapshots without managed workload authority", () => { + expect( + readManagedSnapshotProfileAuthority({ + sandboxName: "legacy", + agentType: "openclaw", + }), + ).toBeNull(); + }); + + it("rejects malformed snapshot authority before consulting the target", () => { + const receipt = { + ...workload("hermes"), + startupProfileSha256: "0".repeat(64), + }; + expect(() => + prepareManagedSnapshotProfileRestore( + { sandboxName: "alpha", agentType: "hermes", workload: receipt }, + sandbox("hermes"), + provider(), + ), + ).toThrow(/invalid managed workload authority/u); + }); + + it("rejects target profile drift and provider refusal", () => { + const receipt = workload("openclaw"); + const source = { sandboxName: "alpha", agentType: "openclaw", workload: receipt }; + expect(() => + prepareManagedSnapshotProfileRestore( + source, + sandbox("openclaw", workload("openclaw", true)), + provider(), + ), + ).toThrow(/requires a managed image or startup-profile rebind/u); + expect(() => + prepareManagedSnapshotProfileRestore(source, sandbox("openclaw", receipt), provider(false)), + ).toThrow(/does not accept the snapshot workload receipt/u); + expect(() => + prepareManagedSnapshotProfileRestore( + source, + sandbox("openclaw", receipt), + provider(true, false), + ), + ).toThrow(/does not support managed-profile restore/u); + }); + + it("fails before a managed cross-sandbox clone can reach image-only creation", () => { + expect(() => + rejectManagedSnapshotCloneUntilRebind( + { + sandboxName: "alpha", + agentType: "langchain-deepagents-code", + workload: workload("langchain-deepagents-code"), + }, + "beta", + ), + ).toThrow(/requires managed-profile clone rebind/u); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot/managed-profile.ts b/src/lib/actions/sandbox/snapshot/managed-profile.ts new file mode 100644 index 00000000000..a35ceaa9062 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/managed-profile.ts @@ -0,0 +1,154 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; +import { fingerprintManagedStartupProfile } from "../../../onboard/managed-startup/profile"; +import type { + RuntimeProviderBundle, + RuntimeProviderManagedProfileRestoreAuthority, +} from "../../../onboard/runtime-provider/contract"; +import { normalizeRuntimeProviderIdentity } from "../../../onboard/runtime-provider/registry"; +import { + type ManagedWorkloadAuthority, + readManagedWorkloadAuthority, +} from "../../../onboard/workload/authority"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../../state/registry/types"; + +export interface ManagedSnapshotProfileSource { + readonly sandboxName: string; + readonly agentType: string; + readonly imageTag?: string | null; + readonly fromDockerfile?: string | null; + readonly workload?: SandboxWorkloadReceipt; +} + +export interface ManagedSnapshotProfileRestorePlan { + readonly schemaVersion: 1; + readonly providerId: string; + readonly sourceSandboxName: string; + readonly targetSandboxName: string; + readonly authority: ManagedWorkloadAuthority; + readonly providerRestoreAuthority: RuntimeProviderManagedProfileRestoreAuthority; +} + +export class ManagedSnapshotProfileRestoreError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(`Managed snapshot profile restore failed: ${message}`, options); + this.name = "ManagedSnapshotProfileRestoreError"; + } +} + +/** + * Reconstruct and validate the secret-free managed startup authority stored in + * a snapshot. No mutable release catalog or current image alias is consulted. + */ +export function readManagedSnapshotProfileAuthority( + source: ManagedSnapshotProfileSource, +): ManagedWorkloadAuthority | null { + try { + return readManagedWorkloadAuthority({ + agent: source.agentType, + fromDockerfile: source.fromDockerfile ?? null, + imageTag: + source.imageTag ?? + (source.workload?.kind === "managed-image" ? source.workload.reference : null), + workload: source.workload, + }); + } catch (error) { + throw new ManagedSnapshotProfileRestoreError( + `snapshot '${source.sandboxName}' has invalid managed workload authority`, + { cause: error }, + ); + } +} + +/** + * Validate an in-place managed-profile restore against the selected provider + * and the current exact workload. PR3.8 restores profile-backed state only + * when no image/profile rebind is required; cross-sandbox rebind and activation + * are intentionally owned by the later clone transaction. + */ +export function prepareManagedSnapshotProfileRestore( + source: ManagedSnapshotProfileSource, + target: SandboxEntry, + provider: RuntimeProviderBundle, +): ManagedSnapshotProfileRestorePlan | null { + const sourceAuthority = readManagedSnapshotProfileAuthority(source); + if (!sourceAuthority) return null; + + const targetProviderId = normalizeRuntimeProviderIdentity(target.openshellDriver); + if ( + targetProviderId !== provider.identity.id || + provider.snapshot.providerId !== provider.identity.id + ) { + throw new ManagedSnapshotProfileRestoreError( + `target '${target.name}' does not belong to provider '${provider.identity.id}'`, + ); + } + if ( + provider.snapshot.supported !== true || + provider.snapshot.capabilities.managedProfileRestore !== true + ) { + throw new ManagedSnapshotProfileRestoreError( + `provider '${provider.identity.id}' does not support managed-profile restore`, + ); + } + if (!provider.workload.acceptsReceipt(sourceAuthority.receipt)) { + throw new ManagedSnapshotProfileRestoreError( + `provider '${provider.identity.id}' does not accept the snapshot workload receipt`, + ); + } + + let targetAuthority: ManagedWorkloadAuthority | null; + try { + targetAuthority = readManagedWorkloadAuthority(target); + } catch (error) { + throw new ManagedSnapshotProfileRestoreError( + `target '${target.name}' has invalid managed workload authority`, + { cause: error }, + ); + } + if (!targetAuthority) { + throw new ManagedSnapshotProfileRestoreError( + `target '${target.name}' is not the snapshot's managed workload`, + ); + } + if ( + targetAuthority.agent !== sourceAuthority.agent || + !isDeepStrictEqual(targetAuthority.receipt, sourceAuthority.receipt) || + !isDeepStrictEqual(targetAuthority.contract, sourceAuthority.contract) || + !isDeepStrictEqual(targetAuthority.profile, sourceAuthority.profile) + ) { + throw new ManagedSnapshotProfileRestoreError( + `target '${target.name}' requires a managed image or startup-profile rebind`, + ); + } + + return Object.freeze({ + schemaVersion: 1 as const, + providerId: provider.identity.id, + sourceSandboxName: source.sandboxName, + targetSandboxName: target.name, + authority: sourceAuthority, + providerRestoreAuthority: { + agent: sourceAuthority.agent, + profileFingerprint: fingerprintManagedStartupProfile(sourceAuthority.profile), + }, + }); +} + +/** + * Managed cross-sandbox restore cannot reuse the legacy image-only create path: + * its startup profile contains sandbox-scoped authority. Validate the source + * first, then stop before deletion or creation until clone/rebind is available. + */ +export function rejectManagedSnapshotCloneUntilRebind( + source: ManagedSnapshotProfileSource, + targetSandboxName: string, +): void { + const authority = readManagedSnapshotProfileAuthority(source); + if (!authority) return; + throw new ManagedSnapshotProfileRestoreError( + `restoring '${source.sandboxName}' as '${targetSandboxName}' requires managed-profile clone rebind`, + ); +} diff --git a/src/lib/actions/sandbox/snapshot/provider-lifecycle.test.ts b/src/lib/actions/sandbox/snapshot/provider-lifecycle.test.ts new file mode 100644 index 00000000000..e4b68e51606 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/provider-lifecycle.test.ts @@ -0,0 +1,272 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { + RuntimeProviderBundle, + RuntimeProviderManagedProfileRestoreAuthority, + RuntimeProviderRuntimeReceipt, +} from "../../../onboard/runtime-provider/contract"; +import type { SandboxEntry } from "../../../state/registry/types"; +import { + captureSandboxRuntimeSnapshot, + confirmSandboxRuntimeRestore, + prepareSandboxRuntimeRestore, +} from "./provider-lifecycle"; + +function sandbox(name = "alpha"): SandboxEntry { + return { name, agent: "openclaw", openshellDriver: "mxc" }; +} + +function runtime(providerId = "mxc"): RuntimeProviderRuntimeReceipt { + return { + schemaVersion: 1, + providerId, + runtime: { kind: "session", handle: `opaque-${providerId}-session` }, + acceleration: { kind: "none" }, + }; +} + +const managedProfile = { + agent: "openclaw", + profileFingerprint: "a".repeat(64), +} as const satisfies RuntimeProviderManagedProfileRestoreAuthority; + +function provider( + options: { + providerId?: string; + preflightProviderId?: string; + runtimeProviderId?: string; + restoreProviderId?: string; + } = {}, +): { + readonly bundle: RuntimeProviderBundle; + readonly preflight: ReturnType; + readonly capture: ReturnType; + readonly validateRestore: ReturnType; + readonly restore: ReturnType; +} { + const providerId = options.providerId ?? "mxc"; + const preflight = vi.fn((operation: "backup" | "restore", entry: SandboxEntry) => ({ + schemaVersion: 1 as const, + providerId: options.preflightProviderId ?? providerId, + operation, + sandboxName: entry.name, + providerHandle: `opaque-${providerId}-preflight`, + lifecycleState: "running" as const, + lifecycleGeneration: "generation-1", + })); + const capture = vi.fn(() => runtime(options.runtimeProviderId ?? providerId)); + const validateRestore = vi.fn(); + const restore = vi.fn( + ( + entry: SandboxEntry, + _preflight: unknown, + _runtime: unknown, + authority: RuntimeProviderManagedProfileRestoreAuthority, + ) => ({ + schemaVersion: 1 as const, + providerId: options.restoreProviderId ?? providerId, + sandboxName: entry.name, + providerHandle: `opaque-${providerId}-restore`, + lifecycleState: "running" as const, + lifecycleGeneration: "generation-1", + runtime: runtime(options.runtimeProviderId ?? providerId), + managedProfile: authority, + }), + ); + return { + bundle: { + identity: { contractVersion: 1, id: providerId, displayName: providerId }, + snapshot: { + providerId, + supported: true, + capabilities: { backup: true, restore: true, managedProfileRestore: true }, + preflight, + capture, + validateRestore, + restore, + }, + } as unknown as RuntimeProviderBundle, + preflight, + capture, + validateRestore, + restore, + }; +} + +describe("snapshot provider lifecycle", () => { + it("captures provider-neutral runtime and lifecycle state behind opaque handles", () => { + const { bundle, preflight, capture } = provider(); + + expect(captureSandboxRuntimeSnapshot(bundle, sandbox())).toEqual({ + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-mxc-preflight", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: runtime(), + }); + expect(preflight).toHaveBeenCalledWith("backup", expect.objectContaining({ name: "alpha" })); + expect(capture).toHaveBeenCalledOnce(); + }); + + it("preflights before restore and revalidates through the same injected facet", () => { + const { bundle, restore, validateRestore } = provider(); + const target = sandbox("target"); + const source = { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-source", + lifecycleState: "running", + lifecycleGeneration: "source-generation", + runtime: runtime(), + }; + + const prepared = prepareSandboxRuntimeRestore(bundle, target, source, managedProfile); + const validated = confirmSandboxRuntimeRestore(bundle, target, prepared); + + expect(prepared.phase).toBe("preflighted"); + expect(validateRestore).toHaveBeenCalledWith( + target, + prepared.preflight, + expect.objectContaining({ providerId: "mxc" }), + managedProfile, + ); + expect(validated.phase).toBe("validated"); + expect(restore).toHaveBeenCalledWith( + target, + prepared.preflight, + expect.objectContaining({ providerId: "mxc" }), + managedProfile, + ); + expect(validated.restoreReceipt).toMatchObject({ + providerId: "mxc", + managedProfile, + }); + }); + + it("rejects provider identity drift before returning snapshot authority", () => { + expect(() => + captureSandboxRuntimeSnapshot(provider({ preflightProviderId: "other" }).bundle, sandbox()), + ).toThrow(/invalid backup preflight authority/u); + expect(() => + captureSandboxRuntimeSnapshot(provider({ runtimeProviderId: "other" }).bundle, sandbox()), + ).toThrow(/unrepresentable runtime state/u); + }); + + it("fails preflight when the target cannot represent snapshot lifecycle state", () => { + const { bundle, restore } = provider(); + expect(() => + prepareSandboxRuntimeRestore( + bundle, + sandbox("target"), + { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-source", + lifecycleState: "paused", + lifecycleGeneration: "source-generation", + runtime: runtime(), + }, + managedProfile, + ), + ).toThrow(/cannot represent the snapshot lifecycle state/u); + expect(restore).not.toHaveBeenCalled(); + }); + + it("propagates provider restore refusal from the read-only preflight edge", () => { + const { bundle, validateRestore, restore } = provider(); + validateRestore.mockImplementationOnce(() => { + throw new Error("source provider handle is invalid"); + }); + + expect(() => + prepareSandboxRuntimeRestore( + bundle, + sandbox("target"), + { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "tampered-source", + lifecycleState: "running", + lifecycleGeneration: "source-generation", + runtime: runtime(), + }, + managedProfile, + ), + ).toThrow(/source provider handle is invalid/u); + expect(restore).not.toHaveBeenCalled(); + }); + + it("rejects stale target authority without calling provider restore", () => { + const { bundle, restore } = provider(); + const prepared = prepareSandboxRuntimeRestore( + bundle, + sandbox("target"), + { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: runtime(), + }, + managedProfile, + ); + + expect(() => confirmSandboxRuntimeRestore(bundle, sandbox("replacement"), prepared)).toThrow( + /restore preflight authority is stale/u, + ); + expect(restore).not.toHaveBeenCalled(); + }); + + it("rejects cross-provider runtime authority before target preflight", () => { + const { bundle, preflight } = provider(); + expect(() => + prepareSandboxRuntimeRestore( + bundle, + sandbox("target"), + { + schemaVersion: 1, + providerId: "other", + providerHandle: "opaque-other", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: runtime("other"), + }, + managedProfile, + ), + ).toThrow(/does not match target provider/u); + expect(preflight).not.toHaveBeenCalled(); + }); + + it("rejects an invalid managed profile authority and provider restore proof", () => { + const source = { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: runtime(), + }; + expect(() => + prepareSandboxRuntimeRestore(provider().bundle, sandbox("target"), source, { + ...managedProfile, + profileFingerprint: "not-a-digest", + }), + ).toThrow(/managed profile restore authority is invalid/u); + + const { bundle } = provider({ restoreProviderId: "other" }); + const prepared = prepareSandboxRuntimeRestore( + bundle, + sandbox("target"), + source, + managedProfile, + ); + expect(() => confirmSandboxRuntimeRestore(bundle, sandbox("target"), prepared)).toThrow( + /invalid managed restore proof/u, + ); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot/provider-lifecycle.ts b/src/lib/actions/sandbox/snapshot/provider-lifecycle.ts new file mode 100644 index 00000000000..3e47c14661e --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/provider-lifecycle.ts @@ -0,0 +1,229 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + RuntimeProviderBundle, + RuntimeProviderManagedProfileRestoreAuthority, + RuntimeProviderRuntimeReceipt, + RuntimeProviderSnapshotPreflightReceipt, + RuntimeProviderSnapshotRestoreReceipt, + RuntimeProviderSnapshotSurface, +} from "../../../onboard/runtime-provider/contract"; +import { + normalizeRuntimeProviderManagedProfileRestoreAuthority, + normalizeRuntimeProviderRuntimeReceipt, + normalizeRuntimeProviderSnapshotPreflightReceipt, + normalizeRuntimeProviderSnapshotRestoreReceipt, +} from "../../../onboard/runtime-provider/registry"; +import { + cloneSandboxRuntimeSnapshot, + SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION, + type SandboxRuntimeSnapshot, +} from "../../../state/registry/runtime-snapshot"; +import type { SandboxEntry } from "../../../state/registry/types"; + +type SupportedSnapshotSurface = Extract; + +export class SandboxSnapshotProviderError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(`Sandbox snapshot provider failed: ${message}`, options); + this.name = "SandboxSnapshotProviderError"; + } +} + +function requireSnapshotSurface( + bundle: RuntimeProviderBundle, + capability: keyof SupportedSnapshotSurface["capabilities"], +): SupportedSnapshotSurface { + const surface = bundle.snapshot; + if ( + surface.supported !== true || + surface.providerId !== bundle.identity.id || + surface.capabilities[capability] !== true + ) { + throw new SandboxSnapshotProviderError( + `runtime provider '${bundle.identity.id}' does not support ${capability}`, + ); + } + return surface; +} + +function requirePreflight( + bundle: RuntimeProviderBundle, + sandbox: SandboxEntry, + operation: "backup" | "restore", + value: unknown, +): RuntimeProviderSnapshotPreflightReceipt { + const preflight = normalizeRuntimeProviderSnapshotPreflightReceipt(value); + if ( + !preflight || + preflight.providerId !== bundle.identity.id || + preflight.operation !== operation || + preflight.sandboxName !== sandbox.name + ) { + throw new SandboxSnapshotProviderError( + `runtime provider '${bundle.identity.id}' returned invalid ${operation} preflight authority`, + ); + } + return preflight; +} + +function requireRuntimeReceipt( + bundle: RuntimeProviderBundle, + value: unknown, +): RuntimeProviderRuntimeReceipt { + const receipt = normalizeRuntimeProviderRuntimeReceipt(value); + if (!receipt || receipt.providerId !== bundle.identity.id) { + throw new SandboxSnapshotProviderError( + `runtime provider '${bundle.identity.id}' returned unrepresentable runtime state`, + ); + } + return receipt; +} + +function requireRestoreReceipt( + bundle: RuntimeProviderBundle, + sandbox: SandboxEntry, + authority: RuntimeProviderManagedProfileRestoreAuthority, + value: unknown, +): RuntimeProviderSnapshotRestoreReceipt { + const receipt = normalizeRuntimeProviderSnapshotRestoreReceipt(value); + if ( + !receipt || + receipt.providerId !== bundle.identity.id || + receipt.sandboxName !== sandbox.name || + receipt.managedProfile.agent !== authority.agent || + receipt.managedProfile.profileFingerprint !== authority.profileFingerprint + ) { + throw new SandboxSnapshotProviderError( + `runtime provider '${bundle.identity.id}' returned invalid managed restore proof`, + ); + } + return receipt; +} + +/** + * Capture the complete provider-neutral snapshot state. Both provider calls + * occur inside the caller's quiescence lock; the provider re-observes runtime + * identity at capture so a stale preflight can never be persisted. + */ +export function captureSandboxRuntimeSnapshot( + bundle: RuntimeProviderBundle, + sandbox: SandboxEntry, +): SandboxRuntimeSnapshot { + const surface = requireSnapshotSurface(bundle, "backup"); + const preflight = requirePreflight( + bundle, + sandbox, + "backup", + surface.preflight("backup", sandbox), + ); + const runtime = requireRuntimeReceipt(bundle, surface.capture(sandbox, preflight)); + return { + schemaVersion: SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION, + providerId: bundle.identity.id, + providerHandle: preflight.providerHandle, + lifecycleState: preflight.lifecycleState, + lifecycleGeneration: preflight.lifecycleGeneration, + runtime, + }; +} + +export interface PreparedSandboxRuntimeRestore { + readonly phase: "preflighted"; + readonly targetProviderId: string; + readonly targetSandboxName: string; + readonly source: SandboxRuntimeSnapshot; + readonly preflight: RuntimeProviderSnapshotPreflightReceipt; + readonly managedProfile: RuntimeProviderManagedProfileRestoreAuthority; +} + +export interface ValidatedSandboxRuntimeRestore { + readonly phase: "validated"; + readonly targetProviderId: string; + readonly targetSandboxName: string; + readonly source: SandboxRuntimeSnapshot; + readonly restoreReceipt: RuntimeProviderSnapshotRestoreReceipt; +} + +/** + * Perform the read-only restore preflight before a force-delete or filesystem + * mutation. Source provider handles remain opaque; PR3.8 self-restore requires + * the exact owning provider, while cross-provider rebinding remains deferred. + */ +export function prepareSandboxRuntimeRestore( + bundle: RuntimeProviderBundle, + target: SandboxEntry, + sourceValue: unknown, + managedProfileValue: unknown, +): PreparedSandboxRuntimeRestore { + const source = cloneSandboxRuntimeSnapshot(sourceValue); + if (!source) { + throw new SandboxSnapshotProviderError("snapshot runtime state is invalid"); + } + if (source.providerId !== bundle.identity.id) { + throw new SandboxSnapshotProviderError( + `snapshot runtime provider '${source.providerId}' does not match target provider '${bundle.identity.id}'`, + ); + } + const surface = requireSnapshotSurface(bundle, "restore"); + const managedProfile = + normalizeRuntimeProviderManagedProfileRestoreAuthority(managedProfileValue); + if (!managedProfile) { + throw new SandboxSnapshotProviderError("managed profile restore authority is invalid"); + } + const preflight = requirePreflight( + bundle, + target, + "restore", + surface.preflight("restore", target), + ); + if (preflight.lifecycleState !== source.lifecycleState) { + throw new SandboxSnapshotProviderError( + `target '${target.name}' cannot represent the snapshot lifecycle state`, + ); + } + surface.validateRestore(target, preflight, source, managedProfile); + return Object.freeze({ + phase: "preflighted" as const, + targetProviderId: bundle.identity.id, + targetSandboxName: target.name, + source, + preflight, + managedProfile, + }); +} + +/** + * Invoke the owning provider after filesystem restoration. The provider + * consumes its exact preflight authority, proves the managed profile is live, + * and returns a normalized runtime/restore receipt; central orchestration + * never interprets either opaque handle. + */ +export function confirmSandboxRuntimeRestore( + bundle: RuntimeProviderBundle, + target: SandboxEntry, + prepared: PreparedSandboxRuntimeRestore, +): ValidatedSandboxRuntimeRestore { + if ( + prepared.phase !== "preflighted" || + prepared.targetProviderId !== bundle.identity.id || + prepared.targetSandboxName !== target.name + ) { + throw new SandboxSnapshotProviderError("restore preflight authority is stale"); + } + const surface = requireSnapshotSurface(bundle, "restore"); + const restoreReceipt = requireRestoreReceipt( + bundle, + target, + prepared.managedProfile, + surface.restore(target, prepared.preflight, prepared.source, prepared.managedProfile), + ); + return Object.freeze({ + phase: "validated" as const, + targetProviderId: bundle.identity.id, + targetSandboxName: target.name, + source: prepared.source, + restoreReceipt, + }); +} diff --git a/src/lib/onboard/runtime-provider/contract.ts b/src/lib/onboard/runtime-provider/contract.ts index d629eb51eed..38124212ce0 100644 --- a/src/lib/onboard/runtime-provider/contract.ts +++ b/src/lib/onboard/runtime-provider/contract.ts @@ -5,6 +5,7 @@ import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/ import type { ManagedImageSelectionPolicy } from "../workload/source"; export const RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION = 1 as const; +export const RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION = 1 as const; export type RuntimeProviderGatewayLauncher = "nemoclaw" | "openshell"; export type RuntimeProviderLifecycleAction = "start" | "stop"; @@ -152,7 +153,7 @@ export interface RuntimeProviderCleanupOperations { } /** - * Provider-neutral, bounded state that later snapshot work may persist. + * Provider-neutral, bounded state persisted by provider-backed snapshots. * Provider handles remain opaque strings; acceleration is normalized so no * action module needs a Docker-, CDI-, or device-specific DTO. */ @@ -174,6 +175,49 @@ export interface RuntimeProviderRuntimeReceipt { }; } +export type RuntimeProviderSnapshotOperation = "backup" | "restore"; +export type RuntimeProviderSnapshotLifecycleState = "running" | "paused" | "stopped"; + +export interface RuntimeProviderSnapshotPreflightReceipt { + readonly schemaVersion: typeof RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION; + readonly providerId: string; + readonly operation: RuntimeProviderSnapshotOperation; + readonly sandboxName: string; + readonly providerHandle: string; + readonly lifecycleState: RuntimeProviderSnapshotLifecycleState; + readonly lifecycleGeneration: string; +} + +export interface RuntimeProviderManagedProfileRestoreAuthority { + readonly agent: string; + readonly profileFingerprint: string; +} + +/** + * Complete normalized source state supplied to the owning restore facet. + * `providerHandle` binds the lifecycle generation and full runtime receipt. + */ +export interface RuntimeProviderSnapshotRestoreSource { + readonly schemaVersion: 1; + readonly providerId: string; + readonly providerHandle: string; + readonly lifecycleState: RuntimeProviderSnapshotLifecycleState; + readonly lifecycleGeneration: string; + readonly runtime: RuntimeProviderRuntimeReceipt; +} + +export interface RuntimeProviderSnapshotRestoreReceipt { + readonly schemaVersion: 1; + readonly providerId: string; + readonly sandboxName: string; + /** Provider-authored proof over preflight, source state, profile, and live runtime. */ + readonly providerHandle: string; + readonly lifecycleState: RuntimeProviderSnapshotLifecycleState; + readonly lifecycleGeneration: string; + readonly runtime: RuntimeProviderRuntimeReceipt; + readonly managedProfile: RuntimeProviderManagedProfileRestoreAuthority; +} + export type RuntimeProviderPreflightDoctorSurface = RuntimeProviderSupportedSurface<{ inspectHost(): RuntimeProviderDoctorCheck; preflightLifecycle( @@ -221,8 +265,31 @@ export type RuntimeProviderBootstrapSurface = export type RuntimeProviderSnapshotSurface = | RuntimeProviderSupportedSurface<{ - capture(sandbox: SandboxEntry): RuntimeProviderRuntimeReceipt; - restore(sandbox: SandboxEntry, receipt: RuntimeProviderRuntimeReceipt): void; + readonly capabilities: { + readonly backup: boolean; + readonly restore: boolean; + readonly managedProfileRestore: boolean; + }; + preflight( + operation: RuntimeProviderSnapshotOperation, + sandbox: SandboxEntry, + ): RuntimeProviderSnapshotPreflightReceipt; + capture( + sandbox: SandboxEntry, + preflight: RuntimeProviderSnapshotPreflightReceipt, + ): RuntimeProviderRuntimeReceipt; + validateRestore( + sandbox: SandboxEntry, + preflight: RuntimeProviderSnapshotPreflightReceipt, + source: RuntimeProviderSnapshotRestoreSource, + managedProfile: RuntimeProviderManagedProfileRestoreAuthority, + ): void; + restore( + sandbox: SandboxEntry, + preflight: RuntimeProviderSnapshotPreflightReceipt, + source: RuntimeProviderSnapshotRestoreSource, + managedProfile: RuntimeProviderManagedProfileRestoreAuthority, + ): RuntimeProviderSnapshotRestoreReceipt; }> | RuntimeProviderUnsupportedSurface; diff --git a/src/lib/onboard/runtime-provider/docker.ts b/src/lib/onboard/runtime-provider/docker.ts index e0bb280ee21..f8481c37c3c 100644 --- a/src/lib/onboard/runtime-provider/docker.ts +++ b/src/lib/onboard/runtime-provider/docker.ts @@ -17,6 +17,7 @@ import { MANAGED_IMAGE_REPOSITORIES, MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, } from "../managed-image/contract"; +import { queryOpenShellDockerSandboxRuntimeSnapshot } from "../openshell-docker-sandbox-containers"; import { RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, type RuntimeProviderBundle, @@ -31,6 +32,7 @@ import { type RuntimeProviderWorkloadCleanupResult, type RuntimeProviderWorkloadProfile, } from "./contract"; +import { createDockerRuntimeProviderSnapshotSurface } from "./snapshot"; type DockerOpResult = { status?: number | null }; type DockerStop = (name: string, options?: Record) => DockerOpResult; @@ -50,6 +52,7 @@ export interface DockerRuntimeProviderDependencies { readonly isRuntimeDown: typeof isDockerRuntimeDown; readonly printRuntimeDownGuidance: typeof printDockerRuntimeDownGuidance; readonly recoverSandbox: typeof recoverDockerDriverSandbox; + readonly queryRuntimeSnapshot: typeof queryOpenShellDockerSandboxRuntimeSnapshot; readonly removeImage: DockerRemoveImage; readonly stopContainer: DockerStop; readonly unpauseContainer: DockerUnpause; @@ -82,6 +85,8 @@ function resolveDependencies( isRuntimeDown: overrides.isRuntimeDown ?? isDockerRuntimeDown, printRuntimeDownGuidance: overrides.printRuntimeDownGuidance ?? printDockerRuntimeDownGuidance, recoverSandbox: overrides.recoverSandbox ?? recoverDockerDriverSandbox, + queryRuntimeSnapshot: + overrides.queryRuntimeSnapshot ?? queryOpenShellDockerSandboxRuntimeSnapshot, removeImage: overrides.removeImage ?? ((reference, options) => loadDockerRemoveImage()(reference, options)), @@ -348,7 +353,10 @@ export function createDockerRuntimeProviderBundle( ], }, bootstrap: unsupported(providerId, futureReason), - snapshot: unsupported(providerId, futureReason), + snapshot: createDockerRuntimeProviderSnapshotSurface(providerId, { + captureHostCommand: deps.captureHostCommand, + queryRuntimeSnapshot: deps.queryRuntimeSnapshot, + }), recovery: unsupported(providerId, futureReason), cleanup: { providerId, diff --git a/src/lib/onboard/runtime-provider/registry.ts b/src/lib/onboard/runtime-provider/registry.ts index 81ee3ba7ae4..4318afe3fd9 100644 --- a/src/lib/onboard/runtime-provider/registry.ts +++ b/src/lib/onboard/runtime-provider/registry.ts @@ -4,12 +4,18 @@ import type { SandboxEntry } from "../../state/registry/types"; import { RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, + RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION, type RuntimeProviderBundle, type RuntimeProviderBundleRegistry, type RuntimeProviderChannelStopTransport, type RuntimeProviderContainerEngineOperation, + type RuntimeProviderManagedProfileRestoreAuthority, type RuntimeProviderMutationOperation, type RuntimeProviderRuntimeReceipt, + type RuntimeProviderSnapshotLifecycleState, + type RuntimeProviderSnapshotPreflightReceipt, + type RuntimeProviderSnapshotRestoreReceipt, + type RuntimeProviderSnapshotRestoreSource, } from "./contract"; const PROVIDER_ID_PATTERN = /^[a-z][a-z0-9-]{0,62}$/u; @@ -30,6 +36,15 @@ const BUNDLE_SURFACES = [ ] as const; const MAX_RECEIPT_HANDLE_BYTES = 4096; const MAX_RECEIPT_DEVICES = 64; +const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]/u; +const SHA256_PATTERN = /^[a-f0-9]{64}$/u; +const MANAGED_PROFILE_AGENT_PATTERN = /^[a-z][a-z0-9-]{0,127}$/u; +const SNAPSHOT_OPERATIONS = new Set(["backup", "restore"]); +const SNAPSHOT_LIFECYCLE_STATES = new Set([ + "running", + "paused", + "stopped", +]); const GATEWAY_LAUNCHERS = new Set(["nemoclaw", "openshell"]); const CHANNEL_STOP_TRANSPORTS: ReadonlySet = new Set([ "docker-kubectl-first", @@ -326,7 +341,13 @@ function validateBootstrapSurface(surface: Record): void { function validateSnapshotSurface(surface: Record): void { if (surface.supported === true) { + const capabilities = requireOwnRecord(surface, "capabilities"); + for (const capability of ["backup", "restore", "managedProfileRestore"] as const) { + requireBoolean(capabilities, capability, "snapshot capabilities"); + } + requireFunction(surface, "preflight", "snapshot"); requireFunction(surface, "capture", "snapshot"); + requireFunction(surface, "validateRestore", "snapshot"); requireFunction(surface, "restore", "snapshot"); } } @@ -583,7 +604,10 @@ export function runtimeProviderContainerEngineIdentity( function boundedString(value: unknown, maxBytes: number): value is string { return ( - typeof value === "string" && value.trim() !== "" && Buffer.byteLength(value, "utf8") <= maxBytes + typeof value === "string" && + value.trim() !== "" && + Buffer.byteLength(value, "utf8") <= maxBytes && + !CONTROL_CHARACTERS.test(value) ); } @@ -632,3 +656,103 @@ export function normalizeRuntimeProviderRuntimeReceipt( }, }; } + +export function normalizeRuntimeProviderManagedProfileRestoreAuthority( + value: unknown, +): RuntimeProviderManagedProfileRestoreAuthority | null { + if ( + !isPlainRecord(value) || + typeof value.agent !== "string" || + !MANAGED_PROFILE_AGENT_PATTERN.test(value.agent) || + typeof value.profileFingerprint !== "string" || + !SHA256_PATTERN.test(value.profileFingerprint) + ) { + return null; + } + return { + agent: value.agent, + profileFingerprint: value.profileFingerprint, + }; +} + +export function normalizeRuntimeProviderSnapshotPreflightReceipt( + value: unknown, +): RuntimeProviderSnapshotPreflightReceipt | null { + if ( + !isPlainRecord(value) || + value.schemaVersion !== RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION || + !validProviderId(value.providerId) || + !SNAPSHOT_OPERATIONS.has(String(value.operation)) || + !boundedString(value.sandboxName, 512) || + !boundedString(value.providerHandle, MAX_RECEIPT_HANDLE_BYTES) || + !SNAPSHOT_LIFECYCLE_STATES.has(value.lifecycleState as RuntimeProviderSnapshotLifecycleState) || + !boundedString(value.lifecycleGeneration, MAX_RECEIPT_HANDLE_BYTES) + ) { + return null; + } + return { + schemaVersion: RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION, + providerId: value.providerId, + operation: value.operation as RuntimeProviderSnapshotPreflightReceipt["operation"], + sandboxName: value.sandboxName, + providerHandle: value.providerHandle, + lifecycleState: value.lifecycleState as RuntimeProviderSnapshotLifecycleState, + lifecycleGeneration: value.lifecycleGeneration, + }; +} + +export function normalizeRuntimeProviderSnapshotRestoreSource( + value: unknown, +): RuntimeProviderSnapshotRestoreSource | null { + if ( + !isPlainRecord(value) || + value.schemaVersion !== 1 || + !validProviderId(value.providerId) || + !boundedString(value.providerHandle, MAX_RECEIPT_HANDLE_BYTES) || + !SNAPSHOT_LIFECYCLE_STATES.has(value.lifecycleState as RuntimeProviderSnapshotLifecycleState) || + !boundedString(value.lifecycleGeneration, MAX_RECEIPT_HANDLE_BYTES) + ) { + return null; + } + const runtime = normalizeRuntimeProviderRuntimeReceipt(value.runtime); + if (!runtime || runtime.providerId !== value.providerId) return null; + return { + schemaVersion: 1, + providerId: value.providerId, + providerHandle: value.providerHandle, + lifecycleState: value.lifecycleState as RuntimeProviderSnapshotLifecycleState, + lifecycleGeneration: value.lifecycleGeneration, + runtime, + }; +} + +export function normalizeRuntimeProviderSnapshotRestoreReceipt( + value: unknown, +): RuntimeProviderSnapshotRestoreReceipt | null { + if ( + !isPlainRecord(value) || + value.schemaVersion !== 1 || + !validProviderId(value.providerId) || + !boundedString(value.sandboxName, 512) || + !boundedString(value.providerHandle, MAX_RECEIPT_HANDLE_BYTES) || + !SNAPSHOT_LIFECYCLE_STATES.has(value.lifecycleState as RuntimeProviderSnapshotLifecycleState) || + !boundedString(value.lifecycleGeneration, MAX_RECEIPT_HANDLE_BYTES) + ) { + return null; + } + const runtime = normalizeRuntimeProviderRuntimeReceipt(value.runtime); + const managedProfile = normalizeRuntimeProviderManagedProfileRestoreAuthority( + value.managedProfile, + ); + if (!runtime || runtime.providerId !== value.providerId || !managedProfile) return null; + return { + schemaVersion: 1, + providerId: value.providerId, + sandboxName: value.sandboxName, + providerHandle: value.providerHandle, + lifecycleState: value.lifecycleState as RuntimeProviderSnapshotLifecycleState, + lifecycleGeneration: value.lifecycleGeneration, + runtime, + managedProfile, + }; +} diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index 4a790f2947b..7fa369f87db 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -31,7 +31,11 @@ import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "./current"; import { createDockerRuntimeProviderBundle } from "./docker"; import { createRuntimeProviderBundleRegistry, + normalizeRuntimeProviderManagedProfileRestoreAuthority, normalizeRuntimeProviderRuntimeReceipt, + normalizeRuntimeProviderSnapshotPreflightReceipt, + normalizeRuntimeProviderSnapshotRestoreReceipt, + normalizeRuntimeProviderSnapshotRestoreSource, RuntimeProviderRegistrationError, resolveRuntimeProviderBundle, } from "./registry"; @@ -125,7 +129,18 @@ describe("RuntimeProviderBundle registry contract", () => { expect(bundle[surface].providerId, `${providerId}.${surface}`).toBe(providerId); } expect(bundle.bootstrap).toMatchObject({ supported: false }); - expect(bundle.snapshot).toMatchObject({ supported: false }); + expect(bundle.snapshot).toMatchObject( + providerId === "docker" + ? { + supported: true, + capabilities: { + backup: true, + restore: true, + managedProfileRestore: true, + }, + } + : { supported: false }, + ); expect(bundle.recovery).toMatchObject({ supported: false }); } }); @@ -430,6 +445,71 @@ describe("RuntimeProviderBundle registry contract", () => { runtime: { ...receipt.runtime, handle: "x".repeat(4097) }, }), ).toBeNull(); + expect( + normalizeRuntimeProviderRuntimeReceipt({ + ...receipt, + runtime: { ...receipt.runtime, handle: "opaque\ninjection" }, + }), + ).toBeNull(); + }); + + it("normalizes snapshot preflight and managed restore proof as one bounded contract", () => { + const managedProfile = { + agent: "openclaw", + profileFingerprint: "f".repeat(64), + }; + const preflight = { + schemaVersion: 1, + providerId: "docker", + operation: "restore", + sandboxName: "alpha", + providerHandle: "opaque-preflight", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + }; + const runtime = { + schemaVersion: 1, + providerId: "docker", + runtime: { kind: "docker-container", handle: "c".repeat(64) }, + acceleration: { kind: "none" }, + }; + const source = { + schemaVersion: 1, + providerId: "docker", + providerHandle: "opaque-source", + lifecycleState: "running", + lifecycleGeneration: "source-generation-1", + runtime, + }; + const restore = { + schemaVersion: 1, + providerId: "docker", + sandboxName: "alpha", + providerHandle: "opaque-restore-proof", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime, + managedProfile, + }; + + expect(normalizeRuntimeProviderManagedProfileRestoreAuthority(managedProfile)).toEqual( + managedProfile, + ); + expect(normalizeRuntimeProviderSnapshotPreflightReceipt(preflight)).toEqual(preflight); + expect(normalizeRuntimeProviderSnapshotRestoreSource(source)).toEqual(source); + expect(normalizeRuntimeProviderSnapshotRestoreReceipt(restore)).toEqual(restore); + expect( + normalizeRuntimeProviderSnapshotPreflightReceipt({ + ...preflight, + lifecycleGeneration: "generation\ninjection", + }), + ).toBeNull(); + expect( + normalizeRuntimeProviderSnapshotRestoreReceipt({ + ...restore, + runtime: { ...runtime, providerId: "other" }, + }), + ).toBeNull(); }); }); diff --git a/src/lib/onboard/runtime-provider/snapshot.test.ts b/src/lib/onboard/runtime-provider/snapshot.test.ts new file mode 100644 index 00000000000..75795695dfe --- /dev/null +++ b/src/lib/onboard/runtime-provider/snapshot.test.ts @@ -0,0 +1,730 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { SandboxEntry } from "../../state/registry/types"; +import type { OpenShellDockerSandboxRuntimeSnapshotQuery } from "../openshell-docker-sandbox-containers"; +import type { + RuntimeProviderManagedProfileRestoreAuthority, + RuntimeProviderRuntimeReceipt, + RuntimeProviderSnapshotPreflightReceipt, +} from "./contract"; +import { + createDockerRuntimeProviderSnapshotSurface, + createRuntimeProviderSnapshotSurface, + observeDockerRuntimeSnapshot, + observeOpenShellRuntimeSnapshot, + RuntimeProviderSnapshotError, + type RuntimeProviderSnapshotObservation, +} from "./snapshot"; + +const managedProfile = { + agent: "openclaw", + profileFingerprint: "f".repeat(64), +} as const satisfies RuntimeProviderManagedProfileRestoreAuthority; + +function sandbox(overrides: Partial = {}): SandboxEntry { + return { + name: "alpha", + agent: "openclaw", + openshellDriver: "mxc", + gatewayName: "nemoclaw-18080", + lifecycleLiveIdentityFingerprint: "a".repeat(64), + sandboxGpuEnabled: false, + sandboxGpuMode: "0", + sandboxGpuDevice: null, + ...overrides, + }; +} + +function observation( + providerId = "mxc", + overrides: Partial = {}, +): RuntimeProviderSnapshotObservation { + return { + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: { + schemaVersion: 1, + providerId, + runtime: { kind: "session", handle: "opaque-mxc-session" }, + acceleration: { kind: "none" }, + }, + ...overrides, + }; +} + +function surfaceDriver( + observe: () => RuntimeProviderSnapshotObservation, + restoreManagedProfile = vi.fn(() => "provider-restore-proof"), +) { + return { observe, restoreManagedProfile }; +} + +function snapshotSource( + preflight: RuntimeProviderSnapshotPreflightReceipt, + runtime: RuntimeProviderRuntimeReceipt, +) { + return { + schemaVersion: 1 as const, + providerId: preflight.providerId, + providerHandle: preflight.providerHandle, + lifecycleState: preflight.lifecycleState, + lifecycleGeneration: preflight.lifecycleGeneration, + runtime, + }; +} + +describe("runtime provider snapshot surface", () => { + it("binds the full runtime and lifecycle generation into opaque backup authority", () => { + const observe = vi.fn(() => observation()); + const surface = createRuntimeProviderSnapshotSurface("mxc", surfaceDriver(observe)); + if (!surface.supported) throw new Error("test surface must be supported"); + + const preflight = surface.preflight("backup", sandbox()); + const receipt = surface.capture(sandbox(), preflight); + + expect(preflight).toMatchObject({ + lifecycleGeneration: "generation-1", + lifecycleState: "running", + }); + expect(preflight.providerHandle).toMatch(/^[a-f0-9]{64}$/u); + expect(preflight.providerHandle).not.toContain("opaque-mxc-session"); + expect(receipt.runtime.handle).toBe("opaque-mxc-session"); + expect(observe).toHaveBeenCalledTimes(2); + }); + + it("invokes the owning provider restore facet and returns managed profile/runtime proof", () => { + const restoreManagedProfile = vi.fn(() => "provider-restore-proof"); + const surface = createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => observation(), restoreManagedProfile), + ); + if (!surface.supported) throw new Error("test surface must be supported"); + const target = sandbox(); + const preflight = surface.preflight("restore", target); + + const receipt = surface.restore( + target, + preflight, + snapshotSource(preflight, observation().runtime), + managedProfile, + ); + + expect(restoreManagedProfile).toHaveBeenCalledWith(target, managedProfile); + expect(receipt).toMatchObject({ + providerId: "mxc", + sandboxName: "alpha", + lifecycleGeneration: "generation-1", + runtime: { providerId: "mxc", runtime: { handle: "opaque-mxc-session" } }, + managedProfile, + }); + expect(receipt.providerHandle).toMatch(/^[a-f0-9]{64}$/u); + }); + + it("restores after a runtime restart and binds proof to current runtime plus source provenance", () => { + const target = sandbox(); + const current = observation("mxc", { + lifecycleGeneration: "current-generation", + runtime: { + ...observation().runtime, + runtime: { kind: "session", handle: "current-session" }, + }, + }); + const restoreFrom = ( + targetObservation: RuntimeProviderSnapshotObservation, + sourceGeneration: string, + sourceHandle: string, + ) => { + const sourceObservation = observation("mxc", { + lifecycleGeneration: sourceGeneration, + runtime: { + ...observation().runtime, + runtime: { kind: "session", handle: sourceHandle }, + }, + }); + const sourceSurface = createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => sourceObservation), + ); + if (!sourceSurface.supported) throw new Error("test surface must be supported"); + const sourcePreflight = sourceSurface.preflight("backup", target); + const source = snapshotSource( + sourcePreflight, + sourceSurface.capture(target, sourcePreflight), + ); + const targetSurface = createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => targetObservation), + ); + if (!targetSurface.supported) throw new Error("test surface must be supported"); + const targetPreflight = targetSurface.preflight("restore", target); + return targetSurface.restore(target, targetPreflight, source, managedProfile); + }; + + const first = restoreFrom(current, "source-generation-1", "source-session-1"); + const changedSource = restoreFrom(current, "source-generation-2", "source-session-2"); + const changedCurrent = restoreFrom( + observation("mxc", { + lifecycleGeneration: "next-current-generation", + runtime: { + ...observation().runtime, + runtime: { kind: "session", handle: "next-current-session" }, + }, + }), + "source-generation-1", + "source-session-1", + ); + + expect(first).toMatchObject({ + lifecycleGeneration: "current-generation", + runtime: { runtime: { handle: "current-session" } }, + }); + expect(changedSource.providerHandle).not.toBe(first.providerHandle); + expect(changedCurrent.providerHandle).not.toBe(first.providerHandle); + }); + + it.each([ + { + label: "before managed-profile proof", + observations: [ + observation(), + observation("mxc", { + lifecycleGeneration: "changed-generation", + runtime: { + ...observation().runtime, + runtime: { kind: "session", handle: "changed-session" }, + }, + }), + ], + expectedRestoreCalls: 0, + }, + { + label: "after managed-profile proof", + observations: [ + observation(), + observation(), + observation("mxc", { + lifecycleGeneration: "changed-generation", + runtime: { + ...observation().runtime, + runtime: { kind: "session", handle: "changed-session" }, + }, + }), + ], + expectedRestoreCalls: 1, + }, + ])("rejects current-runtime changes $label", ({ observations, expectedRestoreCalls }) => { + const restoreManagedProfile = vi.fn(() => "provider-restore-proof"); + const observe = vi.fn<() => RuntimeProviderSnapshotObservation>(); + for (const value of observations) observe.mockReturnValueOnce(value); + const surface = createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(observe, restoreManagedProfile), + ); + if (!surface.supported) throw new Error("test surface must be supported"); + const target = sandbox(); + const preflight = surface.preflight("restore", target); + const source = snapshotSource(preflight, observation().runtime); + + expect(() => surface.restore(target, preflight, source, managedProfile)).toThrow( + /runtime changed after snapshot preflight/u, + ); + expect(restoreManagedProfile).toHaveBeenCalledTimes(expectedRestoreCalls); + }); + + it("fails before provider restore when the target cannot represent source acceleration", () => { + const restoreManagedProfile = vi.fn(() => "provider-restore-proof"); + const targetSurface = createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => observation(), restoreManagedProfile), + ); + const sourceObservation = observation("mxc", { + runtime: { + ...observation().runtime, + acceleration: { kind: "gpu", vendor: "nvidia", devices: ["live-device-0"] }, + }, + }); + const sourceSurface = createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => sourceObservation), + ); + if (!targetSurface.supported || !sourceSurface.supported) { + throw new Error("test surfaces must be supported"); + } + const target = sandbox(); + const sourcePreflight = sourceSurface.preflight("backup", target); + const source = snapshotSource(sourcePreflight, sourceSurface.capture(target, sourcePreflight)); + const preflight = targetSurface.preflight("restore", target); + + expect(() => targetSurface.restore(target, preflight, source, managedProfile)).toThrow( + /cannot represent the snapshot acceleration state/u, + ); + expect(restoreManagedProfile).not.toHaveBeenCalled(); + }); + + it("fails before provider restore when the target cannot represent source lifecycle", () => { + const restoreManagedProfile = vi.fn(() => "provider-restore-proof"); + const targetSurface = createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => observation(), restoreManagedProfile), + ); + const stopped = observation("mxc", { lifecycleState: "stopped" }); + const sourceSurface = createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => stopped), + ); + if (!targetSurface.supported || !sourceSurface.supported) { + throw new Error("test surfaces must be supported"); + } + const target = sandbox(); + const sourcePreflight = sourceSurface.preflight("backup", target); + const source = snapshotSource(sourcePreflight, sourceSurface.capture(target, sourcePreflight)); + const targetPreflight = targetSurface.preflight("restore", target); + + expect(() => targetSurface.restore(target, targetPreflight, source, managedProfile)).toThrow( + /cannot represent the snapshot lifecycle state/u, + ); + expect(restoreManagedProfile).not.toHaveBeenCalled(); + }); + + it.each([ + { + label: "runtime identity/lifecycle", + changed: observation("mxc", { + lifecycleState: "paused", + runtime: { + ...observation().runtime, + runtime: { kind: "session", handle: "replacement-session" }, + }, + }), + }, + { + label: "acceleration", + changed: observation("mxc", { + runtime: { + ...observation().runtime, + acceleration: { + kind: "gpu", + vendor: "nvidia", + devices: ["nvidia.com/gpu=0"], + }, + }, + }), + }, + { + label: "lifecycle generation", + changed: observation("mxc", { lifecycleGeneration: "generation-2" }), + }, + ])("rejects a $label race after preflight", ({ changed }) => { + const observe = vi + .fn<() => RuntimeProviderSnapshotObservation>() + .mockReturnValueOnce(observation()) + .mockReturnValueOnce(changed); + const surface = createRuntimeProviderSnapshotSurface("mxc", surfaceDriver(observe)); + if (!surface.supported) throw new Error("test surface must be supported"); + const target = sandbox(); + const preflight = surface.preflight("backup", target); + + expect(() => surface.capture(target, preflight)).toThrow(/runtime changed after/u); + }); + + it("rejects a preflight receipt from another operation or sandbox", () => { + const surface = createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => observation()), + ); + if (!surface.supported) throw new Error("test surface must be supported"); + const target = sandbox(); + const restorePreflight = surface.preflight("restore", target); + const otherTarget = sandbox({ name: "other" }); + + expect(() => surface.capture(target, restorePreflight)).toThrow( + /stale snapshot preflight authority/u, + ); + expect(() => + surface.restore( + otherTarget, + restorePreflight, + snapshotSource(restorePreflight, observation().runtime), + managedProfile, + ), + ).toThrow(/stale snapshot preflight authority/u); + }); + + it("rejects invalid runtime receipts, restore authority, and provider proof", () => { + const invalidRuntime = createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => observation("other-provider")), + ); + if (!invalidRuntime.supported) throw new Error("test surface must be supported"); + expect(() => invalidRuntime.preflight("backup", sandbox())).toThrow(/invalid runtime receipt/u); + + const invalidProof = createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver( + () => observation(), + vi.fn(() => "proof\ninjection"), + ), + ); + if (!invalidProof.supported) throw new Error("test surface must be supported"); + const preflight = invalidProof.preflight("restore", sandbox()); + const source = snapshotSource(preflight, observation().runtime); + expect(() => invalidProof.restore(sandbox(), preflight, source, managedProfile)).toThrow( + /invalid managed profile restore proof/u, + ); + + expect(() => + invalidProof.restore( + sandbox(), + preflight, + { + ...source, + runtime: { + ...source.runtime, + acceleration: { kind: "gpu", vendor: "nvidia", devices: ["tampered-device"] }, + }, + }, + managedProfile, + ), + ).toThrow(/does not match its provider handle/u); + }); +}); + +describe("OpenShell snapshot observation", () => { + const liveAcceleration = { + kind: "gpu", + vendor: "nvidia", + devices: ["provider-live-device-0"], + } as const satisfies RuntimeProviderRuntimeReceipt["acceleration"]; + + it("requires exact live identity, lifecycle generation, and provider acceleration", () => { + const capture = vi.fn(() => ({ + status: 0, + output: "Name: alpha\nId: openshell-alpha-id\nState: Ready\nGeneration: live-generation-7\n", + stdout: "", + stderr: "", + })); + const observeAcceleration = vi.fn(() => liveAcceleration); + + expect( + observeOpenShellRuntimeSnapshot( + sandbox({ + // Contradictory durable fields must not influence the live receipt. + sandboxGpuEnabled: false, + sandboxGpuMode: "0", + sandboxGpuDevice: null, + }), + "mxc", + { capture: capture as never, observeAcceleration }, + ), + ).toEqual({ + lifecycleState: "running", + lifecycleGeneration: "live-generation-7", + runtime: { + schemaVersion: 1, + providerId: "mxc", + runtime: { kind: "openshell-sandbox", handle: "openshell-alpha-id" }, + acceleration: liveAcceleration, + }, + }); + expect(observeAcceleration).toHaveBeenCalledWith( + expect.objectContaining({ name: "alpha" }), + "openshell-alpha-id", + ); + }); + + it("rejects durable identity and acceleration fallbacks", () => { + const result = { + status: 0, + output: "alpha Ready\nGeneration: live-generation-7\n", + stdout: "", + stderr: "", + }; + expect(() => + observeOpenShellRuntimeSnapshot( + sandbox({ + lifecycleLiveIdentityFingerprint: "a".repeat(64), + sandboxGpuEnabled: true, + sandboxGpuMode: "1", + sandboxGpuDevice: "all", + }), + "mxc", + { + capture: (() => result) as never, + observeAcceleration: () => ({ kind: "none" }), + }, + ), + ).toThrow(/exact live runtime identity/u); + + expect(() => + observeOpenShellRuntimeSnapshot(sandbox(), "mxc", { + capture: (() => ({ + ...result, + output: "Id: sandbox-id\nState: Ready\nGeneration: live-generation-7\n", + })) as never, + }), + ).toThrow(/did not supply live acceleration evidence/u); + }); + + it.each([ + ["Paused", "paused"], + ["Stopped", "stopped"], + ["Exited", "stopped"], + ["Created", "stopped"], + ] as const)("normalizes the OpenShell %s lifecycle as %s", (state, expected) => { + expect( + observeOpenShellRuntimeSnapshot(sandbox(), "mxc", { + capture: (() => ({ + status: 0, + output: `Id: sandbox-id\nState: ${state}\nGeneration: generation-1\n`, + stdout: "", + stderr: "", + })) as never, + observeAcceleration: () => ({ kind: "none" }), + }).lifecycleState, + ).toBe(expected); + }); + + it("rejects mismatched provider, failed inspection, or missing generation", () => { + const capture = vi.fn(); + expect(() => + observeOpenShellRuntimeSnapshot(sandbox({ openshellDriver: "docker" }), "mxc", { + capture: capture as never, + }), + ).toThrow(/belongs to another runtime provider/u); + expect(capture).not.toHaveBeenCalled(); + + for (const result of [ + { status: 1, output: "not found", stdout: "", stderr: "" }, + { + status: 0, + signal: "SIGTERM", + output: "Id: sandbox-id\nState: Ready\nGeneration: generation-1\n", + stdout: "", + stderr: "", + }, + ]) { + expect(() => + observeOpenShellRuntimeSnapshot(sandbox(), "mxc", { + capture: (() => result) as never, + observeAcceleration: () => ({ kind: "none" }), + }), + ).toThrow(/runtime identity could not be inspected/u); + } + expect(() => + observeOpenShellRuntimeSnapshot(sandbox(), "mxc", { + capture: (() => ({ + status: 0, + output: "Id: sandbox-id\nState: Ready\n", + stdout: "", + stderr: "", + })) as never, + observeAcceleration: () => ({ kind: "none" }), + }), + ).toThrow(/lifecycle generation cannot be represented/u); + }); +}); + +function dockerSnapshot( + overrides: Partial> = {}, +): Extract { + return { + ok: true, + imageId: `sha256:${"b".repeat(64)}`, + bookkeepingImageRef: "managed@example", + stateError: "", + deviceRequests: null, + devices: null, + runtime: "runc", + nativeGpuAttachmentState: "absent", + containerId: "c".repeat(64), + ...overrides, + }; +} + +function dockerLifecycleCapture( + containerId = "c".repeat(64), + overrides: { status?: string; paused?: boolean; restartCount?: number } = {}, +) { + return vi.fn(() => ({ + status: 0, + stdout: JSON.stringify([ + containerId, + overrides.status ?? "running", + overrides.paused ?? false, + "2026-07-30T12:00:00Z", + "0001-01-01T00:00:00Z", + overrides.restartCount ?? 0, + ]), + stderr: "", + })); +} + +describe("Docker provider snapshot evidence", () => { + it("captures exact live container, lifecycle, and device selectors", () => { + const queryRuntimeSnapshot = vi.fn(() => + dockerSnapshot({ + deviceRequests: [ + { + Driver: "nvidia", + Count: 0, + DeviceIDs: ["GPU-live-0"], + Capabilities: [["gpu"]], + Options: null, + }, + ], + nativeGpuAttachmentState: "present", + runtime: "nvidia", + }), + ); + const observed = observeDockerRuntimeSnapshot( + sandbox({ + openshellDriver: "docker", + sandboxGpuEnabled: false, + sandboxGpuMode: "0", + sandboxGpuDevice: null, + }), + "docker", + { captureHostCommand: dockerLifecycleCapture(), queryRuntimeSnapshot }, + ); + + expect(observed).toMatchObject({ + lifecycleState: "running", + lifecycleGeneration: expect.stringMatching(/^[a-f0-9]{64}$/u), + runtime: { + providerId: "docker", + runtime: { kind: "docker-container", handle: "c".repeat(64) }, + acceleration: { + kind: "gpu", + vendor: "nvidia", + devices: ["docker-device-id:GPU-live-0"], + }, + }, + }); + }); + + it("accepts Docker's explicit count=-1 selector but rejects inferred or ambiguous GPU state", () => { + const allDevices = observeDockerRuntimeSnapshot( + sandbox({ openshellDriver: "docker" }), + "docker", + { + captureHostCommand: dockerLifecycleCapture(), + queryRuntimeSnapshot: () => + dockerSnapshot({ + deviceRequests: [ + { + Driver: "nvidia", + Count: -1, + DeviceIDs: null, + Capabilities: [["gpu"]], + Options: null, + }, + ], + nativeGpuAttachmentState: "present", + runtime: "nvidia", + }), + }, + ); + expect(allDevices.runtime.acceleration).toMatchObject({ + devices: ["docker-device-request:nvidia:count=-1"], + }); + + for (const snapshot of [ + dockerSnapshot({ + deviceRequests: null, + nativeGpuAttachmentState: "present", + runtime: "nvidia", + }), + dockerSnapshot({ + deviceRequests: [ + { + Driver: "nvidia", + Count: 1, + DeviceIDs: null, + Capabilities: [["gpu"]], + Options: null, + }, + ], + nativeGpuAttachmentState: "present", + runtime: "nvidia", + }), + dockerSnapshot({ nativeGpuAttachmentState: "unknown", runtime: "custom-runtime" }), + ]) { + expect(() => + observeDockerRuntimeSnapshot(sandbox({ openshellDriver: "docker" }), "docker", { + captureHostCommand: dockerLifecycleCapture(), + queryRuntimeSnapshot: () => snapshot, + }), + ).toThrow(/acceleration|exact live device selectors/u); + } + }); + + it("runs the in-sandbox managed-profile verifier during restore and fails closed on refusal", () => { + const captureOpenShell = vi.fn(() => ({ + status: 0, + output: "[managed-startup] verified openclaw profile completion\n", + stdout: "", + stderr: "", + })); + const dependencies = { + captureHostCommand: dockerLifecycleCapture(), + captureOpenShell: captureOpenShell as never, + queryRuntimeSnapshot: () => dockerSnapshot(), + }; + const surface = createDockerRuntimeProviderSnapshotSurface("docker", dependencies); + if (!surface.supported) throw new Error("Docker snapshot surface must be supported"); + const target = sandbox({ openshellDriver: "docker" }); + const preflight = surface.preflight("restore", target); + const source = snapshotSource(preflight, { + schemaVersion: 1, + providerId: "docker", + runtime: { kind: "docker-container", handle: "c".repeat(64) }, + acceleration: { kind: "none" }, + }); + const receipt = surface.restore(target, preflight, source, managedProfile); + expect(receipt.managedProfile).toEqual(managedProfile); + expect(captureOpenShell).toHaveBeenCalledWith( + [ + "sandbox", + "exec", + "--name", + "alpha", + "-g", + "nemoclaw-18080", + "--no-tty", + "--timeout", + "10", + "--", + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", + "--verify-completion", + "--agent", + "openclaw", + "--profile-fingerprint", + managedProfile.profileFingerprint, + ], + expect.objectContaining({ + ignoreError: true, + includeStderr: true, + timeout: 15_000, + }), + ); + + const denied = createDockerRuntimeProviderSnapshotSurface("docker", { + ...dependencies, + captureOpenShell: (() => ({ + status: 1, + output: "profile mismatch", + stdout: "", + stderr: "", + })) as never, + }); + if (!denied.supported) throw new Error("Docker snapshot surface must be supported"); + const deniedPreflight = denied.preflight("restore", target); + const deniedSource = snapshotSource(deniedPreflight, source.runtime); + expect(() => denied.restore(target, deniedPreflight, deniedSource, managedProfile)).toThrow( + /managed profile restoration could not be proven/u, + ); + }); +}); diff --git a/src/lib/onboard/runtime-provider/snapshot.ts b/src/lib/onboard/runtime-provider/snapshot.ts new file mode 100644 index 00000000000..24041a10575 --- /dev/null +++ b/src/lib/onboard/runtime-provider/snapshot.ts @@ -0,0 +1,666 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; + +import { captureOpenshell } from "../../adapters/openshell/runtime"; +import type { SandboxEntry } from "../../state/registry/types"; +import { resolveSandboxGatewayName } from "../gateway-binding"; +import { + type OpenShellDockerSandboxRuntimeSnapshotQuery, + queryOpenShellDockerSandboxRuntimeSnapshot, +} from "../openshell-docker-sandbox-containers"; +import { + RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION, + type RuntimeProviderCommandCapture, + type RuntimeProviderManagedProfileRestoreAuthority, + type RuntimeProviderRuntimeReceipt, + type RuntimeProviderSnapshotLifecycleState, + type RuntimeProviderSnapshotOperation, + type RuntimeProviderSnapshotPreflightReceipt, + type RuntimeProviderSnapshotRestoreReceipt, + type RuntimeProviderSnapshotRestoreSource, + type RuntimeProviderSnapshotSurface, +} from "./contract"; +import { + normalizeRuntimeProviderIdentity, + normalizeRuntimeProviderManagedProfileRestoreAuthority, + normalizeRuntimeProviderRuntimeReceipt, + normalizeRuntimeProviderSnapshotPreflightReceipt, + normalizeRuntimeProviderSnapshotRestoreSource, +} from "./registry"; + +const SANDBOX_ID_PATTERN = /^[A-Za-z0-9._-]{1,512}$/u; +const DOCKER_CONTAINER_ID_PATTERN = /^[a-f0-9]{64}$/u; +const MANAGED_STARTUP_RUNTIME_EXECUTABLE = + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs"; +const LIFECYCLE_GENERATION_PATTERN = /^[A-Za-z0-9._:/=-]{1,512}$/u; +const ANSI_PATTERN = /\u001b\[[0-9;]*m/gu; +const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]/u; + +export interface RuntimeProviderSnapshotObservation { + readonly lifecycleState: RuntimeProviderSnapshotLifecycleState; + readonly lifecycleGeneration: string; + readonly runtime: RuntimeProviderRuntimeReceipt; +} + +export type RuntimeProviderSnapshotObserver = ( + sandbox: SandboxEntry, + providerId: string, +) => RuntimeProviderSnapshotObservation; + +export type RuntimeProviderManagedProfileRestorer = ( + sandbox: SandboxEntry, + authority: RuntimeProviderManagedProfileRestoreAuthority, +) => string; + +export interface RuntimeProviderSnapshotDriver { + readonly observe: RuntimeProviderSnapshotObserver; + readonly restoreManagedProfile: RuntimeProviderManagedProfileRestorer; +} + +export interface OpenShellRuntimeSnapshotDependencies { + readonly capture: typeof captureOpenshell; + /** + * The owning provider must supply acceleration observed from its live + * runtime. Durable registry intent is deliberately not accepted here. + */ + readonly observeAcceleration: ( + sandbox: SandboxEntry, + runtimeId: string, + ) => RuntimeProviderRuntimeReceipt["acceleration"]; +} + +export interface DockerRuntimeSnapshotDependencies { + readonly captureHostCommand: ( + command: string, + args: string[], + timeout?: number, + ) => RuntimeProviderCommandCapture; + readonly captureOpenShell: typeof captureOpenshell; + readonly queryRuntimeSnapshot: ( + sandboxName: string, + ) => OpenShellDockerSandboxRuntimeSnapshotQuery; +} + +export class RuntimeProviderSnapshotError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(`Runtime snapshot provider failed: ${message}`, options); + this.name = "RuntimeProviderSnapshotError"; + } +} + +function gatewayScopedSandboxGetArgs(sandbox: SandboxEntry): string[] { + const gatewayName = resolveSandboxGatewayName(sandbox); + return gatewayName + ? ["sandbox", "get", "-g", gatewayName, sandbox.name] + : ["sandbox", "get", sandbox.name]; +} + +function gatewayScopedManagedProfileVerifyArgs( + sandbox: SandboxEntry, + authority: RuntimeProviderManagedProfileRestoreAuthority, +): string[] { + const args = ["sandbox", "exec", "--name", sandbox.name]; + const gatewayName = resolveSandboxGatewayName(sandbox); + if (gatewayName) args.push("-g", gatewayName); + args.push( + "--no-tty", + "--timeout", + "10", + "--", + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + "--verify-completion", + "--agent", + authority.agent, + "--profile-fingerprint", + authority.profileFingerprint, + ); + return args; +} + +function cleanOutput(value: string): string { + return value.replace(ANSI_PATTERN, ""); +} + +function parseSandboxId(output: string): string | null { + const match = cleanOutput(output).match(/^\s*(?:Id|ID):\s*([A-Za-z0-9._-]+)\s*$/mu); + return match && SANDBOX_ID_PATTERN.test(match[1] ?? "") ? (match[1] ?? null) : null; +} + +function parseLifecycleState( + output: string, + sandboxName: string, +): RuntimeProviderSnapshotLifecycleState | null { + const clean = cleanOutput(output); + const field = clean.match(/^\s*(?:State|Phase|Status):\s*([A-Za-z][A-Za-z0-9_-]*)\s*$/imu)?.[1]; + const row = clean + .split(/\r?\n/u) + .map((line) => line.trim().split(/\s+/u)) + .find((columns) => columns[0] === sandboxName); + const phase = field ?? row?.slice(1).find((value) => /^[A-Za-z][A-Za-z0-9_-]*$/u.test(value)); + if (phase === "Ready" || phase === "Running") return "running"; + if (phase === "Paused") return "paused"; + if (phase === "Stopped" || phase === "Exited" || phase === "Created") return "stopped"; + return null; +} + +function parseLifecycleGeneration(output: string): string | null { + const match = cleanOutput(output).match( + /^\s*(?:Generation|ResourceVersion|Resource Version):\s*([A-Za-z0-9._:/=-]+)\s*$/imu, + ); + const generation = match?.[1] ?? ""; + return LIFECYCLE_GENERATION_PATTERN.test(generation) ? generation : null; +} + +/** + * Observe an OpenShell-owned runtime without exposing its CLI shape to the + * snapshot action. Exact live identity, lifecycle generation, and provider + * acceleration evidence are all mandatory; durable fallbacks fail closed. + */ +export function observeOpenShellRuntimeSnapshot( + sandbox: SandboxEntry, + providerId: string, + dependencies: Partial = {}, +): RuntimeProviderSnapshotObservation { + if (normalizeRuntimeProviderIdentity(sandbox.openshellDriver) !== providerId) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' belongs to another runtime provider`, + ); + } + const capture = dependencies.capture ?? captureOpenshell; + const result = capture(gatewayScopedSandboxGetArgs(sandbox), { + ignoreError: true, + includeStderr: true, + timeout: 10_000, + }); + if (result.status !== 0 || result.error || result.signal) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' runtime identity could not be inspected`, + ); + } + const output = result.output || ""; + const sandboxId = parseSandboxId(output); + if (!sandboxId) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' exact live runtime identity cannot be represented`, + ); + } + const lifecycleState = parseLifecycleState(output, sandbox.name); + const lifecycleGeneration = parseLifecycleGeneration(output); + if (!lifecycleState || !lifecycleGeneration) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' lifecycle generation cannot be represented`, + ); + } + if (!dependencies.observeAcceleration) { + throw new RuntimeProviderSnapshotError( + `provider '${providerId}' did not supply live acceleration evidence`, + ); + } + return { + lifecycleState, + lifecycleGeneration, + runtime: { + schemaVersion: 1, + providerId, + runtime: { + kind: "openshell-sandbox", + handle: sandboxId, + }, + acceleration: dependencies.observeAcceleration(sandbox, sandboxId), + }, + }; +} + +function dockerRequestUsesGpu( + request: NonNullable< + Extract["deviceRequests"] + >[number], +): boolean { + return ( + request.Driver.trim().toLowerCase() === "nvidia" || + request.DeviceIDs?.some((device) => /^nvidia[.]com\/gpu(?:=|$)/iu.test(device.trim())) === + true || + request.Capabilities?.some((group) => + group.some((capability) => capability.trim().toLowerCase() === "gpu"), + ) === true + ); +} + +function dockerGpuSelectors( + snapshot: Extract, +): RuntimeProviderRuntimeReceipt["acceleration"] { + if (snapshot.nativeGpuAttachmentState === "absent") return { kind: "none" }; + if (snapshot.nativeGpuAttachmentState !== "present") { + throw new RuntimeProviderSnapshotError("Docker returned ambiguous live acceleration evidence"); + } + + const selectors: string[] = []; + for (const request of snapshot.deviceRequests ?? []) { + if (!dockerRequestUsesGpu(request)) continue; + if (request.DeviceIDs && request.DeviceIDs.length > 0) { + for (const device of request.DeviceIDs) { + selectors.push(`docker-device-id:${device}`); + } + continue; + } + if (request.Count === -1) { + // Count=-1 is Docker's explicit live all-device selector. Never infer + // this value from a durable "GPU enabled" flag. + selectors.push(`docker-device-request:${request.Driver || "default"}:count=-1`); + continue; + } + throw new RuntimeProviderSnapshotError( + "Docker GPU attachment does not expose exact live device selectors", + ); + } + for (const mapping of snapshot.devices ?? []) { + const rendered = + `docker-device-path:${mapping.PathOnHost}=>${mapping.PathInContainer}` + + `:${mapping.CgroupPermissions}`; + if ( + /^\/dev\/(?:nvidia|dri|nvhost|nvmap|tegra)/iu.test(mapping.PathOnHost.trim()) || + /^\/dev\/(?:nvidia|dri|nvhost|nvmap|tegra)/iu.test(mapping.PathInContainer.trim()) + ) { + selectors.push(rendered); + } + } + const devices = [...new Set(selectors)].sort(); + if ( + devices.length === 0 || + devices.some( + (device) => + device.trim() === "" || + Buffer.byteLength(device, "utf8") > 512 || + CONTROL_CHARACTERS.test(device), + ) + ) { + throw new RuntimeProviderSnapshotError( + "Docker GPU attachment does not expose exact live device selectors", + ); + } + return { kind: "gpu", vendor: "nvidia", devices }; +} + +function parseDockerLifecycle( + result: RuntimeProviderCommandCapture, + expectedContainerId: string, +): { + readonly state: RuntimeProviderSnapshotLifecycleState; + readonly generation: string; +} { + if (result.status !== 0 || result.error) { + throw new RuntimeProviderSnapshotError("Docker lifecycle state could not be inspected"); + } + let fields: unknown; + try { + fields = JSON.parse(result.stdout.trim()); + } catch { + throw new RuntimeProviderSnapshotError("Docker returned malformed lifecycle state"); + } + if ( + !Array.isArray(fields) || + fields.length !== 6 || + fields[0] !== expectedContainerId || + typeof fields[1] !== "string" || + typeof fields[2] !== "boolean" || + typeof fields[3] !== "string" || + typeof fields[4] !== "string" || + !Number.isSafeInteger(fields[5]) || + fields[5] < 0 + ) { + throw new RuntimeProviderSnapshotError("Docker returned malformed lifecycle state"); + } + const status = fields[1].trim().toLowerCase(); + let state: RuntimeProviderSnapshotLifecycleState; + if (status === "running") state = fields[2] ? "paused" : "running"; + else if (["created", "exited", "dead"].includes(status) && fields[2] === false) state = "stopped"; + else { + throw new RuntimeProviderSnapshotError( + `Docker lifecycle '${status || "unknown"}' cannot be represented`, + ); + } + const generation = createHash("sha256") + .update( + JSON.stringify({ + containerId: fields[0], + status, + paused: fields[2], + startedAt: fields[3], + finishedAt: fields[4], + restartCount: fields[5], + }), + "utf8", + ) + .digest("hex"); + return { state, generation }; +} + +export function observeDockerRuntimeSnapshot( + sandbox: SandboxEntry, + providerId: string, + dependencies: Pick< + DockerRuntimeSnapshotDependencies, + "captureHostCommand" | "queryRuntimeSnapshot" + >, +): RuntimeProviderSnapshotObservation { + if (normalizeRuntimeProviderIdentity(sandbox.openshellDriver) !== providerId) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' belongs to another runtime provider`, + ); + } + const snapshot = dependencies.queryRuntimeSnapshot(sandbox.name); + if (!snapshot.ok || !DOCKER_CONTAINER_ID_PATTERN.test(snapshot.containerId)) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' exact Docker runtime identity could not be inspected`, + ); + } + const lifecycle = parseDockerLifecycle( + dependencies.captureHostCommand( + "docker", + [ + "inspect", + "--type", + "container", + "--format", + "[{{json .Id}},{{json .State.Status}},{{json .State.Paused}},{{json .State.StartedAt}},{{json .State.FinishedAt}},{{json .RestartCount}}]", + snapshot.containerId, + ], + 10_000, + ), + snapshot.containerId, + ); + return { + lifecycleState: lifecycle.state, + lifecycleGeneration: lifecycle.generation, + runtime: { + schemaVersion: 1, + providerId, + runtime: { kind: "docker-container", handle: snapshot.containerId }, + acceleration: dockerGpuSelectors(snapshot), + }, + }; +} + +export function verifyOpenShellManagedProfileRestore( + sandbox: SandboxEntry, + authorityValue: RuntimeProviderManagedProfileRestoreAuthority, + dependencies: Pick, +): string { + const authority = normalizeRuntimeProviderManagedProfileRestoreAuthority(authorityValue); + if (!authority) { + throw new RuntimeProviderSnapshotError("managed profile restore authority is invalid"); + } + const result = dependencies.captureOpenShell( + gatewayScopedManagedProfileVerifyArgs(sandbox, authority), + { + ignoreError: true, + includeStderr: true, + timeout: 15_000, + }, + ); + if (result.status !== 0 || result.error || result.signal) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' managed profile restoration could not be proven`, + ); + } + return createHash("sha256") + .update(sandbox.name, "utf8") + .update("\0", "utf8") + .update(authority.agent, "utf8") + .update("\0", "utf8") + .update(authority.profileFingerprint, "utf8") + .update("\0", "utf8") + .update(cleanOutput(result.output || ""), "utf8") + .digest("hex"); +} + +function opaqueProviderHandle( + providerId: string, + observation: RuntimeProviderSnapshotObservation, +): string { + return createHash("sha256") + .update( + JSON.stringify({ + providerId, + lifecycleState: observation.lifecycleState, + lifecycleGeneration: observation.lifecycleGeneration, + runtime: observation.runtime, + }), + "utf8", + ) + .digest("hex"); +} + +function observeAndNormalize( + observer: RuntimeProviderSnapshotObserver, + sandbox: SandboxEntry, + providerId: string, +): RuntimeProviderSnapshotObservation { + const observed = observer(sandbox, providerId); + const runtime = normalizeRuntimeProviderRuntimeReceipt(observed.runtime); + if (!runtime || runtime.providerId !== providerId) { + throw new RuntimeProviderSnapshotError( + `provider '${providerId}' returned an invalid runtime receipt`, + ); + } + if ( + !["running", "paused", "stopped"].includes(observed.lifecycleState) || + !LIFECYCLE_GENERATION_PATTERN.test(observed.lifecycleGeneration) + ) { + throw new RuntimeProviderSnapshotError( + `provider '${providerId}' returned invalid lifecycle authority`, + ); + } + return { + lifecycleState: observed.lifecycleState, + lifecycleGeneration: observed.lifecycleGeneration, + runtime, + }; +} + +function requireStablePreflight( + value: RuntimeProviderSnapshotPreflightReceipt, + providerId: string, + operation: RuntimeProviderSnapshotOperation, + sandbox: SandboxEntry, +): RuntimeProviderSnapshotPreflightReceipt { + const normalized = normalizeRuntimeProviderSnapshotPreflightReceipt(value); + if ( + !normalized || + normalized.providerId !== providerId || + normalized.operation !== operation || + normalized.sandboxName !== sandbox.name + ) { + throw new RuntimeProviderSnapshotError( + `provider '${providerId}' received stale snapshot preflight authority`, + ); + } + return normalized; +} + +function assertUnchanged( + providerId: string, + expected: RuntimeProviderSnapshotPreflightReceipt, + observed: RuntimeProviderSnapshotObservation, +): void { + if ( + opaqueProviderHandle(providerId, observed) !== expected.providerHandle || + observed.lifecycleState !== expected.lifecycleState || + observed.lifecycleGeneration !== expected.lifecycleGeneration + ) { + throw new RuntimeProviderSnapshotError( + `sandbox '${expected.sandboxName}' runtime changed after snapshot preflight`, + ); + } +} + +function restoreProviderHandle( + preflight: RuntimeProviderSnapshotPreflightReceipt, + source: RuntimeProviderSnapshotRestoreSource, + authority: RuntimeProviderManagedProfileRestoreAuthority, + providerProof: string, + observed: RuntimeProviderSnapshotObservation, +): string { + return createHash("sha256") + .update( + JSON.stringify({ + preflight, + source, + authority, + providerProof, + observed, + }), + "utf8", + ) + .digest("hex"); +} + +function validateRestoreRequest( + providerId: string, + driver: RuntimeProviderSnapshotDriver, + sandbox: SandboxEntry, + preflightValue: RuntimeProviderSnapshotPreflightReceipt, + sourceValue: RuntimeProviderSnapshotRestoreSource, + managedProfileValue: RuntimeProviderManagedProfileRestoreAuthority, +): { + readonly expected: RuntimeProviderSnapshotPreflightReceipt; + readonly source: RuntimeProviderSnapshotRestoreSource; + readonly managedProfile: RuntimeProviderManagedProfileRestoreAuthority; +} { + const expected = requireStablePreflight(preflightValue, providerId, "restore", sandbox); + const source = normalizeRuntimeProviderSnapshotRestoreSource(sourceValue); + if (!source || source.providerId !== providerId) { + throw new RuntimeProviderSnapshotError( + "source runtime authority is invalid or belongs to another provider", + ); + } + const sourceObservation = { + lifecycleState: source.lifecycleState, + lifecycleGeneration: source.lifecycleGeneration, + runtime: source.runtime, + }; + if (opaqueProviderHandle(providerId, sourceObservation) !== source.providerHandle) { + throw new RuntimeProviderSnapshotError( + "source runtime receipt does not match its provider handle", + ); + } + // A recovery may legitimately follow a runtime restart. Preserve the exact + // current handle/generation and bind them into the restore receipt rather + // than requiring them to equal the historical source identity. + if (source.lifecycleState !== expected.lifecycleState) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' cannot represent the snapshot lifecycle state`, + ); + } + const managedProfile = + normalizeRuntimeProviderManagedProfileRestoreAuthority(managedProfileValue); + if (!managedProfile) { + throw new RuntimeProviderSnapshotError("managed profile restore authority is invalid"); + } + const observed = observeAndNormalize(driver.observe, sandbox, providerId); + assertUnchanged(providerId, expected, observed); + if (!isDeepStrictEqual(source.runtime.acceleration, observed.runtime.acceleration)) { + throw new RuntimeProviderSnapshotError( + `sandbox '${sandbox.name}' cannot represent the snapshot acceleration state`, + ); + } + return { expected, source, managedProfile }; +} + +export function createRuntimeProviderSnapshotSurface( + providerId: string, + driver: RuntimeProviderSnapshotDriver, +): RuntimeProviderSnapshotSurface { + const capabilities = { + backup: true, + restore: true, + managedProfileRestore: true, + } as const; + return { + providerId, + supported: true, + capabilities, + preflight(operation, sandbox) { + const observed = observeAndNormalize(driver.observe, sandbox, providerId); + return { + schemaVersion: RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION, + providerId, + operation, + sandboxName: sandbox.name, + providerHandle: opaqueProviderHandle(providerId, observed), + lifecycleState: observed.lifecycleState, + lifecycleGeneration: observed.lifecycleGeneration, + }; + }, + capture(sandbox, preflight) { + const expected = requireStablePreflight(preflight, providerId, "backup", sandbox); + const observed = observeAndNormalize(driver.observe, sandbox, providerId); + assertUnchanged(providerId, expected, observed); + return observed.runtime; + }, + validateRestore(sandbox, preflight, source, managedProfile) { + validateRestoreRequest(providerId, driver, sandbox, preflight, source, managedProfile); + }, + restore(sandbox, preflight, sourceValue, managedProfileValue) { + const { expected, source, managedProfile } = validateRestoreRequest( + providerId, + driver, + sandbox, + preflight, + sourceValue, + managedProfileValue, + ); + const providerProof = driver.restoreManagedProfile(sandbox, managedProfile); + if ( + typeof providerProof !== "string" || + providerProof.trim() === "" || + Buffer.byteLength(providerProof, "utf8") > 4096 || + CONTROL_CHARACTERS.test(providerProof) + ) { + throw new RuntimeProviderSnapshotError( + `provider '${providerId}' returned invalid managed profile restore proof`, + ); + } + const after = observeAndNormalize(driver.observe, sandbox, providerId); + assertUnchanged(providerId, expected, after); + const receipt = { + schemaVersion: 1 as const, + providerId, + sandboxName: sandbox.name, + providerHandle: restoreProviderHandle( + expected, + source, + managedProfile, + providerProof, + after, + ), + lifecycleState: after.lifecycleState, + lifecycleGeneration: after.lifecycleGeneration, + runtime: after.runtime, + managedProfile, + } satisfies RuntimeProviderSnapshotRestoreReceipt; + return receipt; + }, + }; +} + +export function createDockerRuntimeProviderSnapshotSurface( + providerId: string, + dependencies: Partial & + Pick, +): RuntimeProviderSnapshotSurface { + const resolved = { + captureHostCommand: dependencies.captureHostCommand, + captureOpenShell: dependencies.captureOpenShell ?? captureOpenshell, + queryRuntimeSnapshot: + dependencies.queryRuntimeSnapshot ?? queryOpenShellDockerSandboxRuntimeSnapshot, + }; + return createRuntimeProviderSnapshotSurface(providerId, { + observe: (sandbox, id) => observeDockerRuntimeSnapshot(sandbox, id, resolved), + restoreManagedProfile: (sandbox, authority) => + verifyOpenShellManagedProfileRestore(sandbox, authority, resolved), + }); +} diff --git a/src/lib/state/registry/runtime-snapshot.test.ts b/src/lib/state/registry/runtime-snapshot.test.ts new file mode 100644 index 00000000000..597decd9fdc --- /dev/null +++ b/src/lib/state/registry/runtime-snapshot.test.ts @@ -0,0 +1,153 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + cloneSandboxRuntimeSnapshot, + SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION, +} from "./runtime-snapshot"; + +function gpuSnapshot() { + return { + schemaVersion: SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION, + providerId: "mxc", + providerHandle: "mxc-snapshot:opaque-123", + lifecycleState: "running", + lifecycleGeneration: "generation-42", + runtime: { + schemaVersion: 1, + providerId: "mxc", + runtime: { kind: "session", handle: "opaque-session-42" }, + acceleration: { + kind: "gpu", + vendor: "nvidia", + devices: ["nvidia.com/gpu=0"], + }, + }, + } as const; +} + +describe("sandbox runtime snapshot normalization", () => { + it("clones opaque provider and runtime handles without interpreting them", () => { + const input = gpuSnapshot(); + const normalized = cloneSandboxRuntimeSnapshot(input); + + expect(normalized).toEqual(input); + expect(normalized).not.toBe(input); + expect(normalized?.runtime).not.toBe(input.runtime); + expect(normalized?.runtime.acceleration).not.toBe(input.runtime.acceleration); + }); + + it("rejects provider identity drift between the wrapper and runtime receipt", () => { + expect( + cloneSandboxRuntimeSnapshot({ + ...gpuSnapshot(), + providerId: "docker", + }), + ).toBeUndefined(); + }); + + it.each([ + { lifecycleState: "restarting" }, + { lifecycleGeneration: "" }, + { providerHandle: "" }, + { providerHandle: "opaque\nhandle" }, + { schemaVersion: 2 }, + ])("rejects an unrepresentable persisted wrapper: %j", (change) => { + expect(cloneSandboxRuntimeSnapshot({ ...gpuSnapshot(), ...change })).toBeUndefined(); + }); + + it("rejects malformed or duplicate normalized acceleration devices", () => { + expect( + cloneSandboxRuntimeSnapshot({ + ...gpuSnapshot(), + runtime: { + ...gpuSnapshot().runtime, + acceleration: { + kind: "gpu", + vendor: "nvidia", + devices: ["nvidia.com/gpu=0", "nvidia.com/gpu=0"], + }, + }, + }), + ).toBeUndefined(); + }); + + it("drops unknown persisted keys instead of widening snapshot authority", () => { + expect( + cloneSandboxRuntimeSnapshot({ + ...gpuSnapshot(), + engine: "podman", + containerName: "must-not-become-authority", + runtime: { + ...gpuSnapshot().runtime, + command: ["delete", "by-name"], + }, + }), + ).toEqual(gpuSnapshot()); + }); + + it.each([ + { + label: "runtime control characters", + runtime: { + ...gpuSnapshot().runtime, + runtime: { kind: "session", handle: "opaque\nsession" }, + }, + }, + { + label: "empty runtime kind", + runtime: { + ...gpuSnapshot().runtime, + runtime: { kind: "", handle: "opaque" }, + }, + }, + { + label: "empty GPU device inventory", + runtime: { + ...gpuSnapshot().runtime, + acceleration: { kind: "gpu", vendor: "nvidia", devices: [] }, + }, + }, + { + label: "unknown acceleration kind", + runtime: { + ...gpuSnapshot().runtime, + acceleration: { kind: "tpu", devices: ["all"] }, + }, + }, + ])("rejects $label in the nested provider receipt", ({ runtime }) => { + expect(cloneSandboxRuntimeSnapshot({ ...gpuSnapshot(), runtime })).toBeUndefined(); + }); + + it("accepts a bounded provider-neutral no-acceleration receipt", () => { + expect( + cloneSandboxRuntimeSnapshot({ + schemaVersion: 1, + providerId: "kubernetes", + providerHandle: "opaque-provider-handle", + lifecycleState: "stopped", + lifecycleGeneration: "generation-1", + runtime: { + schemaVersion: 1, + providerId: "kubernetes", + runtime: { kind: "sandbox", handle: "opaque-runtime-handle" }, + acceleration: { kind: "none" }, + }, + }), + ).toEqual({ + schemaVersion: 1, + providerId: "kubernetes", + providerHandle: "opaque-provider-handle", + lifecycleState: "stopped", + lifecycleGeneration: "generation-1", + runtime: { + schemaVersion: 1, + providerId: "kubernetes", + runtime: { kind: "sandbox", handle: "opaque-runtime-handle" }, + acceleration: { kind: "none" }, + }, + }); + }); +}); diff --git a/src/lib/state/registry/runtime-snapshot.ts b/src/lib/state/registry/runtime-snapshot.ts new file mode 100644 index 00000000000..d4fc90886cb --- /dev/null +++ b/src/lib/state/registry/runtime-snapshot.ts @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { RuntimeProviderRuntimeReceipt } from "../../onboard/runtime-provider/contract"; +import { normalizeRuntimeProviderRuntimeReceipt } from "../../onboard/runtime-provider/registry"; + +export const SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION = 1 as const; + +export type SandboxRuntimeLifecycleState = "running" | "paused" | "stopped"; + +/** + * Provider-neutral runtime state persisted beside a filesystem snapshot. + * + * `providerHandle` and `runtime.handle` remain opaque to the state and action + * layers. Only the owning provider may interpret either value. + */ +export interface SandboxRuntimeSnapshot { + readonly schemaVersion: typeof SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION; + readonly providerId: string; + readonly providerHandle: string; + readonly lifecycleState: SandboxRuntimeLifecycleState; + readonly lifecycleGeneration: string; + readonly runtime: RuntimeProviderRuntimeReceipt; +} + +const LIFECYCLE_STATES = new Set(["running", "paused", "stopped"]); +const MAX_PROVIDER_HANDLE_BYTES = 4096; +const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]/u; + +function isPlainRecord(value: unknown): value is Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function validProviderHandle(value: unknown): value is string { + return ( + typeof value === "string" && + value.trim() !== "" && + Buffer.byteLength(value, "utf8") <= MAX_PROVIDER_HANDLE_BYTES && + !CONTROL_CHARACTERS.test(value) + ); +} + +/** + * Validate and deeply clone an untrusted persisted runtime snapshot. + * Unknown keys are deliberately dropped, while the nested runtime receipt is + * normalized by the sole runtime-provider receipt boundary. + */ +export function cloneSandboxRuntimeSnapshot(value: unknown): SandboxRuntimeSnapshot | undefined { + if ( + !isPlainRecord(value) || + value.schemaVersion !== SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION || + typeof value.providerId !== "string" || + !validProviderHandle(value.providerHandle) || + typeof value.lifecycleState !== "string" || + !LIFECYCLE_STATES.has(value.lifecycleState as SandboxRuntimeLifecycleState) || + !validProviderHandle(value.lifecycleGeneration) + ) { + return undefined; + } + const runtime = normalizeRuntimeProviderRuntimeReceipt(value.runtime); + if (!runtime || runtime.providerId !== value.providerId) return undefined; + return { + schemaVersion: SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION, + providerId: value.providerId, + providerHandle: value.providerHandle, + lifecycleState: value.lifecycleState as SandboxRuntimeLifecycleState, + lifecycleGeneration: value.lifecycleGeneration, + runtime, + }; +} diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 18c6fd449f2..8752009ed00 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -56,6 +56,12 @@ import { parseOpenClawImagePluginInstalls, planOpenClawPluginRestore, } from "./openclaw-plugin-restore.js"; +import { + cloneSandboxRuntimeSnapshot, + type SandboxRuntimeSnapshot, +} from "./registry/runtime-snapshot.js"; +import type { SandboxWorkloadReceipt } from "./registry/types.js"; +import { cloneSandboxWorkloadReceipt } from "./registry/workload.js"; import type { CustomPolicyEntry } from "./registry.js"; import * as registry from "./registry.js"; import { isSshTransportFailure } from "./ssh-transport.js"; @@ -109,6 +115,16 @@ export interface RebuildManifest { * zero-custom snapshot); absent only on legacy manifests. */ customPolicies?: CustomPolicyEntry[]; + /** + * Provider-neutral runtime and acceleration state captured before the + * filesystem copy. Required when `workload` is a managed-image receipt. + */ + runtimeSnapshot?: SandboxRuntimeSnapshot; + /** + * Exact immutable managed workload/profile authority associated with this + * snapshot. Older and explicit Dockerfile snapshots omit this field. + */ + workload?: SandboxWorkloadReceipt; instances?: InstanceBackup[]; // Optional user-provided label for `snapshot restore `. name?: string; @@ -121,6 +137,8 @@ export type SnapshotEntry = RebuildManifest & { snapshotVersion: number }; export interface BackupOptions { name?: string | null; + runtimeSnapshot?: SandboxRuntimeSnapshot; + workload?: SandboxWorkloadReceipt; } export interface InstanceBackup { @@ -265,6 +283,12 @@ export function hasAuthoritativeOpenClawImagePluginProvenance(value: { function isRebuildManifest(value: unknown): value is RebuildManifest { if (!isObjectRecord(value) || !isStateDirArray(value.stateDirs)) return false; const dir = typeof value.dir === "string" ? value.dir : value.writableDir; + const runtimeSnapshot = + value.runtimeSnapshot === undefined + ? undefined + : cloneSandboxRuntimeSnapshot(value.runtimeSnapshot); + const workload = + value.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(value.workload as never); return ( typeof value.version === "number" && typeof value.sandboxName === "string" && @@ -290,6 +314,9 @@ function isRebuildManifest(value: unknown): value is RebuildManifest { typeof value.blueprintDigest === "string") && (value.policyPresets === undefined || isStringArray(value.policyPresets)) && (value.customPolicies === undefined || isCustomPolicyEntryArray(value.customPolicies)) && + (value.runtimeSnapshot === undefined || runtimeSnapshot !== undefined) && + (value.workload === undefined || workload !== undefined) && + (workload?.kind !== "managed-image" || runtimeSnapshot !== undefined) && (value.instances === undefined || (Array.isArray(value.instances) && value.instances.every((entry) => isInstanceBackup(entry)))) && @@ -883,6 +910,32 @@ export { buildStateFileRestoreCommand } from "./state-file-restore.js"; // module. Prefer importing directly from ./ssh-transport in new code. export { isSshTransportFailure }; +function normalizeSnapshotBackupAuthority(options: BackupOptions): { + readonly runtimeSnapshot?: SandboxRuntimeSnapshot; + readonly workload?: SandboxWorkloadReceipt; + readonly error?: string; +} { + const runtimeSnapshot = + options.runtimeSnapshot === undefined + ? undefined + : cloneSandboxRuntimeSnapshot(options.runtimeSnapshot); + const workload = + options.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(options.workload); + if (options.runtimeSnapshot !== undefined && runtimeSnapshot === undefined) { + return { error: "snapshot runtime state is invalid or cannot be represented" }; + } + if (options.workload !== undefined && workload === undefined) { + return { error: "snapshot workload authority is invalid" }; + } + if (workload?.kind === "managed-image" && runtimeSnapshot === undefined) { + return { error: "managed snapshot is missing provider runtime state" }; + } + return { + ...(runtimeSnapshot === undefined ? {} : { runtimeSnapshot }), + ...(workload === undefined ? {} : { workload }), + }; +} + export function backupSandboxState(sandboxName: string, options: BackupOptions = {}): BackupResult { const sb = registry.getSandbox(sandboxName); const agentName = sb?.agent || "openclaw"; @@ -898,6 +951,18 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = `backupSandboxState: agent=${agentName}, dir=${dir}, stateDirs=[${stateDirs.join(",")}], stateFiles=[${stateFiles.map((f) => f.path).join(",")}]`, ); + const snapshotAuthority = normalizeSnapshotBackupAuthority(options); + if (snapshotAuthority.error) { + return { + success: false, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + error: snapshotAuthority.error, + }; + } + const reconcileOpenClawImagePluginProvenance = agentName === "openclaw" && Boolean(sb?.fromDockerfile); let openclawImagePluginInstalls: OpenClawImagePluginInstall[] | undefined; @@ -995,6 +1060,7 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = blueprintDigest: computeBlueprintDigest(), policyPresets, customPolicies, + ...snapshotAuthority, ...(providedName !== null ? { name: providedName } : {}), }; @@ -1853,6 +1919,12 @@ function readManifest(backupPath: string): RebuildManifest | null { const manifest = parsed as RebuildManifest & { dir?: string; writableDir?: string }; const dir = manifest.dir ?? manifest.writableDir; if (!dir) return null; + const runtimeSnapshot = + manifest.runtimeSnapshot === undefined + ? undefined + : cloneSandboxRuntimeSnapshot(manifest.runtimeSnapshot); + const workload = + manifest.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(manifest.workload); return { ...manifest, dir, @@ -1860,6 +1932,8 @@ function readManifest(backupPath: string): RebuildManifest | null { // restore contract can reject them instead of silently de-duplicating. stateFiles: normalizeStateFileSpecsPreservingDuplicates(manifest.stateFiles ?? []), blueprintDigest: manifest.blueprintDigest ?? null, + ...(runtimeSnapshot === undefined ? {} : { runtimeSnapshot }), + ...(workload === undefined ? {} : { workload }), }; } catch { return null; diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index 89be74b5589..94208ca951d 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -5,6 +5,8 @@ // - validateSnapshotName accepts/rejects names // - listBackups computes virtual v versions by timestamp-ascending position // - findBackup resolves selectors (v, name, exact timestamp) + +import { createHash } from "node:crypto"; import fs from "node:fs"; import { syncBuiltinESMExports } from "node:module"; import os from "node:os"; @@ -66,6 +68,39 @@ function writeBackup( fs.writeFileSync(path.join(dir, "rebuild-manifest.json"), JSON.stringify(manifest, null, 2)); return manifest; } +function managedSnapshotAuthority() { + const encodedProfile = Buffer.from('{"schemaVersion":1}', "utf8").toString("base64url"); + return { + workload: { + schemaVersion: 1, + kind: "managed-image", + reference: `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`, + platform: "linux/amd64", + release: "v0.0.97", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true, + }, + runtimeSnapshot: { + schemaVersion: 1, + providerId: "docker", + providerHandle: "opaque-provider-handle", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: { + schemaVersion: 1, + providerId: "docker", + runtime: { kind: "docker-container", handle: "opaque-container-id" }, + acceleration: { kind: "none" }, + }, + }, + } as const; +} afterAll(() => { if (ORIGINAL_HOME === undefined) { delete process.env.HOME; @@ -189,6 +224,38 @@ describe("listBackups computes virtual versions", () => { expect(entry.customPolicies).toEqual(custom); }); + it("round-trips normalized managed workload and provider runtime authority", () => { + const authority = managedSnapshotAuthority(); + writeBackup("test-sandbox", "2026-04-21T14-00-00-000Z", { + workload: { ...authority.workload, ignored: "not-authority" }, + runtimeSnapshot: { + ...authority.runtimeSnapshot, + containerName: "not-authority", + }, + }); + + const [entry] = sandboxState.listBackups("test-sandbox"); + + expect(entry?.workload).toEqual(authority.workload); + expect(entry?.runtimeSnapshot).toEqual(authority.runtimeSnapshot); + }); + + it("rejects a managed snapshot manifest without valid provider runtime authority", () => { + const authority = managedSnapshotAuthority(); + writeBackup("test-sandbox", "2026-04-21T14-00-00-000Z", { + workload: authority.workload, + }); + writeBackup("test-sandbox", "2026-04-21T14-01-00-000Z", { + ...authority, + runtimeSnapshot: { + ...authority.runtimeSnapshot, + lifecycleGeneration: "", + }, + }); + + expect(sandboxState.listBackups("test-sandbox")).toEqual([]); + }); + it("preserves an empty customPolicies array so restore can distinguish zero-custom from legacy snapshots", () => { writeBackup("test-sandbox", "2026-04-21T14-00-00-000Z", { customPolicies: [] }); const [entry] = sandboxState.listBackups("test-sandbox"); From 3a4321dee2e4ba5949f22b5b11828c512208cb11 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 04:25:02 -0700 Subject: [PATCH 047/117] fix(snapshot): enforce managed authority boundaries Signed-off-by: Aaron Erickson --- src/lib/actions/maintenance.test.ts | 8 + src/lib/actions/maintenance.ts | 27 +- .../sandbox/rebuild-flow-helpers.test.ts | 7 +- .../actions/sandbox/rebuild-flow-helpers.ts | 9 +- .../rebuild-restore-forwarding.test.ts | 17 +- .../sandbox/rebuild-restore-phase.test.ts | 16 +- .../actions/sandbox/rebuild-restore-phase.ts | 9 +- ...hot-managed-provider-restore-order.test.ts | 80 ++++- .../snapshot-restore-lifecycle.test.ts | 48 +++ .../sandbox/snapshot-restore-test-fixture.ts | 12 + src/lib/actions/sandbox/snapshot.ts | 130 ++++---- .../sandbox/snapshot/backup-authority.test.ts | 255 +++++++++++++++ .../sandbox/snapshot/backup-authority.ts | 133 ++++++++ .../actions/sandbox/snapshot/dependencies.ts | 1 + .../sandbox/snapshot/managed-profile.test.ts | 1 + .../snapshot/provider-lifecycle.test.ts | 139 ++++++++ .../sandbox/snapshot/provider-lifecycle.ts | 118 +++++-- .../snapshot/restore-authority.test.ts | 247 ++++++++++++++ .../sandbox/snapshot/restore-authority.ts | 155 +++++++++ .../sandbox/stopped-sandbox-backup.test.ts | 12 + .../actions/sandbox/stopped-sandbox-backup.ts | 10 +- .../sandbox-gpu-create-flow.ts | 4 + .../created-sandbox-finalization.test.ts | 46 +++ .../onboard/created-sandbox-finalization.ts | 11 + src/lib/onboard/lifecycle-contracts.md | 28 ++ ...penshell-docker-sandbox-containers.test.ts | 70 +++- .../openshell-docker-sandbox-containers.ts | 80 ++++- src/lib/onboard/runtime-provider/contract.ts | 7 + src/lib/onboard/runtime-provider/registry.ts | 18 +- .../runtime-provider-contract.test.ts | 40 +++ .../onboard/runtime-provider/snapshot.test.ts | 56 +++- src/lib/onboard/runtime-provider/snapshot.ts | 12 + .../onboard/sandbox-gpu-create-flow.test.ts | 4 + src/lib/state/sandbox.ts | 309 ++++++++++++++++-- ...snapshot-managed-restore-authority.test.ts | 163 +++++++++ test/snapshot.test.ts | 49 ++- 36 files changed, 2147 insertions(+), 184 deletions(-) create mode 100644 src/lib/actions/sandbox/snapshot/backup-authority.test.ts create mode 100644 src/lib/actions/sandbox/snapshot/backup-authority.ts create mode 100644 src/lib/actions/sandbox/snapshot/restore-authority.test.ts create mode 100644 src/lib/actions/sandbox/snapshot/restore-authority.ts create mode 100644 test/snapshot-managed-restore-authority.test.ts diff --git a/src/lib/actions/maintenance.test.ts b/src/lib/actions/maintenance.test.ts index 3e4e8288b03..b646d700f4a 100644 --- a/src/lib/actions/maintenance.test.ts +++ b/src/lib/actions/maintenance.test.ts @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ listSandboxes: vi.fn(), + getSandbox: vi.fn(), backupSandboxState: vi.fn(), captureSandboxListWithGatewayPreflightOrExit: vi.fn(), parseReadySandboxNames: vi.fn(), @@ -24,11 +25,18 @@ vi.mock("../state/registry", () => ({ isRouteOnlySandboxReservation: (entry: { pendingRouteReservation?: true; createdAt?: string }) => entry.pendingRouteReservation === true && entry.createdAt === undefined, listSandboxes: mocks.listSandboxes, + getSandbox: mocks.getSandbox, })); vi.mock("../state/sandbox", () => ({ backupSandboxState: mocks.backupSandboxState, BackupResult: {}, })); +vi.mock("../state/mcp-lifecycle-lock", () => ({ + withSandboxMutationLock: vi.fn((_name, callback) => callback()), +})); +vi.mock("./sandbox/snapshot/backup-authority", () => ({ + backupSandboxStateWithManagedAuthority: (name: string) => mocks.backupSandboxState(name), +})); vi.mock("../openshell-sandbox-list", () => ({ captureSandboxListWithGatewayPreflightOrExit: mocks.captureSandboxListWithGatewayPreflightOrExit, })); diff --git a/src/lib/actions/maintenance.ts b/src/lib/actions/maintenance.ts index 9792f2be1c7..227505a5cb7 100644 --- a/src/lib/actions/maintenance.ts +++ b/src/lib/actions/maintenance.ts @@ -22,6 +22,7 @@ import { SANDBOX_IMAGE_REPOS } from "../domain/sandbox/image-tag"; import { resolveGatewayName, resolveSandboxGatewayName } from "../onboard/gateway-binding"; import { captureSandboxListWithGatewayPreflightOrExit } from "../openshell-sandbox-list"; import { parseLiveSandboxNames, parseReadySandboxNames } from "../runtime-recovery"; +import { withSandboxMutationLock } from "../state/mcp-lifecycle-lock"; import * as registry from "../state/registry"; import * as sandboxState from "../state/sandbox"; import { nemoclawStateRoot, resolveHome } from "../state/state-root"; @@ -37,6 +38,7 @@ import { type StartedForBackup, startStoppedSandboxContainerForBackup, } from "./sandbox/stopped-sandbox-backup"; +import * as snapshotBackup from "./sandbox/snapshot/backup-authority"; const useColor = !process.env.NO_COLOR && !!process.stdout.isTTY; const trueColor = @@ -198,7 +200,7 @@ export async function backupAll(): Promise { let unreachableRunning = 0; let notRunningSkipped = 0; const strandedOrphans: string[] = []; - for (const sb of sandboxes) { + const backupRegisteredSandbox = async (sb: (typeof sandboxes)[number]): Promise => { // A registered docker-driver sandbox whose container is merely stopped is // backupable: start it for the duration of the backup and return it to // its stopped state after (#6500). Anything else that is not Ready keeps @@ -211,12 +213,12 @@ export async function backupAll(): Promise { // Tracked separately from `skipped` so the strict gate stays // untripped: there is nothing to back up and nothing to start. strandedOrphans.push(sb.name); - continue; + return; } console.log(` ${D}${notRunningBackupSkipMessage(sb.name)}${R}`); skipped++; notRunningSkipped++; - continue; + return; } console.log(` Starting stopped sandbox '${sb.name}' to back it up...`); } @@ -229,7 +231,13 @@ export async function backupAll(): Promise { const attempt = await backupSandboxWithinShieldsWindow(sb.name, () => startedForBackup ? backupStartedSandboxState(sb.name) - : sandboxState.backupSandboxState(sb.name), + : snapshotBackup.backupSandboxStateWithManagedAuthority( + sb.name, + {}, + { + getSandbox: registry.getSandbox, + }, + ), ); result = attempt.result; orphanManifestMessage = attempt.orphanManifestMessage; @@ -248,17 +256,17 @@ export async function backupAll(): Promise { } if (!returnedToStopped) { failed++; - continue; + return; } if (!shieldsWindowOpened) { console.error(` ${RD}✗${R} ${sb.name}: backup failed (could not safely unlock shields)`); failed++; - continue; + return; } if (orphanManifestMessage) { console.log(` ${YW}⚠${R} Skipped '${sb.name}' (orphan manifest): ${orphanManifestMessage}`); skipped++; - continue; + return; } if (!result) throw new Error(`Backup for '${sb.name}' completed without a result`); if (result.success) { @@ -273,7 +281,7 @@ export async function backupAll(): Promise { ` ${YW}⚠${R} Skipped '${sb.name}' (running but SSH-unreachable; NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP=1 set). Any uncommitted state since the last successful backup will be lost.`, ); skipped++; - continue; + return; } unreachableRunning++; } @@ -284,6 +292,9 @@ export async function backupAll(): Promise { console.error(` ${RD}✗${R} ${sb.name}: backup failed (${failedItems})`); failed++; } + }; + for (const sb of sandboxes) { + await withSandboxMutationLock(sb.name, () => backupRegisteredSandbox(sb)); } // The classification above is only as fresh as the pre-loop listing, and // the backup loop can run for minutes. Confirm with a second pinned listing diff --git a/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts b/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts index 1740d1bbfa8..b480b74facd 100644 --- a/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts +++ b/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts @@ -9,6 +9,7 @@ import * as gatewayRuntime from "../../gateway-runtime-action"; import type { SandboxBaseImageResolutionMetadata } from "../../sandbox-base-image"; import * as sandboxState from "../../state/sandbox"; import * as userManagedFilesProbe from "../../state/user-managed-files-probe"; +import * as snapshotBackup from "./snapshot/backup-authority"; import { backupSandboxStateForRebuild, disposeRebuildAgentBaseImagePreflight, @@ -549,7 +550,7 @@ describe("backupSandboxStateForRebuild with --force", () => { errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); vi.spyOn(console, "log").mockImplementation(() => undefined); - backupSpy = vi.spyOn(sandboxState, "backupSandboxState"); + backupSpy = vi.spyOn(snapshotBackup, "backupSandboxStateWithManagedAuthority"); }); afterEach(() => { @@ -843,7 +844,9 @@ describe("warnUnpreservedUserManagedFiles", () => { logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); - backupSpy = vi.spyOn(sandboxState, "backupSandboxState").mockReturnValue(makeBackupResult()); + backupSpy = vi + .spyOn(snapshotBackup, "backupSandboxStateWithManagedAuthority") + .mockReturnValue(makeBackupResult()); probeSpy = vi.spyOn(userManagedFilesProbe, "probeUserManagedFiles").mockReturnValue({ declared: [], existing: [], diff --git a/src/lib/actions/sandbox/rebuild-flow-helpers.ts b/src/lib/actions/sandbox/rebuild-flow-helpers.ts index b69f2d59054..f30e0ca5247 100644 --- a/src/lib/actions/sandbox/rebuild-flow-helpers.ts +++ b/src/lib/actions/sandbox/rebuild-flow-helpers.ts @@ -49,6 +49,7 @@ import { printWrongGatewayActiveGuidance, } from "./gateway-state"; import { openRebuildShieldsWindow, type RebuildShieldsWindow } from "./rebuild-shields"; +import * as snapshotBackup from "./snapshot/backup-authority"; export type RebuildSandboxEntry = SandboxEntry & { agents?: unknown[] }; @@ -446,7 +447,13 @@ export function backupSandboxStateForRebuild( console.log(" Backing up sandbox state..."); log(`Agent type: ${sb.agent || "openclaw"}, stateDirs from manifest`); - const backup = sandboxState.backupSandboxState(sandboxName); + const backup = snapshotBackup.backupSandboxStateWithManagedAuthority( + sandboxName, + {}, + { + getSandbox: (name) => loadRegistry().sandboxes[name] ?? null, + }, + ); log( `Backup result: success=${backup.success}, backed=${backup.backedUpDirs.join(",")}; files=${backup.backedUpFiles.join(",")}, failed=${backup.failedDirs.join(",")}; failedFiles=${backup.failedFiles.join(",")}`, ); diff --git a/src/lib/actions/sandbox/rebuild-restore-forwarding.test.ts b/src/lib/actions/sandbox/rebuild-restore-forwarding.test.ts index baeef9340ca..84b3e9a3369 100644 --- a/src/lib/actions/sandbox/rebuild-restore-forwarding.test.ts +++ b/src/lib/actions/sandbox/rebuild-restore-forwarding.test.ts @@ -3,8 +3,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import * as sandboxState from "../../state/sandbox"; import { runRebuildRestorePhase } from "./rebuild-restore-phase"; +import * as snapshotRestore from "./snapshot/restore-authority"; afterEach(() => { vi.restoreAllMocks(); @@ -14,7 +14,7 @@ describe("rebuild restore target forwarding", () => { it("forwards the recreated target identity and explicit custom-image capability", () => { vi.spyOn(console, "log").mockImplementation(() => undefined); const restoreRecreatedSandboxState = vi - .spyOn(sandboxState, "restoreRecreatedSandboxState") + .spyOn(snapshotRestore, "restoreRecreatedSandboxStateWithManagedAuthority") .mockReturnValue({ success: true, restoredDirs: [], @@ -34,9 +34,14 @@ describe("rebuild restore target forwarding", () => { log: vi.fn(), }); - expect(restoreRecreatedSandboxState).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backup", { - targetAgentType: "langchain-deepagents-code", - allowCustomImageWholeStateFileRestore: true, - }); + expect(restoreRecreatedSandboxState).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ backupPath: "/tmp/rebuild-backup" }), + { + targetAgentType: "langchain-deepagents-code", + allowCustomImageWholeStateFileRestore: true, + }, + { getSandbox: expect.any(Function) }, + ); }); }); diff --git a/src/lib/actions/sandbox/rebuild-restore-phase.test.ts b/src/lib/actions/sandbox/rebuild-restore-phase.test.ts index dbde96ca34f..831101486d8 100644 --- a/src/lib/actions/sandbox/rebuild-restore-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-restore-phase.test.ts @@ -11,6 +11,7 @@ import { resolveRestoredPolicyRegistryState, } from "./rebuild-post-restore-phase"; import { runRebuildRestorePhase } from "./rebuild-restore-phase"; +import * as snapshotRestore from "./snapshot/restore-authority"; const BUILTIN_OBSERVABILITY_CONTENT = "network_policies:\n observability-otlp-local:\n name: observability-otlp-local\n"; @@ -44,7 +45,7 @@ describe("rebuild policy restore fidelity", () => { vi.spyOn(console, "log").mockImplementation(() => undefined); const consoleError = vi.spyOn(console, "error").mockImplementation(() => undefined); const log = vi.fn(); - vi.spyOn(sandboxState, "restoreRecreatedSandboxState").mockReturnValue({ + vi.spyOn(snapshotRestore, "restoreRecreatedSandboxStateWithManagedAuthority").mockReturnValue({ success: false, restoredDirs: [], restoredFiles: [], @@ -76,7 +77,7 @@ describe("rebuild policy restore fidelity", () => { vi.spyOn(console, "error").mockImplementation(() => undefined); const parsePresetPolicyKeys = vi.spyOn(policies, "parsePresetPolicyKeys"); const restoreRecreatedSandboxState = vi - .spyOn(sandboxState, "restoreRecreatedSandboxState") + .spyOn(snapshotRestore, "restoreRecreatedSandboxStateWithManagedAuthority") .mockReturnValue({ success: true, restoredDirs: [], @@ -104,9 +105,14 @@ describe("rebuild policy restore fidelity", () => { log: vi.fn(), }); - expect(restoreRecreatedSandboxState).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backup", { - targetAgentType: "openclaw", - }); + expect(restoreRecreatedSandboxState).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ backupPath: "/tmp/rebuild-backup" }), + { + targetAgentType: "openclaw", + }, + { getSandbox: expect.any(Function) }, + ); expect(applyPreset).toHaveBeenCalledOnce(); expect(applyPreset).toHaveBeenCalledWith("alpha", "npm"); for (const entry of customPolicies) { diff --git a/src/lib/actions/sandbox/rebuild-restore-phase.ts b/src/lib/actions/sandbox/rebuild-restore-phase.ts index 2ce0640077c..057d7bd9ba9 100644 --- a/src/lib/actions/sandbox/rebuild-restore-phase.ts +++ b/src/lib/actions/sandbox/rebuild-restore-phase.ts @@ -8,11 +8,13 @@ import { OBSERVABILITY_POLICY_BINDING, } from "../../onboard/observability-policy-presets"; import * as policies from "../../policy"; +import { load as loadRegistry } from "../../state/registry/persistence"; import * as sandboxState from "../../state/sandbox"; import { MCP_BRIDGE_POLICY_SOURCE } from "./mcp-bridge-contracts"; import type { RebuildBackupManifest } from "./rebuild-backup-phase"; import type { RebuildLog } from "./rebuild-credential-preflight"; import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; +import * as snapshotRestore from "./snapshot/restore-authority"; export interface RebuildRestorePhaseInput { sandboxName: string; @@ -191,13 +193,16 @@ export function runRebuildRestorePhase(input: RebuildRestorePhaseInput): Rebuild console.log(""); console.log(" Restoring workspace state..."); log(`Restoring from: ${backupManifest.backupPath} into sandbox: ${sandboxName}`); - const restore = sandboxState.restoreRecreatedSandboxState( + const restore = snapshotRestore.restoreRecreatedSandboxStateWithManagedAuthority( sandboxName, - backupManifest.backupPath, + backupManifest, { targetAgentType, ...(targetImageIsCustom ? { allowCustomImageWholeStateFileRestore: true } : {}), }, + { + getSandbox: (name) => loadRegistry().sandboxes[name] ?? null, + }, ); log( `Restore result: success=${restore.success}, restored=${restore.restoredDirs.join(",")}; files=${restore.restoredFiles.join(",")}, failed=${restore.failedDirs.join(",")}; failedFiles=${restore.failedFiles.join(",")}${restore.error ? `; error=${restore.error}` : ""}`, diff --git a/src/lib/actions/sandbox/snapshot-managed-provider-restore-order.test.ts b/src/lib/actions/sandbox/snapshot-managed-provider-restore-order.test.ts index bd9324858a9..f5d617b457f 100644 --- a/src/lib/actions/sandbox/snapshot-managed-provider-restore-order.test.ts +++ b/src/lib/actions/sandbox/snapshot-managed-provider-restore-order.test.ts @@ -1,8 +1,17 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createHash } from "node:crypto"; + import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + MANAGED_IMAGE_REPOSITORIES, + type ShippedManagedImageAgent, +} from "../../onboard/managed-image/contract"; +import { encodeManagedStartupProfile } from "../../onboard/managed-startup/profile"; + import * as fixture from "./snapshot-restore-test-fixture"; const providerRestore = vi.hoisted(() => { @@ -61,6 +70,7 @@ const providerRestore = vi.hoisted(() => { }); vi.mock("./snapshot/dependencies", () => ({ + backupSandboxStateWithManagedAuthority: vi.fn(), captureSandboxRuntimeSnapshot: vi.fn(), confirmSandboxRuntimeRestore: providerRestore.confirmSandboxRuntimeRestore, prepareManagedSnapshotProfileRestore: providerRestore.prepareManagedSnapshotProfileRestore, @@ -70,13 +80,32 @@ vi.mock("./snapshot/dependencies", () => ({ requireCurrentSnapshotRuntimeProvider: providerRestore.requireCurrentSnapshotRuntimeProvider, })); -function managedSnapshot() { +function managedWorkload(agent: ShippedManagedImageAgent = "openclaw") { + const encodedProfile = encodeManagedStartupProfile(managedStartupE2eProfile(agent)); + return { + schemaVersion: 1 as const, + kind: "managed-image" as const, + reference: `${MANAGED_IMAGE_REPOSITORIES[agent]}@sha256:${"a".repeat(64)}`, + platform: "linux/amd64" as const, + release: "v0.0.100", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123-1", + capabilityContractVersion: 1 as const, + startupProfileContractVersion: 1 as const, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true as const, + }; +} + +function managedSnapshot(agent: ShippedManagedImageAgent = "openclaw") { return { snapshotVersion: 4, timestamp: "2026-07-30T00:00:00.000Z", backupPath: "/tmp/backup-alpha", - agentType: "openclaw", - workload: { kind: "managed-image" }, + agentType: agent, + workload: managedWorkload(agent), runtimeSnapshot: providerRestore.source, }; } @@ -95,7 +124,19 @@ beforeEach(() => { agent: "openclaw", openshellDriver: "docker", }); - fixture.restoreSandboxStateMock.mockImplementation(() => { + fixture.restoreSandboxStateMock.mockImplementation((_name, _path, options) => { + try { + options?.validateBeforeMutation?.(); + } catch (error) { + return { + success: false, + restoredDirs: [], + restoredFiles: [], + failedDirs: ["workspace"], + failedFiles: [], + error: error instanceof Error ? error.message : String(error), + }; + } providerRestore.events.push("filesystem-restore"); return { success: true, @@ -113,7 +154,24 @@ afterEach(() => { }); describe("managed snapshot provider restore ordering", () => { - it("refreshes provider authority at the mutation edge and proves the profile afterward", async () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("refreshes %s provider authority at the mutation edge and proves the profile", async (agent) => { + fixture.getLatestBackupMock.mockReturnValue(managedSnapshot(agent)); + fixture.getSandboxMock.mockReturnValue({ + name: "alpha", + agent, + openshellDriver: "docker", + }); + providerRestore.readManagedSnapshotProfileAuthority.mockReturnValue({ agent }); + providerRestore.prepareManagedSnapshotProfileRestore.mockReturnValue({ + providerRestoreAuthority: { + agent, + profileFingerprint: "a".repeat(64), + }, + }); const { runSandboxSnapshot } = await import("./snapshot"); await runSandboxSnapshot("alpha", { kind: "restore" }); @@ -155,7 +213,11 @@ describe("managed snapshot provider restore ordering", () => { }); expect(providerRestore.events).toEqual(["provider-preflight", "provider-preflight-rejected"]); - expect(fixture.restoreSandboxStateMock).not.toHaveBeenCalled(); + expect(fixture.restoreSandboxStateMock).toHaveBeenCalledWith( + "alpha", + "/tmp/backup-alpha", + expect.objectContaining({ validateBeforeMutation: expect.any(Function) }), + ); expect(providerRestore.confirmSandboxRuntimeRestore).not.toHaveBeenCalled(); }); }); @@ -178,7 +240,7 @@ describe("legacy snapshot compatibility gate", () => { name: "alpha", agent: "openclaw", openshellDriver: "docker", - workload: { kind: "managed-image" }, + workload: managedWorkload(), }); const { runSandboxSnapshot } = await import("./snapshot"); @@ -204,7 +266,7 @@ describe("legacy snapshot compatibility gate", () => { agent: "openclaw", openshellDriver: "docker", imageTag: "legacy-source:test", - ...(managedSide === "source" ? { workload: { kind: "managed-image" } } : {}), + ...(managedSide === "source" ? { workload: managedWorkload() } : {}), }; } if (name === "beta" && managedSide === "destination") { @@ -213,7 +275,7 @@ describe("legacy snapshot compatibility gate", () => { agent: "openclaw", openshellDriver: "docker", imageTag: "managed-target@test", - workload: { kind: "managed-image" }, + workload: managedWorkload(), }; } return null; diff --git a/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts b/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts index 2cfbd8af86e..db2c42cf0ae 100644 --- a/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts +++ b/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts @@ -20,6 +20,54 @@ afterEach(() => { } }); describe("runSandboxSnapshot restore: lifecycle and destination safety", () => { + it("holds the per-sandbox mutation lock across snapshot creation", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-snapshot-create-lock-")); + tempHomes.push(tempHome); + vi.stubEnv("HOME", tempHome); + let releaseLock: (() => void) | undefined; + let signalLocked: (() => void) | undefined; + const locked = new Promise((resolve) => { + signalLocked = resolve; + }); + const release = new Promise((resolve) => { + releaseLock = resolve; + }); + const externalMutation = withSandboxMutationLock("alpha", async () => { + signalLocked?.(); + await release; + }); + await locked; + f.backupSandboxStateMock.mockReturnValue({ + success: true, + manifest: { + timestamp: "2026-07-31T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + }, + backedUpDirs: [], + restoredDirs: [], + backedUpFiles: [], + failedDirs: [], + failedFiles: [], + }); + f.findBackupMock.mockReturnValue({ + match: { + snapshotVersion: 4, + timestamp: "2026-07-31T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + }, + }); + const { runSandboxSnapshot } = await import("./snapshot"); + + const create = runSandboxSnapshot("alpha", { kind: "create" }); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(f.backupSandboxStateMock).not.toHaveBeenCalled(); + + releaseLock?.(); + await externalMutation; + await create; + expect(f.backupSandboxStateMock).toHaveBeenCalledWith("alpha", { name: null }); + }); + it("restores the latest snapshot into the source sandbox", async () => { const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); f.getLatestBackupMock.mockReturnValue({ diff --git a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts index f264c5d9880..66819eb0951 100644 --- a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts +++ b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts @@ -128,6 +128,11 @@ const lifecycleMock = vi.hoisted(() => { }); export const backupSandboxStateMock = vi.fn(); +export const captureSnapshotRestoreAuthorityMock = vi.fn(() => ({ + schemaVersion: 1 as const, + backupPath: "/tmp/backup-alpha", + contentSha256: "a".repeat(64), +})); export const loadAgentMock = vi.fn((name: string) => ({ name, policyAdditionsPath: name === "openclaw" ? null : `/repo/agents/${name}/policy-additions.yaml`, @@ -287,6 +292,7 @@ vi.mock("../../state/gateway", () => ({ })); vi.mock("../../state/registry", () => ({ + getBaselineExclusions: vi.fn(() => []), getConfiguredMessagingChannelsFromEntry: vi.fn(() => []), getCustomPolicies: getCustomPoliciesMock, getDisabledMessagingChannelsFromEntry: vi.fn(() => []), @@ -302,6 +308,7 @@ vi.mock("../../state/registry", () => ({ vi.mock("../../state/sandbox", () => ({ backupSandboxState: backupSandboxStateMock, + captureSnapshotRestoreAuthority: captureSnapshotRestoreAuthorityMock, findBackup: findBackupMock, getLatestBackup: getLatestBackupMock, listBackups: listBackupsMock, @@ -332,6 +339,11 @@ vi.mock("./restore-gateway-pairing", () => ({ export function resetSnapshotRestoreMocks(): void { vi.clearAllMocks(); + captureSnapshotRestoreAuthorityMock.mockReturnValue({ + schemaVersion: 1, + backupPath: "/tmp/backup-alpha", + contentSha256: "a".repeat(64), + }); shieldsMock.setIsShieldsDownExport(shieldsMock.isShieldsDownMock); shieldsMock.isShieldsDownMock.mockReturnValue(true); shieldsMock.shieldsUpMock.mockImplementation(() => lifecycleMock.events.push("harden")); diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 6c9b694f5c3..7cb0175de8e 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -80,7 +80,7 @@ import { usesGatewayMetadataProbe, } from "./sandbox-gateway-routing"; import { - captureSandboxRuntimeSnapshot, + backupSandboxStateWithManagedAuthority, confirmSandboxRuntimeRestore, type PreparedSandboxRuntimeRestore, prepareManagedSnapshotProfileRestore, @@ -688,42 +688,13 @@ function runSnapshotCreate( } const label = request.name ? ` (--name ${request.name})` : ""; console.log(` Creating snapshot of '${sandboxName}'${label}...`); - const sourceEntry = registry.getSandbox(sandboxName); - let runtimeSnapshot: ReturnType | undefined; - let workload: NonNullable | undefined; - if (sourceEntry) { - try { - const authority = readManagedSnapshotProfileAuthority({ - sandboxName, - agentType: sourceEntry.agent ?? "", - imageTag: sourceEntry.imageTag, - fromDockerfile: sourceEntry.fromDockerfile, - workload: sourceEntry.workload, - }); - if (authority) { - const provider = requireCurrentSnapshotRuntimeProvider(sourceEntry); - if (!provider.workload.acceptsReceipt(authority.receipt)) { - throw new Error( - `runtime provider '${provider.identity.id}' does not accept the managed workload receipt`, - ); - } - runtimeSnapshot = captureSandboxRuntimeSnapshot(provider, sourceEntry); - workload = authority.receipt; - } - } catch (error) { - console.error( - ` Cannot capture managed snapshot authority: ${ - error instanceof Error ? error.message : String(error) - }.`, - ); - snapshotExit(1); - } - } - const result = sandboxState.backupSandboxState(sandboxName, { - name: request.name ?? null, - ...(runtimeSnapshot === undefined ? {} : { runtimeSnapshot }), - ...(workload === undefined ? {} : { workload }), - }); + const result = backupSandboxStateWithManagedAuthority( + sandboxName, + { + name: request.name ?? null, + }, + { getSandbox: registry.getSandbox }, + ); if (result.success) { const manifest = result.manifest!; const entry = sandboxState.findBackup(sandboxName, manifest.timestamp).match ?? manifest; @@ -1088,6 +1059,7 @@ async function runSnapshotRestoreUnlocked( }; const currentSourceEntry = registry.getSandbox(sandboxName); let hasManagedProfileAuthority = false; + let snapshotRestoreAuthority: sandboxState.SnapshotRestoreAuthority | null = null; try { const snapshotAuthority = readManagedSnapshotProfileAuthority(snapshotProfileSource); hasManagedProfileAuthority = snapshotAuthority !== null; @@ -1107,6 +1079,15 @@ async function runSnapshotRestoreUnlocked( if (isCrossSandboxRestore && hasManagedProfileAuthority) { rejectManagedSnapshotCloneUntilRebind(snapshotProfileSource, targetSandbox); } + if (hasManagedProfileAuthority) { + snapshotRestoreAuthority = sandboxState.captureSnapshotRestoreAuthority( + backupPath, + resolvedSnapshot, + ); + if (!snapshotRestoreAuthority) { + throw new Error("selected snapshot content changed during restore preflight"); + } + } } catch (error) { console.error( ` Cannot restore managed snapshot authority: ${ @@ -1304,48 +1285,45 @@ async function runSnapshotRestoreUnlocked( // reconciliation under the active timer generation. Normal auto-restore // waits; the absolute deadline may preempt this process and reclaim the // token, preventing policy/config mutation after lockdown resumes. - if (preparedRuntimeRestore) { - const currentTarget = registry.getSandbox(targetSandbox); - if (!currentTarget) { - console.error( - ` Cannot revalidate managed snapshot restore: target '${targetSandbox}' is no longer registered.`, - ); - snapshotExit(1); - } - try { - const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); - const profileRestore = prepareManagedSnapshotProfileRestore( - snapshotProfileSource, - currentTarget, - provider, - ); - if (!profileRestore) { - throw new Error("managed profile restore authority is missing"); + const validateManagedRestoreBeforeMutation = preparedRuntimeRestore + ? () => { + const currentTarget = registry.getSandbox(targetSandbox); + if (!currentTarget) { + throw new Error(`target '${targetSandbox}' is no longer registered`); + } + const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); + const profileRestore = prepareManagedSnapshotProfileRestore( + snapshotProfileSource, + currentTarget, + provider, + ); + if (!profileRestore) { + throw new Error("managed profile restore authority is missing"); + } + const prepared = preparedRuntimeRestore; + if (!prepared) throw new Error("managed runtime restore authority is missing"); + // The state layer invokes this after local tar staging and + // immediately before its first remote filesystem mutation. + preparedRuntimeRestore = prepareSandboxRuntimeRestore( + provider, + currentTarget, + prepared.source, + profileRestore.providerRestoreAuthority, + ); } - // Refresh provider authority immediately before filesystem mutation. - // The post-restore facet consumes this exact receipt and proves that - // neither runtime generation nor managed profile changed meanwhile. - preparedRuntimeRestore = prepareSandboxRuntimeRestore( - provider, - currentTarget, - preparedRuntimeRestore.source, - profileRestore.providerRestoreAuthority, - ); - } catch (error) { - console.error( - ` Cannot revalidate managed snapshot restore: ${ - error instanceof Error ? error.message : String(error) - }.`, - ); - snapshotExit(1); - } - } + : null; if (targetSandbox !== sandboxName) { console.log(` Restoring snapshot from '${sandboxName}' into '${targetSandbox}'...`); } else { console.log(` Restoring snapshot into '${sandboxName}'...`); } - const result = sandboxState.restoreSandboxState(targetSandbox, backupPath); + const result = + snapshotRestoreAuthority && validateManagedRestoreBeforeMutation + ? sandboxState.restoreSandboxState(targetSandbox, backupPath, { + authority: snapshotRestoreAuthority, + validateBeforeMutation: validateManagedRestoreBeforeMutation, + }) + : sandboxState.restoreSandboxState(targetSandbox, backupPath); if (result.success) { if (preparedRuntimeRestore) { const currentTarget = registry.getSandbox(targetSandbox); @@ -1364,6 +1342,7 @@ async function runSnapshotRestoreUnlocked( error instanceof Error ? error.message : String(error) }.`, ); + console.error(" Retry this exact snapshot after the runtime provider stabilizes."); snapshotExit(1); } } @@ -1387,6 +1366,9 @@ async function runSnapshotRestoreUnlocked( if (result.failedFiles.length > 0) { console.error(` Failed files: ${result.failedFiles.join(", ")}`); } + if (result.error) { + console.error(` Reason: ${result.error}`); + } snapshotExit(1); } // Post-restore security-state reconciliation is best-effort by design: the @@ -1426,7 +1408,7 @@ export async function runSandboxSnapshot( ) { switch (request.kind) { case "create": { - runSnapshotCreate(sandboxName, request); + await withSandboxMutationLock(sandboxName, () => runSnapshotCreate(sandboxName, request)); break; } case "list": { diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts new file mode 100644 index 00000000000..8221b1535bc --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts @@ -0,0 +1,255 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + MANAGED_IMAGE_REPOSITORIES, + type ShippedManagedImageAgent, +} from "../../../onboard/managed-image/contract"; +import { encodeManagedStartupProfile } from "../../../onboard/managed-startup/profile"; +import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../../state/registry/types"; +import type { BackupOptions, BackupResult } from "../../../state/sandbox"; +import { backupSandboxStateWithManagedAuthority } from "./backup-authority"; + +function workload( + agent: ShippedManagedImageAgent, + changedProfile = false, +): Extract { + const encodedProfile = encodeManagedStartupProfile( + managedStartupE2eProfile(agent, changedProfile), + ); + return { + schemaVersion: 1, + kind: "managed-image", + reference: `${MANAGED_IMAGE_REPOSITORIES[agent]}@sha256:${"a".repeat(64)}`, + platform: "linux/amd64", + release: "v0.0.88", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true, + }; +} + +function sandbox( + agent: ShippedManagedImageAgent, + receipt: SandboxWorkloadReceipt = workload(agent), +): SandboxEntry { + return { + name: "alpha", + agent, + openshellDriver: "mxc", + imageTag: receipt.kind === "managed-image" ? receipt.reference : null, + fromDockerfile: null, + workload: receipt, + }; +} + +function runtime(handle = "session-1") { + return { + schemaVersion: 1, + providerId: "mxc", + providerHandle: `opaque-${handle}`, + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: { + schemaVersion: 1, + providerId: "mxc", + runtime: { kind: "session", handle }, + acceleration: { kind: "none" }, + }, + } as const; +} + +function provider(acceptsReceipt = true): RuntimeProviderBundle { + return { + identity: { contractVersion: 1, id: "mxc", displayName: "MXC" }, + workload: { + providerId: "mxc", + supported: true, + profile: { + support: null, + hostArchitectures: [], + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: false, + }, + acceptsReceipt: () => acceptsReceipt, + }, + } as unknown as RuntimeProviderBundle; +} + +function successfulBackup(options: BackupOptions): BackupResult { + try { + options.validateBeforePublish?.(); + } catch (error) { + return { + success: false, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + error: error instanceof Error ? error.message : String(error), + }; + } + return { + success: true, + manifest: { + version: 1, + sandboxName: "alpha", + timestamp: "2026-07-31T00-00-00-000Z", + agentType: "openclaw", + agentVersion: null, + expectedVersion: null, + stateDirs: [], + dir: "/sandbox", + backupPath: "/tmp/alpha", + blueprintDigest: null, + }, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + }; +} + +describe("managed snapshot backup authority", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("captures and republishes exact %s provider authority", (agent) => { + const entry = sandbox(agent); + const getSandbox = vi.fn(() => entry); + const requireProvider = vi.fn(() => provider()); + const captureRuntime = vi.fn(() => runtime()); + const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); + + const result = backupSandboxStateWithManagedAuthority( + "alpha", + { name: "stable" }, + { getSandbox, requireProvider, captureRuntime, backup }, + ); + + expect(result.success).toBe(true); + expect(backup).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + name: "stable", + workload: entry.workload, + runtimeSnapshot: runtime(), + validateBeforePublish: expect.any(Function), + }), + ); + expect(getSandbox).toHaveBeenCalledTimes(2); + expect(requireProvider).toHaveBeenCalledTimes(2); + expect(captureRuntime).toHaveBeenCalledTimes(2); + }); + + it("keeps explicit Dockerfile backups on the legacy state-only path", () => { + const entry = { + name: "alpha", + agent: "openclaw", + openshellDriver: "mxc", + fromDockerfile: "/tmp/Dockerfile", + } satisfies SandboxEntry; + const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); + const requireProvider = vi.fn(); + const captureRuntime = vi.fn(); + + const result = backupSandboxStateWithManagedAuthority( + "alpha", + { name: "legacy" }, + { + getSandbox: () => entry, + requireProvider, + captureRuntime: captureRuntime as never, + backup, + }, + ); + + expect(result.success).toBe(true); + expect(backup).toHaveBeenCalledWith("alpha", { name: "legacy" }); + expect(requireProvider).not.toHaveBeenCalled(); + expect(captureRuntime).not.toHaveBeenCalled(); + }); + + it("fails before filesystem capture when the provider rejects managed authority", () => { + const entry = sandbox("openclaw"); + const backup = vi.fn(); + + const result = backupSandboxStateWithManagedAuthority( + "alpha", + {}, + { + getSandbox: () => entry, + requireProvider: () => provider(false), + captureRuntime: vi.fn() as never, + backup, + }, + ); + + expect(result).toMatchObject({ + success: false, + error: expect.stringContaining("does not accept the managed workload receipt"), + }); + expect(backup).not.toHaveBeenCalled(); + }); + + it.each([ + { + label: "workload", + secondEntry: sandbox("openclaw", workload("openclaw", true)), + secondRuntime: runtime(), + error: "managed workload changed during backup", + }, + { + label: "runtime", + secondEntry: sandbox("openclaw"), + secondRuntime: runtime("session-2"), + error: "runtime changed during backup", + }, + ])("rejects $label drift before manifest publication", ({ + secondEntry, + secondRuntime, + error, + }) => { + const initialEntry = sandbox("openclaw"); + const getSandbox = vi + .fn<() => SandboxEntry | null>() + .mockReturnValueOnce(initialEntry) + .mockReturnValueOnce(secondEntry); + const captureRuntime = vi + .fn<() => ReturnType>() + .mockReturnValueOnce(runtime()) + .mockReturnValueOnce(secondRuntime); + const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); + + const result = backupSandboxStateWithManagedAuthority( + "alpha", + {}, + { + getSandbox, + requireProvider: () => provider(), + captureRuntime: captureRuntime as ( + bundle: RuntimeProviderBundle, + entry: SandboxEntry, + ) => ReturnType, + backup, + }, + ); + + expect(result).toMatchObject({ + success: false, + error: expect.stringContaining(error), + }); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.ts b/src/lib/actions/sandbox/snapshot/backup-authority.ts new file mode 100644 index 00000000000..bc8ccb371b3 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/backup-authority.ts @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; + +import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; +import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provider/current"; +import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime-provider/registry"; +import type { SandboxEntry } from "../../../state/registry/types"; +import * as sandboxState from "../../../state/sandbox"; +import { readManagedSnapshotProfileAuthority } from "./managed-profile"; +import { captureSandboxRuntimeSnapshot } from "./provider-lifecycle"; + +type SnapshotBackupAuthority = Pick< + sandboxState.BackupOptions, + "runtimeSnapshot" | "workload" | "validateBeforePublish" +>; + +interface SnapshotBackupAuthorityDependencies { + readonly getSandbox: (sandboxName: string) => SandboxEntry | null; + readonly requireProvider: (sandbox: SandboxEntry) => RuntimeProviderBundle; + readonly captureRuntime: typeof captureSandboxRuntimeSnapshot; + readonly backup: typeof sandboxState.backupSandboxState; +} + +const defaultDependencies: Omit = { + requireProvider: (sandbox) => + requireRuntimeProviderBundleForSandbox(sandbox, CURRENT_RUNTIME_PROVIDER_BUNDLES), + captureRuntime: captureSandboxRuntimeSnapshot, + // Keep the call late-bound so tests and alternative state stores can replace + // the module export without this adapter retaining an import-time reference. + backup: (...args) => sandboxState.backupSandboxState(...args), +}; + +function failure(error: unknown): sandboxState.BackupResult { + const detail = error instanceof Error ? error.message : String(error); + return { + success: false, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + error: `Cannot capture managed snapshot authority: ${detail}.`, + }; +} + +function backupStateOnly( + dependencies: SnapshotBackupAuthorityDependencies, + sandboxName: string, + options: Pick, +): sandboxState.BackupResult { + return options.name === undefined + ? dependencies.backup(sandboxName) + : dependencies.backup(sandboxName, options); +} + +function readAuthority(entry: SandboxEntry) { + return readManagedSnapshotProfileAuthority({ + sandboxName: entry.name, + agentType: entry.agent ?? "", + imageTag: entry.imageTag, + fromDockerfile: entry.fromDockerfile, + workload: entry.workload, + }); +} + +function captureManagedAuthority( + entry: SandboxEntry, + dependencies: SnapshotBackupAuthorityDependencies, +): SnapshotBackupAuthority | null { + const authority = readAuthority(entry); + if (!authority) return null; + const provider = dependencies.requireProvider(entry); + if (!provider.workload.acceptsReceipt(authority.receipt)) { + throw new Error( + `runtime provider '${provider.identity.id}' does not accept the managed workload receipt`, + ); + } + const runtimeSnapshot = dependencies.captureRuntime(provider, entry); + const workload = authority.receipt; + + return { + runtimeSnapshot, + workload, + validateBeforePublish: () => { + const current = dependencies.getSandbox(entry.name); + if (!current) { + throw new Error(`sandbox '${entry.name}' is no longer registered`); + } + const currentAuthority = readAuthority(current); + if (!currentAuthority || !isDeepStrictEqual(currentAuthority.receipt, workload)) { + throw new Error(`sandbox '${entry.name}' managed workload changed during backup`); + } + const currentProvider = dependencies.requireProvider(current); + if ( + currentProvider.identity.id !== provider.identity.id || + !currentProvider.workload.acceptsReceipt(currentAuthority.receipt) + ) { + throw new Error(`sandbox '${entry.name}' runtime provider changed during backup`); + } + const currentRuntime = dependencies.captureRuntime(currentProvider, current); + if (!isDeepStrictEqual(currentRuntime, runtimeSnapshot)) { + throw new Error(`sandbox '${entry.name}' runtime changed during backup`); + } + }, + }; +} + +/** + * Capture one managed workload and runtime authority pair around the complete + * filesystem copy. The state layer publishes the manifest only after the + * final callback confirms that the same provider authority remains live. + */ +export function backupSandboxStateWithManagedAuthority( + sandboxName: string, + options: Pick = {}, + overrides: Pick & + Partial>, +): sandboxState.BackupResult { + const dependencies = { ...defaultDependencies, ...overrides }; + const entry = dependencies.getSandbox(sandboxName); + if (!entry) return backupStateOnly(dependencies, sandboxName, options); + + let authority: SnapshotBackupAuthority | null; + try { + authority = captureManagedAuthority(entry, dependencies); + } catch (error) { + return failure(error); + } + return authority + ? dependencies.backup(sandboxName, { ...options, ...authority }) + : backupStateOnly(dependencies, sandboxName, options); +} diff --git a/src/lib/actions/sandbox/snapshot/dependencies.ts b/src/lib/actions/sandbox/snapshot/dependencies.ts index e3d573177f6..e428143b209 100644 --- a/src/lib/actions/sandbox/snapshot/dependencies.ts +++ b/src/lib/actions/sandbox/snapshot/dependencies.ts @@ -22,6 +22,7 @@ export { prepareSandboxRuntimeRestore, SandboxSnapshotProviderError, } from "./provider-lifecycle"; +export { backupSandboxStateWithManagedAuthority } from "./backup-authority"; /** * Resolve the one already-registered provider bundle for a durable sandbox. diff --git a/src/lib/actions/sandbox/snapshot/managed-profile.test.ts b/src/lib/actions/sandbox/snapshot/managed-profile.test.ts index 74f42d070ab..29596cb5275 100644 --- a/src/lib/actions/sandbox/snapshot/managed-profile.test.ts +++ b/src/lib/actions/sandbox/snapshot/managed-profile.test.ts @@ -74,6 +74,7 @@ function provider(accepted = true, managedProfileRestore = true): RuntimeProvide snapshot: { providerId: "mxc", supported: true, + contractVersion: 1, capabilities: { backup: true, restore: true, diff --git a/src/lib/actions/sandbox/snapshot/provider-lifecycle.test.ts b/src/lib/actions/sandbox/snapshot/provider-lifecycle.test.ts index e4b68e51606..0770debc921 100644 --- a/src/lib/actions/sandbox/snapshot/provider-lifecycle.test.ts +++ b/src/lib/actions/sandbox/snapshot/provider-lifecycle.test.ts @@ -82,6 +82,7 @@ function provider( snapshot: { providerId, supported: true, + contractVersion: 1, capabilities: { backup: true, restore: true, managedProfileRestore: true }, preflight, capture, @@ -147,6 +148,47 @@ describe("snapshot provider lifecycle", () => { }); }); + it("leaves opaque provider and runtime handles under provider ownership", () => { + const { bundle, restore } = provider(); + const target = sandbox("target"); + const prepared = prepareSandboxRuntimeRestore( + bundle, + target, + { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-source-provider", + lifecycleState: "running", + lifecycleGeneration: "source-generation", + runtime: { + ...runtime(), + runtime: { kind: "session", handle: "opaque-source-runtime" }, + }, + }, + managedProfile, + ); + restore.mockReturnValueOnce({ + schemaVersion: 1, + providerId: "mxc", + sandboxName: "target", + providerHandle: "opaque-provider-owned-restore", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: { + ...runtime(), + runtime: { kind: "replacement-session", handle: "opaque-provider-owned-runtime" }, + }, + managedProfile, + }); + + expect(confirmSandboxRuntimeRestore(bundle, target, prepared).restoreReceipt).toMatchObject({ + providerHandle: "opaque-provider-owned-restore", + runtime: { + runtime: { kind: "replacement-session", handle: "opaque-provider-owned-runtime" }, + }, + }); + }); + it("rejects provider identity drift before returning snapshot authority", () => { expect(() => captureSandboxRuntimeSnapshot(provider({ preflightProviderId: "other" }).bundle, sandbox()), @@ -269,4 +311,101 @@ describe("snapshot provider lifecycle", () => { /invalid managed restore proof/u, ); }); + + it.each([ + { field: "lifecycle state", lifecycleState: "stopped", lifecycleGeneration: "generation-1" }, + { field: "lifecycle generation", lifecycleState: "running", lifecycleGeneration: "changed" }, + ] as const)("rejects restore proof with changed $field", ({ + lifecycleState, + lifecycleGeneration, + }) => { + const { bundle, restore } = provider(); + const target = sandbox("target"); + const prepared = prepareSandboxRuntimeRestore( + bundle, + target, + { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-source", + lifecycleState: "running", + lifecycleGeneration: "source-generation", + runtime: runtime(), + }, + managedProfile, + ); + restore.mockReturnValueOnce({ + schemaVersion: 1, + providerId: "mxc", + sandboxName: "target", + providerHandle: "provider-owned-restore-handle", + lifecycleState, + lifecycleGeneration, + runtime: runtime(), + managedProfile, + }); + + expect(() => confirmSandboxRuntimeRestore(bundle, target, prepared)).toThrow( + /invalid managed restore proof/u, + ); + }); + + it("rejects restore proof that changes acceleration authority", () => { + const { bundle } = provider(); + const target = sandbox("target"); + const prepared = prepareSandboxRuntimeRestore( + bundle, + target, + { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-source", + lifecycleState: "running", + lifecycleGeneration: "source-generation", + runtime: { + ...runtime(), + acceleration: { kind: "gpu", vendor: "nvidia", devices: ["GPU-0"] }, + }, + }, + managedProfile, + ); + + expect(() => confirmSandboxRuntimeRestore(bundle, target, prepared)).toThrow( + /invalid managed restore proof/u, + ); + }); + + it("isolates central authority from a hostile provider that mutates backup inputs", () => { + const { bundle, capture } = provider(); + capture.mockImplementationOnce((_entry, preflight) => { + (preflight as { providerHandle: string }).providerHandle = "mutated"; + return runtime(); + }); + + expect(() => captureSandboxRuntimeSnapshot(bundle, sandbox())).toThrow(TypeError); + }); + + it("isolates central authority from a hostile MXC-style restore facet", () => { + const { bundle, validateRestore } = provider(); + validateRestore.mockImplementationOnce((_entry, _preflight, source, authority) => { + (source as { providerHandle: string }).providerHandle = "mutated"; + (authority as { agent: string }).agent = "other"; + }); + + expect(() => + prepareSandboxRuntimeRestore( + bundle, + sandbox("target"), + { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-source", + lifecycleState: "running", + lifecycleGeneration: "source-generation", + runtime: runtime(), + }, + managedProfile, + ), + ).toThrow(TypeError); + }); }); diff --git a/src/lib/actions/sandbox/snapshot/provider-lifecycle.ts b/src/lib/actions/sandbox/snapshot/provider-lifecycle.ts index 3e47c14661e..623de52a1ed 100644 --- a/src/lib/actions/sandbox/snapshot/provider-lifecycle.ts +++ b/src/lib/actions/sandbox/snapshot/provider-lifecycle.ts @@ -1,6 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { isDeepStrictEqual } from "node:util"; + +import { cloneAndDeepFreeze } from "../../../core/immutable"; import type { RuntimeProviderBundle, RuntimeProviderManagedProfileRestoreAuthority, @@ -31,6 +34,12 @@ export class SandboxSnapshotProviderError extends Error { } } +/** + * Provider facets are extension points, so readonly TypeScript annotations are + * not a runtime trust boundary. Give every provider a detached, deeply frozen + * copy and retain only separately normalized values in central orchestration. + */ + function requireSnapshotSurface( bundle: RuntimeProviderBundle, capability: keyof SupportedSnapshotSurface["capabilities"], @@ -85,6 +94,8 @@ function requireRestoreReceipt( bundle: RuntimeProviderBundle, sandbox: SandboxEntry, authority: RuntimeProviderManagedProfileRestoreAuthority, + preflight: RuntimeProviderSnapshotPreflightReceipt, + source: SandboxRuntimeSnapshot, value: unknown, ): RuntimeProviderSnapshotRestoreReceipt { const receipt = normalizeRuntimeProviderSnapshotRestoreReceipt(value); @@ -93,7 +104,10 @@ function requireRestoreReceipt( receipt.providerId !== bundle.identity.id || receipt.sandboxName !== sandbox.name || receipt.managedProfile.agent !== authority.agent || - receipt.managedProfile.profileFingerprint !== authority.profileFingerprint + receipt.managedProfile.profileFingerprint !== authority.profileFingerprint || + receipt.lifecycleState !== preflight.lifecycleState || + receipt.lifecycleGeneration !== preflight.lifecycleGeneration || + !isDeepStrictEqual(receipt.runtime.acceleration, source.runtime.acceleration) ) { throw new SandboxSnapshotProviderError( `runtime provider '${bundle.identity.id}' returned invalid managed restore proof`, @@ -112,21 +126,26 @@ export function captureSandboxRuntimeSnapshot( sandbox: SandboxEntry, ): SandboxRuntimeSnapshot { const surface = requireSnapshotSurface(bundle, "backup"); + const providerSandbox = cloneAndDeepFreeze(sandbox); const preflight = requirePreflight( bundle, sandbox, "backup", - surface.preflight("backup", sandbox), + surface.preflight("backup", providerSandbox), ); - const runtime = requireRuntimeReceipt(bundle, surface.capture(sandbox, preflight)); - return { + const immutablePreflight = cloneAndDeepFreeze(preflight); + const runtime = requireRuntimeReceipt( + bundle, + surface.capture(providerSandbox, immutablePreflight), + ); + return cloneAndDeepFreeze({ schemaVersion: SANDBOX_RUNTIME_SNAPSHOT_SCHEMA_VERSION, providerId: bundle.identity.id, - providerHandle: preflight.providerHandle, - lifecycleState: preflight.lifecycleState, - lifecycleGeneration: preflight.lifecycleGeneration, - runtime, - }; + providerHandle: immutablePreflight.providerHandle, + lifecycleState: immutablePreflight.lifecycleState, + lifecycleGeneration: immutablePreflight.lifecycleGeneration, + runtime: cloneAndDeepFreeze(runtime), + }); } export interface PreparedSandboxRuntimeRestore { @@ -167,6 +186,8 @@ export function prepareSandboxRuntimeRestore( ); } const surface = requireSnapshotSurface(bundle, "restore"); + const providerTarget = cloneAndDeepFreeze(target); + const immutableSource = cloneAndDeepFreeze(source); const managedProfile = normalizeRuntimeProviderManagedProfileRestoreAuthority(managedProfileValue); if (!managedProfile) { @@ -176,21 +197,62 @@ export function prepareSandboxRuntimeRestore( bundle, target, "restore", - surface.preflight("restore", target), + surface.preflight("restore", providerTarget), ); if (preflight.lifecycleState !== source.lifecycleState) { throw new SandboxSnapshotProviderError( `target '${target.name}' cannot represent the snapshot lifecycle state`, ); } - surface.validateRestore(target, preflight, source, managedProfile); - return Object.freeze({ + const immutablePreflight = cloneAndDeepFreeze(preflight); + const immutableManagedProfile = cloneAndDeepFreeze(managedProfile); + surface.validateRestore( + providerTarget, + immutablePreflight, + immutableSource, + immutableManagedProfile, + ); + return cloneAndDeepFreeze({ phase: "preflighted" as const, targetProviderId: bundle.identity.id, targetSandboxName: target.name, - source, - preflight, - managedProfile, + source: immutableSource, + preflight: immutablePreflight, + managedProfile: immutableManagedProfile, + }); +} + +function normalizePreparedRestore( + bundle: RuntimeProviderBundle, + target: SandboxEntry, + prepared: PreparedSandboxRuntimeRestore, +): PreparedSandboxRuntimeRestore { + const source = cloneSandboxRuntimeSnapshot(prepared.source); + const preflight = normalizeRuntimeProviderSnapshotPreflightReceipt(prepared.preflight); + const managedProfile = normalizeRuntimeProviderManagedProfileRestoreAuthority( + prepared.managedProfile, + ); + if ( + prepared.phase !== "preflighted" || + prepared.targetProviderId !== bundle.identity.id || + prepared.targetSandboxName !== target.name || + !source || + source.providerId !== bundle.identity.id || + !preflight || + preflight.providerId !== bundle.identity.id || + preflight.operation !== "restore" || + preflight.sandboxName !== target.name || + !managedProfile + ) { + throw new SandboxSnapshotProviderError("restore preflight authority is stale"); + } + return cloneAndDeepFreeze({ + phase: "preflighted" as const, + targetProviderId: bundle.identity.id, + targetSandboxName: target.name, + source: cloneAndDeepFreeze(source), + preflight: cloneAndDeepFreeze(preflight), + managedProfile: cloneAndDeepFreeze(managedProfile), }); } @@ -205,25 +267,27 @@ export function confirmSandboxRuntimeRestore( target: SandboxEntry, prepared: PreparedSandboxRuntimeRestore, ): ValidatedSandboxRuntimeRestore { - if ( - prepared.phase !== "preflighted" || - prepared.targetProviderId !== bundle.identity.id || - prepared.targetSandboxName !== target.name - ) { - throw new SandboxSnapshotProviderError("restore preflight authority is stale"); - } + const authority = normalizePreparedRestore(bundle, target, prepared); const surface = requireSnapshotSurface(bundle, "restore"); + const providerTarget = cloneAndDeepFreeze(target); const restoreReceipt = requireRestoreReceipt( bundle, target, - prepared.managedProfile, - surface.restore(target, prepared.preflight, prepared.source, prepared.managedProfile), + authority.managedProfile, + authority.preflight, + authority.source, + surface.restore( + providerTarget, + authority.preflight, + authority.source, + authority.managedProfile, + ), ); - return Object.freeze({ + return cloneAndDeepFreeze({ phase: "validated" as const, targetProviderId: bundle.identity.id, targetSandboxName: target.name, - source: prepared.source, - restoreReceipt, + source: authority.source, + restoreReceipt: cloneAndDeepFreeze(restoreReceipt), }); } diff --git a/src/lib/actions/sandbox/snapshot/restore-authority.test.ts b/src/lib/actions/sandbox/snapshot/restore-authority.test.ts new file mode 100644 index 00000000000..719b42a3099 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/restore-authority.test.ts @@ -0,0 +1,247 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + MANAGED_IMAGE_REPOSITORIES, + type ShippedManagedImageAgent, +} from "../../../onboard/managed-image/contract"; +import { encodeManagedStartupProfile } from "../../../onboard/managed-startup/profile"; +import type { + RuntimeProviderBundle, + RuntimeProviderManagedProfileRestoreAuthority, +} from "../../../onboard/runtime-provider/contract"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../../state/registry/types"; +import type { + RebuildManifest, + RecreatedSandboxRestoreOptions, + RestoreResult, +} from "../../../state/sandbox"; +import { restoreRecreatedSandboxStateWithManagedAuthority } from "./restore-authority"; + +function workload( + agent: ShippedManagedImageAgent, +): Extract { + const encodedProfile = encodeManagedStartupProfile(managedStartupE2eProfile(agent)); + return { + schemaVersion: 1, + kind: "managed-image", + reference: `${MANAGED_IMAGE_REPOSITORIES[agent]}@sha256:${"a".repeat(64)}`, + platform: "linux/amd64", + release: "v0.0.88", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true, + }; +} + +function runtimeSnapshot() { + return { + schemaVersion: 1, + providerId: "mxc", + providerHandle: "opaque-preflight", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: { + schemaVersion: 1, + providerId: "mxc", + runtime: { kind: "session", handle: "session-1" }, + acceleration: { kind: "none" }, + }, + } as const; +} + +function manifest(agent: ShippedManagedImageAgent): RebuildManifest { + return { + version: 1, + sandboxName: "alpha", + timestamp: "2026-07-31T00-00-00-000Z", + agentType: agent, + agentVersion: null, + expectedVersion: null, + stateDirs: [], + dir: "/sandbox", + backupPath: "/tmp/alpha", + blueprintDigest: null, + workload: workload(agent), + runtimeSnapshot: runtimeSnapshot(), + }; +} + +function sandbox(agent: ShippedManagedImageAgent): SandboxEntry { + const receipt = workload(agent); + return { + name: "alpha", + agent, + openshellDriver: "mxc", + imageTag: receipt.reference, + fromDockerfile: null, + workload: receipt, + }; +} + +function provider(agent: ShippedManagedImageAgent) { + const preflight = vi.fn((operation: "backup" | "restore", entry: SandboxEntry) => ({ + schemaVersion: 1 as const, + providerId: "mxc", + operation, + sandboxName: entry.name, + providerHandle: "opaque-preflight", + lifecycleState: "running" as const, + lifecycleGeneration: "generation-1", + })); + const validateRestore = vi.fn(); + const restore = vi.fn( + ( + entry: SandboxEntry, + _preflight: unknown, + _source: unknown, + authority: RuntimeProviderManagedProfileRestoreAuthority, + ) => ({ + schemaVersion: 1 as const, + providerId: "mxc", + sandboxName: entry.name, + providerHandle: "opaque-restore", + lifecycleState: "running" as const, + lifecycleGeneration: "generation-1", + runtime: runtimeSnapshot().runtime, + managedProfile: authority, + }), + ); + const bundle = { + identity: { contractVersion: 1, id: "mxc", displayName: "MXC" }, + workload: { + providerId: "mxc", + supported: true, + profile: { + support: null, + hostArchitectures: [], + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: false, + }, + acceptsReceipt: (receipt: SandboxWorkloadReceipt | undefined) => + receipt?.kind === "managed-image" && receipt.reference === workload(agent).reference, + }, + snapshot: { + providerId: "mxc", + supported: true, + contractVersion: 1, + capabilities: { backup: true, restore: true, managedProfileRestore: true }, + preflight, + capture: () => runtimeSnapshot().runtime, + validateRestore, + restore, + }, + } as unknown as RuntimeProviderBundle; + return { bundle, preflight, validateRestore, restore }; +} + +describe("managed rebuild restore authority", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("revalidates %s content and provider authority at the mutation edge", (agent) => { + const target = sandbox(agent); + const runtimeProvider = provider(agent); + const restore = vi.fn( + (_name: string, _path: string, options: RecreatedSandboxRestoreOptions): RestoreResult => { + options.validateBeforeMutation?.(); + return { + success: true, + restoredDirs: ["workspace"], + failedDirs: [], + restoredFiles: [], + failedFiles: [], + }; + }, + ); + + const result = restoreRecreatedSandboxStateWithManagedAuthority( + "alpha", + manifest(agent), + { targetAgentType: agent }, + { + getSandbox: () => target, + requireProvider: () => runtimeProvider.bundle, + captureContentAuthority: () => ({ + schemaVersion: 1, + backupPath: "/tmp/alpha", + contentSha256: "c".repeat(64), + }), + restore, + }, + ); + + expect(result.success).toBe(true); + expect(restore).toHaveBeenCalledWith( + "alpha", + "/tmp/alpha", + expect.objectContaining({ + authority: expect.objectContaining({ contentSha256: "c".repeat(64) }), + validateBeforeMutation: expect.any(Function), + }), + ); + expect(runtimeProvider.preflight).toHaveBeenCalledTimes(2); + expect(runtimeProvider.validateRestore).toHaveBeenCalledTimes(2); + expect(runtimeProvider.restore).toHaveBeenCalledOnce(); + }); + + it("keeps legacy rebuild manifests on the state-only restore path", () => { + const legacy = { ...manifest("openclaw"), workload: undefined, runtimeSnapshot: undefined }; + const restore = vi.fn(() => ({ + success: true, + restoredDirs: [], + failedDirs: [], + restoredFiles: [], + failedFiles: [], + })); + + expect( + restoreRecreatedSandboxStateWithManagedAuthority( + "alpha", + legacy, + { targetAgentType: "openclaw" }, + { + getSandbox: vi.fn(), + requireProvider: vi.fn() as never, + captureContentAuthority: vi.fn(), + restore, + }, + ).success, + ).toBe(true); + expect(restore).toHaveBeenCalledWith("alpha", "/tmp/alpha", { + targetAgentType: "openclaw", + }); + }); + + it("rejects a managed manifest without provider runtime authority", () => { + const restore = vi.fn(); + const result = restoreRecreatedSandboxStateWithManagedAuthority( + "alpha", + { ...manifest("hermes"), runtimeSnapshot: undefined }, + { targetAgentType: "hermes" }, + { + getSandbox: vi.fn(), + requireProvider: vi.fn() as never, + captureContentAuthority: vi.fn(), + restore, + }, + ); + + expect(result).toMatchObject({ + success: false, + error: expect.stringContaining("missing provider runtime authority"), + }); + expect(restore).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot/restore-authority.ts b/src/lib/actions/sandbox/snapshot/restore-authority.ts new file mode 100644 index 00000000000..8d5b2198e58 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/restore-authority.ts @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; +import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provider/current"; +import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime-provider/registry"; +import type { SandboxEntry } from "../../../state/registry/types"; +import * as sandboxState from "../../../state/sandbox"; +import { + prepareManagedSnapshotProfileRestore, + readManagedSnapshotProfileAuthority, +} from "./managed-profile"; +import { + confirmSandboxRuntimeRestore, + type PreparedSandboxRuntimeRestore, + prepareSandboxRuntimeRestore, +} from "./provider-lifecycle"; + +interface ManagedRestoreAuthorityDependencies { + readonly getSandbox: (sandboxName: string) => SandboxEntry | null; + readonly requireProvider: (sandbox: SandboxEntry) => RuntimeProviderBundle; + readonly captureContentAuthority: typeof sandboxState.captureSnapshotRestoreAuthority; + readonly restore: typeof sandboxState.restoreRecreatedSandboxState; +} + +const defaultDependencies: Omit = { + requireProvider: (sandbox) => + requireRuntimeProviderBundleForSandbox(sandbox, CURRENT_RUNTIME_PROVIDER_BUNDLES), + captureContentAuthority: (...args) => sandboxState.captureSnapshotRestoreAuthority(...args), + restore: (...args) => sandboxState.restoreRecreatedSandboxState(...args), +}; + +function failure(error: unknown): sandboxState.RestoreResult { + const detail = error instanceof Error ? error.message : String(error); + return { + success: false, + restoredDirs: [], + failedDirs: ["manifest"], + restoredFiles: [], + failedFiles: [], + error: `Cannot restore managed snapshot authority: ${detail}.`, + }; +} + +/** + * Restore a rebuild backup through the same provider and content authority + * boundary as an explicit snapshot restore. Legacy/custom-image manifests + * retain their existing state-only path. + */ +export function restoreRecreatedSandboxStateWithManagedAuthority( + sandboxName: string, + manifest: sandboxState.RebuildManifest, + options: sandboxState.RecreatedSandboxRestoreOptions, + overrides: Pick & + Partial>, +): sandboxState.RestoreResult { + const dependencies = { ...defaultDependencies, ...overrides }; + let snapshotProfile; + try { + snapshotProfile = readManagedSnapshotProfileAuthority({ + sandboxName: manifest.sandboxName, + agentType: manifest.agentType, + workload: manifest.workload, + }); + } catch (error) { + return failure(error); + } + if (!snapshotProfile) { + return dependencies.restore(sandboxName, manifest.backupPath, options); + } + if (!manifest.runtimeSnapshot) { + return failure("managed snapshot is missing provider runtime authority"); + } + + let prepared: PreparedSandboxRuntimeRestore; + let providerId: string; + let contentAuthority: sandboxState.SnapshotRestoreAuthority; + try { + const target = dependencies.getSandbox(sandboxName); + if (!target) throw new Error(`target '${sandboxName}' is not registered`); + const provider = dependencies.requireProvider(target); + providerId = provider.identity.id; + const profileRestore = prepareManagedSnapshotProfileRestore( + { + sandboxName: manifest.sandboxName, + agentType: manifest.agentType, + workload: manifest.workload, + }, + target, + provider, + ); + if (!profileRestore) throw new Error("managed profile restore authority is missing"); + const captured = dependencies.captureContentAuthority(manifest.backupPath, manifest); + if (!captured) throw new Error("selected snapshot content changed during restore preflight"); + contentAuthority = captured; + prepared = prepareSandboxRuntimeRestore( + provider, + target, + manifest.runtimeSnapshot, + profileRestore.providerRestoreAuthority, + ); + } catch (error) { + return failure(error); + } + + const restore = dependencies.restore(sandboxName, manifest.backupPath, { + ...options, + authority: contentAuthority, + validateBeforeMutation: () => { + const current = dependencies.getSandbox(sandboxName); + if (!current) throw new Error(`target '${sandboxName}' is no longer registered`); + const provider = dependencies.requireProvider(current); + if (provider.identity.id !== providerId) { + throw new Error(`target '${sandboxName}' runtime provider changed before restore`); + } + const profileRestore = prepareManagedSnapshotProfileRestore( + { + sandboxName: manifest.sandboxName, + agentType: manifest.agentType, + workload: manifest.workload, + }, + current, + provider, + ); + if (!profileRestore) throw new Error("managed profile restore authority is missing"); + prepared = prepareSandboxRuntimeRestore( + provider, + current, + prepared.source, + profileRestore.providerRestoreAuthority, + ); + }, + }); + if (!restore.success) return restore; + + try { + const current = dependencies.getSandbox(sandboxName); + if (!current) throw new Error(`target '${sandboxName}' is no longer registered`); + const provider = dependencies.requireProvider(current); + if (provider.identity.id !== providerId) { + throw new Error(`target '${sandboxName}' runtime provider changed during restore`); + } + confirmSandboxRuntimeRestore(provider, current, prepared); + return restore; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + return { + ...restore, + success: false, + error: + `State was restored, but managed runtime proof failed: ${detail}. ` + + `Retry this exact snapshot after the runtime stabilizes.`, + }; + } +} diff --git a/src/lib/actions/sandbox/stopped-sandbox-backup.test.ts b/src/lib/actions/sandbox/stopped-sandbox-backup.test.ts index e13fc7264ff..ff73c8101a4 100644 --- a/src/lib/actions/sandbox/stopped-sandbox-backup.test.ts +++ b/src/lib/actions/sandbox/stopped-sandbox-backup.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it, vi } from "vitest"; const adapterMocks = vi.hoisted(() => ({ dockerRun: vi.fn(), dockerCapture: vi.fn(), + backupWithAuthority: vi.fn(), })); vi.mock("../../adapters/docker/run", () => ({ @@ -19,6 +20,9 @@ vi.mock("../../state/registry", () => ({ vi.mock("../../state/sandbox", () => ({ backupSandboxState: vi.fn(), })); +vi.mock("./snapshot/backup-authority", () => ({ + backupSandboxStateWithManagedAuthority: (name: string) => adapterMocks.backupWithAuthority(name), +})); import * as registry from "../../state/registry"; import { @@ -225,6 +229,14 @@ describe("backupStartedSandboxState", () => { const unreachable = { ...ok, success: false, unreachable: true }; const denied = { ...ok, success: false }; + it("uses managed provider authority through the default stopped-backup path", async () => { + adapterMocks.backupWithAuthority.mockReturnValueOnce(ok); + + await expect(backupStartedSandboxState("my-sb")).resolves.toEqual(ok); + + expect(adapterMocks.backupWithAuthority).toHaveBeenCalledWith("my-sb"); + }); + it("retries while the just-started container's SSH endpoint is unreachable (#6500)", async () => { const backup = vi .fn() diff --git a/src/lib/actions/sandbox/stopped-sandbox-backup.ts b/src/lib/actions/sandbox/stopped-sandbox-backup.ts index b1a9c9d6e7f..667584104ad 100644 --- a/src/lib/actions/sandbox/stopped-sandbox-backup.ts +++ b/src/lib/actions/sandbox/stopped-sandbox-backup.ts @@ -12,6 +12,7 @@ import { import * as registry from "../../state/registry"; import * as sandboxState from "../../state/sandbox"; import { resolveSandboxContainerOwner } from "./sandbox-container-owner"; +import * as snapshotBackup from "./snapshot/backup-authority"; /** Read a registered sandbox's OpenShell driver, treating registry read * failure as unknown so callers fail closed on driver-gated decisions. */ @@ -182,7 +183,14 @@ interface BackupRetryDeps { } const defaultBackupRetryDeps: BackupRetryDeps = { - backup: (name) => sandboxState.backupSandboxState(name), + backup: (name) => + snapshotBackup.backupSandboxStateWithManagedAuthority( + name, + {}, + { + getSandbox: registry.getSandbox, + }, + ), sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), attempts: 5, delayMs: 2000, diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index f0224759b04..7061252f12c 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -97,6 +97,10 @@ export function setupGpuFlowMocks(mocks: Record imageId: GPU_IMAGE_ID, bookkeepingImageRef: "openshell/sandbox-from:test", stateError: "", + deviceRequests: null, + devices: null, + runtime: "nvidia", + nvidiaVisibleDevices: "all", nativeGpuAttachmentState: "present", containerId: "container-a", }); diff --git a/src/lib/onboard/created-sandbox-finalization.test.ts b/src/lib/onboard/created-sandbox-finalization.test.ts index f470719fa59..bb068a9b454 100644 --- a/src/lib/onboard/created-sandbox-finalization.test.ts +++ b/src/lib/onboard/created-sandbox-finalization.test.ts @@ -504,6 +504,52 @@ describe("created OpenClaw sandbox finalization", () => { expect(register).toHaveBeenCalledWith(pluginInstalls); }); + it("defers managed restore before unregistered target authority can be bound", () => { + const register = vi.fn(); + const error = vi.fn(); + + expect(() => + finalizeCreatedSandbox( + { + sandboxName: "openclaw", + restoreBackupPath: "/tmp/managed-openclaw-backup", + preUpgradeBackup: false, + targetAgentType: "openclaw", + validateManagedDcode: false, + provider: "compatible-endpoint", + model: "demo", + preferredInferenceApi: "openai-completions", + }, + { + discoverFreshOpenClawImagePluginInstalls: vi.fn(), + restoreRecreatedSandboxState: () => ({ + success: false, + restoredDirs: [], + failedDirs: ["manifest"], + restoredFiles: [], + failedFiles: [], + error: sandboxState.MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR, + }), + getDcodeSelectionDrift: vi.fn(), + register, + note: vi.fn(), + error, + exitProcess: (code): never => { + throw new Error(`exit ${code}`); + }, + }, + ), + ).toThrow("exit 1"); + + expect(register).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledWith(expect.stringContaining("restore is deferred")); + expect(error).toHaveBeenCalledWith( + " State was not restored and registry metadata was not updated.", + ); + expect(error).toHaveBeenCalledWith(' openshell sandbox delete "openclaw"'); + expect(error).toHaveBeenCalledWith(" Manual recovery: /tmp/managed-openclaw-backup"); + }); + it("fails closed before restore and registration when provenance discovery fails", () => { const restoreRecreatedSandboxState = vi.fn(); const register = vi.fn(); diff --git a/src/lib/onboard/created-sandbox-finalization.ts b/src/lib/onboard/created-sandbox-finalization.ts index f95dc3c22ee..c0efd29d416 100644 --- a/src/lib/onboard/created-sandbox-finalization.ts +++ b/src/lib/onboard/created-sandbox-finalization.ts @@ -6,6 +6,7 @@ import type { OpenClawManagedExtensionDiscoveryResult, } from "../state/openclaw-plugin-restore"; import { + MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR, OPENCLAW_IMAGE_PLUGIN_PROVENANCE_RESTORE_ERROR, type RecreatedSandboxRestoreOptions, type RestoreResult, @@ -90,6 +91,16 @@ export function finalizeCreatedSandbox( ` ✓ State restored (${restore.restoredDirs.length} directories, ${restore.restoredFiles.length} files)`, ); } else { + if (restore.error === MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR) { + deps.error( + ` Managed snapshot restore is deferred for newly created sandbox '${options.sandboxName}' until its runtime authority can be bound before registry publication.`, + ); + deps.error(" State was not restored and registry metadata was not updated."); + deps.error(" Remove the unregistered sandbox before retrying:"); + deps.error(` openshell sandbox delete ${JSON.stringify(options.sandboxName)}`); + deps.error(` Manual recovery: ${options.restoreBackupPath}`); + return deps.exitProcess(1); + } if (restore.error === OPENCLAW_IMAGE_PLUGIN_PROVENANCE_RESTORE_ERROR) { deps.error( ` OpenClaw image plugin provenance validation failed for sandbox '${options.sandboxName}': ${restore.error}`, diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 544b5170984..c55e537932c 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -140,6 +140,33 @@ The handler clears the journal after it records both create and registration rec This slice covers resumed onboard replacement, including not-ready repair and non-default gateways. Rebuild and non-resumed re-onboard remain under #6492. +## Managed snapshot and rebuild restore authority + +Managed-image backups use one provider-neutral authority path for explicit snapshot creation, `backup-all`, stopped-sandbox backup retries, and rebuild backups. +The contract applies to OpenClaw, Hermes, and Deep Agents Code. +The path records the exact managed workload receipt and a versioned runtime receipt from the sandbox's registered provider. +The state layer copies and sanitizes the backup before it invokes the provider fence. +The fence re-reads the registry row and re-observes the provider runtime. +The state layer publishes the manifest atomically only when the workload, provider, lifecycle generation, runtime identity, and acceleration receipt still match. +It removes the unpublished backup directory when the fence rejects publication. + +A managed restore binds the selected manifest and every backup payload to one content digest. +The state layer recomputes that digest after local restore staging and before its first remote filesystem mutation. +At the same mutation edge, central orchestration asks the registered provider to revalidate the target runtime and managed startup profile. +After state restoration, the provider must prove the managed profile and runtime state again. +Explicit snapshot restore and rebuild restore use this same boundary. + +The snapshot provider facet has its own contract version. +Provider inputs are detached and deeply frozen at the extension boundary, and central orchestration retains only normalized receipts. +Docker lifecycle inspection and GPU inspection remain inside the Docker provider adapter. +The provider-neutral receipt can represent another provider, including an MXC-style implementation, without adding provider switches to snapshot or rebuild orchestration. + +Legacy and custom-image snapshots retain their state-only backup and restore path. +This contract does not activate another runtime provider or managed-image onboarding path. +Ordinary onboard recreation and create finalization remain deferred because the replacement target is not registered when that restore currently runs; the raw state layer rejects a managed manifest unless both exact content authority and a runtime-validation fence are present. +Cross-provider clone and rebind, durable interrupted-restore recovery, ordinary recreate integration, and user-visible runtime activation remain separate review units. +If provider proof fails after filesystem restoration, NemoClaw reports that state changed and requires the operator to retry the exact snapshot after the runtime stabilizes. + ## Agent-specific differences | Agent | Lifecycle difference | @@ -208,5 +235,6 @@ PR #5955 moved the rebuild messaging conflict check before destruction. | Session sanitation, sandbox prompt checkpoints, and no-secret persistence | `src/lib/state/onboard-session-sandbox-prompts.test.ts`, `src/lib/state/onboard-checkpoint.test.ts`, `machine/handlers/sandbox-create-intent-boundary.test.ts` | Tri-state decisions remain scoped to checkpointed sandbox choices. | | Versioned checkpoint schema, tri-state decisions, migration, and unknown-future fail-safe | `src/lib/state/onboard-checkpoint.test.ts`, `src/lib/state/onboard-checkpoint-migrate.test.ts` | Live decision reads still use legacy fields | | Resumable create replay, durable identity, and stale-binding fail-closed | `src/lib/onboard/checkpoint-replay.test.ts`, `src/lib/onboard/checkpoint-resume-guard.test.ts`, `machine/handlers/sandbox-checkpoint-crash-recovery.test.ts` | None at the sandbox-handler boundary. | +| Managed snapshot workload, content, and provider authority across explicit and rebuild flows | `src/lib/actions/sandbox/snapshot/backup-authority.test.ts`, `restore-authority.test.ts`, `managed-profile.test.ts`, `provider-lifecycle.test.ts`, and `snapshot-managed-provider-restore-order.test.ts` | Cross-provider clone and rebind, durable interrupted-restore recovery, and user-visible runtime activation remain separate review units. | When lifecycle behavior changes one of these contracts, update the map and the narrow owning test in that same PR. Do not add source-text scans or production scaffolding solely to preserve current orchestration order. diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.test.ts b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts index e26eb9b8ec5..57b49ccd7fa 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.test.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts @@ -8,11 +8,21 @@ const IMAGE_ID = `sha256:${"a".repeat(64)}`; const BOOKKEEPING_IMAGE_REF = "openshell/sandbox-from:alpha"; const EMPTY_RUNTIME_FIELDS = [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", null, [], "runc"]; -function querySnapshot(fields: unknown) { +function querySnapshot(fields: unknown, nvidiaVisibleDevices?: string) { const dockerRun = vi .fn() .mockReturnValueOnce({ status: 0, stdout: "container-a\n", stderr: "" }) .mockReturnValueOnce({ status: 0, stdout: JSON.stringify(fields), stderr: "" }); + if (Array.isArray(fields) && String(fields[5]).toLowerCase() === "nvidia") { + dockerRun.mockReturnValueOnce({ + status: 0, + stdout: + nvidiaVisibleDevices === undefined + ? "" + : `NVIDIA_VISIBLE_DEVICES=${nvidiaVisibleDevices}\n`, + stderr: "", + }); + } return { dockerRun, result: queryOpenShellDockerSandboxRuntimeSnapshot("alpha", { dockerRun }), @@ -31,6 +41,7 @@ describe("queryOpenShellDockerSandboxRuntimeSnapshot", () => { deviceRequests: null, devices: [], runtime: "runc", + nvidiaVisibleDevices: null, nativeGpuAttachmentState: "absent", containerId: "container-a", }); @@ -114,14 +125,10 @@ describe("queryOpenShellDockerSandboxRuntimeSnapshot", () => { ], ["NVIDIA runtime", null, [], "nvidia"], ])("detects a host-configured GPU attachment from %s", (_label, requests, devices, runtime) => { - const { result } = querySnapshot([ - IMAGE_ID, - BOOKKEEPING_IMAGE_REF, - "", - requests, - devices, - runtime, - ]); + const { result } = querySnapshot( + [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", requests, devices, runtime], + runtime === "nvidia" ? "all" : undefined, + ); expect(result).toMatchObject({ ok: true, @@ -129,6 +136,51 @@ describe("queryOpenShellDockerSandboxRuntimeSnapshot", () => { }); }); + it.each([ + ["all devices", "all", "present"], + ["an exact device list", "0,GPU-live-1", "present"], + ["no devices", "none", "absent"], + ["runtime bypass", "void", "absent"], + ] as const)("reads only NVIDIA_VISIBLE_DEVICES for %s", (_label, value, expectedState) => { + const { dockerRun, result } = querySnapshot( + [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", null, [], "nvidia"], + value, + ); + + expect(result).toMatchObject({ + ok: true, + nvidiaVisibleDevices: value, + nativeGpuAttachmentState: expectedState, + }); + expect(dockerRun).toHaveBeenLastCalledWith( + [ + "inspect", + "--type", + "container", + "--format", + '{{range .Config.Env}}{{if eq (index (split . "=") 0) "NVIDIA_VISIBLE_DEVICES"}}{{println .}}{{end}}{{end}}', + "container-a", + ], + expect.objectContaining({ suppressOutput: true }), + ); + }); + + it.each([ + "all,0", + "0,0", + "GPU-0 with-space", + ])("rejects ambiguous NVIDIA_VISIBLE_DEVICES value %s", (value) => { + const { result } = querySnapshot( + [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", null, [], "nvidia"], + value, + ); + + expect(result).toEqual({ + ok: false, + error: "docker inspect returned invalid NVIDIA_VISIBLE_DEVICES", + }); + }); + it.each([ ["unknown runtime", null, [], "nvidia-container-runtime"], [ diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.ts b/src/lib/onboard/openshell-docker-sandbox-containers.ts index 2c045cdb086..a8934027fd6 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.ts @@ -107,6 +107,11 @@ export type OpenShellDockerSandboxRuntimeSnapshotQuery = deviceRequests: OpenShellDockerDeviceRequest[] | null; devices: OpenShellDockerDeviceMapping[] | null; runtime: string; + /** + * Exact allowlisted NVIDIA Container Runtime selector. Other container + * environment entries are deliberately never returned by this query. + */ + nvidiaVisibleDevices: string | null; /** Closed-world classification of host-owned Docker GPU configuration. */ nativeGpuAttachmentState: OpenShellDockerGpuAttachmentState; containerId: string; @@ -196,9 +201,9 @@ function classifyGpuAttachment( deviceRequests: OpenShellDockerDeviceRequest[] | null, devices: OpenShellDockerDeviceMapping[] | null, runtime: string, + nvidiaVisibleDevices: string | null, ): OpenShellDockerGpuAttachmentState { const normalizedRuntime = runtime.trim().toLowerCase(); - if (normalizedRuntime === "nvidia") return "present"; if ( deviceRequests?.some( (request) => @@ -211,6 +216,10 @@ function classifyGpuAttachment( ) { return "present"; } + if (normalizedRuntime === "nvidia") { + if (nvidiaVisibleDevices === null) return "unknown"; + return ["", "none", "void"].includes(nvidiaVisibleDevices) ? "absent" : "present"; + } if ( devices?.some( (mapping) => @@ -228,6 +237,45 @@ function classifyGpuAttachment( return noDeviceRequests && noDeviceMappings && knownNonGpuRuntime ? "absent" : "unknown"; } +function parseNvidiaVisibleDevices(result: { + stdout?: string | Buffer | null; + stderr?: string | Buffer | null; + status?: number | null; +}): { ok: true; value: string | null } | { ok: false; error: string } { + if (Number(result.status ?? 1) !== 0) { + return { + ok: false, + error: + commandResultText(result) || "docker inspect could not read NVIDIA_VISIBLE_DEVICES safely", + }; + } + const lines = String(result.stdout ?? "") + .split(/\r?\n/u) + .filter((line) => line.length > 0); + if (lines.length === 0) return { ok: true, value: null }; + if (lines.length !== 1 || !lines[0]?.startsWith("NVIDIA_VISIBLE_DEVICES=")) { + return { ok: false, error: "docker inspect returned ambiguous NVIDIA_VISIBLE_DEVICES" }; + } + const value = lines[0].slice("NVIDIA_VISIBLE_DEVICES=".length); + const identifiers = value.split(","); + const validKeyword = ["", "all", "none", "void"].includes(value); + const validIdentifiers = + !validKeyword && + identifiers.length > 0 && + identifiers.length <= 64 && + identifiers.every( + (identifier) => + !["all", "none", "void"].includes(identifier) && + /^[A-Za-z0-9._:/=-]{1,256}$/u.test(identifier) && + Buffer.byteLength(identifier, "utf8") <= 256, + ) && + new Set(identifiers).size === identifiers.length; + if (!validKeyword && !validIdentifiers) { + return { ok: false, error: "docker inspect returned invalid NVIDIA_VISIBLE_DEVICES" }; + } + return { ok: true, value }; +} + /** * Inspect the one exactly labeled native container before deletion. * @@ -295,6 +343,28 @@ export function queryOpenShellDockerSandboxRuntimeSnapshot( const deviceRequests = fields[3]; const devices = fields[4]; const runtime = fields[5]; + let nvidiaVisibleDevices: string | null = null; + if (runtime.trim().toLowerCase() === "nvidia") { + const visibleDevices = parseNvidiaVisibleDevices( + run( + [ + "inspect", + "--type", + "container", + "--format", + '{{range .Config.Env}}{{if eq (index (split . "=") 0) "NVIDIA_VISIBLE_DEVICES"}}{{println .}}{{end}}{{end}}', + containerId, + ], + { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_SANDBOX_QUERY_TIMEOUT_MS, + }, + ), + ); + if (!visibleDevices.ok) return visibleDevices; + nvidiaVisibleDevices = visibleDevices.value; + } return { ok: true, imageId: fields[0].toLowerCase(), @@ -303,7 +373,13 @@ export function queryOpenShellDockerSandboxRuntimeSnapshot( deviceRequests, devices, runtime, - nativeGpuAttachmentState: classifyGpuAttachment(deviceRequests, devices, runtime), + nvidiaVisibleDevices, + nativeGpuAttachmentState: classifyGpuAttachment( + deviceRequests, + devices, + runtime, + nvidiaVisibleDevices, + ), containerId, }; } diff --git a/src/lib/onboard/runtime-provider/contract.ts b/src/lib/onboard/runtime-provider/contract.ts index 38124212ce0..644239b3a7a 100644 --- a/src/lib/onboard/runtime-provider/contract.ts +++ b/src/lib/onboard/runtime-provider/contract.ts @@ -5,6 +5,7 @@ import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/ import type { ManagedImageSelectionPolicy } from "../workload/source"; export const RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION = 1 as const; +export const RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION = 1 as const; export const RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION = 1 as const; export type RuntimeProviderGatewayLauncher = "nemoclaw" | "openshell"; @@ -265,6 +266,12 @@ export type RuntimeProviderBootstrapSurface = export type RuntimeProviderSnapshotSurface = | RuntimeProviderSupportedSurface<{ + /** + * Version the snapshot facet independently so providers can reject a + * central contract they do not implement without forcing unrelated + * bundle surfaces to rev in lockstep. + */ + readonly contractVersion: typeof RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION; readonly capabilities: { readonly backup: boolean; readonly restore: boolean; diff --git a/src/lib/onboard/runtime-provider/registry.ts b/src/lib/onboard/runtime-provider/registry.ts index 4318afe3fd9..765f1feed9f 100644 --- a/src/lib/onboard/runtime-provider/registry.ts +++ b/src/lib/onboard/runtime-provider/registry.ts @@ -4,6 +4,7 @@ import type { SandboxEntry } from "../../state/registry/types"; import { RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, + RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION, RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION, type RuntimeProviderBundle, type RuntimeProviderBundleRegistry, @@ -339,8 +340,13 @@ function validateBootstrapSurface(surface: Record): void { } } -function validateSnapshotSurface(surface: Record): void { +function validateSnapshotSurface(providerId: string, surface: Record): void { if (surface.supported === true) { + if (surface.contractVersion !== RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION) { + throw new RuntimeProviderRegistrationError( + `snapshot for '${providerId}' has an unsupported contract version`, + ); + } const capabilities = requireOwnRecord(surface, "capabilities"); for (const capability of ["backup", "restore", "managedProfileRestore"] as const) { requireBoolean(capabilities, capability, "snapshot capabilities"); @@ -349,6 +355,11 @@ function validateSnapshotSurface(surface: Record): void { requireFunction(surface, "capture", "snapshot"); requireFunction(surface, "validateRestore", "snapshot"); requireFunction(surface, "restore", "snapshot"); + if (capabilities.managedProfileRestore === true && capabilities.restore !== true) { + throw new RuntimeProviderRegistrationError( + `snapshot for '${providerId}' cannot restore managed profiles without restore support`, + ); + } } } @@ -417,7 +428,7 @@ function validateSupportedSurfaceSchemas( validateLifecycleSurface(providerId, surfaces.lifecycle); validateMutationAuthoritySurface(providerId, surfaces.mutationAuthority); validateBootstrapSurface(surfaces.bootstrap); - validateSnapshotSurface(surfaces.snapshot); + validateSnapshotSurface(providerId, surfaces.snapshot); validateRecoverySurface(surfaces.recovery); validateCleanupSurface(surfaces.cleanup); validateContainerEngineSurface(providerId, surfaces.containerEngine); @@ -682,7 +693,8 @@ export function normalizeRuntimeProviderSnapshotPreflightReceipt( !isPlainRecord(value) || value.schemaVersion !== RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION || !validProviderId(value.providerId) || - !SNAPSHOT_OPERATIONS.has(String(value.operation)) || + typeof value.operation !== "string" || + !SNAPSHOT_OPERATIONS.has(value.operation) || !boundedString(value.sandboxName, 512) || !boundedString(value.providerHandle, MAX_RECEIPT_HANDLE_BYTES) || !SNAPSHOT_LIFECYCLE_STATES.has(value.lifecycleState as RuntimeProviderSnapshotLifecycleState) || diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index 7fa369f87db..bf2a47fe4b4 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -425,6 +425,40 @@ describe("RuntimeProviderBundle registry contract", () => { ).toThrow(/duplicate operation identities/u); }); + it("versions supported snapshot facets and enforces managed-profile capability dependencies", () => { + const docker = CURRENT_RUNTIME_PROVIDER_BUNDLES.docker!; + const snapshot = docker.snapshot; + expectSupportedSurface(snapshot); + expect(snapshot.contractVersion).toBe(1); + + expect(() => + createRuntimeProviderBundleRegistry([ + [ + "docker", + replaceSurface(docker, "snapshot", { + ...snapshot, + contractVersion: 2, + }), + ], + ]), + ).toThrow(/unsupported contract version/u); + expect(() => + createRuntimeProviderBundleRegistry([ + [ + "docker", + replaceSurface(docker, "snapshot", { + ...snapshot, + capabilities: { + ...snapshot.capabilities, + restore: false, + managedProfileRestore: true, + }, + }), + ], + ]), + ).toThrow(/cannot restore managed profiles/u); + }); + it("normalizes bounded opaque runtime receipts and rejects duplicate GPU devices", () => { const receipt = { schemaVersion: 1, @@ -504,6 +538,12 @@ describe("RuntimeProviderBundle registry contract", () => { lifecycleGeneration: "generation\ninjection", }), ).toBeNull(); + expect( + normalizeRuntimeProviderSnapshotPreflightReceipt({ + ...preflight, + operation: { toString: () => "restore" }, + }), + ).toBeNull(); expect( normalizeRuntimeProviderSnapshotRestoreReceipt({ ...restore, diff --git a/src/lib/onboard/runtime-provider/snapshot.test.ts b/src/lib/onboard/runtime-provider/snapshot.test.ts index 75795695dfe..3ec35633b4c 100644 --- a/src/lib/onboard/runtime-provider/snapshot.test.ts +++ b/src/lib/onboard/runtime-provider/snapshot.test.ts @@ -538,6 +538,7 @@ function dockerSnapshot( deviceRequests: null, devices: null, runtime: "runc", + nvidiaVisibleDevices: null, nativeGpuAttachmentState: "absent", containerId: "c".repeat(64), ...overrides, @@ -577,6 +578,7 @@ describe("Docker provider snapshot evidence", () => { ], nativeGpuAttachmentState: "present", runtime: "nvidia", + nvidiaVisibleDevices: "GPU-live-0", }), ); const observed = observeDockerRuntimeSnapshot( @@ -599,7 +601,7 @@ describe("Docker provider snapshot evidence", () => { acceleration: { kind: "gpu", vendor: "nvidia", - devices: ["docker-device-id:GPU-live-0"], + devices: ["docker-device-id:GPU-live-0", "docker-nvidia-visible-device:GPU-live-0"], }, }, }); @@ -624,11 +626,12 @@ describe("Docker provider snapshot evidence", () => { ], nativeGpuAttachmentState: "present", runtime: "nvidia", + nvidiaVisibleDevices: "all", }), }, ); expect(allDevices.runtime.acceleration).toMatchObject({ - devices: ["docker-device-request:nvidia:count=-1"], + devices: ["docker-device-request:nvidia:count=-1", "docker-nvidia-visible-devices:all"], }); for (const snapshot of [ @@ -636,6 +639,7 @@ describe("Docker provider snapshot evidence", () => { deviceRequests: null, nativeGpuAttachmentState: "present", runtime: "nvidia", + nvidiaVisibleDevices: null, }), dockerSnapshot({ deviceRequests: [ @@ -661,10 +665,42 @@ describe("Docker provider snapshot evidence", () => { } }); - it("runs the in-sandbox managed-profile verifier during restore and fails closed on refusal", () => { + it("captures the NVIDIA Container Runtime selector used by Jetson", () => { + const observed = observeDockerRuntimeSnapshot( + sandbox({ openshellDriver: "docker" }), + "docker", + { + captureHostCommand: dockerLifecycleCapture(), + queryRuntimeSnapshot: () => + dockerSnapshot({ + deviceRequests: null, + devices: null, + nativeGpuAttachmentState: "present", + runtime: "nvidia", + nvidiaVisibleDevices: "0,GPU-live-1", + }), + }, + ); + + expect(observed.runtime.acceleration).toEqual({ + kind: "gpu", + vendor: "nvidia", + devices: ["docker-nvidia-visible-device:0", "docker-nvidia-visible-device:GPU-live-1"], + }); + }); + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("runs the in-sandbox %s profile verifier and fails closed on refusal", (agent) => { + const authority = { + agent, + profileFingerprint: managedProfile.profileFingerprint, + }; const captureOpenShell = vi.fn(() => ({ status: 0, - output: "[managed-startup] verified openclaw profile completion\n", + output: `[managed-startup] verified ${agent} profile completion\n`, stdout: "", stderr: "", })); @@ -675,7 +711,7 @@ describe("Docker provider snapshot evidence", () => { }; const surface = createDockerRuntimeProviderSnapshotSurface("docker", dependencies); if (!surface.supported) throw new Error("Docker snapshot surface must be supported"); - const target = sandbox({ openshellDriver: "docker" }); + const target = sandbox({ agent, openshellDriver: "docker" }); const preflight = surface.preflight("restore", target); const source = snapshotSource(preflight, { schemaVersion: 1, @@ -683,8 +719,8 @@ describe("Docker provider snapshot evidence", () => { runtime: { kind: "docker-container", handle: "c".repeat(64) }, acceleration: { kind: "none" }, }); - const receipt = surface.restore(target, preflight, source, managedProfile); - expect(receipt.managedProfile).toEqual(managedProfile); + const receipt = surface.restore(target, preflight, source, authority); + expect(receipt.managedProfile).toEqual(authority); expect(captureOpenShell).toHaveBeenCalledWith( [ "sandbox", @@ -700,9 +736,9 @@ describe("Docker provider snapshot evidence", () => { "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", "--verify-completion", "--agent", - "openclaw", + agent, "--profile-fingerprint", - managedProfile.profileFingerprint, + authority.profileFingerprint, ], expect.objectContaining({ ignoreError: true, @@ -723,7 +759,7 @@ describe("Docker provider snapshot evidence", () => { if (!denied.supported) throw new Error("Docker snapshot surface must be supported"); const deniedPreflight = denied.preflight("restore", target); const deniedSource = snapshotSource(deniedPreflight, source.runtime); - expect(() => denied.restore(target, deniedPreflight, deniedSource, managedProfile)).toThrow( + expect(() => denied.restore(target, deniedPreflight, deniedSource, authority)).toThrow( /managed profile restoration could not be proven/u, ); }); diff --git a/src/lib/onboard/runtime-provider/snapshot.ts b/src/lib/onboard/runtime-provider/snapshot.ts index 24041a10575..8eac713502a 100644 --- a/src/lib/onboard/runtime-provider/snapshot.ts +++ b/src/lib/onboard/runtime-provider/snapshot.ts @@ -13,6 +13,7 @@ import { } from "../openshell-docker-sandbox-containers"; import { RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION, + RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION, type RuntimeProviderCommandCapture, type RuntimeProviderManagedProfileRestoreAuthority, type RuntimeProviderRuntimeReceipt, @@ -238,6 +239,16 @@ function dockerGpuSelectors( } const selectors: string[] = []; + if (snapshot.runtime.trim().toLowerCase() === "nvidia") { + const visibleDevices = snapshot.nvidiaVisibleDevices; + if (visibleDevices === "all") { + selectors.push("docker-nvidia-visible-devices:all"); + } else if (visibleDevices && !["none", "void"].includes(visibleDevices)) { + for (const device of visibleDevices.split(",")) { + selectors.push(`docker-nvidia-visible-device:${device}`); + } + } + } for (const request of snapshot.deviceRequests ?? []) { if (!dockerRequestUsesGpu(request)) continue; if (request.DeviceIDs && request.DeviceIDs.length > 0) { @@ -582,6 +593,7 @@ export function createRuntimeProviderSnapshotSurface( return { providerId, supported: true, + contractVersion: RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION, capabilities, preflight(operation, sandbox) { const observed = observeAndNormalize(driver.observe, sandbox, providerId); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index f11e1e21142..c7671ff4e60 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -84,6 +84,10 @@ const DEFAULT_RUNTIME_SNAPSHOT = { imageId: IMAGE_ID, bookkeepingImageRef: "openshell/sandbox-from:test", stateError: "", + deviceRequests: null, + devices: null, + runtime: "runc", + nvidiaVisibleDevices: null, nativeGpuAttachmentState: "absent" as const, containerId: "container-a", }; diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 8752009ed00..d05dc3e7fc5 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -14,6 +14,7 @@ import { chmodSync, closeSync, existsSync, + fstatSync, lstatSync, mkdirSync, mkdtempSync, @@ -21,6 +22,7 @@ import { readdirSync, readFileSync, readlinkSync, + readSync, renameSync, rmSync, statSync, @@ -28,6 +30,7 @@ import { } from "node:fs"; import os from "node:os"; import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; import { spawnSync } from "child_process"; import { captureSandboxSshConfigCommand } from "../adapters/openshell/client.js"; @@ -60,7 +63,7 @@ import { cloneSandboxRuntimeSnapshot, type SandboxRuntimeSnapshot, } from "./registry/runtime-snapshot.js"; -import type { SandboxWorkloadReceipt } from "./registry/types.js"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "./registry/types.js"; import { cloneSandboxWorkloadReceipt } from "./registry/workload.js"; import type { CustomPolicyEntry } from "./registry.js"; import * as registry from "./registry.js"; @@ -75,6 +78,8 @@ const REBUILD_BACKUPS_DIR = path.join(nemoclawStateRoot(HOME_DIR, GATEWAY_PORT), const MANIFEST_VERSION = 1; export const OPENCLAW_IMAGE_PLUGIN_PROVENANCE_RESTORE_ERROR = "custom-image OpenClaw plugin provenance is missing or invalid"; +export const MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR = + "managed snapshot restore requires exact content and runtime authority"; function parseJson(text: string): T { return JSON.parse(text); @@ -139,6 +144,12 @@ export interface BackupOptions { name?: string | null; runtimeSnapshot?: SandboxRuntimeSnapshot; workload?: SandboxWorkloadReceipt; + /** + * Internal publication fence for provider-backed backups. The callback + * runs after data capture and sanitization but before the manifest becomes + * visible to restore and rebuild flows. + */ + validateBeforePublish?: () => void; } export interface InstanceBackup { @@ -190,7 +201,24 @@ export interface RestoreResult { error?: string; } -export interface RecreatedSandboxRestoreOptions { +export interface SnapshotRestoreAuthority { + readonly schemaVersion: 1; + readonly backupPath: string; + readonly contentSha256: string; +} + +export interface SnapshotRestoreOptions { + /** + * Content identity captured from the selected manifest and every backup + * payload. The state layer revalidates it after local staging and before + * the first remote filesystem mutation. + */ + readonly authority?: SnapshotRestoreAuthority; + /** Internal provider fence invoked at the same last-safe mutation edge. */ + readonly validateBeforeMutation?: () => void; +} + +export interface RecreatedSandboxRestoreOptions extends SnapshotRestoreOptions { /** Agent in the newly created target image, not the backup manifest agent. */ targetAgentType: string; /** Explicit capability for custom images whose config must be restored wholesale. */ @@ -204,6 +232,8 @@ interface InternalRestoreOptions { allowCustomImageWholeStateFileRestore?: true; discoverFreshOpenClawImagePluginInstalls?: true; freshOpenClawImagePluginInstalls?: readonly OpenClawImagePluginInstall[]; + authority?: SnapshotRestoreAuthority; + validateBeforeMutation?: () => void; } export interface TarValidationResult { @@ -936,6 +966,63 @@ function normalizeSnapshotBackupAuthority(options: BackupOptions): { }; } +function validateSnapshotPublication( + backupPath: string, + validateBeforePublish: BackupOptions["validateBeforePublish"], +): string | null { + if (!validateBeforePublish) return null; + try { + validateBeforePublish(); + return null; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + try { + rmSync(backupPath, { recursive: true, force: true }); + return `Snapshot authority changed during backup: ${detail}`; + } catch (cleanupError) { + const cleanupDetail = + cleanupError instanceof Error ? cleanupError.message : String(cleanupError); + return ( + `Snapshot authority changed during backup: ${detail}. ` + + `The unpublished backup at '${backupPath}' could not be removed: ${cleanupDetail}` + ); + } + } +} + +function resolveOpenClawBackupMetadata( + agentName: string, + sandbox: SandboxEntry | null, + configDir: string, +): { + readonly reconcileImagePluginProvenance: boolean; + readonly pluginInstalls?: OpenClawImagePluginInstall[]; + readonly error?: string; +} { + const reconcileImagePluginProvenance = + agentName === "openclaw" && Boolean(sandbox?.fromDockerfile); + if ( + agentName !== "openclaw" || + (!reconcileImagePluginProvenance && sandbox?.openclawImagePluginInstalls === undefined) + ) { + return { reconcileImagePluginProvenance }; + } + const provenance = parseOpenClawImagePluginInstalls( + sandbox?.openclawImagePluginInstalls, + configDir, + ); + if (!provenance.ok) { + return { + reconcileImagePluginProvenance, + error: "registered OpenClaw image plugin provenance is missing or invalid", + }; + } + return { + reconcileImagePluginProvenance, + pluginInstalls: cloneOpenClawImagePluginInstalls(provenance.pluginInstalls), + }; +} + export function backupSandboxState(sandboxName: string, options: BackupOptions = {}): BackupResult { const sb = registry.getSandbox(sandboxName); const agentName = sb?.agent || "openclaw"; @@ -963,25 +1050,16 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = }; } - const reconcileOpenClawImagePluginProvenance = - agentName === "openclaw" && Boolean(sb?.fromDockerfile); - let openclawImagePluginInstalls: OpenClawImagePluginInstall[] | undefined; - if ( - agentName === "openclaw" && - (reconcileOpenClawImagePluginProvenance || sb?.openclawImagePluginInstalls !== undefined) - ) { - const provenance = parseOpenClawImagePluginInstalls(sb?.openclawImagePluginInstalls, dir); - if (!provenance.ok) { - return { - success: false, - backedUpDirs: [], - failedDirs: [], - backedUpFiles: [], - failedFiles: [], - error: "registered OpenClaw image plugin provenance is missing or invalid", - }; - } - openclawImagePluginInstalls = cloneOpenClawImagePluginInstalls(provenance.pluginInstalls); + const openClawMetadata = resolveOpenClawBackupMetadata(agentName, sb, dir); + if (openClawMetadata.error) { + return { + success: false, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + error: openClawMetadata.error, + }; } // Validate user-supplied name and check for conflicts BEFORE creating any @@ -1018,6 +1096,16 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const backupPath = path.join(REBUILD_BACKUPS_DIR, sandboxName, timestamp); + if (existsSync(backupPath)) { + return { + success: false, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + error: `Snapshot path '${backupPath}' already exists; retry the backup.`, + }; + } // SECURITY: Verify backup destination ancestors are not symlinks. // Without this check, an attacker who plants ~/.nemoclaw/rebuild-backups @@ -1048,8 +1136,10 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = agentType: agentName, agentVersion: sb?.agentVersion || null, expectedVersion: agent.expectedVersion, - ...(openclawImagePluginInstalls !== undefined ? { openclawImagePluginInstalls } : {}), - ...(reconcileOpenClawImagePluginProvenance + ...(openClawMetadata.pluginInstalls !== undefined + ? { openclawImagePluginInstalls: openClawMetadata.pluginInstalls } + : {}), + ...(openClawMetadata.reconcileImagePluginProvenance ? { reconcileOpenClawImagePluginProvenance: true } : {}), stateDirs, @@ -1073,6 +1163,17 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = if (stateDirs.length === 0 && stateFiles.length === 0) { _log("WARNING: Agent manifest declares no state_dirs or state_files — nothing to back up"); + const publicationError = validateSnapshotPublication(backupPath, options.validateBeforePublish); + if (publicationError) { + return { + success: false, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + error: publicationError, + }; + } writeManifest(backupPath, manifest); return { success: true, manifest, backedUpDirs, failedDirs, backedUpFiles, failedFiles }; } @@ -1398,6 +1499,17 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = manifest.stateDirs.includes(failedDir), ); + const publicationError = validateSnapshotPublication(backupPath, options.validateBeforePublish); + if (publicationError) { + return { + success: false, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + error: publicationError, + }; + } writeManifest(backupPath, manifest); manifest.backupPath = backupPath; @@ -1415,10 +1527,136 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = // ── Restore ──────────────────────────────────────────────────────── +function snapshotManifestAuthority(manifest: RebuildManifest): RebuildManifest { + const normalized = { + ...manifest, + backupPath: path.resolve(manifest.backupPath), + } as RebuildManifest & { snapshotVersion?: unknown }; + // snapshotVersion is a list-time cursor, not persisted restore authority. + // Every other normalized manifest field can affect restore behavior and + // therefore remains bound to the operator's selected snapshot. + delete normalized.snapshotVersion; + return normalized; +} + +function hashSnapshotTree(backupPath: string): string { + const hash = createHash("sha256"); + const visit = (directory: string, relativeDirectory: string): void => { + const entries = readdirSync(directory, { withFileTypes: true }).sort((left, right) => + left.name.localeCompare(right.name), + ); + for (const entry of entries) { + const fullPath = path.join(directory, entry.name); + const relativePath = path.posix.join( + relativeDirectory.split(path.sep).join(path.posix.sep), + entry.name, + ); + const stat = lstatSync(fullPath); + if (stat.isDirectory()) { + hash.update(JSON.stringify(["directory", relativePath]), "utf8"); + visit(fullPath, relativePath); + continue; + } + if (stat.isSymbolicLink()) { + hash.update(JSON.stringify(["symlink", relativePath, readlinkSync(fullPath)]), "utf8"); + continue; + } + if (!stat.isFile()) { + throw new Error(`snapshot contains unsupported entry '${relativePath}'`); + } + hash.update(JSON.stringify(["file", relativePath, stat.size]), "utf8"); + const descriptor = openSync(fullPath, "r"); + try { + const opened = fstatSync(descriptor); + if (!opened.isFile() || opened.dev !== stat.dev || opened.ino !== stat.ino) { + throw new Error(`snapshot entry '${relativePath}' changed while it was opened`); + } + const buffer = Buffer.allocUnsafe(64 * 1024); + for (;;) { + const bytesRead = readSync(descriptor, buffer, 0, buffer.byteLength, null); + if (bytesRead === 0) break; + hash.update(buffer.subarray(0, bytesRead)); + } + const after = fstatSync(descriptor); + if (after.size !== opened.size || after.mtimeMs !== opened.mtimeMs) { + throw new Error(`snapshot entry '${relativePath}' changed while it was read`); + } + } finally { + closeSync(descriptor); + } + } + }; + visit(backupPath, ""); + return hash.digest("hex"); +} + +/** + * Bind a selected, validated manifest to all bytes that restore can consume. + * Returns null for an unsafe path, malformed manifest, selection drift, or a + * payload that changes while it is being hashed. + */ +export function captureSnapshotRestoreAuthority( + backupPath: string, + expectedManifest?: RebuildManifest, +): SnapshotRestoreAuthority | null { + try { + const root = path.resolve(REBUILD_BACKUPS_DIR); + const candidate = path.resolve(backupPath); + if (candidate === root || !isWithinRoot(candidate, root)) return null; + rejectSymlinksOnPath(candidate); + if (!lstatSync(path.join(candidate, "rebuild-manifest.json")).isFile()) return null; + const manifest = readManifest(candidate); + if (!manifest || path.resolve(manifest.backupPath) !== candidate) return null; + if ( + expectedManifest && + !isDeepStrictEqual( + snapshotManifestAuthority(manifest), + snapshotManifestAuthority(expectedManifest), + ) + ) { + return null; + } + return { + schemaVersion: 1, + backupPath: candidate, + contentSha256: hashSnapshotTree(candidate), + }; + } catch { + return null; + } +} + +function validateSnapshotRestoreMutation( + backupPath: string, + options: Pick, +): string | null { + if (options.authority) { + const current = captureSnapshotRestoreAuthority(backupPath); + if ( + !current || + current.backupPath !== options.authority.backupPath || + current.contentSha256 !== options.authority.contentSha256 + ) { + return "Selected snapshot content changed before filesystem mutation"; + } + } + try { + options.validateBeforeMutation?.(); + return null; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + return `Runtime authority changed before filesystem mutation: ${detail}`; + } +} + /** * Restore state directories into a sandbox from a prior backup. */ -export function restoreSandboxState(sandboxName: string, backupPath: string): RestoreResult { +export function restoreSandboxState( + sandboxName: string, + backupPath: string, + options: SnapshotRestoreOptions = {}, +): RestoreResult { const target = registry.getSandbox(sandboxName); if (!target) { return { @@ -1433,6 +1671,10 @@ export function restoreSandboxState(sandboxName: string, backupPath: string): Re return restoreSandboxStateInternal(sandboxName, backupPath, { targetAgentType: String(target.agent || "openclaw"), ...(target.fromDockerfile ? { allowCustomImageWholeStateFileRestore: true } : {}), + ...(options.authority ? { authority: options.authority } : {}), + ...(options.validateBeforeMutation + ? { validateBeforeMutation: options.validateBeforeMutation } + : {}), }); } @@ -1451,6 +1693,10 @@ export function restoreRecreatedSandboxState( ? { discoverFreshOpenClawImagePluginInstalls: true } : {}), freshOpenClawImagePluginInstalls: options.freshOpenClawImagePluginInstalls, + ...(options.authority ? { authority: options.authority } : {}), + ...(options.validateBeforeMutation + ? { validateBeforeMutation: options.validateBeforeMutation } + : {}), }); } @@ -1517,6 +1763,12 @@ function restoreSandboxStateInternal( error, }; }; + if ( + manifest.workload?.kind === "managed-image" && + (!options.authority || !options.validateBeforeMutation) + ) { + return failRestoreContract(MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR); + } if (options.targetAgentType !== manifest.agentType) { return failRestoreContract( `Backup agent '${manifest.agentType}' does not match target agent '${options.targetAgentType}'`, @@ -1625,6 +1877,10 @@ function restoreSandboxStateInternal( } if (cleanupStateDirs.length === 0 && localFiles.length === 0) { + const mutationAuthorityError = validateSnapshotRestoreMutation(backupPath, options); + if (mutationAuthorityError) { + return failRestoreContract(mutationAuthorityError); + } _log("No dirs or files to restore"); return { success: true, restoredDirs, failedDirs, restoredFiles, failedFiles }; } @@ -1716,6 +1972,11 @@ function restoreSandboxStateInternal( restoreTar = tarResult.stdout; } + const mutationAuthorityError = validateSnapshotRestoreMutation(backupPath, options); + if (mutationAuthorityError) { + return failRestoreContract(mutationAuthorityError); + } + // Remove existing state dirs before extracting so stale files from later // snapshots don't persist after restoring an earlier one. OpenClaw's // image-managed extensions are preserved from the freshly built image and diff --git a/test/snapshot-managed-restore-authority.test.ts b/test/snapshot-managed-restore-authority.test.ts new file mode 100644 index 00000000000..cf4c9df3f87 --- /dev/null +++ b/test/snapshot-managed-restore-authority.test.ts @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { encodeManagedStartupProfile } from "../src/lib/onboard/managed-startup/profile"; + +const ORIGINAL_HOME = process.env.HOME; +const TMP_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-snapshot-authority-")); +process.env.HOME = TMP_HOME; +const sandboxState = await import("../src/lib/state/sandbox.js"); +const BACKUPS_ROOT = path.join(TMP_HOME, ".nemoclaw", "rebuild-backups"); + +afterAll(() => { + if (ORIGINAL_HOME === undefined) { + delete process.env.HOME; + } else { + process.env.HOME = ORIGINAL_HOME; + } + fs.rmSync(TMP_HOME, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(BACKUPS_ROOT, { recursive: true, force: true }); +}); + +function managedAuthority() { + const encodedProfile = encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")); + return { + workload: { + schemaVersion: 1, + kind: "managed-image", + reference: `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`, + platform: "linux/amd64", + release: "v0.0.97", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true, + }, + runtimeSnapshot: { + schemaVersion: 1, + providerId: "docker", + providerHandle: "opaque-provider-handle", + lifecycleState: "running", + lifecycleGeneration: "generation-1", + runtime: { + schemaVersion: 1, + providerId: "docker", + runtime: { kind: "docker-container", handle: "opaque-container-id" }, + acceleration: { kind: "none" }, + }, + }, + } as const; +} + +function writeBackup(overrides: Record = {}) { + const timestamp = "2026-04-21T14-00-00-000Z"; + const backupPath = path.join(BACKUPS_ROOT, "alpha", timestamp); + fs.mkdirSync(backupPath, { recursive: true }); + const manifest = { + version: 1, + sandboxName: "alpha", + timestamp, + agentType: "openclaw", + agentVersion: null, + expectedVersion: null, + stateDirs: [], + dir: "/sandbox/.openclaw", + backupPath, + blueprintDigest: null, + ...overrides, + }; + fs.writeFileSync( + path.join(backupPath, "rebuild-manifest.json"), + JSON.stringify(manifest, null, 2), + ); + return manifest; +} + +function writeOpenClawRegistry(): void { + fs.mkdirSync(path.join(TMP_HOME, ".nemoclaw"), { recursive: true }); + fs.writeFileSync( + path.join(TMP_HOME, ".nemoclaw", "sandboxes.json"), + JSON.stringify({ + defaultSandbox: "alpha", + sandboxes: { + alpha: { + name: "alpha", + model: "demo", + provider: "compatible-endpoint", + gpuEnabled: false, + policies: [], + agent: "openclaw", + }, + }, + }), + ); +} + +describe("managed snapshot restore authority", () => { + it("binds every normalized restore-relevant manifest field selected by the operator", () => { + const manifest = writeBackup({ backedUpDirs: ["workspace"], stateDirs: ["workspace"] }); + const selected = sandboxState.getLatestBackup("alpha"); + expect(selected).not.toBeNull(); + + fs.writeFileSync( + path.join(manifest.backupPath, "rebuild-manifest.json"), + JSON.stringify({ ...manifest, stateDirs: ["workspace", "agents"] }, null, 2), + ); + + expect(sandboxState.captureSnapshotRestoreAuthority(manifest.backupPath, selected!)).toBeNull(); + }); + + it("requires both content and runtime fences at each raw state entry point", () => { + const manifest = writeBackup(managedAuthority()); + const contentAuthority = sandboxState.captureSnapshotRestoreAuthority(manifest.backupPath); + expect(contentAuthority).not.toBeNull(); + + for (const partialAuthority of [ + {}, + { authority: contentAuthority! }, + { validateBeforeMutation: vi.fn() }, + ]) { + expect( + sandboxState.restoreRecreatedSandboxState("alpha", manifest.backupPath, { + targetAgentType: "openclaw", + ...partialAuthority, + }), + ).toMatchObject({ + success: false, + error: sandboxState.MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR, + }); + } + + writeOpenClawRegistry(); + expect(sandboxState.restoreSandboxState("alpha", manifest.backupPath)).toMatchObject({ + success: false, + error: sandboxState.MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR, + }); + + const validateBeforeMutation = vi.fn(); + expect( + sandboxState.restoreRecreatedSandboxState("alpha", manifest.backupPath, { + targetAgentType: "openclaw", + freshOpenClawImagePluginInstalls: [], + authority: contentAuthority!, + validateBeforeMutation, + }), + ).toMatchObject({ success: true }); + expect(validateBeforeMutation).toHaveBeenCalledOnce(); + }); +}); diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index 94208ca951d..26f7f05ee33 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -13,6 +13,8 @@ import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { afterAll, beforeEach, describe, expect, it } from "vitest"; +import { managedStartupE2eProfile } from "../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { encodeManagedStartupProfile } from "../src/lib/onboard/managed-startup/profile"; // Override HOME BEFORE importing sandbox-state — it reads process.env.HOME // at module-load time to compute REBUILD_BACKUPS_DIR. Captured original is @@ -69,7 +71,7 @@ function writeBackup( return manifest; } function managedSnapshotAuthority() { - const encodedProfile = Buffer.from('{"schemaVersion":1}', "utf8").toString("base64url"); + const encodedProfile = encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")); return { workload: { schemaVersion: 1, @@ -361,6 +363,32 @@ describe("listBackups computes virtual versions", () => { }); }); +describe("snapshot restore content authority", () => { + it("binds the selected manifest and payload bytes to one digest", () => { + const manifest = writeBackup("alpha", "2026-04-21T14-00-00-000Z", { + backedUpDirs: ["workspace"], + stateDirs: ["workspace"], + }); + const backupPath = String(manifest.backupPath); + fs.mkdirSync(path.join(backupPath, "workspace")); + fs.writeFileSync(path.join(backupPath, "workspace", "state.txt"), "before\n"); + const selected = sandboxState.getLatestBackup("alpha"); + expect(selected).not.toBeNull(); + + const authority = sandboxState.captureSnapshotRestoreAuthority(backupPath, selected!); + expect(authority).toMatchObject({ + schemaVersion: 1, + backupPath, + contentSha256: expect.stringMatching(/^[a-f0-9]{64}$/u), + }); + + fs.writeFileSync(path.join(backupPath, "workspace", "state.txt"), "after\n"); + expect(sandboxState.captureSnapshotRestoreAuthority(backupPath)?.contentSha256).not.toBe( + authority?.contentSha256, + ); + }); +}); + describe("findBackup", () => { it("matches v against the computed version", () => { writeBackup("test-sandbox", "2026-04-21T14-00-00-000Z"); // v1 (oldest) @@ -604,6 +632,25 @@ process.exit(0); expect(backup.manifest?.reconcileOpenClawImagePluginProvenance).toBe(true); expect(backup.manifest?.openclawImagePluginInstalls).toEqual([]); expect(fs.readdirSync(stagingRoot)).toEqual([]); + + const rejected = sandboxState.backupSandboxState("alpha", { + validateBeforePublish: () => { + throw new Error("runtime generation changed"); + }, + }); + expect(rejected).toMatchObject({ + success: false, + error: expect.stringContaining( + "Snapshot authority changed during backup: runtime generation changed", + ), + }); + const published = fs + .readdirSync(path.join(BACKUPS_ROOT, "alpha")) + .filter((entry) => + fs.existsSync(path.join(BACKUPS_ROOT, "alpha", entry, "rebuild-manifest.json")), + ); + expect(published).toHaveLength(1); + expect(fs.readdirSync(stagingRoot)).toEqual([]); } finally { restoreEnv("NEMOCLAW_OPENSHELL_BIN", oldOpenshell); restoreEnv("TMPDIR", oldTmpdir); From 8b195817a7e910471a29fa3cc798e47f91f55f7a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 19:19:00 -0700 Subject: [PATCH 048/117] feat(snapshot): prepare immutable managed clone handoff Signed-off-by: Aaron Erickson --- ...hot-managed-clone-handoff-dormancy.test.ts | 24 + src/lib/messaging/clone-rebind.ts | 166 ++++++ .../engines/credential-binding-engine.ts | 5 +- .../compiler/engines/host-forward-engine.ts | 3 +- src/lib/messaging/hydration.ts | 17 +- src/lib/messaging/index.ts | 1 + src/lib/messaging/persistence.ts | 36 +- src/lib/messaging/plan-validation.ts | 7 +- src/lib/name-validation.ts | 10 + src/lib/onboard/inference-route.ts | 20 +- src/lib/onboard/lifecycle-contracts.md | 1 + .../managed-startup-clone-rebinder.test.ts | 418 +++++++++++++++ .../onboard/managed-startup/clone-rebinder.ts | 502 ++++++++++++++++++ .../managed-workload-clone-handoff.test.ts | 417 +++++++++++++++ src/lib/onboard/runtime-provider/contract.ts | 1 + src/lib/onboard/runtime-provider/docker.ts | 1 + src/lib/onboard/runtime-provider/registry.ts | 1 + src/lib/onboard/workload/clone.ts | 390 ++++++++++++++ src/lib/state/registry/types.ts | 2 + test/helpers/runtime-provider-bundle.ts | 1 + 20 files changed, 2008 insertions(+), 15 deletions(-) create mode 100644 src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts create mode 100644 src/lib/messaging/clone-rebind.ts create mode 100644 src/lib/onboard/managed-startup-clone-rebinder.test.ts create mode 100644 src/lib/onboard/managed-startup/clone-rebinder.ts create mode 100644 src/lib/onboard/managed-workload-clone-handoff.test.ts create mode 100644 src/lib/onboard/workload/clone.ts diff --git a/src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts b/src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts new file mode 100644 index 00000000000..53647e6bfb2 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readFileSync } from "node:fs"; + +import { describe, expect, it } from "vitest"; + +describe("managed snapshot clone handoff activation boundary", () => { + it("keeps the PR3.9 contract unwired while production restore stays fail-closed", () => { + const dependencies = readFileSync( + new URL("./snapshot/dependencies.ts", import.meta.url), + "utf8", + ); + const productionAction = readFileSync(new URL("./snapshot.ts", import.meta.url), "utf8"); + + expect(dependencies).not.toContain("prepareManagedWorkloadCloneHandoff"); + expect(dependencies).not.toContain("ManagedWorkloadCloneHandoff"); + expect(productionAction).toContain("rejectManagedSnapshotCloneUntilRebind"); + expect(productionAction).not.toContain("prepareManagedWorkloadCloneHandoff"); + expect(productionAction).not.toContain("ManagedWorkloadCloneHandoff"); + expect(productionAction).not.toContain("prepareManagedCloneProviders"); + expect(productionAction).not.toContain("provisionManagedCloneProviders"); + }); +}); diff --git a/src/lib/messaging/clone-rebind.ts b/src/lib/messaging/clone-rebind.ts new file mode 100644 index 00000000000..4f156533486 --- /dev/null +++ b/src/lib/messaging/clone-rebind.ts @@ -0,0 +1,166 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { cloneAndDeepFreeze } from "../core/immutable"; +import { isValidName } from "../name-validation"; +import { createBuiltInChannelManifestRegistry } from "./channels/built-ins"; +import { hydrateDerivedSandboxMessagingPlanFields } from "./hydration"; +import type { MessagingAgentId, SandboxMessagingPlan } from "./manifest"; +import { compactSandboxMessagingPlanForPersistence } from "./persistence"; +import { parseSandboxMessagingPlan } from "./plan-validation"; + +export interface SandboxMessagingCloneRebindInput { + readonly sourceSandboxName: string; + readonly destinationSandboxName: string; + readonly agent: MessagingAgentId; + readonly sourcePlan: unknown; + /** + * Explicit non-secret inputs used by manifest renderers. Clone rebinding + * never consults process.env, so an unrelated host credential cannot change + * the destination plan or enter its fingerprints. + */ + readonly environment?: Readonly>; +} + +export class SandboxMessagingCloneRebindError extends Error { + constructor(message: string) { + super(`Cannot rebind managed messaging plan: ${message}`); + this.name = "SandboxMessagingCloneRebindError"; + } +} + +function fail(message: string): never { + throw new SandboxMessagingCloneRebindError(message); +} + +function requireSandboxName(value: string, label: string): string { + if (!isValidName(value)) fail(`${label} sandbox name is invalid`); + return value; +} + +/** + * Recompile one secret-free managed messaging plan for a destination sandbox. + * + * Compact persistence data is the retained intent boundary. All target-bound + * provider names and executable derived fields are rebuilt from current + * built-in manifests with an explicit, credential-free environment. + */ +export function rebindSandboxMessagingPlanForClone( + input: SandboxMessagingCloneRebindInput, +): SandboxMessagingPlan { + const sourceSandboxName = requireSandboxName(input.sourceSandboxName, "source"); + const destinationSandboxName = requireSandboxName(input.destinationSandboxName, "destination"); + if (sourceSandboxName === destinationSandboxName) { + fail("source and destination sandbox names must differ"); + } + + const manifestRegistry = createBuiltInChannelManifestRegistry(); + const supportedChannelIds = manifestRegistry + .listAvailable({ agent: input.agent }) + .map((manifest) => manifest.id); + const environment = Object.freeze({ ...(input.environment ?? {}) }); + const sourcePlan = parseSandboxMessagingPlan(input.sourcePlan, { + sandboxName: sourceSandboxName, + agent: input.agent, + supportedChannelIds, + environment, + }); + if (!sourcePlan) fail("source plan is invalid or uses a non-built-in channel"); + const sourceChannelIds = new Set(sourcePlan.channels.map((channel) => channel.channelId)); + if (sourcePlan.disabledChannels.some((channelId) => !sourceChannelIds.has(channelId))) { + fail("source plan disables a channel that is not configured"); + } + for (const channel of sourcePlan.channels) { + const manifest = manifestRegistry.get(channel.channelId); + if (!manifest || !manifest.supportedAgents.includes(input.agent)) { + fail(`source channel ${channel.channelId} is not a supported built-in`); + } + const listedDisabled = sourcePlan.disabledChannels.includes(channel.channelId); + if (channel.disabled !== listedDisabled || (channel.active && !channel.configured)) { + fail(`source channel ${channel.channelId} has inconsistent lifecycle state`); + } + const seenInputIds = new Set(); + for (const planInput of channel.inputs) { + if (seenInputIds.has(planInput.inputId)) { + fail(`source channel ${channel.channelId} repeats input ${planInput.inputId}`); + } + seenInputIds.add(planInput.inputId); + const manifestInput = manifest.inputs.find((candidate) => candidate.id === planInput.inputId); + if (!manifestInput || manifestInput.kind !== planInput.kind) { + fail( + `source channel ${channel.channelId} input ${planInput.inputId} is not manifest-owned`, + ); + } + if (planInput.kind === "secret" && planInput.value !== undefined) { + fail("source plan contains a raw secret input value"); + } + } + if (channel.active && !channel.disabled) { + for (const requiredInput of manifest.inputs.filter((candidate) => candidate.required)) { + const retained = channel.inputs.find((candidate) => candidate.inputId === requiredInput.id); + const available = + requiredInput.kind === "secret" + ? retained?.credentialAvailable === true + : retained?.value !== undefined; + if (!available) { + fail( + `active source channel ${channel.channelId} is missing required input ${requiredInput.id}`, + ); + } + } + } + } + + const compact = compactSandboxMessagingPlanForPersistence(sourcePlan); + const credentialBindings = (compact.credentialBindings ?? []).map( + ({ credentialHash: _sourceCredentialHash, ...binding }) => binding, + ); + const targetIntent = { + ...compact, + // A source hash describes the source gateway credential, not the explicit + // credential that clone provisioning will write into the destination. + credentialBindings, + sandboxName: destinationSandboxName, + // These fields are executable output, not retained clone intent. + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + runtimeSetup: { nodePreloads: [], envAliases: [], secretScans: [] }, + stateUpdates: [], + healthChecks: [], + } as const; + const normalized = parseSandboxMessagingPlan(targetIntent, { + sandboxName: destinationSandboxName, + agent: input.agent, + supportedChannelIds, + environment, + }); + if (!normalized) fail("destination intent could not be normalized"); + const hydrated = hydrateDerivedSandboxMessagingPlanFields(normalized, { environment }); + const rebound = parseSandboxMessagingPlan(hydrated, { + sandboxName: destinationSandboxName, + agent: input.agent, + supportedChannelIds, + environment, + }); + if (!rebound) fail("destination plan could not be validated after manifest hydration"); + const secretProvenanceNeutralRebound = { + ...rebound, + credentialBindings: rebound.credentialBindings.map( + ({ credentialHash: _ambientCredentialHash, ...binding }) => binding, + ), + }; + for (const binding of secretProvenanceNeutralRebound.credentialBindings) { + const credential = manifestRegistry + .get(binding.channelId) + ?.credentials.find((candidate) => candidate.id === binding.credentialId); + const expectedProviderName = credential?.providerName.replaceAll( + "{sandboxName}", + destinationSandboxName, + ); + if (!expectedProviderName || binding.providerName !== expectedProviderName) { + fail(`destination provider identity for ${binding.channelId} could not be proven`); + } + } + return cloneAndDeepFreeze(secretProvenanceNeutralRebound); +} diff --git a/src/lib/messaging/compiler/engines/credential-binding-engine.ts b/src/lib/messaging/compiler/engines/credential-binding-engine.ts index 1a87227e597..11030a0cb28 100644 --- a/src/lib/messaging/compiler/engines/credential-binding-engine.ts +++ b/src/lib/messaging/compiler/engines/credential-binding-engine.ts @@ -1,19 +1,20 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { hashCredential } from "../../../security/credential-hash"; import type { ChannelManifest, SandboxMessagingCredentialBindingPlan, SandboxMessagingInputReference, } from "../../manifest"; import type { ManifestCompilerContext } from "../types"; -import { hashCredential } from "../../../security/credential-hash"; import { resolveSandboxNameTemplate } from "./template"; export function planCredentialBindings( manifest: ChannelManifest, context: ManifestCompilerContext, inputs: readonly SandboxMessagingInputReference[], + environment: Readonly> = process.env, ): SandboxMessagingCredentialBindingPlan[] { return manifest.credentials.map((credential) => { const sourceInput = inputs.find((input) => input.inputId === credential.sourceInput); @@ -24,7 +25,7 @@ export function planCredentialBindings( const envKey = sourceInput?.sourceEnv ?? credential.providerEnvKey; const credentialHash = credentialAvailable - ? (hashCredential(process.env[envKey]) ?? undefined) + ? (hashCredential(environment[envKey]) ?? undefined) : undefined; return { diff --git a/src/lib/messaging/compiler/engines/host-forward-engine.ts b/src/lib/messaging/compiler/engines/host-forward-engine.ts index 91bc8c796ff..3408c08bc66 100644 --- a/src/lib/messaging/compiler/engines/host-forward-engine.ts +++ b/src/lib/messaging/compiler/engines/host-forward-engine.ts @@ -17,10 +17,11 @@ export function planHostForward( inputs: readonly SandboxMessagingInputReference[], active: boolean, referenceResolver?: RenderTemplateReferenceResolver, + environment: Readonly> = process.env, ): SandboxMessagingHostForwardPlan | undefined { if (!active || !manifest.hostForward) return undefined; - const context = { inputs, env: process.env, referenceResolver }; + const context = { inputs, env: environment, referenceResolver }; if (!isTruthyRenderTemplate(manifest.hostForward.when, context)) return undefined; const portValue = resolveRenderTemplatesInValue(manifest.hostForward.port, context); diff --git a/src/lib/messaging/hydration.ts b/src/lib/messaging/hydration.ts index 605f1017f98..07d79f3dcfd 100644 --- a/src/lib/messaging/hydration.ts +++ b/src/lib/messaging/hydration.ts @@ -42,12 +42,19 @@ import { normalizePersistedInputs, } from "./persistence"; +export interface SandboxMessagingHydrationOptions { + /** Explicit environment seam for deterministic rehydration without ambient credentials. */ + readonly environment?: Readonly>; +} + export function hydrateDerivedSandboxMessagingPlanFields( plan: SandboxMessagingPlan, + options: SandboxMessagingHydrationOptions = {}, ): SandboxMessagingPlan { + const environment = options.environment ?? process.env; const manifestRegistry = createBuiltInChannelManifestRegistry(); const channels = plan.channels.map((channel) => - hydrateChannelFromManifest(plan, channel, manifestRegistry.get(channel.channelId)), + hydrateChannelFromManifest(plan, channel, manifestRegistry.get(channel.channelId), environment), ); const hydratedPlan = { ...plan, channels }; const manifests = channels.flatMap((channel) => { @@ -63,7 +70,7 @@ export function hydrateDerivedSandboxMessagingPlanFields( agentRender: plan.agentRender.length > 0 ? plan.agentRender - : agentRenderFromManifests(hydratedPlan, manifestRegistry), + : agentRenderFromManifests(hydratedPlan, manifestRegistry, environment), buildSteps: plan.buildSteps.length > 0 ? plan.buildSteps @@ -82,6 +89,7 @@ function hydrateChannelFromManifest( plan: SandboxMessagingPlan, channel: SandboxMessagingChannelPlan, manifest: ChannelManifest | undefined, + environment: Readonly>, ): SandboxMessagingChannelPlan { const { hostForward: _oldHostForward, ...channelWithoutHostForward } = channel; const disabled = channel.disabled || plan.disabledChannels.includes(channel.channelId); @@ -91,7 +99,7 @@ function hydrateChannelFromManifest( const configured = channel.configured; const active = channel.active && !disabled; const hostForward = manifest - ? planHostForward(manifest, inputs, active, createBuiltInRenderTemplateResolver()) + ? planHostForward(manifest, inputs, active, createBuiltInRenderTemplateResolver(), environment) : undefined; return { ...channelWithoutHostForward, @@ -231,6 +239,7 @@ function runtimeSetupHasEntries(setup: SandboxMessagingRuntimeSetupPlan | undefi function agentRenderFromManifests( plan: SandboxMessagingPlan, manifestRegistry: ReturnType, + environment: Readonly>, ): SandboxMessagingAgentRenderPlan[] { const render: SandboxMessagingAgentRenderPlan[] = []; const referenceResolver = createBuiltInRenderTemplateResolver(); @@ -239,7 +248,7 @@ function agentRenderFromManifests( if (!manifest) continue; const context = { inputs: channel.inputs, - env: process.env, + env: environment, referenceResolver, }; diff --git a/src/lib/messaging/index.ts b/src/lib/messaging/index.ts index 3ad95871443..1da55c0c870 100644 --- a/src/lib/messaging/index.ts +++ b/src/lib/messaging/index.ts @@ -3,6 +3,7 @@ export * from "./applier"; export * from "./channels"; +export * from "./clone-rebind"; export * from "./compiler"; export * from "./diagnostics"; export * from "./hooks"; diff --git a/src/lib/messaging/persistence.ts b/src/lib/messaging/persistence.ts index 1d07305ca4e..d189f480a6c 100644 --- a/src/lib/messaging/persistence.ts +++ b/src/lib/messaging/persistence.ts @@ -131,6 +131,7 @@ export function compactSandboxMessagingPlanForPersistence( export function normalizePersistedSandboxMessagingPlanShape( plan: MaybeCompactMessagingPlan, + environment: Readonly> = process.env, ): SandboxMessagingPlan { const manifestRegistry = createBuiltInChannelManifestRegistry(); const disabledChannels = plan.disabledChannels.filter( @@ -138,9 +139,19 @@ export function normalizePersistedSandboxMessagingPlanShape( ); const disabledSet = new Set(disabledChannels); const channels = plan.channels.map((channel) => - normalizePersistedChannel(channel, disabledSet, manifestRegistry.get(channel.channelId)), + normalizePersistedChannel( + channel, + disabledSet, + manifestRegistry.get(channel.channelId), + environment, + ), + ); + const credentialBindings = normalizePersistedCredentialBindings( + plan, + channels, + manifestRegistry, + environment, ); - const credentialBindings = normalizePersistedCredentialBindings(plan, channels, manifestRegistry); const normalizedPlan: SandboxMessagingPlan = { ...plan, channels, @@ -184,6 +195,7 @@ function normalizePersistedChannel( channel: MaybeCompactMessagingChannelPlan, disabledSet: ReadonlySet, manifest: ChannelManifest | undefined, + environment: Readonly>, ): SandboxMessagingChannelPlan { const disabled = channel.disabled ?? disabledSet.has(channel.channelId); const configured = channel.configured ?? true; @@ -194,7 +206,13 @@ function normalizePersistedChannel( const active = channel.active ?? (configured && !disabled && requiredInputsAvailable(manifest, inputs)); const hostForward = manifest - ? planHostForward(manifest, inputs, active && !disabled, createBuiltInRenderTemplateResolver()) + ? planHostForward( + manifest, + inputs, + active && !disabled, + createBuiltInRenderTemplateResolver(), + environment, + ) : undefined; return { @@ -308,6 +326,7 @@ function normalizePersistedCredentialBindings( plan: MaybeCompactMessagingPlan, channels: readonly SandboxMessagingChannelPlan[], manifestRegistry: ReturnType, + environment: Readonly>, ): SandboxMessagingCredentialBindingPlan[] { const persisted = plan.credentialBindings ?? []; if ( @@ -337,6 +356,7 @@ function normalizePersistedCredentialBindings( planForBindings, manifests, new Map(channels.map((channel) => [channel.channelId, channel.inputs] as const)), + environment, ); return generated.map((binding) => overlayPersistedCredentialBinding(binding, persisted)); } @@ -345,12 +365,16 @@ function credentialBindingsFromManifests( plan: SandboxMessagingPlan, manifests: readonly ChannelManifest[], inputRegistry: ReadonlyMap, + environment: Readonly>, ): SandboxMessagingCredentialBindingPlan[] { const context = compilerContext(plan); return manifests.flatMap((manifest) => - planCredentialBindings(manifest, context, inputRegistry.get(manifest.id) ?? []).map((binding) => - overlayPersistedCredentialBinding(binding, plan.credentialBindings), - ), + planCredentialBindings( + manifest, + context, + inputRegistry.get(manifest.id) ?? [], + environment, + ).map((binding) => overlayPersistedCredentialBinding(binding, plan.credentialBindings)), ); } diff --git a/src/lib/messaging/plan-validation.ts b/src/lib/messaging/plan-validation.ts index 422d0a61ae3..bdf411e286c 100644 --- a/src/lib/messaging/plan-validation.ts +++ b/src/lib/messaging/plan-validation.ts @@ -19,6 +19,8 @@ export interface SandboxMessagingPlanParseOptions { sandboxName?: string | null; agent?: MessagingAgentId | string | null; supportedChannelIds?: readonly MessagingChannelId[] | readonly string[] | null; + /** Explicit environment seam for deterministic rehydration without ambient credentials. */ + environment?: Readonly>; } export function parseSandboxMessagingPlan( @@ -114,7 +116,10 @@ export function parseSandboxMessagingPlan( } return cloneSandboxMessagingPlan( - normalizePersistedSandboxMessagingPlanShape(value as MaybeCompactMessagingPlan), + normalizePersistedSandboxMessagingPlanShape( + value as MaybeCompactMessagingPlan, + options.environment, + ), ); } diff --git a/src/lib/name-validation.ts b/src/lib/name-validation.ts index a3b49850278..5af96f5bc83 100644 --- a/src/lib/name-validation.ts +++ b/src/lib/name-validation.ts @@ -5,6 +5,11 @@ import { NAME_ALLOWED_FORMAT as CANONICAL_NAME_ALLOWED_FORMAT, NAME_MAX_LENGTH as CANONICAL_NAME_MAX_LENGTH, NAME_VALID_PATTERN as CANONICAL_NAME_VALID_PATTERN, + PROVIDER_NAME_ALLOWED_FORMAT as CANONICAL_PROVIDER_NAME_ALLOWED_FORMAT, + PROVIDER_NAME_MAX_LENGTH as CANONICAL_PROVIDER_NAME_MAX_LENGTH, + PROVIDER_NAME_VALID_PATTERN as CANONICAL_PROVIDER_NAME_VALID_PATTERN, + isValidName as isCanonicalValidName, + isValidProviderName as isCanonicalValidProviderName, } from "../../nemoclaw/dist/shared/sandbox-name.cjs"; // sourceOfTruth: nemoclaw/src/shared/sandbox-name.cts @@ -15,6 +20,11 @@ import { export const NAME_MAX_LENGTH = CANONICAL_NAME_MAX_LENGTH; export const NAME_ALLOWED_FORMAT = CANONICAL_NAME_ALLOWED_FORMAT; export const NAME_VALID_PATTERN = CANONICAL_NAME_VALID_PATTERN; +export const PROVIDER_NAME_MAX_LENGTH = CANONICAL_PROVIDER_NAME_MAX_LENGTH; +export const PROVIDER_NAME_ALLOWED_FORMAT = CANONICAL_PROVIDER_NAME_ALLOWED_FORMAT; +export const PROVIDER_NAME_VALID_PATTERN = CANONICAL_PROVIDER_NAME_VALID_PATTERN; +export const isValidName = isCanonicalValidName; +export const isValidProviderName = isCanonicalValidProviderName; function validationSubject(label: string): string { const normalized = label.trim().toLowerCase(); diff --git a/src/lib/onboard/inference-route.ts b/src/lib/onboard/inference-route.ts index 810f2464c5c..347ee7121a5 100644 --- a/src/lib/onboard/inference-route.ts +++ b/src/lib/onboard/inference-route.ts @@ -1,7 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { parseGatewayInference } from "../inference/config"; +import { + getSandboxInferenceConfig, + parseGatewayInference, + resolveAgentInferenceApi, +} from "../inference/config"; import { type CurrentGatewayRouteCompatibilityCheck, type CurrentGatewayRouteDiscoveryPreflight, @@ -12,6 +16,20 @@ import { listSandboxes } from "../state/registry"; type RunCaptureOpenshell = (args: string[], options?: { ignoreError?: boolean }) => string | null; +/** Resolve the exact portable inference route shared by managed rebuild and clone paths. */ +export function resolveManagedStartupInferenceRoute( + agentName: string, + provider: string, + model: string, + preferredInferenceApi: string | null, +) { + const api = + agentName === "langchain-deepagents-code" + ? "openai-completions" + : resolveAgentInferenceApi(agentName, provider, preferredInferenceApi); + return getSandboxInferenceConfig(model, provider, api); +} + export function createInferenceRouteHelpers( runCaptureOpenshell: RunCaptureOpenshell, listSandboxesFn: typeof listSandboxes = listSandboxes, diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index c55e537932c..19c6ebd8de6 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -117,6 +117,7 @@ runtime mutation | **`--fresh` onboard** — `resolveOnboardEntryOptions`, `prepareFreshSession`, `createBaseImageResolutionContext` | Current flags/environment/prompts replace resumable intent. `--fresh` disables auto-resume and forces base-image resolution; it does not prove that the selected sandbox name is unused. | The first destructive effect is local: the prior onboard session is cleared before a new session is saved. A matching live sandbox can later reuse or recreate through the normal sandbox decision; `--fresh` does not itself delete it. | The new session and machine snapshot replace the old resume checkpoint. Credential and effect boundaries then match new onboard or live recreate. | The discarded resume checkpoint is not restored on later failure. Covered by `entry-options.test.ts`, `session-bootstrap.test.ts`, and base-image resolution tests. | | **Resume, re-onboard, or recreate** — `onboard()`, `prepareOnboardSession`, `decideSandboxResume`, live-sandbox handling in `createSandbox` | For `--resume`, the recorded session is authoritative and conflicting current name/provider/model/image/tool-disclosure hints are rejected. A new re-onboard run takes current flags, environment, and prompts as intent while registry/gateway state provides drift evidence. The machine resolves a complete secret-free create intent, including policy, messaging/provider, GPU, resource, disabled-channel, and agent inputs, before repair/removal or live recreation. | Ordinary live recreation conditionally backs up before provider cleanup, **delete**, image removal, and registry removal. A selected pre-upgrade backup suppresses a new one; an explicit override permits recreation without backup. Resume registry removal and `repair-and-recreate` now follow complete intent validation. Temporary policy/build artifacts remain materialization effects after the delete boundary. | Resume continues the recorded session/machine snapshot; non-resume re-onboard writes a new session first. OpenClaw records completed sandbox name, web search, messaging, and resource choices with explicit progress markers, including explicit `null` choices, while the complete create intent stays process-local and is not persisted or emitted. Raw credential values remain outside the session. A missing process value can be rebound only when the same OpenClaw session recorded successfully registering that provider and its live name, type, and credential-key binding still match; otherwise interactive resume requests it again and non-interactive resume exits with environment-variable guidance. Credentials are checked before mutation and again immediately before materialization. | Restore failures warn and can still publish the replacement; managed-DCode live-selection failure leaves a running, unregistered sandbox with manual-delete guidance. Checkpoint replay reuses an exact live sandbox after an interrupted create and backfills missing create/register receipts. Cancel rollback is not armed and there is no rebuild-style receipt rollback. Coverage: transition traces, create-intent characterization, checkpoint replay and resume guards, and sandbox-handler crash recovery. Gaps: early backup asymmetry and no rebuild-style cross-effect rollback. | | **Rebuild or installer-driven upgrade** — `rebuildSandbox` in `rebuild-pipeline.ts`; `upgradeSandboxes` | Registry state is authoritative. A matching session may fill guarded legacy gaps only when its selection agrees; an unrelated/global session is never used. Ambient provider/model selection is quarantined by `isolateAmbientRecreateEnv`, apart from narrowly scoped legacy recovery. Preflight assembles target config, messaging/policy/runtime inputs, recovery inputs, and a retained replacement context. Generic agents use `preflightRebuildImage`; DCode uses its specialized managed-context preflight instead and proves the live route only for normal live rebuild. Resource profile is not part of preflight. | Consent persistence, target-gateway selection/recovery, and target-preflight registry updates can precede disposable image build/probes. Backup is the first durable recovery checkpoint when available. Shields unlock, MCP detach/scrub, and NIM stop are destructive in-place effects before the **sandbox delete** boundary. Prepared context and mutation-edge conditions are rechecked before delete, proving buildability/input identity but not replacement health or atomic swap. | Durable checkpoints are the backup/recovery manifest when one exists and the rewritten recreate session; stale recovery can reach deletion without a manifest, making that session its first new durable checkpoint. Rollback receipts/snapshots are process-local. Credential metadata comes from the target or guarded fallback; raw credentials/providers are checked against current process/gateway state, while prepared installer recovery may reconstruct a missing gateway provider from a validated host credential. | In-process rollback best-effort restores registry/MCP retry metadata, but process death after non-MCP delete can still lose it. The inner onboard selects resource profile after deletion from non-quarantined ambient input. Covered by rebuild, image-preflight, DCode, and messaging tests. Gaps: post-delete resource intent plus health-before-delete and atomic swap. Closed issue #5801 records the original gap; #6835 fixed only the printed recovery path. | +| **Managed snapshot clone handoff (internal and dormant)** — `prepareManagedWorkloadCloneHandoff` | The current source registry row owns mutable operator intent; the selected snapshot owns immutable managed-workload and provider-runtime history. Preparation proves the selected runtime provider and its `clone` capability, exact current registry generation and live-identity fingerprint, snapshot/source workload equivalence, snapshot runtime generation, and the state layer's selected-manifest/payload digest. It then rebinds the secret-free startup profile, messaging intent, dashboard identity, and any provider-owned Hermes inference name for OpenClaw, Hermes, or DCode without a central Podman-specific switch. | None. The handoff is an inert planning artifact and performs no provider, sandbox, registry, filesystem, credential, or broker effect. Production snapshot restore continues to reject cross-sandbox managed-image restore through `rejectManagedSnapshotCloneUntilRebind`; no user-visible clone support is advertised by this slice. | The returned handoff is a deeply frozen, locally owned value carrying exact source registry compare-and-swap authority, immutable workload authority, provider runtime generation evidence, `SnapshotRestoreAuthority` content identity, rebound managed profile, and destination registry intent. It contains credential-presence metadata and provider names, never raw credential values or live handles. | There is intentionally no compensation because preparation has no effects. All-agent and Docker/MXC-style provider tests cover the dormant contract and canonical name boundaries. Provider materialization, destination creation/bootstrap, mutation-edge content/provider authority revalidation, rollback, recovery, protected E2E, and activation remain later slices. | | **Channel add/remove/start/stop** — `addSandboxChannel`, `removeSandboxChannel`, `sandboxChannelsSetEnabled` in `policy-channel.ts` | Add compiles and merges a manifest-derived channel delta with `MessagingWorkflowPlanner`. Start, stop, and remove transform the registry plan and rehydrate executable render/build/runtime/forward details from current manifests. | Token-backed add can mutate gateway credentials before policy and plan persistence; QR/in-sandbox-auth add skips that credential upsert. Start persists the enabled plan before policy; stop persists the disabled plan before the rebuild prompt. Remove clears QR-backed durable state when applicable, detaches gateway/bridge state, removes policy, then persists the plan. A queued rebuild has a separate delete boundary. | The compact registry messaging plan is authoritative; render/build/runtime/state/health entries and nested host-forward details are rehydrated rather than persisted. Session policy-preset sync is best-effort, and channel mutations do not rewrite `Session.messagingPlan`. Raw tokens stay in process/gateway bindings. | `rollbackChannelAdd`, re-disable after failed start, and fail-closed QR-state cleanup provide partial compensation. Covered by `policy-channel*.test.ts`, `workflow-planner.test.ts`, and channel integration tests. Gaps: channel add has a separate `--force` conflict policy; add/remove effects can precede plan persistence, and persistence failures are not fully rolled back. | | **Provider, model, or credential-binding change** — `runInferenceSet` | CLI intent plus registry/session metadata. Target resolution and OpenShell preparation occur before locking. The target is re-resolved in the mutating phase under the sandbox lifecycle and timer-bound shields locks; that phase validates provider/model syntax, selected agent, shields state, and local reachability before the first write. | First mutation is the gateway route, then a minimal registry write, API-family/config resolution, registry refresh, best-effort config/hash sync, matching-session update, and audit. An OpenClaw API-family change can then restart the managed gateway after the shields lock is released but while the outer sandbox lock remains held. No sandbox deletion. | Registry and matching session store logical provider/model/credential-environment metadata. Audit records the action, sandbox, and reason rather than credentials; raw values remain gateway-bound. | Forward-only; no rollback. `rebuild` is the repair path for degraded state. Covered by `inference-set*.test.ts`. Gap: several stores can diverge after a mid-sequence failure. | | **Credential rotation** — `configRotateToken` in `src/lib/sandbox/config.ts` | A session with `credentialEnv` selects the provider and binding. A non-null different `sandboxName` is rejected, but a legacy/null session name is accepted for the requested sandbox. The new value comes from a named environment variable, stdin, or a secret prompt; it is trimmed, then rejected when empty or still containing internal whitespace. | `saveCredential` first stages the value in the current process. OpenShell provider update is the first external mutation, with provider create as a fallback; audit follows. No sandbox deletion. | The logical binding is unchanged, so session and registry are not rewritten. The raw value exists only in process memory/environment and the gateway provider; audit records action/sandbox/reason without the value. | No rollback after a successful provider update; an audit failure can report failure after the credential is already active. Covered by the rotate-token cases in `test/config-set-nested-ssrf.test.ts`. Gap: a null-name legacy session is not strongly bound to the requested sandbox. | diff --git a/src/lib/onboard/managed-startup-clone-rebinder.test.ts b/src/lib/onboard/managed-startup-clone-rebinder.test.ts new file mode 100644 index 00000000000..24603b4a9cf --- /dev/null +++ b/src/lib/onboard/managed-startup-clone-rebinder.test.ts @@ -0,0 +1,418 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import type { SandboxMessagingPlan } from "../messaging/manifest"; +import { PEM } from "./__test-helpers__/corporate-ca-fixtures"; +import { + type ManagedStartupCloneCurrentState, + ManagedStartupCloneRebindError, + rebindManagedStartupProfileForClone, +} from "./managed-startup/clone-rebinder"; +import { + buildManagedStartupProfile, + type ManagedStartupProfileBuilderInput, +} from "./managed-startup/profile-builder"; + +function messagingPlan(agent: "openclaw" | "hermes", sandboxName = "source"): SandboxMessagingPlan { + return { + schemaVersion: 1, + sandboxName, + agent, + workflow: "onboard", + channels: [ + { + channelId: "telegram", + configured: true, + active: true, + disabled: false, + inputs: [ + { inputId: "botToken", credentialAvailable: true }, + { inputId: "allowedIds", value: ["123456"] }, + ], + }, + ], + disabledChannels: [], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + stateUpdates: [], + healthChecks: [], + } as unknown as SandboxMessagingPlan; +} + +function openClawInput(): ManagedStartupProfileBuilderInput { + return { + agent: "openclaw", + inference: { + routeProvider: "inference", + upstreamProvider: "openai-api", + model: "gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses", + primaryModelRef: "inference/gpt-5.4", + compatibility: {}, + }, + dashboard: { + agent: "openclaw", + mode: "loopback", + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: messagingPlan("openclaw"), + dcodeAutoApprovalMode: null, + observabilityEnabled: null, + environment: {}, + corporateCa: null, + }; +} + +function hermesInput(): ManagedStartupProfileBuilderInput { + return { + agent: "hermes", + inference: { + routeProvider: "inference", + upstreamProvider: "compatible-anthropic-endpoint", + model: "claude-sonnet-4-6", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + dashboard: { + agent: "hermes", + mode: "loopback-forwarded", + url: "http://127.0.0.1:19189", + publicPort: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: messagingPlan("hermes"), + dcodeAutoApprovalMode: null, + observabilityEnabled: null, + environment: {}, + corporateCa: null, + }; +} + +function dcodeInput(): ManagedStartupProfileBuilderInput { + return { + agent: "langchain-deepagents-code", + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + model: "openai/gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + dashboard: { agent: "langchain-deepagents-code", mode: "disabled" }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: "disabled", + observabilityEnabled: false, + environment: {}, + corporateCa: null, + }; +} + +function rebind( + built: ReturnType, + expectedAgent: ManagedStartupProfileBuilderInput["agent"], + destinationDashboardPort: number | null, + currentOverrides: Partial = {}, + names: { + readonly sourceSandboxName?: string; + readonly destinationSandboxName?: string; + } = {}, +) { + const profile = built.profile; + const webSearch = + profile.agentConfig.agent === "langchain-deepagents-code" + ? null + : profile.agentConfig.webSearch; + const hermesDashboard = profile.dashboard.agent === "hermes" ? profile.dashboard : null; + const dcodeConfig = + profile.agentConfig.agent === "langchain-deepagents-code" ? profile.agentConfig : null; + return rebindManagedStartupProfileForClone({ + sourceSandboxName: names.sourceSandboxName ?? "source", + destinationSandboxName: names.destinationSandboxName ?? "destination", + expectedAgent, + destinationDashboardPort, + ...(expectedAgent === "hermes" && profile.tools.enabledGateways.length > 0 + ? { destinationHermesInferenceProvider: "destination-hermes-inference" } + : {}), + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + ...(built.corporateCaB64 === undefined ? {} : { corporateCaB64: built.corporateCaB64 }), + currentSource: { + provider: profile.inference.upstreamProvider, + model: profile.inference.model, + endpointUrl: profile.inference.upstreamEndpointUrl, + preferredInferenceApi: profile.inference.api, + compatibleEndpointReasoning: + profile.agent === "openclaw" && profile.inference.upstreamProvider === "compatible-endpoint" + ? profile.tuning.reasoning + ? "true" + : "false" + : null, + compatibleEndpointReasoningEffort: + profile.agent === "openclaw" && + profile.inference.upstreamProvider === "compatible-endpoint" && + profile.tuning.reasoningEffort !== "default" + ? profile.tuning.reasoningEffort + : null, + toolDisclosure: profile.tools.disclosure, + webSearchEnabled: webSearch?.enabled, + webSearchProvider: webSearch?.provider, + messaging: + profile.messaging.plan === null + ? undefined + : { schemaVersion: 1, plan: profile.messaging.plan }, + hermesToolGateways: profile.tools.enabledGateways, + hermesDashboardEnabled: hermesDashboard?.mode === "loopback-forwarded", + hermesDashboardPort: + hermesDashboard?.mode === "loopback-forwarded" ? hermesDashboard.publicPort : undefined, + hermesDashboardInternalPort: + hermesDashboard?.mode === "loopback-forwarded" ? hermesDashboard.internalPort : undefined, + hermesDashboardTui: + hermesDashboard?.mode === "loopback-forwarded" ? hermesDashboard.tuiEnabled : undefined, + dashboardPort: + profile.dashboard.agent === "openclaw" + ? profile.dashboard.port + : hermesDashboard?.mode === "loopback-forwarded" + ? hermesDashboard.publicPort + : undefined, + dashboardRemoteBindPrepared: + profile.dashboard.agent === "openclaw" + ? profile.dashboard.bindAddress === "0.0.0.0" + : undefined, + dcodeAutoApprovalMode: dcodeConfig?.autoApprovalMode, + observabilityEnabled: dcodeConfig?.observabilityEnabled, + ...currentOverrides, + }, + }); +} + +describe("rebindManagedStartupProfileForClone", () => { + it("rebinds OpenClaw dashboard and manifest-derived provider identity without ambient tokens", () => { + const built = buildManagedStartupProfile(openClawInput()); + const previousToken = process.env.TELEGRAM_BOT_TOKEN; + process.env.TELEGRAM_BOT_TOKEN = "ambient-token-must-not-be-read"; + try { + const rebound = rebind(built, "openclaw", 20_789); + expect(rebound.profile.dashboard).toMatchObject({ + agent: "openclaw", + url: "http://127.0.0.1:20789", + port: 20_789, + }); + expect(rebound.profile.messaging.plan).toMatchObject({ + sandboxName: "destination", + credentialBindings: [ + { + providerName: "destination-telegram-bridge", + credentialAvailable: true, + }, + ], + }); + expect(JSON.stringify(rebound.profile.messaging.plan)).not.toContain( + "ambient-token-must-not-be-read", + ); + expect( + (rebound.profile.messaging.plan as unknown as SandboxMessagingPlan).credentialBindings[0], + ).not.toHaveProperty("credentialHash"); + expect(rebound.startupProfileSha256).not.toBe(built.startupProfileSha256); + expect(Object.isFrozen(rebound)).toBe(true); + expect(Object.isFrozen(rebound.profile)).toBe(true); + expect(Object.isFrozen(rebound.profile.messaging.plan)).toBe(true); + expect(Object.isFrozen(rebound.profile.tools.enabledGateways)).toBe(true); + } finally { + previousToken === undefined + ? Reflect.deleteProperty(process.env, "TELEGRAM_BOT_TOKEN") + : Reflect.set(process.env, "TELEGRAM_BOT_TOKEN", previousToken); + } + }); + + it("rebinds the current compatible-endpoint reasoning effort instead of stale receipt tuning", () => { + const built = buildManagedStartupProfile({ + ...openClawInput(), + inference: { + ...openClawInput().inference, + upstreamProvider: "compatible-endpoint", + api: "openai-completions", + }, + environment: { + NEMOCLAW_REASONING: "true", + NEMOCLAW_REASONING_EFFORT: "low", + }, + }); + + const rebound = rebind(built, "openclaw", 20_789, { + compatibleEndpointReasoning: "true", + compatibleEndpointReasoningEffort: "high", + }); + + expect(built.profile.tuning.reasoningEffort).toBe("low"); + expect(rebound.profile.tuning).toMatchObject({ + reasoning: true, + reasoningEffort: "high", + }); + }); + + it("rebinds Hermes public dashboard and provider identity while retaining its internal port", () => { + const rebound = rebind(buildManagedStartupProfile(hermesInput()), "hermes", 21_189); + + expect(rebound.profile.dashboard).toEqual({ + agent: "hermes", + mode: "loopback-forwarded", + url: "http://127.0.0.1:21189", + publicPort: 21_189, + internalPort: 29_189, + tuiEnabled: true, + }); + expect(rebound.profile.messaging.plan).toMatchObject({ + sandboxName: "destination", + credentialBindings: [{ providerName: "destination-telegram-bridge" }], + }); + }); + + it("rebinds managed-tool Hermes inference to the destination provider identity", () => { + const built = buildManagedStartupProfile({ + ...hermesInput(), + hermesToolGateways: ["nous-web"], + }); + + const rebound = rebind(built, "hermes", 21_189); + + expect(rebound.profile.inference).toMatchObject({ + routeProvider: "inference", + upstreamProvider: "destination-hermes-inference", + }); + expect(rebound.profile.tools.enabledGateways).toEqual(["nous-web"]); + }); + + it("rebinds DCode without inventing dashboard or messaging state", () => { + const rebound = rebind( + buildManagedStartupProfile(dcodeInput()), + "langchain-deepagents-code", + null, + ); + + expect(rebound.profile.dashboard).toEqual({ + agent: "langchain-deepagents-code", + mode: "disabled", + }); + expect(rebound.profile.messaging).toEqual({ plan: null }); + expect(rebound.encodedProfile).toBe(buildManagedStartupProfile(dcodeInput()).encodedProfile); + }); + + it("retains and revalidates the exact public corporate CA transport", () => { + const built = buildManagedStartupProfile({ + ...openClawInput(), + corporateCa: { + pem: PEM, + sourcePath: "/public/corporate-ca.pem", + sourceEnv: "NEMOCLAW_CORPORATE_CA_BUNDLE", + }, + }); + + const rebound = rebind(built, "openclaw", 20_789); + + expect(rebound.corporateCaB64).toBe(built.corporateCaB64); + expect(rebound.profile.corporateCa).toEqual(built.profile.corporateCa); + }); + + it("fails closed on receipt hash, source messaging identity, and unexpected CA transport", () => { + const built = buildManagedStartupProfile(openClawInput()); + expect(() => + rebindManagedStartupProfileForClone({ + sourceSandboxName: "source", + destinationSandboxName: "destination", + expectedAgent: "openclaw", + destinationDashboardPort: 20_789, + encodedProfile: built.encodedProfile, + startupProfileSha256: "0".repeat(64), + currentSource: { + provider: built.profile.inference.upstreamProvider, + model: built.profile.inference.model, + }, + }), + ).toThrow(ManagedStartupCloneRebindError); + + const wrongIdentity = buildManagedStartupProfile({ + ...openClawInput(), + messagingPlan: messagingPlan("openclaw", "other-source"), + }); + expect(() => rebind(wrongIdentity, "openclaw", 20_789)).toThrow(/source plan is invalid/u); + + expect(() => + rebindManagedStartupProfileForClone({ + sourceSandboxName: "source", + destinationSandboxName: "destination", + expectedAgent: "openclaw", + destinationDashboardPort: 20_789, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + corporateCaB64: "eA==", + currentSource: { + provider: built.profile.inference.upstreamProvider, + model: built.profile.inference.model, + }, + }), + ).toThrow(/corporate CA transport/u); + }); + + it("uses the canonical sandbox grammar and rejects an identity-preserving clone", () => { + const built = buildManagedStartupProfile(dcodeInput()); + + expect(() => + rebind( + built, + "langchain-deepagents-code", + null, + {}, + { + sourceSandboxName: "1source", + }, + ), + ).toThrow(/source sandbox name is invalid/u); + expect(() => + rebind( + built, + "langchain-deepagents-code", + null, + {}, + { + destinationSandboxName: "1destination", + }, + ), + ).toThrow(/destination sandbox name is invalid/u); + expect(() => + rebind( + built, + "langchain-deepagents-code", + null, + {}, + { + destinationSandboxName: "source", + }, + ), + ).toThrow(/source and destination sandbox names must differ/u); + }); +}); diff --git a/src/lib/onboard/managed-startup/clone-rebinder.ts b/src/lib/onboard/managed-startup/clone-rebinder.ts new file mode 100644 index 00000000000..11efd1b5801 --- /dev/null +++ b/src/lib/onboard/managed-startup/clone-rebinder.ts @@ -0,0 +1,502 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { cloneAndDeepFreeze } from "../../core/immutable"; +import { resolveContextWindowForModel } from "../../inference/context-window"; +import { rebindSandboxMessagingPlanForClone } from "../../messaging/clone-rebind"; +import { isValidName } from "../../name-validation"; +import { DEFAULT_TOOL_DISCLOSURE } from "../../tool-disclosure"; +import { resolveManagedStartupInferenceRoute } from "../inference-route"; +import { validateManagedStartupCorporateCaTransport } from "./application"; +import { + decodeManagedStartupProfile, + encodeManagedStartupProfile, + type ManagedStartupAgent, + type ManagedStartupDashboard, + type ManagedStartupJsonObject, + type ManagedStartupProfile, + validateManagedStartupProfile, +} from "./profile"; + +const SHA256_PATTERN = /^[a-f0-9]{64}$/u; +const MANAGED_INFERENCE_API_SET = new Set([ + "openai-completions", + "openai-responses", + "anthropic-messages", +]); + +/** + * Current durable state for the source sandbox. The managed-image receipt owns + * immutable image affordances; mutable operator intent is re-read from this + * state before a clone handoff can be prepared. + */ +export interface ManagedStartupCloneCurrentState { + readonly provider?: string | null; + readonly model?: string | null; + readonly endpointUrl?: string | null; + readonly preferredInferenceApi?: string | null; + readonly compatibleEndpointReasoning?: "true" | "false" | string | null; + readonly compatibleEndpointReasoningEffort?: "low" | "medium" | "high" | string | null; + readonly toolDisclosure?: "progressive" | "direct" | string; + readonly webSearchEnabled?: boolean; + readonly webSearchProvider?: "brave" | "tavily" | string | null; + readonly messaging?: { readonly schemaVersion?: number; readonly plan?: unknown } | null; + readonly hermesToolGateways?: readonly string[]; + readonly hermesDashboardEnabled?: boolean; + readonly hermesDashboardPort?: number | null; + readonly hermesDashboardInternalPort?: number | null; + readonly hermesDashboardTui?: boolean; + readonly dashboardPort?: number | null; + readonly dashboardRemoteBindPrepared?: boolean; + readonly dcodeAutoApprovalMode?: "disabled" | "thread-opt-in" | string; + readonly observabilityEnabled?: boolean; +} + +export interface ManagedStartupCloneRebindInput { + readonly sourceSandboxName: string; + readonly destinationSandboxName: string; + readonly expectedAgent: ManagedStartupAgent; + readonly destinationDashboardPort: number | null; + /** Destination-scoped OpenShell identity for Hermes' host-minted inference key. */ + readonly destinationHermesInferenceProvider?: string; + readonly encodedProfile: string; + readonly startupProfileSha256: string; + readonly corporateCaB64?: string; + readonly currentSource: ManagedStartupCloneCurrentState; +} + +export interface ReboundManagedStartupClone { + readonly profile: ManagedStartupProfile; + readonly encodedProfile: string; + readonly startupProfileSha256: string; + readonly corporateCaB64?: string; +} + +export class ManagedStartupCloneRebindError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(`Cannot prepare managed snapshot clone: ${message}`, options); + this.name = "ManagedStartupCloneRebindError"; + } +} + +function fail(message: string, cause?: unknown): never { + throw new ManagedStartupCloneRebindError(message, cause === undefined ? undefined : { cause }); +} + +function requireSandboxName(value: string, label: string): string { + if (!isValidName(value)) fail(`${label} sandbox name is invalid`); + return value; +} + +function requireDestinationPort(port: number | null, agent: ManagedStartupAgent): number { + if (!Number.isInteger(port) || port === null || port < 1024 || port > 65_535) { + fail(`${agent} requires an allocated destination dashboard port`); + } + return port; +} + +function urlAtPort(raw: string, port: number): string { + let parsed: URL; + try { + parsed = new URL(raw); + } catch (error) { + fail("source dashboard URL is invalid", error); + } + parsed.port = String(port); + return parsed.toString(); +} + +function requireCurrentString(value: unknown, label: string): string { + if (typeof value !== "string" || value.trim() === "" || value !== value.trim()) { + fail(`current source ${label} is missing or invalid`); + } + return value; +} + +function optionalCurrentString(value: unknown, label: string): string | null { + if (value === null || value === undefined) return null; + return requireCurrentString(value, label); +} + +function currentInference( + profile: ManagedStartupProfile, + current: ManagedStartupCloneCurrentState, +): ManagedStartupProfile["inference"] { + const provider = requireCurrentString(current.provider, "inference provider"); + const model = requireCurrentString(current.model, "inference model"); + const preferredApi = optionalCurrentString( + current.preferredInferenceApi, + "preferred inference API", + ); + if (preferredApi !== null && !MANAGED_INFERENCE_API_SET.has(preferredApi)) { + fail("current source preferred inference API is unsupported"); + } + const resolved = resolveManagedStartupInferenceRoute( + profile.agent, + provider, + model, + preferredApi, + ); + if (!MANAGED_INFERENCE_API_SET.has(resolved.inferenceApi)) { + fail("current source inference route resolved an unsupported API"); + } + const upstreamEndpointUrl = + profile.agent === "langchain-deepagents-code" + ? optionalCurrentString(current.endpointUrl, "upstream endpoint URL") + : null; + return { + routeProvider: resolved.providerKey, + upstreamProvider: provider, + model, + routedBaseUrl: resolved.inferenceBaseUrl, + upstreamEndpointUrl, + api: resolved.inferenceApi as ManagedStartupProfile["inference"]["api"], + primaryModelRef: profile.agent === "openclaw" ? resolved.primaryModelRef : null, + compatibility: + profile.agent === "openclaw" + ? (JSON.parse(JSON.stringify(resolved.inferenceCompat ?? {})) as ManagedStartupJsonObject) + : null, + inputModalities: profile.agent === "openclaw" ? profile.inference.inputModalities : null, + }; +} + +function currentToolDisclosure( + current: ManagedStartupCloneCurrentState, +): ManagedStartupProfile["tools"]["disclosure"] { + const value = current.toolDisclosure ?? DEFAULT_TOOL_DISCLOSURE; + if (value !== "progressive" && value !== "direct") { + fail("current source tool disclosure is invalid"); + } + return value; +} + +function currentWebSearch( + profile: ManagedStartupProfile, + current: ManagedStartupCloneCurrentState, +): Extract["webSearch"] { + if (profile.agentConfig.agent === "langchain-deepagents-code") { + fail("DCode cannot carry web-search state"); + } + const enabled = current.webSearchEnabled === true; + const configuredProvider = current.webSearchProvider; + if ( + configuredProvider !== undefined && + configuredProvider !== null && + configuredProvider !== "brave" && + configuredProvider !== "tavily" + ) { + fail("current source web-search provider is invalid"); + } + const provider = enabled + ? configuredProvider + : (configuredProvider ?? profile.agentConfig.webSearch.provider); + if (provider !== "brave" && provider !== "tavily") { + fail("enabled current source web search has no valid provider"); + } + if (profile.agent === "hermes" && provider !== "tavily") { + fail("current Hermes web search must use Tavily"); + } + return { enabled, provider }; +} + +function currentAgentConfig( + profile: ManagedStartupProfile, + current: ManagedStartupCloneCurrentState, +): ManagedStartupProfile["agentConfig"] { + if (profile.agentConfig.agent === "openclaw") { + return { + ...profile.agentConfig, + webSearch: currentWebSearch(profile, current), + }; + } + if (profile.agentConfig.agent === "hermes") { + return { + ...profile.agentConfig, + webSearch: currentWebSearch(profile, current), + }; + } + const autoApprovalMode = current.dcodeAutoApprovalMode ?? "disabled"; + if (autoApprovalMode !== "disabled" && autoApprovalMode !== "thread-opt-in") { + fail("current DCode auto-approval mode is invalid"); + } + return { + agent: "langchain-deepagents-code", + autoApprovalMode, + observabilityEnabled: current.observabilityEnabled === true, + }; +} + +function currentSourceDashboard( + profile: ManagedStartupProfile, + current: ManagedStartupCloneCurrentState, +): ManagedStartupDashboard { + if (profile.dashboard.agent === "openclaw") { + const port = + current.dashboardPort === undefined || current.dashboardPort === null + ? profile.dashboard.port + : requireDestinationPort(current.dashboardPort, profile.agent); + const remoteBind = current.dashboardRemoteBindPrepared === true; + if (remoteBind !== (profile.dashboard.bindAddress === "0.0.0.0")) { + fail("current OpenClaw dashboard bind state conflicts with its managed receipt"); + } + return { + ...profile.dashboard, + url: urlAtPort(profile.dashboard.url, port), + port, + }; + } + if (profile.dashboard.agent === "hermes") { + if (current.hermesDashboardEnabled !== true) { + return { + agent: "hermes", + mode: "disabled", + url: profile.dashboard.url, + publicPort: null, + internalPort: null, + tuiEnabled: false, + }; + } + const publicPort = requireDestinationPort( + current.hermesDashboardPort ?? current.dashboardPort ?? null, + profile.agent, + ); + const internalPort = requireDestinationPort( + current.hermesDashboardInternalPort ?? null, + profile.agent, + ); + return { + agent: "hermes", + mode: "loopback-forwarded", + url: urlAtPort(profile.dashboard.url, publicPort), + publicPort, + internalPort, + tuiEnabled: current.hermesDashboardTui === true, + }; + } + return profile.dashboard; +} + +function currentMessagingPlan(current: ManagedStartupCloneCurrentState): unknown | null { + if (current.messaging === undefined || current.messaging === null) return null; + if (current.messaging.schemaVersion !== 1 || current.messaging.plan === undefined) { + fail("current source messaging state is invalid"); + } + return current.messaging.plan; +} + +function reconcileCurrentSourceProfile( + profile: ManagedStartupProfile, + current: ManagedStartupCloneCurrentState, +): ManagedStartupProfile { + const inference = currentInference(profile, current); + const hermesToolGateways = current.hermesToolGateways ?? []; + if ( + !Array.isArray(hermesToolGateways) || + !hermesToolGateways.every((value) => typeof value === "string") + ) { + fail("current Hermes tool gateways are invalid"); + } + let reasoning = profile.tuning.reasoning; + let reasoningEffort = profile.tuning.reasoningEffort; + let contextWindow = profile.tuning.contextWindow; + if (profile.agent === "openclaw") { + const currentReasoning = current.compatibleEndpointReasoning; + if ( + currentReasoning !== undefined && + currentReasoning !== null && + currentReasoning !== "true" && + currentReasoning !== "false" + ) { + fail("current source compatible-endpoint reasoning state is invalid"); + } + reasoning = current.provider === "compatible-endpoint" ? currentReasoning === "true" : false; + const currentReasoningEffort = current.compatibleEndpointReasoningEffort; + if ( + currentReasoningEffort !== undefined && + currentReasoningEffort !== null && + currentReasoningEffort !== "low" && + currentReasoningEffort !== "medium" && + currentReasoningEffort !== "high" + ) { + fail("current source compatible-endpoint reasoning-effort state is invalid"); + } + reasoningEffort = + inference.upstreamProvider === "compatible-endpoint" && inference.api === "openai-completions" + ? (currentReasoningEffort ?? "default") + : "default"; + if ( + profile.inference.upstreamProvider !== current.provider || + profile.inference.model !== current.model + ) { + contextWindow = resolveContextWindowForModel( + requireCurrentString(current.provider, "inference provider"), + requireCurrentString(current.model, "inference model"), + ); + if (contextWindow === null) { + fail("current OpenClaw inference route has no verifiable context window"); + } + } + } + return validateManagedStartupProfile({ + ...profile, + agentConfig: currentAgentConfig(profile, current), + inference, + dashboard: currentSourceDashboard(profile, current), + tools: { + disclosure: currentToolDisclosure(current), + enabledGateways: profile.agent === "hermes" ? [...hermesToolGateways] : [], + }, + messaging: { plan: currentMessagingPlan(current) }, + tuning: { ...profile.tuning, contextWindow, reasoning, reasoningEffort }, + }); +} + +function destinationDashboard( + profile: ManagedStartupProfile, + destinationDashboardPort: number | null, +): ManagedStartupDashboard { + const dashboard = profile.dashboard; + if (dashboard.agent === "openclaw") { + const port = requireDestinationPort(destinationDashboardPort, profile.agent); + return { + ...dashboard, + url: urlAtPort(dashboard.url, port), + port, + }; + } + if (dashboard.agent === "hermes") { + if (dashboard.mode === "disabled") { + if (destinationDashboardPort === null) { + return { + ...dashboard, + url: "http://127.0.0.1/", + }; + } + return { + ...dashboard, + url: urlAtPort(dashboard.url, destinationDashboardPort), + }; + } + const port = requireDestinationPort(destinationDashboardPort, profile.agent); + return { + ...dashboard, + url: urlAtPort(dashboard.url, port), + publicPort: port, + }; + } + if (destinationDashboardPort !== null) { + fail("langchain-deepagents-code cannot accept a destination dashboard port"); + } + return dashboard; +} + +function destinationMessagingPlan( + profile: ManagedStartupProfile, + sourceSandboxName: string, + destinationSandboxName: string, +): ManagedStartupJsonObject | null { + if (profile.messaging.plan === null) return null; + if (profile.agent === "langchain-deepagents-code") { + fail("langchain-deepagents-code cannot carry a messaging plan"); + } + const rebound = rebindSandboxMessagingPlanForClone({ + sourceSandboxName, + destinationSandboxName, + agent: profile.agent, + sourcePlan: profile.messaging.plan, + environment: { + NEMOCLAW_PROXY_HOST: profile.proxy.managedHost, + NEMOCLAW_PROXY_PORT: String(profile.proxy.managedPort), + }, + }); + return JSON.parse(JSON.stringify(rebound)) as ManagedStartupJsonObject; +} + +function destinationInference( + profile: ManagedStartupProfile, + input: ManagedStartupCloneRebindInput, +): ManagedStartupProfile["inference"] { + if (profile.agent !== "hermes" || profile.tools.enabledGateways.length === 0) { + return profile.inference; + } + const provider = requireCurrentString( + input.destinationHermesInferenceProvider, + "destination Hermes inference provider", + ); + return { + ...profile.inference, + upstreamProvider: provider, + }; +} + +/** + * Verify a source managed receipt transport and bind its secret-free intent to + * a newly allocated destination identity before any snapshot mutation occurs. + */ +export function rebindManagedStartupProfileForClone( + input: ManagedStartupCloneRebindInput, +): ReboundManagedStartupClone { + const sourceSandboxName = requireSandboxName(input.sourceSandboxName, "source"); + const destinationSandboxName = requireSandboxName(input.destinationSandboxName, "destination"); + if (sourceSandboxName === destinationSandboxName) { + fail("source and destination sandbox names must differ"); + } + if ( + !SHA256_PATTERN.test(input.startupProfileSha256) || + createHash("sha256").update(input.encodedProfile, "utf8").digest("hex") !== + input.startupProfileSha256 + ) { + fail("source profile transport does not match its receipt SHA-256 digest"); + } + + let sourceProfile: ManagedStartupProfile; + try { + sourceProfile = decodeManagedStartupProfile(input.encodedProfile); + } catch (error) { + fail("source profile transport is not canonical and valid", error); + } + if (sourceProfile.agent !== input.expectedAgent) { + fail(`source profile targets ${sourceProfile.agent}, expected ${input.expectedAgent}`); + } + try { + validateManagedStartupCorporateCaTransport(input.corporateCaB64, sourceProfile); + } catch (error) { + fail("source corporate CA transport does not match the profile", error); + } + + let profile: ManagedStartupProfile; + try { + const currentSourceProfile = reconcileCurrentSourceProfile(sourceProfile, input.currentSource); + profile = validateManagedStartupProfile({ + ...currentSourceProfile, + inference: destinationInference(currentSourceProfile, input), + dashboard: destinationDashboard(currentSourceProfile, input.destinationDashboardPort), + messaging: { + plan: destinationMessagingPlan( + currentSourceProfile, + sourceSandboxName, + destinationSandboxName, + ), + }, + }); + } catch (error) { + if (error instanceof ManagedStartupCloneRebindError) throw error; + const detail = error instanceof Error ? `: ${error.message}` : ""; + fail(`destination profile could not be validated${detail}`, error); + } + + const encodedProfile = encodeManagedStartupProfile(profile); + const startupProfileSha256 = createHash("sha256").update(encodedProfile, "utf8").digest("hex"); + return cloneAndDeepFreeze( + input.corporateCaB64 === undefined + ? { profile, encodedProfile, startupProfileSha256 } + : { + profile, + encodedProfile, + startupProfileSha256, + corporateCaB64: input.corporateCaB64, + }, + ); +} diff --git a/src/lib/onboard/managed-workload-clone-handoff.test.ts b/src/lib/onboard/managed-workload-clone-handoff.test.ts new file mode 100644 index 00000000000..c002f0eb2f3 --- /dev/null +++ b/src/lib/onboard/managed-workload-clone-handoff.test.ts @@ -0,0 +1,417 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { describe, expect, it, vi } from "vitest"; +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { createInMemoryRuntimeProviderBundle } from "../../../test/helpers/runtime-provider-bundle"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../state/registry/types"; +import { + MANAGED_IMAGE_REPOSITORIES, + type ShippedManagedImageAgent, +} from "./managed-image/contract"; +import { + encodeManagedStartupProfile, + type ManagedStartupProfile, + validateManagedStartupProfile, +} from "./managed-startup/profile"; +import type { + RuntimeProviderBundle, + RuntimeProviderWorkloadProfile, +} from "./runtime-provider/contract"; +import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "./runtime-provider/current"; +import { ManagedWorkloadCloneError, prepareManagedWorkloadCloneHandoff } from "./workload/clone"; + +const PORTABLE_PROFILE = { + support: { + exactDigestReferences: true, + platforms: ["linux/amd64", "linux/arm64"], + startupProfileContractVersions: [1], + capabilityContractVersions: [1], + }, + hostArchitectures: ["amd64", "arm64"], + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, +} as const satisfies RuntimeProviderWorkloadProfile; + +function provider(providerId: "docker" | "mxc"): RuntimeProviderBundle { + if (providerId === "docker") return CURRENT_RUNTIME_PROVIDER_BUNDLES.docker!; + return createInMemoryRuntimeProviderBundle({ + providerId, + workloadProfile: PORTABLE_PROFILE, + }); +} + +function receipt( + agent: ShippedManagedImageAgent, + profile: ManagedStartupProfile, +): Extract { + const encodedProfile = encodeManagedStartupProfile(profile); + return { + schemaVersion: 1, + kind: "managed-image", + reference: `${MANAGED_IMAGE_REPOSITORIES[agent]}@sha256:${"a".repeat(64)}`, + platform: "linux/amd64", + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: true, + shared: true, + }; +} + +function source( + agent: ShippedManagedImageAgent, + providerId: "docker" | "mxc", + profile = managedStartupE2eProfile(agent), +): SandboxEntry { + const workload = receipt(agent, profile); + return { + name: "source", + agent, + openshellDriver: providerId, + imageTag: workload.reference, + workload, + lifecycleGeneration: "generation-source-current", + lifecycleLiveIdentityFingerprint: "f".repeat(64), + provider: profile.inference.upstreamProvider, + model: profile.inference.model, + endpointUrl: profile.inference.upstreamEndpointUrl, + endpointSource: profile.inference.upstreamEndpointUrl ? "onboard" : null, + credentialEnv: "NVIDIA_API_KEY", + preferredInferenceApi: profile.inference.api, + compatibleEndpointReasoning: null, + compatibleEndpointReasoningEffort: null, + toolDisclosure: profile.tools.disclosure, + webSearchEnabled: + profile.agentConfig.agent === "langchain-deepagents-code" + ? false + : profile.agentConfig.webSearch.enabled, + webSearchProvider: + profile.agentConfig.agent === "langchain-deepagents-code" + ? null + : profile.agentConfig.webSearch.provider, + ...(profile.messaging.plan === null + ? {} + : { + messaging: { + schemaVersion: 1 as const, + plan: profile.messaging.plan as unknown as NonNullable< + SandboxEntry["messaging"] + >["plan"], + }, + }), + ...(profile.dashboard.agent === "openclaw" + ? { + dashboardPort: profile.dashboard.port, + dashboardRemoteBindPrepared: profile.dashboard.bindAddress === "0.0.0.0", + } + : {}), + ...(profile.agent === "hermes" && profile.tools.enabledGateways.length > 0 + ? { hermesToolGateways: [...profile.tools.enabledGateways] } + : {}), + ...(profile.agentConfig.agent === "langchain-deepagents-code" + ? { + dcodeAutoApprovalMode: profile.agentConfig.autoApprovalMode, + observabilityEnabled: profile.agentConfig.observabilityEnabled, + } + : {}), + }; +} + +function runtimeSnapshot(providerId: string) { + return { + schemaVersion: 1 as const, + providerId, + providerHandle: `${providerId}:snapshot:source`, + lifecycleState: "running" as const, + lifecycleGeneration: "generation-source-1", + runtime: { + schemaVersion: 1 as const, + providerId, + runtime: { kind: `${providerId}-workload`, handle: `${providerId}:runtime:source` }, + acceleration: { + kind: "gpu" as const, + vendor: "nvidia", + devices: ["nvidia.com/gpu=0"], + }, + }, + }; +} + +function restoreAuthority() { + return { + schemaVersion: 1 as const, + backupPath: "/tmp/nemoclaw-managed-clone-source", + contentSha256: "c".repeat(64), + }; +} + +function prepare( + entry: SandboxEntry, + selectedProvider: RuntimeProviderBundle, + getHermesInferenceProviderName = vi.fn( + (sandboxName: string) => `${sandboxName}-hermes-inference`, + ), + destinationSandboxName = "destination", +) { + return prepareManagedWorkloadCloneHandoff({ + source: entry, + snapshot: { + sandboxName: entry.name, + agentType: entry.agent!, + workload: entry.workload, + runtimeSnapshot: runtimeSnapshot(selectedProvider.identity.id), + restoreAuthority: restoreAuthority(), + }, + destinationSandboxName, + destinationDashboardPort: entry.agent === "openclaw" ? 20_789 : null, + provider: selectedProvider, + getHermesInferenceProviderName, + }); +} + +describe("prepareManagedWorkloadCloneHandoff", () => { + it.each([ + ["docker", "openclaw"], + ["docker", "hermes"], + ["docker", "langchain-deepagents-code"], + ["mxc", "openclaw"], + ["mxc", "hermes"], + ["mxc", "langchain-deepagents-code"], + ] as const)("keeps %s clone handoff provider-bound for %s", (providerId, agent) => { + const selectedProvider = provider(providerId); + const entry = source(agent, providerId); + + const handoff = prepare(entry, selectedProvider); + + expect(handoff).toMatchObject({ + schemaVersion: 1, + phase: "rebound", + providerId, + sourceSandboxName: "source", + destinationSandboxName: "destination", + sourceRegistryAuthority: { + providerId, + lifecycleGeneration: "generation-source-current", + liveIdentityFingerprint: "f".repeat(64), + }, + runtimeSnapshot: { + providerId, + runtime: { + providerId, + acceleration: { kind: "gpu", vendor: "nvidia" }, + }, + }, + snapshotRestoreAuthority: restoreAuthority(), + workload: { + kind: "managed-image", + reference: entry.imageTag, + platform: "linux/amd64", + }, + registryFields: { + model: entry.model, + preferredInferenceApi: "openai-completions", + }, + }); + expect(handoff.rebound.profile.agent).toBe(agent); + expect(handoff.workload.encodedProfile).toBe(handoff.rebound.encodedProfile); + expect(JSON.stringify(handoff)).not.toContain("podman"); + expect(Object.isFrozen(handoff)).toBe(true); + expect(Object.isFrozen(handoff.sourceAuthority.profile)).toBe(true); + expect(Object.isFrozen(handoff.runtimeSnapshot.runtime.acceleration)).toBe(true); + expect(Object.isFrozen(handoff.rebound.profile.tools.enabledGateways)).toBe(true); + }); + + it("rebinds managed-tool Hermes to an injected destination provider identity", () => { + const profile = validateManagedStartupProfile({ + ...managedStartupE2eProfile("hermes"), + tools: { + disclosure: "direct", + enabledGateways: ["nous-web"], + }, + }); + const entry = source("hermes", "mxc", profile); + const resolver = vi.fn(() => "mxc-destination-inference"); + + const handoff = prepare(entry, provider("mxc"), resolver); + + expect(resolver).toHaveBeenCalledWith("destination"); + expect(handoff.rebound.profile).toMatchObject({ + inference: { upstreamProvider: "mxc-destination-inference" }, + tools: { disclosure: "direct", enabledGateways: ["nous-web"] }, + }); + expect(handoff.registryFields).toMatchObject({ + provider: "nvidia", + hermesInferenceProvider: "mxc-destination-inference", + hermesToolGateways: ["nous-web"], + }); + }); + + it("carries the destination-bound messaging plan as typed registry state", () => { + const profile = validateManagedStartupProfile({ + ...managedStartupE2eProfile("openclaw"), + messaging: { + plan: { + schemaVersion: 1, + sandboxName: "source", + agent: "openclaw", + workflow: "onboard", + channels: [ + { + channelId: "telegram", + configured: true, + active: true, + disabled: false, + inputs: [ + { inputId: "botToken", credentialAvailable: true }, + { inputId: "allowedIds", value: ["123456"] }, + ], + }, + ], + disabledChannels: [], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + runtimeSetup: { nodePreloads: [], envAliases: [], secretScans: [] }, + stateUpdates: [], + healthChecks: [], + }, + }, + }); + const entry = source("openclaw", "docker", profile); + + const handoff = prepare(entry, provider("docker")); + + expect(handoff.messaging).toMatchObject({ + schemaVersion: 1, + plan: { + sandboxName: "destination", + credentialBindings: [{ providerName: "destination-telegram-bridge" }], + }, + }); + expect(JSON.stringify(handoff.messaging)).not.toContain("credentialHash"); + }); + + it("never gives provider receipt validation a mutable authority value", () => { + const base = provider("mxc"); + const observedReceipts: SandboxWorkloadReceipt[] = []; + const mutationResults: boolean[] = []; + const hostileProvider: RuntimeProviderBundle = { + ...base, + workload: { + ...base.workload, + acceptsReceipt: (candidate) => { + if (candidate?.kind === "managed-image") { + observedReceipts.push(candidate); + mutationResults.push(Reflect.set(candidate, "reference", "mutated-by-provider")); + } + return true; + }, + }, + }; + const entry = source("openclaw", "mxc"); + + const handoff = prepare(entry, hostileProvider); + + expect(observedReceipts).toHaveLength(2); + expect(observedReceipts.every((candidate) => Object.isFrozen(candidate))).toBe(true); + expect(mutationResults).toEqual([false, false]); + expect(handoff.workload.reference).toBe(entry.imageTag); + }); + + it("fails closed on stale managed authority, provider drift, and missing clone authority", () => { + const entry = source("openclaw", "mxc"); + const selectedProvider = provider("mxc"); + expect(() => + prepareManagedWorkloadCloneHandoff({ + source: entry, + snapshot: { + sandboxName: "source", + agentType: "openclaw", + workload: { + ...(entry.workload as Extract< + SandboxWorkloadReceipt, + { readonly kind: "managed-image" } + >), + sourceRevision: "c".repeat(40), + }, + runtimeSnapshot: runtimeSnapshot("mxc"), + restoreAuthority: restoreAuthority(), + }, + destinationSandboxName: "destination", + destinationDashboardPort: 20_789, + provider: selectedProvider, + getHermesInferenceProviderName: vi.fn(), + }), + ).toThrow(/no longer matches/u); + + expect(() => prepare(entry, provider("docker"))).toThrow(/does not match selected provider/u); + + const unauthorized = { + ...selectedProvider, + mutationAuthority: { + ...selectedProvider.mutationAuthority, + operations: + selectedProvider.mutationAuthority.supported === true + ? selectedProvider.mutationAuthority.operations.filter( + (operation) => operation !== "clone", + ) + : [], + }, + } as RuntimeProviderBundle; + expect(() => prepare(entry, unauthorized)).toThrow(ManagedWorkloadCloneError); + }); + + it("uses the canonical sandbox and provider grammars at the exported boundary", () => { + const profile = validateManagedStartupProfile({ + ...managedStartupE2eProfile("hermes"), + tools: { disclosure: "direct", enabledGateways: ["nous-web"] }, + }); + const entry = source("hermes", "docker", profile); + const longestSandboxName = `a${"b".repeat(62)}`; + + expect(() => prepare(entry, provider("docker"), undefined, longestSandboxName)).not.toThrow(); + expect(() => prepare(entry, provider("docker"), undefined, "1destination")).toThrow( + /destination sandbox name is invalid/u, + ); + expect(() => + prepare( + entry, + provider("docker"), + vi.fn(() => "1provider"), + ), + ).toThrow(/destination Hermes inference provider name is invalid/u); + }); + + it("rejects malformed snapshot content authority before exposing a handoff", () => { + const entry = source("openclaw", "docker"); + + expect(() => + prepareManagedWorkloadCloneHandoff({ + source: entry, + snapshot: { + sandboxName: entry.name, + agentType: entry.agent!, + workload: entry.workload, + runtimeSnapshot: runtimeSnapshot("docker"), + restoreAuthority: { + schemaVersion: 1, + backupPath: "relative/snapshot", + contentSha256: "c".repeat(64), + }, + }, + destinationSandboxName: "destination", + destinationDashboardPort: 20_789, + provider: provider("docker"), + getHermesInferenceProviderName: vi.fn(), + }), + ).toThrow(/snapshot content authority is invalid/u); + }); +}); diff --git a/src/lib/onboard/runtime-provider/contract.ts b/src/lib/onboard/runtime-provider/contract.ts index 644239b3a7a..8888d95f83a 100644 --- a/src/lib/onboard/runtime-provider/contract.ts +++ b/src/lib/onboard/runtime-provider/contract.ts @@ -17,6 +17,7 @@ export type RuntimeProviderMutationOperation = | "stop" | "inference-set" | "rebuild" + | "clone" | "provider-cleanup" | "destroy" | "workload-cleanup"; diff --git a/src/lib/onboard/runtime-provider/docker.ts b/src/lib/onboard/runtime-provider/docker.ts index f8481c37c3c..63945928895 100644 --- a/src/lib/onboard/runtime-provider/docker.ts +++ b/src/lib/onboard/runtime-provider/docker.ts @@ -347,6 +347,7 @@ export function createDockerRuntimeProviderBundle( "stop", "inference-set", "rebuild", + "clone", "provider-cleanup", "destroy", "workload-cleanup", diff --git a/src/lib/onboard/runtime-provider/registry.ts b/src/lib/onboard/runtime-provider/registry.ts index 765f1feed9f..332792d2dc2 100644 --- a/src/lib/onboard/runtime-provider/registry.ts +++ b/src/lib/onboard/runtime-provider/registry.ts @@ -59,6 +59,7 @@ const MUTATION_OPERATIONS = new Set([ "stop", "inference-set", "rebuild", + "clone", "provider-cleanup", "destroy", "workload-cleanup", diff --git a/src/lib/onboard/workload/clone.ts b/src/lib/onboard/workload/clone.ts new file mode 100644 index 00000000000..b3d28ed6783 --- /dev/null +++ b/src/lib/onboard/workload/clone.ts @@ -0,0 +1,390 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isAbsolute, resolve } from "node:path"; +import { isDeepStrictEqual } from "node:util"; + +import { cloneAndDeepFreeze } from "../../core/immutable"; +import { createBuiltInChannelManifestRegistry } from "../../messaging/channels/built-ins"; +import type { MessagingAgentId } from "../../messaging/manifest"; +import { parseSandboxMessagingPlan } from "../../messaging/plan-validation"; +import { isValidName, isValidProviderName } from "../../name-validation"; +import { + captureSandboxRebuildAuthority, + type SandboxRebuildAuthority, +} from "../../state/registry/rebuild-authority"; +import { + cloneSandboxRuntimeSnapshot, + type SandboxRuntimeSnapshot, +} from "../../state/registry/runtime-snapshot"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; +import { cloneSandboxWorkloadReceipt } from "../../state/registry/workload"; +import type { SandboxMessagingState } from "../../state/registry-messaging"; +import type { SnapshotRestoreAuthority } from "../../state/sandbox"; +import { + type ReboundManagedStartupClone, + rebindManagedStartupProfileForClone, +} from "../managed-startup/clone-rebinder"; +import type { ManagedStartupProfile } from "../managed-startup/profile"; +import type { RuntimeProviderBundle } from "../runtime-provider/contract"; +import { + normalizeRuntimeProviderIdentity, + requireRuntimeProviderMutationAuthority, +} from "../runtime-provider/registry"; +import { type ManagedWorkloadAuthority, readManagedWorkloadAuthority } from "./authority"; + +export interface ManagedWorkloadCloneSnapshot { + readonly sandboxName: string; + readonly agentType: string; + readonly workload?: SandboxWorkloadReceipt; + readonly runtimeSnapshot?: SandboxRuntimeSnapshot; + /** Exact selected manifest and payload identity captured by the state layer. */ + readonly restoreAuthority: SnapshotRestoreAuthority; +} + +export interface ManagedWorkloadCloneRegistryFields { + readonly provider: SandboxEntry["provider"]; + readonly model: SandboxEntry["model"]; + readonly endpointUrl: SandboxEntry["endpointUrl"]; + readonly endpointSource: SandboxEntry["endpointSource"]; + readonly credentialEnv: SandboxEntry["credentialEnv"]; + readonly preferredInferenceApi: SandboxEntry["preferredInferenceApi"]; + readonly compatibleEndpointReasoning: SandboxEntry["compatibleEndpointReasoning"]; + readonly compatibleEndpointReasoningEffort: SandboxEntry["compatibleEndpointReasoningEffort"]; + readonly toolDisclosure: SandboxEntry["toolDisclosure"]; + readonly webSearchEnabled: SandboxEntry["webSearchEnabled"]; + readonly webSearchProvider: SandboxEntry["webSearchProvider"]; + readonly observabilityEnabled: SandboxEntry["observabilityEnabled"]; + readonly dcodeAutoApprovalMode?: SandboxEntry["dcodeAutoApprovalMode"]; + readonly hermesToolGateways?: readonly string[]; + readonly hermesInferenceProvider?: string; + readonly hermesDashboardEnabled?: true; + readonly hermesDashboardPort?: number; + readonly hermesDashboardInternalPort?: number; + readonly hermesDashboardTui?: true; + readonly dashboardPort?: number; + readonly dashboardRemoteBindPrepared: boolean; +} + +export interface PreparedManagedWorkloadCloneHandoff { + readonly schemaVersion: 1; + readonly phase: "rebound"; + readonly providerId: string; + readonly sourceSandboxName: string; + readonly destinationSandboxName: string; + /** Exact current registry row authority to revalidate at the mutation edge. */ + readonly sourceRegistryAuthority: SandboxRebuildAuthority; + readonly sourceAuthority: ManagedWorkloadAuthority; + readonly runtimeSnapshot: SandboxRuntimeSnapshot; + readonly snapshotRestoreAuthority: SnapshotRestoreAuthority; + readonly rebound: ReboundManagedStartupClone; + readonly workload: Extract; + readonly messaging?: SandboxMessagingState; + readonly registryFields: ManagedWorkloadCloneRegistryFields; +} + +export interface PrepareManagedWorkloadCloneHandoffInput { + readonly source: SandboxEntry; + readonly snapshot: ManagedWorkloadCloneSnapshot; + readonly destinationSandboxName: string; + readonly destinationDashboardPort: number | null; + readonly provider: RuntimeProviderBundle; + /** + * Provider-name ownership stays outside central clone orchestration. Hermes + * supplies this resolver only when managed tools require a destination key. + */ + readonly getHermesInferenceProviderName: (sandboxName: string) => string; +} + +export class ManagedWorkloadCloneError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(`Managed workload clone preflight failed: ${message}`, options); + this.name = "ManagedWorkloadCloneError"; + } +} + +const SHA256_PATTERN = /^[a-f0-9]{64}$/u; +const MAX_AUTHORITY_PATH_BYTES = 4096; + +function fail(message: string, cause?: unknown): never { + throw new ManagedWorkloadCloneError(message, cause === undefined ? undefined : { cause }); +} + +function requireSandboxName(value: string, label: string): string { + if (!isValidName(value)) fail(`${label} sandbox name is invalid`); + return value; +} + +function requireProviderName(value: string, label: string): string { + if (!isValidProviderName(value)) fail(`${label} provider name is invalid`); + return value; +} + +function cloneSnapshotRestoreAuthority(value: SnapshotRestoreAuthority): SnapshotRestoreAuthority { + if ( + value?.schemaVersion !== 1 || + typeof value.backupPath !== "string" || + !isAbsolute(value.backupPath) || + resolve(value.backupPath) !== value.backupPath || + value.backupPath.includes("\0") || + Buffer.byteLength(value.backupPath, "utf8") > MAX_AUTHORITY_PATH_BYTES || + typeof value.contentSha256 !== "string" || + !SHA256_PATTERN.test(value.contentSha256) + ) { + fail("snapshot content authority is invalid"); + } + return { + schemaVersion: 1, + backupPath: value.backupPath, + contentSha256: value.contentSha256, + }; +} + +function readSnapshotAuthority(snapshot: ManagedWorkloadCloneSnapshot): ManagedWorkloadAuthority { + let authority: ManagedWorkloadAuthority | null; + try { + authority = readManagedWorkloadAuthority({ + agent: snapshot.agentType, + fromDockerfile: null, + imageTag: + snapshot.workload?.kind === "managed-image" ? snapshot.workload.reference : undefined, + workload: snapshot.workload, + }); + } catch (error) { + fail(`snapshot '${snapshot.sandboxName}' has invalid managed workload authority`, error); + } + if (!authority) fail(`snapshot '${snapshot.sandboxName}' is not a managed workload`); + return authority; +} + +function registryFields( + profile: ManagedStartupProfile, + source: SandboxEntry, +): ManagedWorkloadCloneRegistryFields { + const webSearch = + profile.agentConfig.agent === "langchain-deepagents-code" + ? null + : profile.agentConfig.webSearch; + const hermesDashboard = profile.dashboard.agent === "hermes" ? profile.dashboard : null; + const dcodeConfig = + profile.agentConfig.agent === "langchain-deepagents-code" ? profile.agentConfig : null; + const hermesInferenceProvider = + profile.agent === "hermes" && profile.tools.enabledGateways.length > 0 + ? profile.inference.upstreamProvider + : undefined; + const dashboardPort = + profile.dashboard.agent === "openclaw" + ? profile.dashboard.port + : hermesDashboard?.mode === "loopback-forwarded" + ? hermesDashboard.publicPort + : undefined; + return { + // Snapshot peers retain the source gateway route. The isolated Hermes + // provider owns only the destination sandbox's rotating runtime key. + provider: + hermesInferenceProvider === undefined ? profile.inference.upstreamProvider : source.provider, + model: profile.inference.model, + endpointUrl: source.endpointUrl ?? null, + endpointSource: source.endpointSource ?? null, + credentialEnv: source.credentialEnv ?? null, + preferredInferenceApi: profile.inference.api, + compatibleEndpointReasoning: + profile.agent === "openclaw" && profile.inference.upstreamProvider === "compatible-endpoint" + ? profile.tuning.reasoning + ? "true" + : "false" + : null, + compatibleEndpointReasoningEffort: + profile.agent === "openclaw" && + profile.inference.upstreamProvider === "compatible-endpoint" && + profile.inference.api === "openai-completions" && + profile.tuning.reasoningEffort !== "default" + ? profile.tuning.reasoningEffort + : null, + toolDisclosure: profile.tools.disclosure, + webSearchEnabled: webSearch?.enabled === true, + webSearchProvider: webSearch?.enabled === true ? webSearch.provider : null, + observabilityEnabled: dcodeConfig?.observabilityEnabled === true, + ...(dcodeConfig ? { dcodeAutoApprovalMode: dcodeConfig.autoApprovalMode } : {}), + ...(profile.agent === "hermes" && profile.tools.enabledGateways.length > 0 + ? { hermesToolGateways: [...profile.tools.enabledGateways] } + : {}), + ...(hermesInferenceProvider === undefined ? {} : { hermesInferenceProvider }), + ...(hermesDashboard?.mode === "loopback-forwarded" + ? { + hermesDashboardEnabled: true as const, + hermesDashboardPort: hermesDashboard.publicPort, + hermesDashboardInternalPort: hermesDashboard.internalPort, + ...(hermesDashboard.tuiEnabled ? { hermesDashboardTui: true as const } : {}), + } + : {}), + ...(dashboardPort === undefined ? {} : { dashboardPort }), + dashboardRemoteBindPrepared: + profile.dashboard.agent === "openclaw" && profile.dashboard.bindAddress === "0.0.0.0", + }; +} + +function reboundWorkload( + source: ManagedWorkloadAuthority, + rebound: ReboundManagedStartupClone, + provider: RuntimeProviderBundle, +): Extract { + const candidate = { + ...source.receipt, + encodedProfile: rebound.encodedProfile, + startupProfileSha256: rebound.startupProfileSha256, + ...(rebound.corporateCaB64 === undefined ? {} : { corporateCaB64: rebound.corporateCaB64 }), + } as const; + const normalized = cloneSandboxWorkloadReceipt(candidate); + if (normalized?.kind !== "managed-image") { + fail("rebound managed workload receipt is invalid"); + } + const immutable = cloneAndDeepFreeze(normalized); + if (!provider.workload.acceptsReceipt(immutable)) { + fail(`provider '${provider.identity.id}' rejected the rebound workload receipt`); + } + return immutable; +} + +function reboundMessaging( + profile: ManagedStartupProfile, + destinationSandboxName: string, +): SandboxMessagingState | undefined { + if (profile.messaging.plan === null) return undefined; + if (profile.agent === "langchain-deepagents-code") { + fail("DCode clone unexpectedly produced a messaging plan"); + } + const agent = profile.agent as MessagingAgentId; + const manifestRegistry = createBuiltInChannelManifestRegistry(); + const plan = parseSandboxMessagingPlan(profile.messaging.plan, { + sandboxName: destinationSandboxName, + agent, + supportedChannelIds: manifestRegistry.listAvailable({ agent }).map((manifest) => manifest.id), + environment: { + NEMOCLAW_PROXY_HOST: profile.proxy.managedHost, + NEMOCLAW_PROXY_PORT: String(profile.proxy.managedPort), + }, + }); + if (!plan) fail("rebound messaging plan is invalid"); + return { schemaVersion: 1, plan }; +} + +/** + * Build the provider-bound, secret-free handoff consumed by the later held + * create/bootstrap transaction. This function performs no provider, registry, + * filesystem, credential, or sandbox mutation. + */ +export function prepareManagedWorkloadCloneHandoff( + input: PrepareManagedWorkloadCloneHandoffInput, +): PreparedManagedWorkloadCloneHandoff { + const sourceSandboxName = requireSandboxName(input.source.name, "source"); + const snapshotSandboxName = requireSandboxName(input.snapshot.sandboxName, "snapshot source"); + const destinationSandboxName = requireSandboxName(input.destinationSandboxName, "destination"); + if (sourceSandboxName !== snapshotSandboxName) { + fail("snapshot source identity does not match the current source sandbox"); + } + if (sourceSandboxName === destinationSandboxName) { + fail("source and destination sandbox names must differ"); + } + + const providerId = normalizeRuntimeProviderIdentity(input.source.openshellDriver); + if ( + providerId !== input.provider.identity.id || + input.provider.workload.providerId !== input.provider.identity.id + ) { + fail( + `source provider '${providerId}' does not match selected provider ` + + `'${input.provider.identity.id}'`, + ); + } + try { + requireRuntimeProviderMutationAuthority(input.provider, "clone"); + } catch (error) { + fail(`provider '${input.provider.identity.id}' does not authorize clone handoff`, error); + } + + let sourceRegistryAuthority: SandboxRebuildAuthority; + try { + sourceRegistryAuthority = captureSandboxRebuildAuthority( + input.source, + input.provider.identity.id, + ); + } catch (error) { + fail(`source '${sourceSandboxName}' has no exact registry generation authority`, error); + } + + let currentAuthority: ManagedWorkloadAuthority | null; + try { + currentAuthority = readManagedWorkloadAuthority(input.source); + } catch (error) { + fail(`source '${sourceSandboxName}' has invalid managed workload authority`, error); + } + if (!currentAuthority) fail(`source '${sourceSandboxName}' is not a managed workload`); + const snapshotAuthority = readSnapshotAuthority(input.snapshot); + if (!isDeepStrictEqual(currentAuthority, snapshotAuthority)) { + fail("current source managed authority no longer matches the selected snapshot"); + } + if (!input.provider.workload.acceptsReceipt(snapshotAuthority.receipt)) { + fail(`provider '${input.provider.identity.id}' rejected the snapshot workload receipt`); + } + + const runtimeSnapshot = cloneSandboxRuntimeSnapshot(input.snapshot.runtimeSnapshot); + if ( + !runtimeSnapshot || + runtimeSnapshot.providerId !== input.provider.identity.id || + runtimeSnapshot.runtime.providerId !== input.provider.identity.id + ) { + fail("snapshot runtime authority does not belong to the selected provider"); + } + const snapshotRestoreAuthority = cloneSnapshotRestoreAuthority(input.snapshot.restoreAuthority); + + const needsHermesInferenceProvider = + snapshotAuthority.agent === "hermes" && + Array.isArray(input.source.hermesToolGateways) && + input.source.hermesToolGateways.length > 0; + const destinationHermesInferenceProvider = needsHermesInferenceProvider + ? requireProviderName( + input.getHermesInferenceProviderName(destinationSandboxName), + "destination Hermes inference", + ) + : undefined; + + let rebound: ReboundManagedStartupClone; + try { + rebound = rebindManagedStartupProfileForClone({ + sourceSandboxName, + destinationSandboxName, + expectedAgent: snapshotAuthority.agent, + destinationDashboardPort: input.destinationDashboardPort, + ...(destinationHermesInferenceProvider === undefined + ? {} + : { destinationHermesInferenceProvider }), + encodedProfile: snapshotAuthority.receipt.encodedProfile, + startupProfileSha256: snapshotAuthority.receipt.startupProfileSha256, + ...(snapshotAuthority.receipt.corporateCaB64 === undefined + ? {} + : { corporateCaB64: snapshotAuthority.receipt.corporateCaB64 }), + currentSource: input.source, + }); + } catch (error) { + fail("managed startup profile could not be rebound", error); + } + const workload = reboundWorkload(snapshotAuthority, rebound, input.provider); + const messaging = reboundMessaging(rebound.profile, destinationSandboxName); + + return cloneAndDeepFreeze({ + schemaVersion: 1 as const, + phase: "rebound" as const, + providerId: input.provider.identity.id, + sourceSandboxName, + destinationSandboxName, + sourceRegistryAuthority, + sourceAuthority: snapshotAuthority, + runtimeSnapshot, + snapshotRestoreAuthority, + rebound, + workload, + ...(messaging === undefined ? {} : { messaging }), + registryFields: registryFields(rebound.profile, input.source), + }); +} diff --git a/src/lib/state/registry/types.ts b/src/lib/state/registry/types.ts index f50d21d962d..386bb16ff54 100644 --- a/src/lib/state/registry/types.ts +++ b/src/lib/state/registry/types.ts @@ -128,6 +128,8 @@ export interface SandboxEntry extends Partial { messaging?: SandboxMessagingState; mcp?: SandboxMcpState; hermesToolGateways?: string[]; + /** Destination-scoped provider holding the host-minted Hermes inference key. */ + hermesInferenceProvider?: string; hermesDashboardEnabled?: boolean; hermesDashboardPort?: number | null; hermesDashboardInternalPort?: number | null; diff --git a/test/helpers/runtime-provider-bundle.ts b/test/helpers/runtime-provider-bundle.ts index ba3e4c66811..512b544cdbc 100644 --- a/test/helpers/runtime-provider-bundle.ts +++ b/test/helpers/runtime-provider-bundle.ts @@ -150,6 +150,7 @@ export function createInMemoryRuntimeProviderBundle({ "stop", "inference-set", "rebuild", + "clone", "provider-cleanup", "destroy", "workload-cleanup", From 1a91ef28713b92d056a73f4afe6f7e89f5ffd0d5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 19:24:10 -0700 Subject: [PATCH 049/117] test(snapshot): keep provider authority cases linear Signed-off-by: Aaron Erickson --- ...hot-managed-provider-restore-order.test.ts | 41 +++-- ...penshell-docker-sandbox-containers.test.ts | 6 +- .../onboard/runtime-provider/snapshot.test.ts | 145 ++++++++++-------- ...snapshot-managed-restore-authority.test.ts | 8 +- 4 files changed, 106 insertions(+), 94 deletions(-) diff --git a/src/lib/actions/sandbox/snapshot-managed-provider-restore-order.test.ts b/src/lib/actions/sandbox/snapshot-managed-provider-restore-order.test.ts index f5d617b457f..f5082a04cbb 100644 --- a/src/lib/actions/sandbox/snapshot-managed-provider-restore-order.test.ts +++ b/src/lib/actions/sandbox/snapshot-managed-provider-restore-order.test.ts @@ -259,27 +259,26 @@ describe("legacy snapshot compatibility gate", () => { "source", "destination", ] as const)("rejects cross-clone when the current %s is managed", async (managedSide) => { - fixture.getSandboxMock.mockImplementation((name) => { - if (name === "alpha") { - return { - name, - agent: "openclaw", - openshellDriver: "docker", - imageTag: "legacy-source:test", - ...(managedSide === "source" ? { workload: managedWorkload() } : {}), - }; - } - if (name === "beta" && managedSide === "destination") { - return { - name, - agent: "openclaw", - openshellDriver: "docker", - imageTag: "managed-target@test", - workload: managedWorkload(), - }; - } - return null; - }); + const source = { + name: "alpha", + agent: "openclaw" as const, + openshellDriver: "docker", + imageTag: "legacy-source:test", + ...(managedSide === "source" ? { workload: managedWorkload() } : {}), + }; + const destination = + managedSide === "destination" + ? { + name: "beta", + agent: "openclaw" as const, + openshellDriver: "docker", + imageTag: "managed-target@test", + workload: managedWorkload(), + } + : null; + fixture.getSandboxMock.mockImplementation((name) => + name === "alpha" ? source : name === "beta" ? destination : null, + ); fixture.parseLiveSandboxNamesMock.mockReturnValue( new Set(managedSide === "destination" ? ["alpha", "beta"] : ["alpha"]), ); diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.test.ts b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts index 57b49ccd7fa..60bc55fef2c 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.test.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts @@ -12,9 +12,8 @@ function querySnapshot(fields: unknown, nvidiaVisibleDevices?: string) { const dockerRun = vi .fn() .mockReturnValueOnce({ status: 0, stdout: "container-a\n", stderr: "" }) - .mockReturnValueOnce({ status: 0, stdout: JSON.stringify(fields), stderr: "" }); - if (Array.isArray(fields) && String(fields[5]).toLowerCase() === "nvidia") { - dockerRun.mockReturnValueOnce({ + .mockReturnValueOnce({ status: 0, stdout: JSON.stringify(fields), stderr: "" }) + .mockReturnValueOnce({ status: 0, stdout: nvidiaVisibleDevices === undefined @@ -22,7 +21,6 @@ function querySnapshot(fields: unknown, nvidiaVisibleDevices?: string) { : `NVIDIA_VISIBLE_DEVICES=${nvidiaVisibleDevices}\n`, stderr: "", }); - } return { dockerRun, result: queryOpenShellDockerSandboxRuntimeSnapshot("alpha", { dockerRun }), diff --git a/src/lib/onboard/runtime-provider/snapshot.test.ts b/src/lib/onboard/runtime-provider/snapshot.test.ts index 3ec35633b4c..68de9202464 100644 --- a/src/lib/onboard/runtime-provider/snapshot.test.ts +++ b/src/lib/onboard/runtime-provider/snapshot.test.ts @@ -76,11 +76,19 @@ function snapshotSource( }; } +function requireSupportedSurface( + surface: T, +): Extract { + expect(surface.supported).toBe(true); + return surface as Extract; +} + describe("runtime provider snapshot surface", () => { it("binds the full runtime and lifecycle generation into opaque backup authority", () => { const observe = vi.fn(() => observation()); - const surface = createRuntimeProviderSnapshotSurface("mxc", surfaceDriver(observe)); - if (!surface.supported) throw new Error("test surface must be supported"); + const surface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface("mxc", surfaceDriver(observe)), + ); const preflight = surface.preflight("backup", sandbox()); const receipt = surface.capture(sandbox(), preflight); @@ -97,11 +105,12 @@ describe("runtime provider snapshot surface", () => { it("invokes the owning provider restore facet and returns managed profile/runtime proof", () => { const restoreManagedProfile = vi.fn(() => "provider-restore-proof"); - const surface = createRuntimeProviderSnapshotSurface( - "mxc", - surfaceDriver(() => observation(), restoreManagedProfile), + const surface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => observation(), restoreManagedProfile), + ), ); - if (!surface.supported) throw new Error("test surface must be supported"); const target = sandbox(); const preflight = surface.preflight("restore", target); @@ -144,21 +153,23 @@ describe("runtime provider snapshot surface", () => { runtime: { kind: "session", handle: sourceHandle }, }, }); - const sourceSurface = createRuntimeProviderSnapshotSurface( - "mxc", - surfaceDriver(() => sourceObservation), + const sourceSurface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => sourceObservation), + ), ); - if (!sourceSurface.supported) throw new Error("test surface must be supported"); const sourcePreflight = sourceSurface.preflight("backup", target); const source = snapshotSource( sourcePreflight, sourceSurface.capture(target, sourcePreflight), ); - const targetSurface = createRuntimeProviderSnapshotSurface( - "mxc", - surfaceDriver(() => targetObservation), + const targetSurface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => targetObservation), + ), ); - if (!targetSurface.supported) throw new Error("test surface must be supported"); const targetPreflight = targetSurface.preflight("restore", target); return targetSurface.restore(target, targetPreflight, source, managedProfile); }; @@ -219,11 +230,9 @@ describe("runtime provider snapshot surface", () => { const restoreManagedProfile = vi.fn(() => "provider-restore-proof"); const observe = vi.fn<() => RuntimeProviderSnapshotObservation>(); for (const value of observations) observe.mockReturnValueOnce(value); - const surface = createRuntimeProviderSnapshotSurface( - "mxc", - surfaceDriver(observe, restoreManagedProfile), + const surface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface("mxc", surfaceDriver(observe, restoreManagedProfile)), ); - if (!surface.supported) throw new Error("test surface must be supported"); const target = sandbox(); const preflight = surface.preflight("restore", target); const source = snapshotSource(preflight, observation().runtime); @@ -236,9 +245,11 @@ describe("runtime provider snapshot surface", () => { it("fails before provider restore when the target cannot represent source acceleration", () => { const restoreManagedProfile = vi.fn(() => "provider-restore-proof"); - const targetSurface = createRuntimeProviderSnapshotSurface( - "mxc", - surfaceDriver(() => observation(), restoreManagedProfile), + const targetSurface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => observation(), restoreManagedProfile), + ), ); const sourceObservation = observation("mxc", { runtime: { @@ -246,13 +257,12 @@ describe("runtime provider snapshot surface", () => { acceleration: { kind: "gpu", vendor: "nvidia", devices: ["live-device-0"] }, }, }); - const sourceSurface = createRuntimeProviderSnapshotSurface( - "mxc", - surfaceDriver(() => sourceObservation), + const sourceSurface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => sourceObservation), + ), ); - if (!targetSurface.supported || !sourceSurface.supported) { - throw new Error("test surfaces must be supported"); - } const target = sandbox(); const sourcePreflight = sourceSurface.preflight("backup", target); const source = snapshotSource(sourcePreflight, sourceSurface.capture(target, sourcePreflight)); @@ -266,18 +276,19 @@ describe("runtime provider snapshot surface", () => { it("fails before provider restore when the target cannot represent source lifecycle", () => { const restoreManagedProfile = vi.fn(() => "provider-restore-proof"); - const targetSurface = createRuntimeProviderSnapshotSurface( - "mxc", - surfaceDriver(() => observation(), restoreManagedProfile), + const targetSurface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => observation(), restoreManagedProfile), + ), ); const stopped = observation("mxc", { lifecycleState: "stopped" }); - const sourceSurface = createRuntimeProviderSnapshotSurface( - "mxc", - surfaceDriver(() => stopped), + const sourceSurface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => stopped), + ), ); - if (!targetSurface.supported || !sourceSurface.supported) { - throw new Error("test surfaces must be supported"); - } const target = sandbox(); const sourcePreflight = sourceSurface.preflight("backup", target); const source = snapshotSource(sourcePreflight, sourceSurface.capture(target, sourcePreflight)); @@ -322,8 +333,9 @@ describe("runtime provider snapshot surface", () => { .fn<() => RuntimeProviderSnapshotObservation>() .mockReturnValueOnce(observation()) .mockReturnValueOnce(changed); - const surface = createRuntimeProviderSnapshotSurface("mxc", surfaceDriver(observe)); - if (!surface.supported) throw new Error("test surface must be supported"); + const surface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface("mxc", surfaceDriver(observe)), + ); const target = sandbox(); const preflight = surface.preflight("backup", target); @@ -331,11 +343,12 @@ describe("runtime provider snapshot surface", () => { }); it("rejects a preflight receipt from another operation or sandbox", () => { - const surface = createRuntimeProviderSnapshotSurface( - "mxc", - surfaceDriver(() => observation()), + const surface = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => observation()), + ), ); - if (!surface.supported) throw new Error("test surface must be supported"); const target = sandbox(); const restorePreflight = surface.preflight("restore", target); const otherTarget = sandbox({ name: "other" }); @@ -354,21 +367,23 @@ describe("runtime provider snapshot surface", () => { }); it("rejects invalid runtime receipts, restore authority, and provider proof", () => { - const invalidRuntime = createRuntimeProviderSnapshotSurface( - "mxc", - surfaceDriver(() => observation("other-provider")), + const invalidRuntime = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver(() => observation("other-provider")), + ), ); - if (!invalidRuntime.supported) throw new Error("test surface must be supported"); expect(() => invalidRuntime.preflight("backup", sandbox())).toThrow(/invalid runtime receipt/u); - const invalidProof = createRuntimeProviderSnapshotSurface( - "mxc", - surfaceDriver( - () => observation(), - vi.fn(() => "proof\ninjection"), + const invalidProof = requireSupportedSurface( + createRuntimeProviderSnapshotSurface( + "mxc", + surfaceDriver( + () => observation(), + vi.fn(() => "proof\ninjection"), + ), ), ); - if (!invalidProof.supported) throw new Error("test surface must be supported"); const preflight = invalidProof.preflight("restore", sandbox()); const source = snapshotSource(preflight, observation().runtime); expect(() => invalidProof.restore(sandbox(), preflight, source, managedProfile)).toThrow( @@ -709,8 +724,9 @@ describe("Docker provider snapshot evidence", () => { captureOpenShell: captureOpenShell as never, queryRuntimeSnapshot: () => dockerSnapshot(), }; - const surface = createDockerRuntimeProviderSnapshotSurface("docker", dependencies); - if (!surface.supported) throw new Error("Docker snapshot surface must be supported"); + const surface = requireSupportedSurface( + createDockerRuntimeProviderSnapshotSurface("docker", dependencies), + ); const target = sandbox({ agent, openshellDriver: "docker" }); const preflight = surface.preflight("restore", target); const source = snapshotSource(preflight, { @@ -747,16 +763,17 @@ describe("Docker provider snapshot evidence", () => { }), ); - const denied = createDockerRuntimeProviderSnapshotSurface("docker", { - ...dependencies, - captureOpenShell: (() => ({ - status: 1, - output: "profile mismatch", - stdout: "", - stderr: "", - })) as never, - }); - if (!denied.supported) throw new Error("Docker snapshot surface must be supported"); + const denied = requireSupportedSurface( + createDockerRuntimeProviderSnapshotSurface("docker", { + ...dependencies, + captureOpenShell: (() => ({ + status: 1, + output: "profile mismatch", + stdout: "", + stderr: "", + })) as never, + }), + ); const deniedPreflight = denied.preflight("restore", target); const deniedSource = snapshotSource(deniedPreflight, source.runtime); expect(() => denied.restore(target, deniedPreflight, deniedSource, authority)).toThrow( diff --git a/test/snapshot-managed-restore-authority.test.ts b/test/snapshot-managed-restore-authority.test.ts index cf4c9df3f87..c195699751d 100644 --- a/test/snapshot-managed-restore-authority.test.ts +++ b/test/snapshot-managed-restore-authority.test.ts @@ -18,11 +18,9 @@ const sandboxState = await import("../src/lib/state/sandbox.js"); const BACKUPS_ROOT = path.join(TMP_HOME, ".nemoclaw", "rebuild-backups"); afterAll(() => { - if (ORIGINAL_HOME === undefined) { - delete process.env.HOME; - } else { - process.env.HOME = ORIGINAL_HOME; - } + void (ORIGINAL_HOME === undefined + ? Reflect.deleteProperty(process.env, "HOME") + : Reflect.set(process.env, "HOME", ORIGINAL_HOME)); fs.rmSync(TMP_HOME, { recursive: true, force: true }); }); From fabad32581afd4573192de1daa7904614ba29ace Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 19:30:02 -0700 Subject: [PATCH 050/117] test(snapshot): keep clone authority cases linear Signed-off-by: Aaron Erickson --- .../managed-workload-clone-handoff.test.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/lib/onboard/managed-workload-clone-handoff.test.ts b/src/lib/onboard/managed-workload-clone-handoff.test.ts index c002f0eb2f3..98bb6230a57 100644 --- a/src/lib/onboard/managed-workload-clone-handoff.test.ts +++ b/src/lib/onboard/managed-workload-clone-handoff.test.ts @@ -36,11 +36,12 @@ const PORTABLE_PROFILE = { } as const satisfies RuntimeProviderWorkloadProfile; function provider(providerId: "docker" | "mxc"): RuntimeProviderBundle { - if (providerId === "docker") return CURRENT_RUNTIME_PROVIDER_BUNDLES.docker!; - return createInMemoryRuntimeProviderBundle({ - providerId, - workloadProfile: PORTABLE_PROFILE, - }); + return providerId === "docker" + ? CURRENT_RUNTIME_PROVIDER_BUNDLES.docker! + : createInMemoryRuntimeProviderBundle({ + providerId, + workloadProfile: PORTABLE_PROFILE, + }); } function receipt( @@ -308,9 +309,10 @@ describe("prepareManagedWorkloadCloneHandoff", () => { workload: { ...base.workload, acceptsReceipt: (candidate) => { - if (candidate?.kind === "managed-image") { - observedReceipts.push(candidate); - mutationResults.push(Reflect.set(candidate, "reference", "mutated-by-provider")); + const managedCandidates = candidate?.kind === "managed-image" ? [candidate] : []; + for (const managedCandidate of managedCandidates) { + observedReceipts.push(managedCandidate); + mutationResults.push(Reflect.set(managedCandidate, "reference", "mutated-by-provider")); } return true; }, From 25aa3043c8c9cbeaa79bb5cb5b753be9c1ecca1a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 19:27:00 -0700 Subject: [PATCH 051/117] feat(snapshot): add managed clone provider transaction Signed-off-by: Aaron Erickson --- ...hot-managed-clone-handoff-dormancy.test.ts | 10 +- .../snapshot-managed-clone-providers.test.ts | 596 ++++++++++++++++ .../actions/sandbox/snapshot/dependencies.ts | 28 +- .../snapshot/managed-clone-providers.ts | 666 ++++++++++++++++++ src/lib/onboard/lifecycle-contracts.md | 2 +- 5 files changed, 1297 insertions(+), 5 deletions(-) create mode 100644 src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts create mode 100644 src/lib/actions/sandbox/snapshot/managed-clone-providers.ts diff --git a/src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts b/src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts index 53647e6bfb2..5e07f125a1f 100644 --- a/src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts +++ b/src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts @@ -6,18 +6,22 @@ import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; describe("managed snapshot clone handoff activation boundary", () => { - it("keeps the PR3.9 contract unwired while production restore stays fail-closed", () => { + it("exposes the PR3.9 dependency seam while production restore stays fail-closed", () => { const dependencies = readFileSync( new URL("./snapshot/dependencies.ts", import.meta.url), "utf8", ); const productionAction = readFileSync(new URL("./snapshot.ts", import.meta.url), "utf8"); - expect(dependencies).not.toContain("prepareManagedWorkloadCloneHandoff"); - expect(dependencies).not.toContain("ManagedWorkloadCloneHandoff"); + expect(dependencies).toContain("prepareManagedWorkloadCloneHandoff"); + expect(dependencies).toContain("prepareManagedCloneProviderTransaction"); + expect(dependencies).toContain("revalidateManagedCloneMutationAuthority"); expect(productionAction).toContain("rejectManagedSnapshotCloneUntilRebind"); expect(productionAction).not.toContain("prepareManagedWorkloadCloneHandoff"); expect(productionAction).not.toContain("ManagedWorkloadCloneHandoff"); + expect(productionAction).not.toContain("prepareManagedCloneProviderTransaction"); + expect(productionAction).not.toContain("provisionManagedCloneProviderTransaction"); + expect(productionAction).not.toContain("revalidateManagedCloneMutationAuthority"); expect(productionAction).not.toContain("prepareManagedCloneProviders"); expect(productionAction).not.toContain("provisionManagedCloneProviders"); }); diff --git a/src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts b/src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts new file mode 100644 index 00000000000..7fc2df7a9c1 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts @@ -0,0 +1,596 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { describe, expect, it, vi } from "vitest"; +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import type { SandboxMessagingPlan } from "../../messaging/manifest"; +import { + encodeManagedStartupProfile, + type ManagedStartupAgent, + type ManagedStartupProfile, +} from "../../onboard/managed-startup/profile"; +import { + captureSandboxRebuildAuthority, + type SandboxRebuildAuthority, +} from "../../state/registry/rebuild-authority"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; +import type { SnapshotRestoreAuthority } from "../../state/sandbox"; +import { + cleanupManagedCloneProviderTransaction, + type ManagedCloneProviderBinding, + ManagedCloneProviderTransactionError, + prepareManagedCloneProviderTransaction, + provisionManagedCloneProviderTransaction, + revalidateManagedCloneMutationAuthority, +} from "./snapshot/managed-clone-providers"; + +const CONTENT_AUTHORITY = { + schemaVersion: 1, + backupPath: "/tmp/nemoclaw-managed-clone-source", + contentSha256: "c".repeat(64), +} as const satisfies SnapshotRestoreAuthority; + +const TOKEN_BINDING = { + providerName: "destination-runtime-token", + providerType: "generic", + providerEnvKey: "RUNTIME_TOKEN", + source: "runtime-extension", +} as const satisfies ManagedCloneProviderBinding; + +function receipt(profile: ManagedStartupProfile) { + const encodedProfile = encodeManagedStartupProfile(profile); + return { + schemaVersion: 1 as const, + kind: "managed-image" as const, + reference: `ghcr.io/nvidia/nemoclaw/${profile.agent}-sandbox@sha256:${"a".repeat(64)}`, + platform: "linux/amd64" as const, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: true, + shared: true, + } satisfies Extract; +} + +function entry( + name: string, + profile: ManagedStartupProfile, + overrides: Partial = {}, +): SandboxEntry { + const workload = receipt(profile); + return { + name, + agent: profile.agent, + openshellDriver: "docker", + imageTag: workload.reference, + workload, + lifecycleGeneration: `generation-${name}`, + lifecycleLiveIdentityFingerprint: createHash("sha256").update(name).digest("hex"), + provider: profile.inference.upstreamProvider, + model: profile.inference.model, + ...overrides, + }; +} + +function messagingPlan(sandboxName: string): SandboxMessagingPlan { + return { + schemaVersion: 1, + sandboxName, + agent: "openclaw", + workflow: "onboard", + channels: [ + { + channelId: "telegram", + configured: true, + active: true, + disabled: false, + inputs: [{ inputId: "botToken", credentialAvailable: true }], + }, + ], + disabledChannels: [], + credentialBindings: [ + { + channelId: "telegram", + credentialId: "botToken", + providerName: `${sandboxName}-telegram-bridge`, + providerEnvKey: "TELEGRAM_BOT_TOKEN", + credentialAvailable: true, + }, + ], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + runtimeSetup: { nodePreloads: [], envAliases: [], secretScans: [] }, + stateUpdates: [], + healthChecks: [], + } as SandboxMessagingPlan; +} + +function handoff( + profile: ManagedStartupProfile, + source: SandboxEntry, + messaging?: SandboxMessagingPlan, +) { + return { + providerId: "docker", + sourceSandboxName: "source", + destinationSandboxName: "destination", + sourceRegistryAuthority: captureSandboxRebuildAuthority(source, "docker"), + snapshotRestoreAuthority: CONTENT_AUTHORITY, + rebound: { + profile, + encodedProfile: + source.workload?.kind === "managed-image" ? source.workload.encodedProfile : "", + startupProfileSha256: + source.workload?.kind === "managed-image" ? source.workload.startupProfileSha256 : "", + }, + ...(messaging === undefined + ? {} + : { messaging: { schemaVersion: 1 as const, plan: messaging } }), + }; +} + +type LiveBinding = Pick< + ManagedCloneProviderBinding, + "providerName" | "providerType" | "providerEnvKey" +>; + +function providerMetadata(binding: LiveBinding): string { + return [ + `Name: ${binding.providerName}`, + `Type: ${binding.providerType}`, + `Credential keys: ${binding.providerEnvKey}`, + "Config keys: ", + "", + ].join("\n"); +} + +function providerRunner(initial: readonly LiveBinding[] = []) { + const live = new Map(initial.map((binding) => [binding.providerName, { ...binding }] as const)); + const commands: string[] = []; + let createBehavior: + | ((binding: LiveBinding) => { readonly materialize?: LiveBinding; readonly status: number }) + | undefined; + let failDelete = false; + const run = vi.fn((args: string[]) => { + commands.push(args.join(" ")); + if (args[0] === "provider" && args[1] === "get") { + const name = args[2] ?? ""; + const binding = live.get(name); + return binding + ? { status: 0, stdout: providerMetadata(binding), stderr: "" } + : { status: 1, stdout: "", stderr: `provider '${name}' not found` }; + } + if (args[0] === "provider" && args[1] === "create") { + const binding = { + providerName: args[3] ?? "", + providerType: args[5] ?? "", + providerEnvKey: args[7] ?? "", + }; + const outcome = createBehavior?.(binding) ?? { status: 0, materialize: binding }; + if (outcome.materialize) live.set(binding.providerName, { ...outcome.materialize }); + return { status: outcome.status, stdout: "", stderr: "" }; + } + if (args[0] === "provider" && args[1] === "delete") { + if (failDelete) return { status: 1, stdout: "", stderr: "gateway unavailable" }; + live.delete(args[2] ?? ""); + return { status: 0, stdout: "", stderr: "" }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "detach") { + return { status: 0, stdout: "", stderr: "" }; + } + return { status: 1, stdout: "", stderr: "unsupported test command" }; + }); + return { + commands, + live, + run, + setCreateBehavior(value: typeof createBehavior) { + createBehavior = value; + }, + setFailDelete(value: boolean) { + failDelete = value; + }, + }; +} + +function authorityDeps( + source: SandboxEntry, + destination: SandboxEntry | null = null, + content: SnapshotRestoreAuthority | null = CONTENT_AUTHORITY, +) { + return { + readSandbox: (name: string) => + name === source.name ? source : name === destination?.name ? destination : null, + captureSnapshotRestoreAuthority: vi.fn(() => content), + }; +} + +function prepareWithBinding(input: { + readonly agent?: ManagedStartupAgent; + readonly binding?: ManagedCloneProviderBinding; + readonly destination?: SandboxEntry | null; + readonly environment?: NodeJS.ProcessEnv; + readonly runner?: ReturnType; +}) { + const profile = managedStartupE2eProfile(input.agent ?? "openclaw"); + const source = entry("source", profile); + const runner = input.runner ?? providerRunner(); + const destination = input.destination ?? null; + const prepared = prepareManagedCloneProviderTransaction({ + handoff: handoff(profile, source), + destination, + additionalBindings: [input.binding ?? TOKEN_BINDING], + environment: input.environment ?? { RUNTIME_TOKEN: "test-only-runtime-token" }, + runOpenshell: runner.run, + transactionId: "1".repeat(32), + }); + return { destination, prepared, profile, runner, source }; +} + +describe("managed clone provider transaction", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("keeps the %s transaction provider-neutral, secret-free, and deeply frozen", (agent) => { + const { prepared } = prepareWithBinding({ agent }); + + expect(prepared).toMatchObject({ + providerId: "docker", + sourceSandboxName: "source", + destinationSandboxName: "destination", + snapshotRestoreAuthority: CONTENT_AUTHORITY, + providers: [{ binding: TOKEN_BINDING, action: "create" }], + }); + expect(JSON.stringify(prepared)).not.toContain("test-only-runtime-token"); + expect(Object.isFrozen(prepared)).toBe(true); + expect(Object.isFrozen(prepared.snapshotRestoreAuthority)).toBe(true); + expect(Object.isFrozen(prepared.sourceRegistryAuthority.workload)).toBe(true); + expect(Object.isFrozen(prepared.providers[0]?.binding)).toBe(true); + }); + + it("resolves active messaging providers from the handoff", () => { + const profile = managedStartupE2eProfile("openclaw"); + const source = entry("source", profile); + const runner = providerRunner(); + const plan = messagingPlan("destination"); + + const prepared = prepareManagedCloneProviderTransaction({ + handoff: handoff(profile, source, plan), + destination: null, + environment: { TELEGRAM_BOT_TOKEN: "test-only-telegram-token" }, + runOpenshell: runner.run, + transactionId: "2".repeat(32), + }); + + expect(prepared.providers).toEqual([ + { + binding: { + providerName: "destination-telegram-bridge", + providerType: "generic", + providerEnvKey: "TELEGRAM_BOT_TOKEN", + source: "messaging", + }, + action: "create", + }, + ]); + }); + + it("reuses an exact provider only with exact destination registry ownership", () => { + const profile = managedStartupE2eProfile("openclaw"); + const source = entry("source", profile); + const plan = messagingPlan("destination"); + const destination = entry("destination", profile, { messaging: { schemaVersion: 1, plan } }); + const liveBinding = { + providerName: "destination-telegram-bridge", + providerType: "generic", + providerEnvKey: "TELEGRAM_BOT_TOKEN", + }; + const runner = providerRunner([liveBinding]); + const prepared = prepareManagedCloneProviderTransaction({ + handoff: handoff(profile, source, plan), + destination, + environment: { TELEGRAM_BOT_TOKEN: "test-only-telegram-token" }, + runOpenshell: runner.run, + transactionId: "3".repeat(32), + }); + + expect(prepared.providers[0]?.action).toBe("reuse-destination-owned"); + const receipt = provisionManagedCloneProviderTransaction(prepared, { + ...authorityDeps(source, destination), + environment: { TELEGRAM_BOT_TOKEN: "test-only-telegram-token" }, + runOpenshell: runner.run, + }); + expect(receipt.providers[0]?.disposition).toBe("reused-destination-owned"); + expect( + runner.commands.some((command) => /provider (create|delete|update)/u.test(command)), + ).toBe(false); + expect(cleanupManagedCloneProviderTransaction(receipt, runner.run)).toMatchObject({ + status: "complete", + providers: [{ outcome: "reused-preserved" }], + }); + }); + + it("rejects an exact same-name provider without destination ownership", () => { + const runner = providerRunner([TOKEN_BINDING]); + + expect(() => prepareWithBinding({ runner })).toThrow(/without exact destination ownership/u); + expect( + runner.commands.some((command) => /provider (create|delete|update)/u.test(command)), + ).toBe(false); + }); + + it("fails closed on indeterminate provider inspection with bounded diagnostics", () => { + const profile = managedStartupE2eProfile("openclaw"); + const source = entry("source", profile); + const runOpenshell = vi.fn(() => ({ + status: 1, + stdout: "", + stderr: "gateway transport unavailable", + })); + + expect(() => + prepareManagedCloneProviderTransaction({ + handoff: handoff(profile, source), + destination: null, + additionalBindings: [TOKEN_BINDING], + environment: { RUNTIME_TOKEN: "test-only-runtime-token" }, + runOpenshell, + transactionId: "7".repeat(32), + }), + ).toThrow(/could not prove whether provider/u); + expect(runOpenshell).toHaveBeenCalledWith( + ["provider", "get", TOKEN_BINDING.providerName], + expect.objectContaining({ + maxBuffer: 64 * 1024, + suppressOutput: true, + timeout: 5_000, + }), + ); + expect(runOpenshell).toHaveBeenCalledOnce(); + }); + + it("rejects an incompatible provider collision during read-only preflight", () => { + const runner = providerRunner([{ ...TOKEN_BINDING, providerType: "other" }]); + + expect(() => prepareWithBinding({ runner })).toThrow(/incompatible live binding/u); + expect( + runner.commands.some((command) => /provider (create|delete|update)/u.test(command)), + ).toBe(false); + }); + + it("revalidates snapshot, source, and destination authority before provider mutation", () => { + const { prepared, runner, source } = prepareWithBinding({}); + const changedContent = { ...CONTENT_AUTHORITY, contentSha256: "d".repeat(64) }; + + expect(() => + provisionManagedCloneProviderTransaction(prepared, { + ...authorityDeps(source, null, changedContent), + environment: { RUNTIME_TOKEN: "test-only-runtime-token" }, + runOpenshell: runner.run, + }), + ).toThrow(/snapshot content changed before mutation/u); + expect(runner.commands.some((command) => command.startsWith("provider create"))).toBe(false); + + expect(() => + revalidateManagedCloneMutationAuthority(prepared, { + ...authorityDeps({ ...source, model: "changed-model" }), + }), + ).toThrow(/source registry authority changed/u); + expect(() => + revalidateManagedCloneMutationAuthority(prepared, { + ...authorityDeps(source, entry("destination", managedStartupE2eProfile("openclaw"))), + }), + ).toThrow(/destination appeared after clone preflight/u); + }); + + it("revalidates content authority for an agent with no credential providers", () => { + const profile = managedStartupE2eProfile("langchain-deepagents-code"); + const source = entry("source", profile); + const runner = providerRunner(); + const prepared = prepareManagedCloneProviderTransaction({ + handoff: handoff(profile, source), + destination: null, + environment: {}, + runOpenshell: runner.run, + transactionId: "6".repeat(32), + }); + + expect(prepared.providers).toEqual([]); + expect(() => + provisionManagedCloneProviderTransaction(prepared, { + ...authorityDeps(source, null, { + ...CONTENT_AUTHORITY, + contentSha256: "d".repeat(64), + }), + environment: {}, + runOpenshell: runner.run, + }), + ).toThrow(/snapshot content changed before mutation/u); + expect(runner.commands).toEqual([]); + }); + + it("creates with an exact receipt and makes cleanup idempotent against name reuse", () => { + const { prepared, runner, source } = prepareWithBinding({}); + const receipt = provisionManagedCloneProviderTransaction(prepared, { + ...authorityDeps(source), + environment: { RUNTIME_TOKEN: "test-only-runtime-token" }, + runOpenshell: runner.run, + }); + + expect(receipt.providers).toEqual([{ binding: TOKEN_BINDING, disposition: "created" }]); + expect(Object.isFrozen(receipt.providers[0]?.binding)).toBe(true); + expect(cleanupManagedCloneProviderTransaction(receipt, runner.run)).toMatchObject({ + status: "complete", + providers: [{ outcome: "deleted" }], + }); + runner.live.set(TOKEN_BINDING.providerName, { ...TOKEN_BINDING, providerType: "other" }); + const deletesBeforeRetry = runner.commands.filter((command) => + command.startsWith("provider delete"), + ).length; + expect(cleanupManagedCloneProviderTransaction(receipt, runner.run)).toMatchObject({ + status: "complete", + providers: [{ outcome: "already-cleaned" }], + }); + expect(runner.commands.filter((command) => command.startsWith("provider delete"))).toHaveLength( + deletesBeforeRetry, + ); + expect(runner.live.get(TOKEN_BINDING.providerName)?.providerType).toBe("other"); + }); + + it("rolls back confirmed providers when a later credential disappears", () => { + const first = { ...TOKEN_BINDING, providerName: "destination-first-token" }; + const second = { + ...TOKEN_BINDING, + providerName: "destination-second-token", + providerEnvKey: "SECOND_TOKEN", + }; + const profile = managedStartupE2eProfile("openclaw"); + const source = entry("source", profile); + const runner = providerRunner(); + const prepared = prepareManagedCloneProviderTransaction({ + handoff: handoff(profile, source), + destination: null, + additionalBindings: [first, second], + environment: { + RUNTIME_TOKEN: "test-only-runtime-token", + SECOND_TOKEN: "test-only-second-token", + }, + runOpenshell: runner.run, + transactionId: "4".repeat(32), + }); + + let failure: ManagedCloneProviderTransactionError | null = null; + try { + provisionManagedCloneProviderTransaction(prepared, { + ...authorityDeps(source), + environment: { RUNTIME_TOKEN: "test-only-runtime-token" }, + runOpenshell: runner.run, + }); + } catch (error) { + failure = error as ManagedCloneProviderTransactionError; + } + expect(failure).toBeInstanceOf(ManagedCloneProviderTransactionError); + expect(failure?.partialReceipt?.providers).toEqual([ + { binding: first, disposition: "created" }, + ]); + expect(failure?.rollback?.status).toBe("complete"); + expect(runner.live.has(first.providerName)).toBe(false); + expect(runner.commands).toContain(`provider delete ${first.providerName}`); + }); + + it.each([ + ["nonzero and missing", 1, undefined], + ["nonzero and exact", 1, TOKEN_BINDING], + [ + "zero and collision", + 0, + { ...TOKEN_BINDING, providerType: "other" } satisfies ManagedCloneProviderBinding, + ], + ] as const)("preserves an unowned provider after an ambiguous create: %s", (_name, status, materialize) => { + const runner = providerRunner(); + runner.setCreateBehavior(() => ({ status, ...(materialize ? { materialize } : {}) })); + const { prepared, source } = prepareWithBinding({ runner }); + + expect(() => + provisionManagedCloneProviderTransaction(prepared, { + ...authorityDeps(source), + environment: { RUNTIME_TOKEN: "test-only-runtime-token" }, + runOpenshell: runner.run, + }), + ).toThrow(/preserving the observed/u); + expect(runner.commands).not.toContain(`provider delete ${TOKEN_BINDING.providerName}`); + expect(runner.live.has(TOKEN_BINDING.providerName)).toBe(Boolean(materialize)); + }); + + it("reconciles and preserves an exact provider when the create adapter throws", () => { + const runner = providerRunner(); + runner.setCreateBehavior((binding) => { + runner.live.set(binding.providerName, binding); + throw new Error("synthetic child-process transport loss"); + }); + const { prepared, source } = prepareWithBinding({ runner }); + + expect(() => + provisionManagedCloneProviderTransaction(prepared, { + ...authorityDeps(source), + environment: { RUNTIME_TOKEN: "test-only-runtime-token" }, + runOpenshell: runner.run, + }), + ).toThrow(/exact but unowned/u); + expect(runner.commands).not.toContain(`provider delete ${TOKEN_BINDING.providerName}`); + expect(runner.live.has(TOKEN_BINDING.providerName)).toBe(true); + }); + + it("reports cleanup failure without discarding its exact retry receipt", () => { + const { prepared, runner, source } = prepareWithBinding({}); + const receipt = provisionManagedCloneProviderTransaction(prepared, { + ...authorityDeps(source), + environment: { RUNTIME_TOKEN: "test-only-runtime-token" }, + runOpenshell: runner.run, + }); + runner.setFailDelete(true); + + expect(cleanupManagedCloneProviderTransaction(receipt, runner.run)).toMatchObject({ + status: "partial", + providers: [{ outcome: "delete-failed" }], + }); + expect(runner.live.has(TOKEN_BINDING.providerName)).toBe(true); + }); + + it("rejects a cloned or fabricated cleanup receipt", () => { + const { prepared, runner, source } = prepareWithBinding({}); + const receipt = provisionManagedCloneProviderTransaction(prepared, { + ...authorityDeps(source), + environment: { RUNTIME_TOKEN: "test-only-runtime-token" }, + runOpenshell: runner.run, + }); + + expect(() => + cleanupManagedCloneProviderTransaction(structuredClone(receipt), runner.run), + ).toThrow(/exact process-local ownership receipt/u); + expect(runner.live.has(TOKEN_BINDING.providerName)).toBe(true); + }); + + it("fails a force-replace transaction when destination authority becomes stale", () => { + const profile = managedStartupE2eProfile("openclaw"); + const source = entry("source", profile); + const destination = entry("destination", profile); + const runner = providerRunner(); + const prepared = prepareManagedCloneProviderTransaction({ + handoff: handoff(profile, source), + destination, + additionalBindings: [TOKEN_BINDING], + environment: { RUNTIME_TOKEN: "test-only-runtime-token" }, + runOpenshell: runner.run, + transactionId: "5".repeat(32), + }); + const staleDestination = { ...destination, model: "changed-model" }; + + expect(() => + provisionManagedCloneProviderTransaction(prepared, { + ...authorityDeps(source, staleDestination), + environment: { RUNTIME_TOKEN: "test-only-runtime-token" }, + runOpenshell: runner.run, + }), + ).toThrow(/destination registry authority changed/u); + expect(runner.commands.some((command) => command.startsWith("provider create"))).toBe(false); + }); + + it("captures the complete destination row in the reuse authority receipt", () => { + const profile = managedStartupE2eProfile("openclaw"); + const destination = entry("destination", profile); + const { prepared } = prepareWithBinding({ destination }); + + expect(prepared.destinationRegistryAuthority).toEqual( + captureSandboxRebuildAuthority(destination, "docker") as SandboxRebuildAuthority, + ); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot/dependencies.ts b/src/lib/actions/sandbox/snapshot/dependencies.ts index e428143b209..d4e13ec60f3 100644 --- a/src/lib/actions/sandbox/snapshot/dependencies.ts +++ b/src/lib/actions/sandbox/snapshot/dependencies.ts @@ -6,6 +6,33 @@ import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provi import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime-provider/registry"; import type { SandboxEntry } from "../../../state/registry/types"; +export type { + ManagedWorkloadCloneSnapshot, + PreparedManagedWorkloadCloneHandoff, + PrepareManagedWorkloadCloneHandoffInput, +} from "../../../onboard/workload/clone"; +export { + ManagedWorkloadCloneError, + prepareManagedWorkloadCloneHandoff, +} from "../../../onboard/workload/clone"; +export { backupSandboxStateWithManagedAuthority } from "./backup-authority"; +export type { + ManagedCloneProviderBinding, + ManagedCloneProviderCleanupResult, + ManagedCloneProviderCommandResult, + ManagedCloneProviderOwnershipReceipt, + ManagedCloneProviderRunner, + ManagedCloneProviderTransactionReceipt, + PreparedManagedCloneProvider, + PreparedManagedCloneProviderTransaction, +} from "./managed-clone-providers"; +export { + cleanupManagedCloneProviderTransaction, + ManagedCloneProviderTransactionError, + prepareManagedCloneProviderTransaction, + provisionManagedCloneProviderTransaction, + revalidateManagedCloneMutationAuthority, +} from "./managed-clone-providers"; export { ManagedSnapshotProfileRestoreError, prepareManagedSnapshotProfileRestore, @@ -22,7 +49,6 @@ export { prepareSandboxRuntimeRestore, SandboxSnapshotProviderError, } from "./provider-lifecycle"; -export { backupSandboxStateWithManagedAuthority } from "./backup-authority"; /** * Resolve the one already-registered provider bundle for a durable sandbox. diff --git a/src/lib/actions/sandbox/snapshot/managed-clone-providers.ts b/src/lib/actions/sandbox/snapshot/managed-clone-providers.ts new file mode 100644 index 00000000000..f87189523da --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/managed-clone-providers.ts @@ -0,0 +1,666 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { randomBytes } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; + +import { cloneAndDeepFreeze } from "../../../core/immutable"; +import type { SandboxMessagingPlan } from "../../../messaging/manifest"; +import { isValidName, isValidProviderName } from "../../../name-validation"; +import { reportsExactProviderNotFound } from "../../../onboard/extra-provider-diagnostic-parser"; +import { + matchesGatewayCredentialOnlyProviderBinding, + parseGatewayProviderMetadata, +} from "../../../onboard/gateway-provider-metadata"; +import type { ManagedStartupProfile } from "../../../onboard/managed-startup/profile"; +import { normalizeRuntimeProviderIdentity } from "../../../onboard/runtime-provider/registry"; +import { deleteProviderWithRecovery } from "../../../onboard/sandbox-provider-cleanup"; +import type { PreparedManagedWorkloadCloneHandoff } from "../../../onboard/workload/clone"; +import { + captureSandboxRebuildAuthority, + type SandboxRebuildAuthority, + sandboxRebuildAuthorityMatchesEntry, +} from "../../../state/registry/rebuild-authority"; +import type { SandboxEntry } from "../../../state/registry/types"; +import * as sandboxState from "../../../state/sandbox"; + +const PROVIDER_PROBE_DIAGNOSTIC_LIMIT = 64 * 1024; +const PROVIDER_PROBE_TIMEOUT_MS = 5_000; +const PROVIDER_TYPE_PATTERN = /^[A-Za-z0-9._:-]{1,64}$/u; +const PROVIDER_ENV_KEY_PATTERN = /^[A-Z_][A-Z0-9_]{0,127}$/u; +const TRANSACTION_ID_PATTERN = /^[a-f0-9]{32}$/u; + +export type ManagedCloneProviderCommandResult = { + readonly status: number | null; + readonly stdout?: string | Buffer | null; + readonly stderr?: string | Buffer | null; + readonly error?: unknown; + readonly signal?: NodeJS.Signals | string | null; +}; + +export type ManagedCloneProviderRunner = ( + args: string[], + options?: { + readonly [key: string]: unknown; + readonly ignoreError?: boolean; + readonly env?: NodeJS.ProcessEnv; + readonly maxBuffer?: number; + readonly suppressOutput?: boolean; + readonly stdio?: ["ignore", "ignore" | "pipe", "ignore" | "pipe"]; + readonly timeout?: number; + }, +) => ManagedCloneProviderCommandResult; + +export interface ManagedCloneProviderBinding { + readonly providerName: string; + readonly providerType: string; + readonly providerEnvKey: string; + /** Provider-neutral contribution owner, for diagnostics only. */ + readonly source: string; +} + +export interface PreparedManagedCloneProvider { + readonly binding: ManagedCloneProviderBinding; + readonly action: "create" | "reuse-destination-owned"; +} + +export interface PreparedManagedCloneProviderTransaction { + readonly schemaVersion: 1; + readonly phase: "prepared"; + readonly transactionId: string; + readonly providerId: string; + readonly sourceSandboxName: string; + readonly destinationSandboxName: string; + readonly sourceRegistryAuthority: SandboxRebuildAuthority; + readonly snapshotRestoreAuthority: sandboxState.SnapshotRestoreAuthority; + readonly destinationRegistryAuthority?: SandboxRebuildAuthority; + readonly providers: readonly PreparedManagedCloneProvider[]; +} + +export interface ManagedCloneProviderOwnershipReceipt { + readonly binding: ManagedCloneProviderBinding; + readonly disposition: "created" | "reused-destination-owned"; +} + +export interface ManagedCloneProviderTransactionReceipt { + readonly schemaVersion: 1; + readonly phase: "materialized"; + readonly transactionId: string; + readonly providerId: string; + readonly destinationSandboxName: string; + readonly providers: readonly ManagedCloneProviderOwnershipReceipt[]; +} + +export type ManagedCloneProviderCleanupOutcome = + | "already-cleaned" + | "already-missing" + | "deleted" + | "delete-failed" + | "drift-preserved" + | "inspection-failed" + | "reused-preserved"; + +export interface ManagedCloneProviderCleanupResult { + readonly status: "complete" | "partial"; + readonly providers: readonly { + readonly providerName: string; + readonly outcome: ManagedCloneProviderCleanupOutcome; + }[]; +} + +type CaptureSnapshotRestoreAuthority = typeof sandboxState.captureSnapshotRestoreAuthority; +type ReadSandbox = (sandboxName: string) => SandboxEntry | null; + +export class ManagedCloneProviderTransactionError extends Error { + readonly partialReceipt?: ManagedCloneProviderTransactionReceipt; + readonly rollback?: ManagedCloneProviderCleanupResult; + + constructor( + message: string, + options: ErrorOptions & { + readonly partialReceipt?: ManagedCloneProviderTransactionReceipt; + readonly rollback?: ManagedCloneProviderCleanupResult; + } = {}, + ) { + super(`Managed clone provider transaction failed: ${message}`, options); + this.name = "ManagedCloneProviderTransactionError"; + this.partialReceipt = options.partialReceipt; + this.rollback = options.rollback; + } +} + +const issuedReceipts = new WeakSet(); +const completedCleanup = new WeakMap>(); + +function fail(message: string, cause?: unknown): never { + throw new ManagedCloneProviderTransactionError( + message, + cause === undefined ? undefined : { cause }, + ); +} + +function commandStreamText(value: string | Buffer | null | undefined): string { + return Buffer.isBuffer(value) ? value.toString("utf8") : (value ?? ""); +} + +type ProviderInspection = + | { readonly kind: "collision" } + | { readonly kind: "exact" } + | { readonly kind: "missing" }; + +function inspectProvider( + binding: ManagedCloneProviderBinding, + runOpenshell: ManagedCloneProviderRunner, +): ProviderInspection { + const result = runOpenshell(["provider", "get", binding.providerName], { + ignoreError: true, + maxBuffer: PROVIDER_PROBE_DIAGNOSTIC_LIMIT, + stdio: ["ignore", "pipe", "pipe"], + suppressOutput: true, + timeout: PROVIDER_PROBE_TIMEOUT_MS, + }); + if (result.error || result.signal || result.status !== 0) { + const output = `${commandStreamText(result.stdout)}\n${commandStreamText(result.stderr)}`; + if ( + !result.error && + !result.signal && + result.status === 1 && + reportsExactProviderNotFound(output, binding.providerName, PROVIDER_PROBE_DIAGNOSTIC_LIMIT) + ) { + return { kind: "missing" }; + } + fail( + `could not prove whether provider '${binding.providerName}' exists; ` + + "refusing destination mutation", + ); + } + + const metadata = parseGatewayProviderMetadata( + `${commandStreamText(result.stdout)}\n${commandStreamText(result.stderr)}`, + ); + return matchesGatewayCredentialOnlyProviderBinding(metadata, { + name: binding.providerName, + type: binding.providerType, + credentialKey: binding.providerEnvKey, + }) + ? { kind: "exact" } + : { kind: "collision" }; +} + +function validatedBinding(binding: ManagedCloneProviderBinding): ManagedCloneProviderBinding { + if (!isValidProviderName(binding.providerName)) { + fail(`provider name '${binding.providerName}' is invalid`); + } + if (!PROVIDER_TYPE_PATTERN.test(binding.providerType)) { + fail(`provider '${binding.providerName}' has an invalid type`); + } + if (!PROVIDER_ENV_KEY_PATTERN.test(binding.providerEnvKey)) { + fail(`provider '${binding.providerName}' has an invalid credential binding`); + } + if ( + typeof binding.source !== "string" || + binding.source.trim() === "" || + binding.source !== binding.source.trim() || + Buffer.byteLength(binding.source, "utf8") > 128 + ) { + fail(`provider '${binding.providerName}' has an invalid contribution owner`); + } + return { ...binding }; +} + +function mergeBindings( + bindings: readonly ManagedCloneProviderBinding[], +): readonly ManagedCloneProviderBinding[] { + const merged = new Map(); + for (const candidate of bindings) { + const binding = validatedBinding(candidate); + const existing = merged.get(binding.providerName); + if ( + existing && + (existing.providerType !== binding.providerType || + existing.providerEnvKey !== binding.providerEnvKey) + ) { + fail(`provider '${binding.providerName}' has conflicting desired bindings`); + } + if (!existing) merged.set(binding.providerName, binding); + } + return [...merged.values()]; +} + +function activeMessagingCredentialBindings( + plan: SandboxMessagingPlan | null | undefined, + expectedSandboxName: string, + expectedAgent: string | null | undefined, +): readonly SandboxMessagingPlan["credentialBindings"][number][] { + if (!plan) return []; + if ( + plan.sandboxName !== expectedSandboxName || + (expectedAgent !== null && expectedAgent !== undefined && plan.agent !== expectedAgent) + ) { + fail(`messaging provider ownership does not belong to '${expectedSandboxName}'`); + } + const activeChannels = new Set( + plan.channels + .filter( + (channel) => + channel.active && !channel.disabled && !plan.disabledChannels.includes(channel.channelId), + ) + .map((channel) => channel.channelId), + ); + return plan.credentialBindings.filter((binding) => activeChannels.has(binding.channelId)); +} + +function applicationBindings(input: { + readonly profile: ManagedStartupProfile; + readonly messagingPlan: SandboxMessagingPlan | null | undefined; + readonly sandboxName: string; +}): readonly ManagedCloneProviderBinding[] { + const bindings: ManagedCloneProviderBinding[] = activeMessagingCredentialBindings( + input.messagingPlan, + input.sandboxName, + input.profile.agent, + ).map((binding) => ({ + providerName: binding.providerName, + providerType: "generic", + providerEnvKey: binding.providerEnvKey, + source: "messaging", + })); + if (input.profile.agentConfig.agent !== "langchain-deepagents-code") { + const webSearch = input.profile.agentConfig.webSearch; + if (webSearch.enabled) { + bindings.push({ + providerName: `${input.sandboxName}-${webSearch.provider}-search`, + providerType: + input.profile.agent === "hermes" && webSearch.provider === "tavily" + ? "nemoclaw-hermes-tavily" + : webSearch.provider, + providerEnvKey: webSearch.provider === "tavily" ? "TAVILY_API_KEY" : "BRAVE_API_KEY", + source: "web-search", + }); + } + } + return mergeBindings(bindings); +} + +function destinationOwnedBindings(entry: SandboxEntry): readonly ManagedCloneProviderBinding[] { + const bindings: ManagedCloneProviderBinding[] = activeMessagingCredentialBindings( + entry.messaging?.plan, + entry.name, + entry.agent, + ).map((binding) => ({ + providerName: binding.providerName, + providerType: "generic", + providerEnvKey: binding.providerEnvKey, + source: "messaging", + })); + if (entry.webSearchEnabled === true && entry.webSearchProvider) { + bindings.push({ + providerName: `${entry.name}-${entry.webSearchProvider}-search`, + providerType: + entry.agent === "hermes" && entry.webSearchProvider === "tavily" + ? "nemoclaw-hermes-tavily" + : entry.webSearchProvider, + providerEnvKey: entry.webSearchProvider === "tavily" ? "TAVILY_API_KEY" : "BRAVE_API_KEY", + source: "web-search", + }); + } + return mergeBindings(bindings); +} + +function sameBinding( + left: ManagedCloneProviderBinding, + right: ManagedCloneProviderBinding, +): boolean { + return ( + left.providerName === right.providerName && + left.providerType === right.providerType && + left.providerEnvKey === right.providerEnvKey + ); +} + +function hasCredential(environment: NodeJS.ProcessEnv, envKey: string): boolean { + const value = environment[envKey]; + return typeof value === "string" && value.replace(/\r/gu, "").trim().length > 0; +} + +function requireTransactionId(value: string | undefined): string { + const transactionId = value ?? randomBytes(16).toString("hex"); + if (!TRANSACTION_ID_PATTERN.test(transactionId)) fail("transaction identity is invalid"); + return transactionId; +} + +type CloneProviderHandoff = Pick< + PreparedManagedWorkloadCloneHandoff, + | "destinationSandboxName" + | "messaging" + | "providerId" + | "rebound" + | "snapshotRestoreAuthority" + | "sourceRegistryAuthority" + | "sourceSandboxName" +>; + +/** + * Build one inert provider transaction from the deep-frozen clone handoff. + * Existing exact providers are reusable only when the destination registry + * independently proves the same logical binding. They are never rewritten: + * credential rotation remains a separate explicit operation with its own + * recovery contract. + */ +export function prepareManagedCloneProviderTransaction(input: { + readonly handoff: CloneProviderHandoff; + readonly destination: SandboxEntry | null; + readonly additionalBindings?: readonly ManagedCloneProviderBinding[]; + readonly resolveAdditionalDestinationOwnedBindings?: ( + destination: Readonly, + ) => readonly ManagedCloneProviderBinding[]; + readonly environment?: NodeJS.ProcessEnv; + readonly runOpenshell: ManagedCloneProviderRunner; + readonly transactionId?: string; +}): PreparedManagedCloneProviderTransaction { + const destinationSandboxName = input.handoff.destinationSandboxName; + if ( + !isValidName(input.handoff.sourceSandboxName) || + !isValidName(destinationSandboxName) || + input.handoff.sourceSandboxName === destinationSandboxName + ) { + fail("clone sandbox identity is invalid"); + } + if ( + input.handoff.sourceRegistryAuthority.sandboxName !== input.handoff.sourceSandboxName || + input.handoff.sourceRegistryAuthority.providerId !== input.handoff.providerId + ) { + fail("clone handoff registry authority does not match its provider and source identity"); + } + if (input.destination && input.destination.name !== destinationSandboxName) { + fail("destination registry authority names a different sandbox"); + } + const desired = mergeBindings([ + ...applicationBindings({ + profile: input.handoff.rebound.profile, + messagingPlan: input.handoff.messaging?.plan, + sandboxName: destinationSandboxName, + }), + ...(input.additionalBindings ?? []), + ]); + const owned = input.destination + ? mergeBindings([ + ...destinationOwnedBindings(input.destination), + ...(input.resolveAdditionalDestinationOwnedBindings?.(input.destination) ?? []), + ]) + : []; + const environment = input.environment ?? process.env; + const providers: PreparedManagedCloneProvider[] = []; + for (const binding of desired) { + if (!hasCredential(environment, binding.providerEnvKey)) { + fail( + `${binding.source} provider '${binding.providerName}' requires an explicit clone ` + + `credential in ${binding.providerEnvKey}`, + ); + } + const inspection = inspectProvider(binding, input.runOpenshell); + if (inspection.kind === "collision") { + fail(`provider '${binding.providerName}' has an incompatible live binding`); + } + if (inspection.kind === "exact") { + if (!input.destination || !owned.some((candidate) => sameBinding(candidate, binding))) { + fail( + `provider '${binding.providerName}' exists without exact destination ownership; ` + + "refusing credential reuse", + ); + } + providers.push({ binding, action: "reuse-destination-owned" }); + continue; + } + providers.push({ binding, action: "create" }); + } + + let destinationRegistryAuthority: SandboxRebuildAuthority | undefined; + if (input.destination) { + try { + destinationRegistryAuthority = captureSandboxRebuildAuthority( + input.destination, + normalizeRuntimeProviderIdentity(input.destination.openshellDriver), + ); + } catch (error) { + fail("destination has no exact managed registry authority", error); + } + } + + return cloneAndDeepFreeze({ + schemaVersion: 1 as const, + phase: "prepared" as const, + transactionId: requireTransactionId(input.transactionId), + providerId: input.handoff.providerId, + sourceSandboxName: input.handoff.sourceSandboxName, + destinationSandboxName, + sourceRegistryAuthority: structuredClone(input.handoff.sourceRegistryAuthority), + snapshotRestoreAuthority: structuredClone(input.handoff.snapshotRestoreAuthority), + ...(destinationRegistryAuthority === undefined ? {} : { destinationRegistryAuthority }), + providers, + }); +} + +/** Revalidate every durable authority at the last safe edge before mutation. */ +export function revalidateManagedCloneMutationAuthority( + prepared: PreparedManagedCloneProviderTransaction, + input: { + readonly readSandbox: ReadSandbox; + readonly captureSnapshotRestoreAuthority?: CaptureSnapshotRestoreAuthority; + }, +): void { + const source = input.readSandbox(prepared.sourceSandboxName); + if (!sandboxRebuildAuthorityMatchesEntry(prepared.sourceRegistryAuthority, source)) { + fail("source registry authority changed before mutation"); + } + const currentDestination = input.readSandbox(prepared.destinationSandboxName); + if (prepared.destinationRegistryAuthority) { + if ( + !sandboxRebuildAuthorityMatchesEntry( + prepared.destinationRegistryAuthority, + currentDestination, + ) + ) { + fail("destination registry authority changed before mutation"); + } + } else if (currentDestination !== null) { + fail("destination appeared after clone preflight"); + } + const capture = + input.captureSnapshotRestoreAuthority ?? sandboxState.captureSnapshotRestoreAuthority; + const content = capture(prepared.snapshotRestoreAuthority.backupPath); + if (!content || !isDeepStrictEqual(content, prepared.snapshotRestoreAuthority)) { + fail("selected snapshot content changed before mutation"); + } +} + +function issueReceipt( + prepared: PreparedManagedCloneProviderTransaction, + providers: readonly ManagedCloneProviderOwnershipReceipt[], +): ManagedCloneProviderTransactionReceipt { + const receipt = cloneAndDeepFreeze({ + schemaVersion: 1 as const, + phase: "materialized" as const, + transactionId: prepared.transactionId, + providerId: prepared.providerId, + destinationSandboxName: prepared.destinationSandboxName, + providers: providers.map((provider) => ({ + binding: { ...provider.binding }, + disposition: provider.disposition, + })), + }); + issuedReceipts.add(receipt); + completedCleanup.set(receipt, new Set()); + return receipt; +} + +/** + * Materialize missing providers under one process-local ownership ledger. + * A non-zero create followed by an exact provider is explicitly ambiguous: + * it is preserved and never claimed by this transaction. + */ +export function provisionManagedCloneProviderTransaction( + prepared: PreparedManagedCloneProviderTransaction, + input: { + readonly environment?: NodeJS.ProcessEnv; + readonly runOpenshell: ManagedCloneProviderRunner; + readonly readSandbox: ReadSandbox; + readonly captureSnapshotRestoreAuthority?: CaptureSnapshotRestoreAuthority; + }, +): ManagedCloneProviderTransactionReceipt { + const environment = input.environment ?? process.env; + const confirmed: ManagedCloneProviderOwnershipReceipt[] = []; + try { + // The transaction boundary must still fence a clone with no credential + // providers (for example DCode) before a later caller proceeds to sandbox + // or filesystem mutation. + revalidateManagedCloneMutationAuthority(prepared, input); + for (const provider of prepared.providers) { + revalidateManagedCloneMutationAuthority(prepared, input); + const current = inspectProvider(provider.binding, input.runOpenshell); + if (provider.action === "reuse-destination-owned") { + if (current.kind !== "exact") { + fail(`destination-owned provider '${provider.binding.providerName}' changed before use`); + } + confirmed.push({ + binding: provider.binding, + disposition: "reused-destination-owned", + }); + continue; + } + if (current.kind !== "missing") { + fail(`provider '${provider.binding.providerName}' appeared after preflight`); + } + const credential = environment[provider.binding.providerEnvKey]?.replace(/\r/gu, "").trim(); + if (!credential) { + fail(`credential ${provider.binding.providerEnvKey} disappeared before provider creation`); + } + let result: ManagedCloneProviderCommandResult; + try { + result = input.runOpenshell( + [ + "provider", + "create", + "--name", + provider.binding.providerName, + "--type", + provider.binding.providerType, + "--credential", + provider.binding.providerEnvKey, + ], + { + ignoreError: true, + env: { [provider.binding.providerEnvKey]: credential }, + stdio: ["ignore", "pipe", "pipe"], + suppressOutput: true, + }, + ); + } catch (error) { + // A thrown child-process adapter can still mean the gateway committed + // the create. Reconcile by exact metadata and preserve it as unowned. + result = { status: null, error }; + } + const reconciled = inspectProvider(provider.binding, input.runOpenshell); + if (result.status !== 0 || result.error || result.signal) { + const state = reconciled.kind === "exact" ? "exact but unowned" : reconciled.kind; + fail( + `create for provider '${provider.binding.providerName}' had an ambiguous result ` + + `(${state}); preserving the observed provider`, + ); + } + if (reconciled.kind !== "exact") { + fail( + `provider '${provider.binding.providerName}' was not exact after successful create; ` + + "preserving the observed state", + ); + } + confirmed.push({ binding: provider.binding, disposition: "created" }); + } + return issueReceipt(prepared, confirmed); + } catch (cause) { + const partialReceipt = issueReceipt(prepared, confirmed); + const rollback = cleanupManagedCloneProviderTransaction(partialReceipt, input.runOpenshell); + const detail = cause instanceof Error ? cause.message : String(cause); + throw new ManagedCloneProviderTransactionError(detail, { + cause, + partialReceipt, + rollback, + }); + } +} + +/** + * Idempotently clean only providers confirmed by this exact in-process + * receipt. Once a name is cleaned, the ledger never re-inspects it, preventing + * a repeated cleanup from deleting a later same-name provider. + */ +export function cleanupManagedCloneProviderTransaction( + receipt: ManagedCloneProviderTransactionReceipt, + runOpenshell: ManagedCloneProviderRunner, +): ManagedCloneProviderCleanupResult { + if (!issuedReceipts.has(receipt)) { + fail("cleanup requires the exact process-local ownership receipt"); + } + const cleaned = completedCleanup.get(receipt); + if (!cleaned) fail("cleanup ownership ledger is unavailable"); + const outcomes: Array<{ + providerName: string; + outcome: ManagedCloneProviderCleanupOutcome; + }> = []; + for (const provider of [...receipt.providers].reverse()) { + const providerName = provider.binding.providerName; + if (provider.disposition === "reused-destination-owned") { + outcomes.push({ providerName, outcome: "reused-preserved" }); + continue; + } + if (cleaned.has(providerName)) { + outcomes.push({ providerName, outcome: "already-cleaned" }); + continue; + } + let inspection: ProviderInspection; + try { + inspection = inspectProvider(provider.binding, runOpenshell); + } catch { + outcomes.push({ providerName, outcome: "inspection-failed" }); + continue; + } + if (inspection.kind === "missing") { + cleaned.add(providerName); + outcomes.push({ providerName, outcome: "already-missing" }); + continue; + } + if (inspection.kind === "collision") { + outcomes.push({ providerName, outcome: "drift-preserved" }); + continue; + } + const deletion = deleteProviderWithRecovery(providerName, { + runOpenshell, + allowedSandboxes: [receipt.destinationSandboxName], + }); + if (!deletion.ok) { + outcomes.push({ providerName, outcome: "delete-failed" }); + continue; + } + try { + if (inspectProvider(provider.binding, runOpenshell).kind !== "missing") { + outcomes.push({ providerName, outcome: "delete-failed" }); + continue; + } + } catch { + outcomes.push({ providerName, outcome: "inspection-failed" }); + continue; + } + cleaned.add(providerName); + outcomes.push({ providerName, outcome: "deleted" }); + } + return cloneAndDeepFreeze({ + status: outcomes.every((result) => + ["already-cleaned", "already-missing", "deleted", "reused-preserved"].includes( + result.outcome, + ), + ) + ? ("complete" as const) + : ("partial" as const), + providers: outcomes, + }); +} diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 19c6ebd8de6..54197352d28 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -117,7 +117,7 @@ runtime mutation | **`--fresh` onboard** — `resolveOnboardEntryOptions`, `prepareFreshSession`, `createBaseImageResolutionContext` | Current flags/environment/prompts replace resumable intent. `--fresh` disables auto-resume and forces base-image resolution; it does not prove that the selected sandbox name is unused. | The first destructive effect is local: the prior onboard session is cleared before a new session is saved. A matching live sandbox can later reuse or recreate through the normal sandbox decision; `--fresh` does not itself delete it. | The new session and machine snapshot replace the old resume checkpoint. Credential and effect boundaries then match new onboard or live recreate. | The discarded resume checkpoint is not restored on later failure. Covered by `entry-options.test.ts`, `session-bootstrap.test.ts`, and base-image resolution tests. | | **Resume, re-onboard, or recreate** — `onboard()`, `prepareOnboardSession`, `decideSandboxResume`, live-sandbox handling in `createSandbox` | For `--resume`, the recorded session is authoritative and conflicting current name/provider/model/image/tool-disclosure hints are rejected. A new re-onboard run takes current flags, environment, and prompts as intent while registry/gateway state provides drift evidence. The machine resolves a complete secret-free create intent, including policy, messaging/provider, GPU, resource, disabled-channel, and agent inputs, before repair/removal or live recreation. | Ordinary live recreation conditionally backs up before provider cleanup, **delete**, image removal, and registry removal. A selected pre-upgrade backup suppresses a new one; an explicit override permits recreation without backup. Resume registry removal and `repair-and-recreate` now follow complete intent validation. Temporary policy/build artifacts remain materialization effects after the delete boundary. | Resume continues the recorded session/machine snapshot; non-resume re-onboard writes a new session first. OpenClaw records completed sandbox name, web search, messaging, and resource choices with explicit progress markers, including explicit `null` choices, while the complete create intent stays process-local and is not persisted or emitted. Raw credential values remain outside the session. A missing process value can be rebound only when the same OpenClaw session recorded successfully registering that provider and its live name, type, and credential-key binding still match; otherwise interactive resume requests it again and non-interactive resume exits with environment-variable guidance. Credentials are checked before mutation and again immediately before materialization. | Restore failures warn and can still publish the replacement; managed-DCode live-selection failure leaves a running, unregistered sandbox with manual-delete guidance. Checkpoint replay reuses an exact live sandbox after an interrupted create and backfills missing create/register receipts. Cancel rollback is not armed and there is no rebuild-style receipt rollback. Coverage: transition traces, create-intent characterization, checkpoint replay and resume guards, and sandbox-handler crash recovery. Gaps: early backup asymmetry and no rebuild-style cross-effect rollback. | | **Rebuild or installer-driven upgrade** — `rebuildSandbox` in `rebuild-pipeline.ts`; `upgradeSandboxes` | Registry state is authoritative. A matching session may fill guarded legacy gaps only when its selection agrees; an unrelated/global session is never used. Ambient provider/model selection is quarantined by `isolateAmbientRecreateEnv`, apart from narrowly scoped legacy recovery. Preflight assembles target config, messaging/policy/runtime inputs, recovery inputs, and a retained replacement context. Generic agents use `preflightRebuildImage`; DCode uses its specialized managed-context preflight instead and proves the live route only for normal live rebuild. Resource profile is not part of preflight. | Consent persistence, target-gateway selection/recovery, and target-preflight registry updates can precede disposable image build/probes. Backup is the first durable recovery checkpoint when available. Shields unlock, MCP detach/scrub, and NIM stop are destructive in-place effects before the **sandbox delete** boundary. Prepared context and mutation-edge conditions are rechecked before delete, proving buildability/input identity but not replacement health or atomic swap. | Durable checkpoints are the backup/recovery manifest when one exists and the rewritten recreate session; stale recovery can reach deletion without a manifest, making that session its first new durable checkpoint. Rollback receipts/snapshots are process-local. Credential metadata comes from the target or guarded fallback; raw credentials/providers are checked against current process/gateway state, while prepared installer recovery may reconstruct a missing gateway provider from a validated host credential. | In-process rollback best-effort restores registry/MCP retry metadata, but process death after non-MCP delete can still lose it. The inner onboard selects resource profile after deletion from non-quarantined ambient input. Covered by rebuild, image-preflight, DCode, and messaging tests. Gaps: post-delete resource intent plus health-before-delete and atomic swap. Closed issue #5801 records the original gap; #6835 fixed only the printed recovery path. | -| **Managed snapshot clone handoff (internal and dormant)** — `prepareManagedWorkloadCloneHandoff` | The current source registry row owns mutable operator intent; the selected snapshot owns immutable managed-workload and provider-runtime history. Preparation proves the selected runtime provider and its `clone` capability, exact current registry generation and live-identity fingerprint, snapshot/source workload equivalence, snapshot runtime generation, and the state layer's selected-manifest/payload digest. It then rebinds the secret-free startup profile, messaging intent, dashboard identity, and any provider-owned Hermes inference name for OpenClaw, Hermes, or DCode without a central Podman-specific switch. | None. The handoff is an inert planning artifact and performs no provider, sandbox, registry, filesystem, credential, or broker effect. Production snapshot restore continues to reject cross-sandbox managed-image restore through `rejectManagedSnapshotCloneUntilRebind`; no user-visible clone support is advertised by this slice. | The returned handoff is a deeply frozen, locally owned value carrying exact source registry compare-and-swap authority, immutable workload authority, provider runtime generation evidence, `SnapshotRestoreAuthority` content identity, rebound managed profile, and destination registry intent. It contains credential-presence metadata and provider names, never raw credential values or live handles. | There is intentionally no compensation because preparation has no effects. All-agent and Docker/MXC-style provider tests cover the dormant contract and canonical name boundaries. Provider materialization, destination creation/bootstrap, mutation-edge content/provider authority revalidation, rollback, recovery, protected E2E, and activation remain later slices. | +| **Managed snapshot clone handoff and provider transaction (internal and dormant)** — `prepareManagedWorkloadCloneHandoff`; `prepareManagedCloneProviderTransaction` | The current source registry row owns mutable operator intent; the selected snapshot owns immutable managed-workload and provider-runtime history. Handoff preparation proves the selected runtime provider and its `clone` capability, exact current registry generation and live-identity fingerprint, snapshot/source workload equivalence, snapshot runtime generation, and the state layer's selected-manifest/payload digest. It rebinds the secret-free startup profile, messaging intent, dashboard identity, and provider-owned contributions for OpenClaw, Hermes, or DCode without a central Podman-specific switch. Provider preparation then resolves active application bindings plus provider-contributed bindings, treating a live exact provider as reusable only when the destination registry independently proves that same logical binding. | The handoff and provider plan are inert. The internal materializer can create only bindings proven absent at preflight; it never updates or deletes an existing destination-owned provider. Immediately before each create it revalidates the source and optional destination registry rows plus the exact `SnapshotRestoreAuthority`. Production snapshot restore does not invoke this transaction and continues to reject cross-sandbox managed-image restore through `rejectManagedSnapshotCloneUntilRebind`; no user-visible clone support is advertised. | Both plans are deeply frozen and secret-free. A successful create produces an exact process-local ownership receipt; a non-zero create reconciled to an exact provider remains ambiguous and unowned. The receipt ledger remembers completed cleanup so a repeated cleanup cannot delete a later same-name provider. Raw credentials exist only in the explicit apply environment and one OpenShell child environment. | Failure rolls back only providers confirmed created by the exact in-process receipt, preserves collisions and ambiguous creates, reports incomplete cleanup for retry, and never rewrites a reused provider. All-agent and Docker/MXC-style handoff tests plus provider race, force-replace, disappearing-credential, rollback, and idempotent-cleanup tests cover the dormant contract. Destination creation/bootstrap, filesystem mutation-edge invocation, Hermes broker activation, durable recovery, protected E2E, and user-visible activation remain later slices. | | **Channel add/remove/start/stop** — `addSandboxChannel`, `removeSandboxChannel`, `sandboxChannelsSetEnabled` in `policy-channel.ts` | Add compiles and merges a manifest-derived channel delta with `MessagingWorkflowPlanner`. Start, stop, and remove transform the registry plan and rehydrate executable render/build/runtime/forward details from current manifests. | Token-backed add can mutate gateway credentials before policy and plan persistence; QR/in-sandbox-auth add skips that credential upsert. Start persists the enabled plan before policy; stop persists the disabled plan before the rebuild prompt. Remove clears QR-backed durable state when applicable, detaches gateway/bridge state, removes policy, then persists the plan. A queued rebuild has a separate delete boundary. | The compact registry messaging plan is authoritative; render/build/runtime/state/health entries and nested host-forward details are rehydrated rather than persisted. Session policy-preset sync is best-effort, and channel mutations do not rewrite `Session.messagingPlan`. Raw tokens stay in process/gateway bindings. | `rollbackChannelAdd`, re-disable after failed start, and fail-closed QR-state cleanup provide partial compensation. Covered by `policy-channel*.test.ts`, `workflow-planner.test.ts`, and channel integration tests. Gaps: channel add has a separate `--force` conflict policy; add/remove effects can precede plan persistence, and persistence failures are not fully rolled back. | | **Provider, model, or credential-binding change** — `runInferenceSet` | CLI intent plus registry/session metadata. Target resolution and OpenShell preparation occur before locking. The target is re-resolved in the mutating phase under the sandbox lifecycle and timer-bound shields locks; that phase validates provider/model syntax, selected agent, shields state, and local reachability before the first write. | First mutation is the gateway route, then a minimal registry write, API-family/config resolution, registry refresh, best-effort config/hash sync, matching-session update, and audit. An OpenClaw API-family change can then restart the managed gateway after the shields lock is released but while the outer sandbox lock remains held. No sandbox deletion. | Registry and matching session store logical provider/model/credential-environment metadata. Audit records the action, sandbox, and reason rather than credentials; raw values remain gateway-bound. | Forward-only; no rollback. `rebuild` is the repair path for degraded state. Covered by `inference-set*.test.ts`. Gap: several stores can diverge after a mid-sequence failure. | | **Credential rotation** — `configRotateToken` in `src/lib/sandbox/config.ts` | A session with `credentialEnv` selects the provider and binding. A non-null different `sandboxName` is rejected, but a legacy/null session name is accepted for the requested sandbox. The new value comes from a named environment variable, stdin, or a secret prompt; it is trimmed, then rejected when empty or still containing internal whitespace. | `saveCredential` first stages the value in the current process. OpenShell provider update is the first external mutation, with provider create as a fallback; audit follows. No sandbox deletion. | The logical binding is unchanged, so session and registry are not rewritten. The raw value exists only in process memory/environment and the gateway provider; audit records action/sandbox/reason without the value. | No rollback after a successful provider update; an audit failure can report failure after the credential is already active. Covered by the rotate-token cases in `test/config-set-nested-ssrf.test.ts`. Gap: a null-name legacy session is not strongly bound to the requested sandbox. | From 6d4d858a7fe10ab62025e77a0e083e234f339883 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 19:28:46 -0700 Subject: [PATCH 052/117] test(snapshot): satisfy provider transaction CLI typings Signed-off-by: Aaron Erickson --- .../actions/sandbox/snapshot-managed-clone-providers.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts b/src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts index 7fc2df7a9c1..b187c2ba1c0 100644 --- a/src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts +++ b/src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts @@ -109,7 +109,7 @@ function messagingPlan(sandboxName: string): SandboxMessagingPlan { runtimeSetup: { nodePreloads: [], envAliases: [], secretScans: [] }, stateUpdates: [], healthChecks: [], - } as SandboxMessagingPlan; + } as unknown as SandboxMessagingPlan; } function handoff( From cbecf7b22bb5f7907c06b1c37abbc83469f4354a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 03:57:03 -0700 Subject: [PATCH 053/117] feat(snapshot): stage Hermes clone broker activation Signed-off-by: Aaron Erickson --- .../host/runtime-refresh-credentials.ts | 50 ++ agents/hermes/host/tool-gateway-broker.ts | 537 +++++++++++++++++- .../host/tool-gateway-control-contract.ts | 59 ++ ...apshot-hermes-managed-clone-broker.test.ts | 314 ++++++++++ .../snapshot/hermes-managed-clone-broker.ts | 283 +++++++++ .../snapshot/managed-clone-providers.ts | 12 +- src/lib/hermes-tool-gateway-broker.ts | 531 ++++++++++++++++- src/lib/hermes-tool-gateway-clone-broker.ts | 55 ++ .../hermes-tool-gateway-broker-fixture.ts | 41 ++ test/hermes-tool-gateway-broker.test.ts | 445 ++++++++++++++- ...s-tool-gateway-runtime-credentials.test.ts | 56 ++ 11 files changed, 2343 insertions(+), 40 deletions(-) create mode 100644 agents/hermes/host/runtime-refresh-credentials.ts create mode 100644 agents/hermes/host/tool-gateway-control-contract.ts create mode 100644 src/lib/actions/sandbox/snapshot-hermes-managed-clone-broker.test.ts create mode 100644 src/lib/actions/sandbox/snapshot/hermes-managed-clone-broker.ts create mode 100644 src/lib/hermes-tool-gateway-clone-broker.ts create mode 100644 test/helpers/hermes-tool-gateway-broker-fixture.ts create mode 100644 test/hermes-tool-gateway-runtime-credentials.test.ts diff --git a/agents/hermes/host/runtime-refresh-credentials.ts b/agents/hermes/host/runtime-refresh-credentials.ts new file mode 100644 index 00000000000..f257a5a40ab --- /dev/null +++ b/agents/hermes/host/runtime-refresh-credentials.ts @@ -0,0 +1,50 @@ +// @ts-nocheck +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Process-memory-only refresh credentials for the shared Hermes tool broker. + * + * Durable state carries only hashes. Each sandbox identity owns one in-memory + * value, so adding or removing a clone cannot replace another sandbox's + * credential even when every sandbox shares the same broker listener. + */ +class RuntimeRefreshCredentialStore { + constructor(hashCredential) { + this.hashCredential = hashCredential; + this.credentials = new Map(); + } + + register(state, refreshToken) { + const sandbox = String(state?.sandbox || "").trim(); + const expectedHash = String(state?.refresh_token_sha256 || "").trim(); + const normalized = String(refreshToken || "").trim(); + if (!sandbox || !expectedHash || !normalized) return false; + if (this.hashCredential(normalized) !== expectedHash) return false; + this.credentials.set(sandbox, normalized); + return true; + } + + resolve(state) { + const sandbox = String(state?.sandbox || "").trim(); + const expectedHash = String(state?.refresh_token_sha256 || "").trim(); + const refreshToken = this.credentials.get(sandbox); + if (!sandbox || !expectedHash || !refreshToken) return null; + if (this.hashCredential(refreshToken) !== expectedHash) { + this.credentials.delete(sandbox); + return null; + } + return refreshToken; + } + + rotate(state, nextRefreshToken) { + return this.register(state, nextRefreshToken); + } + + unregister(sandboxName) { + const sandbox = String(sandboxName || "").trim(); + return sandbox ? this.credentials.delete(sandbox) : false; + } +} + +module.exports = { RuntimeRefreshCredentialStore }; diff --git a/agents/hermes/host/tool-gateway-broker.ts b/agents/hermes/host/tool-gateway-broker.ts index 9cc52cd5b02..d2da24381ad 100755 --- a/agents/hermes/host/tool-gateway-broker.ts +++ b/agents/hermes/host/tool-gateway-broker.ts @@ -9,8 +9,9 @@ * * Hermes managed tools need a Nous subscription credential, but the sandbox * must not own raw Nous OAuth state. NemoClaw stores the refresh credential in - * OpenShell provider storage, gives the sandbox only an OpenShell resolver - * placeholder, and keeps the raw refresh token in this host process after OAuth + * host-broker memory, gives the sandbox only an opaque per-sandbox broker + * credential through OpenShell provider storage, and keeps the raw refresh + * token in this host process after OAuth * onboarding. The broker refreshes on the host with x-nous-refresh-token, * injects a short-lived access token upstream, and persists only credential * hashes so rotated refresh tokens can update OpenShell without writing raw @@ -22,6 +23,15 @@ const fs = require("fs"); const http = require("http"); const path = require("path"); const { spawnSync } = require("child_process"); +const { RuntimeRefreshCredentialStore } = require("./runtime-refresh-credentials.ts"); +const { + boundedControlDeadline, + isValidActivationToken, + isValidControlRequestId, + isValidName, + isValidProviderName, + remainingControlTime, +} = require("./tool-gateway-control-contract.ts"); const PORT = parseInt(process.env.HERMES_TOOL_GATEWAY_PORT || "11436", 10); const STATE_DIR = process.env.HERMES_TOOL_GATEWAY_STATE_DIR; @@ -36,6 +46,8 @@ const OPENSHELL_BIN = process.env.NEMOCLAW_OPENSHELL_BIN || "openshell"; const CREDENTIAL_ENV = process.env.HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV || "NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN"; +const CONTROL_SOCKET_PATH = process.env.HERMES_TOOL_GATEWAY_CONTROL_SOCKET || ""; +const PREFLIGHT_PROBE = process.env.HERMES_TOOL_GATEWAY_PREFLIGHT_PROBE === "1"; const HERMES_INFERENCE_PROVIDER_NAME = process.env.HERMES_INFERENCE_PROVIDER_NAME || "hermes-provider"; const HERMES_INFERENCE_CREDENTIAL_ENV = @@ -62,6 +74,7 @@ const UPSTREAM_REQUEST_TIMEOUT_MS = readPositiveIntEnv( 60_000, 1000, ); +const STAGED_CLONE_BINDING_TTL_MS = 5 * 60 * 1000; const DEFAULT_INFERENCE_BASE_URL = "https://inference-api.nousresearch.com/v1"; const TRUSTED_INFERENCE_BASE_URLS = new Set([DEFAULT_INFERENCE_BASE_URL]); @@ -101,11 +114,15 @@ const TOKEN_HEADERS = [ ]; const accessTokenCache = new Map(); +const stagedCloneBindings = new Map(); +const stagedCloneRequests = new Map(); function sha256(value) { return crypto.createHash("sha256").update(String(value)).digest("hex"); } +const runtimeRefreshCredentials = new RuntimeRefreshCredentialStore(sha256); + function loadMatrix() { try { const matrix = JSON.parse(fs.readFileSync(MATRIX_PATH, "utf8")); @@ -147,6 +164,12 @@ function loadStateFile(file) { } } +function loadStateForSandbox(sandboxName) { + const sandbox = String(sandboxName || "").trim(); + if (!isValidName(sandbox)) return null; + return loadStateFile(path.join(STATE_DIR, `${sandbox}.json`)); +} + function findStateByRefreshToken(refreshToken) { const digest = sha256(refreshToken); for (const file of stateFiles()) { @@ -208,17 +231,30 @@ function extractRefreshToken(req) { } function resolveRuntimeRefreshToken(loaded) { + return runtimeRefreshCredentials.resolve(loaded?.state); +} + +function registerInitialRuntimeRefreshCredential() { const refreshToken = String(process.env[CREDENTIAL_ENV] || "").trim(); - if (!refreshToken) { - return null; - } - const expectedHash = String(loaded?.state?.refresh_token_sha256 || ""); - if (!expectedHash || !timingSafeEqualString(expectedHash, sha256(refreshToken))) { - return null; + if (!refreshToken) return; + const exactSandbox = String(process.env.HERMES_TOOL_GATEWAY_INITIAL_SANDBOX || "").trim(); + const digest = sha256(refreshToken); + for (const file of stateFiles()) { + const loaded = loadStateFile(file); + if ( + loaded && + (!exactSandbox || loaded.state.sandbox === exactSandbox) && + timingSafeEqualString(String(loaded.state.refresh_token_sha256 || ""), digest) + ) { + runtimeRefreshCredentials.register(loaded.state, refreshToken); + } } - return refreshToken; + delete process.env[CREDENTIAL_ENV]; + delete process.env.HERMES_TOOL_GATEWAY_INITIAL_SANDBOX; } +registerInitialRuntimeRefreshCredential(); + function parseRoute(reqUrl) { const url = new URL(reqUrl || "/", "http://broker.local"); const parts = url.pathname.split("/").filter(Boolean); @@ -263,13 +299,17 @@ function atomicWriteJson(file, value) { function updateOpenshellRefreshProvider(state, refreshToken) { const providerName = String(state.provider_name || ""); if (!providerName) return; + const providerCredential = + typeof state.broker_token === "string" && state.broker_token + ? state.broker_token + : refreshToken; const result = spawnSync( OPENSHELL_BIN, ["provider", "update", providerName, "--credential", CREDENTIAL_ENV], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], - env: { ...process.env, [CREDENTIAL_ENV]: refreshToken }, + env: { ...process.env, [CREDENTIAL_ENV]: providerCredential }, timeout: 30_000, }, ); @@ -280,11 +320,23 @@ function updateOpenshellRefreshProvider(state, refreshToken) { } } -function updateOpenshellInferenceProvider(apiKey, baseUrl) { +function operationTimeout(deadlineAtMs, capMs = UPSTREAM_REQUEST_TIMEOUT_MS) { + if (deadlineAtMs === undefined || deadlineAtMs === null) return capMs; + const timeout = remainingControlTime(deadlineAtMs, capMs); + if (timeout === 0) { + throw Object.assign(new Error("clone_control_deadline_exceeded"), { + code: "clone_control_deadline_exceeded", + }); + } + return timeout; +} + +function updateOpenshellInferenceProvider(state, apiKey, baseUrl, deadlineAtMs = null) { + const providerName = String(state.inference_provider_name || HERMES_INFERENCE_PROVIDER_NAME); const args = [ "provider", "update", - HERMES_INFERENCE_PROVIDER_NAME, + providerName, "--credential", HERMES_INFERENCE_CREDENTIAL_ENV, ]; @@ -295,7 +347,7 @@ function updateOpenshellInferenceProvider(apiKey, baseUrl) { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, [HERMES_INFERENCE_CREDENTIAL_ENV]: apiKey }, - timeout: 30_000, + timeout: operationTimeout(deadlineAtMs, 30_000), }); if (result.status !== 0) { throw Object.assign(new Error("openshell_inference_provider_update_failed"), { @@ -304,7 +356,7 @@ function updateOpenshellInferenceProvider(apiKey, baseUrl) { } } -async function refreshAccessToken(refreshToken, loaded) { +async function refreshAccessToken(refreshToken, loaded, deadlineAtMs = null) { const digest = sha256(refreshToken); const cached = accessTokenCache.get(digest); if (cached?.accessToken && !tokenExpiresSoon(cached)) { @@ -323,6 +375,7 @@ async function refreshAccessToken(refreshToken, loaded) { "x-nous-refresh-token": refreshToken, }, body, + signal: AbortSignal.timeout(operationTimeout(deadlineAtMs)), }); if (!resp.ok) { @@ -362,8 +415,8 @@ async function refreshAccessToken(refreshToken, loaded) { }; atomicWriteJson(loaded.file, nextState); loaded.state = nextState; + runtimeRefreshCredentials.rotate(nextState, nextRefreshToken); } - process.env[CREDENTIAL_ENV] = nextRefreshToken; return payload.access_token; } @@ -372,6 +425,228 @@ function agentKeyExpiresAt() { return new Date(Date.now() + AGENT_KEY_MIN_TTL_SECONDS * 1000).toISOString(); } +function stageRequestMatches(request, sandbox, refreshToken, inferenceProviderName) { + return ( + request.sandbox === sandbox && + request.original_refresh_token_sha256 === sha256(refreshToken) && + request.inference_provider_name === inferenceProviderName + ); +} + +async function stageCloneBinding( + sandbox, + refreshToken, + inferenceProviderName, + requestId, + deadlineAtMs, +) { + if (!isValidName(sandbox)) { + throw Object.assign(new Error("invalid_stage_sandbox"), { code: "invalid_stage_sandbox" }); + } + if (!isValidProviderName(inferenceProviderName)) { + throw Object.assign(new Error("invalid_stage_provider"), { code: "invalid_stage_provider" }); + } + if (!isValidControlRequestId(requestId)) { + throw Object.assign(new Error("invalid_stage_request_id"), { + code: "invalid_stage_request_id", + }); + } + const boundedDeadline = boundedControlDeadline(deadlineAtMs); + if (boundedDeadline === null) { + throw Object.assign(new Error("invalid_stage_deadline"), { code: "invalid_stage_deadline" }); + } + const existing = stagedCloneRequests.get(requestId); + if (existing) { + if (!stageRequestMatches(existing, sandbox, refreshToken, inferenceProviderName)) { + throw Object.assign(new Error("stage_request_identity_mismatch"), { + code: "stage_request_identity_mismatch", + }); + } + if (existing.state === "discarded") { + throw Object.assign(new Error("stage_request_already_discarded"), { + code: "stage_request_already_discarded", + }); + } + if (existing.state === "pending") return existing.promise; + return { + activationToken: existing.activation_token, + brokerToken: existing.broker_token, + state: existing.state, + }; + } + + const request = { + request_id: requestId, + sandbox, + original_refresh_token_sha256: sha256(refreshToken), + inference_provider_name: inferenceProviderName, + expires_at_ms: Date.now() + STAGED_CLONE_BINDING_TTL_MS, + state: "pending", + promise: null, + }; + const operation = (async () => { + const body = new URLSearchParams({ + grant_type: "refresh_token", + client_id: CLIENT_ID, + }); + const refreshResponse = await fetch(`${PORTAL_BASE_URL}/api/oauth/token`, { + method: "POST", + headers: { + Accept: "application/json", + "Content-Type": "application/x-www-form-urlencoded", + "x-nous-refresh-token": refreshToken, + }, + body, + signal: AbortSignal.timeout(operationTimeout(boundedDeadline)), + }); + if (!refreshResponse.ok) { + const code = + refreshResponse.status === 400 || refreshResponse.status === 401 + ? "reauth_required" + : "refresh_failed"; + throw Object.assign(new Error(`refresh_failed_http_${refreshResponse.status}`), { code }); + } + const refreshed = await refreshResponse.json(); + if (!refreshed?.access_token) { + throw Object.assign(new Error("token_response_missing_access_token"), { + code: "refresh_failed", + }); + } + const nextRefreshToken = + typeof refreshed.refresh_token === "string" && refreshed.refresh_token + ? refreshed.refresh_token + : refreshToken; + const agentKey = await mintAgentKey(refreshed.access_token, boundedDeadline); + const activationToken = `nc_activate_${crypto.randomBytes(32).toString("base64url")}`; + const brokerToken = `nc_broker_${crypto.randomBytes(32).toString("base64url")}`; + const runtime_credential_state = { + sandbox: `staged:${activationToken}`, + refresh_token_sha256: sha256(nextRefreshToken), + }; + if (!runtimeRefreshCredentials.register(runtime_credential_state, nextRefreshToken)) { + throw Object.assign(new Error("staged_runtime_registration_failed"), { + code: "staged_runtime_registration_failed", + }); + } + Object.assign(request, { + state: "staged", + activation_token: activationToken, + broker_token: brokerToken, + runtime_credential_state, + inference_api_key: agentKey.api_key, + inference_base_url: trustedInferenceBaseUrl(agentKey.inference_base_url), + inference_agent_key_expires_at: agentKeyExpiresAt(), + }); + stagedCloneBindings.set(activationToken, request); + const expiryTimer = setTimeout(() => { + if (request.state === "staged") discardStagedCloneBinding(activationToken); + stagedCloneBindings.delete(activationToken); + stagedCloneRequests.delete(requestId); + }, STAGED_CLONE_BINDING_TTL_MS); + expiryTimer.unref?.(); + return { activationToken, brokerToken, state: request.state }; + })(); + request.promise = operation; + stagedCloneRequests.set(requestId, request); + try { + return await operation; + } catch (error) { + if (stagedCloneRequests.get(requestId) === request) stagedCloneRequests.delete(requestId); + throw error; + } +} + +function stagedCloneBinding(activationToken, sandbox) { + const staged = stagedCloneBindings.get(activationToken); + if ( + !staged || + staged.sandbox !== sandbox || + staged.state !== "staged" || + staged.expires_at_ms <= Date.now() + ) { + if (staged?.state === "staged") discardStagedCloneBinding(activationToken); + return null; + } + return staged; +} + +function discardStagedCloneBinding(activationToken) { + const staged = stagedCloneBindings.get(activationToken); + if (!staged) return false; + if (staged.state === "discarded") return true; + if (staged.state === "activated") return false; + if (staged?.runtime_credential_state?.sandbox) { + runtimeRefreshCredentials.unregister(staged.runtime_credential_state.sandbox); + } + staged.state = "discarded"; + return true; +} + +function activateStagedCloneBinding(sandbox, activationToken, deadlineAtMs) { + const known = stagedCloneBindings.get(activationToken); + if (known?.sandbox === sandbox && known.state === "activated") return true; + const staged = stagedCloneBinding(activationToken, sandbox); + const loaded = loadStateForSandbox(sandbox); + const stagedRefreshToken = runtimeRefreshCredentials.resolve(staged?.runtime_credential_state); + if ( + !staged || + !stagedRefreshToken || + !loaded || + loaded.state.refresh_token_sha256 !== staged.original_refresh_token_sha256 || + loaded.state.broker_token !== staged.broker_token || + loaded.state.inference_provider_name !== staged.inference_provider_name + ) { + throw Object.assign(new Error("staged_binding_mismatch"), { + code: "staged_binding_mismatch", + }); + } + const nextState = { + ...loaded.state, + refresh_token_sha256: sha256(stagedRefreshToken), + inference_credential_env: HERMES_INFERENCE_CREDENTIAL_ENV, + inference_base_url: staged.inference_base_url, + inference_agent_key_expires_at: staged.inference_agent_key_expires_at, + inference_agent_key_rotated_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + }; + if (!runtimeRefreshCredentials.register(nextState, stagedRefreshToken)) { + throw Object.assign(new Error("staged_runtime_registration_failed"), { + code: "staged_runtime_registration_failed", + }); + } + try { + updateOpenshellInferenceProvider( + loaded.state, + staged.inference_api_key, + staged.inference_base_url, + deadlineAtMs, + ); + atomicWriteJson(loaded.file, nextState); + loaded.state = nextState; + } catch (error) { + runtimeRefreshCredentials.unregister(sandbox); + throw error; + } + runtimeRefreshCredentials.unregister(staged.runtime_credential_state.sandbox); + staged.state = "activated"; + return true; +} + +function cloneBindingStatus(requestId, activationToken) { + const request = isValidControlRequestId(requestId) + ? stagedCloneRequests.get(requestId) + : isValidActivationToken(activationToken) + ? stagedCloneBindings.get(activationToken) + : null; + if (!request) return null; + return { + request_id: request.request_id, + activation_token: request.activation_token, + broker_token: request.broker_token, + state: request.state, + }; +} + function trustedInferenceBaseUrl(value) { const normalized = String(value || "") .trim() @@ -382,7 +657,7 @@ function trustedInferenceBaseUrl(value) { return DEFAULT_INFERENCE_BASE_URL; } -async function mintAgentKey(accessToken) { +async function mintAgentKey(accessToken, deadlineAtMs = null) { const resp = await fetch(`${PORTAL_BASE_URL}/api/oauth/agent-key`, { method: "POST", headers: { @@ -391,6 +666,7 @@ async function mintAgentKey(accessToken) { "Content-Type": "application/json", }, body: JSON.stringify({ min_ttl_seconds: AGENT_KEY_MIN_TTL_SECONDS }), + signal: AbortSignal.timeout(operationTimeout(deadlineAtMs)), }); if (!resp.ok) { const code = @@ -413,10 +689,10 @@ async function ensureInferenceAgentKey(loaded, refreshToken, options = {}) { const accessToken = await refreshAccessToken(refreshToken, loaded); const agentKey = await mintAgentKey(accessToken); const inferenceBaseUrl = trustedInferenceBaseUrl(agentKey.inference_base_url); - updateOpenshellInferenceProvider(agentKey.api_key, inferenceBaseUrl); + updateOpenshellInferenceProvider(loaded.state, agentKey.api_key, inferenceBaseUrl); const nextState = { ...loaded.state, - inference_provider_name: HERMES_INFERENCE_PROVIDER_NAME, + inference_provider_name: loaded.state.inference_provider_name || HERMES_INFERENCE_PROVIDER_NAME, inference_credential_env: HERMES_INFERENCE_CREDENTIAL_ENV, inference_base_url: inferenceBaseUrl, inference_agent_key_expires_at: agentKeyExpiresAt(), @@ -501,6 +777,123 @@ function sendText(res, status, text) { res.end(text); } +function readControlJson(req) { + return new Promise((resolve, reject) => { + const chunks = []; + let size = 0; + req.on("data", (chunk) => { + size += chunk.length; + if (size > 16_384) { + reject(new Error("control_request_too_large")); + req.destroy(); + return; + } + chunks.push(chunk); + }); + req.on("end", () => { + try { + resolve(JSON.parse(Buffer.concat(chunks).toString("utf8"))); + } catch { + reject(new Error("control_request_invalid")); + } + }); + req.on("error", reject); + }); +} + +async function handleControlRequest(req, res) { + if (req.method !== "POST") { + sendText(res, 405, "method not allowed"); + return; + } + const payload = await readControlJson(req); + const sandbox = String(payload?.sandbox || "").trim(); + if (PREFLIGHT_PROBE && req.url === "/preflight") { + sendJson(res, 200, { ok: true }); + return; + } + if (req.url === "/credentials/stage") { + const refreshToken = String(payload?.refresh_token || "").trim(); + const inferenceProviderName = String(payload?.inference_provider_name || "").trim(); + const requestId = String(payload?.request_id || "").trim(); + if (!refreshToken) { + sendText(res, 409, "staged credential is empty"); + return; + } + const staged = await stageCloneBinding( + sandbox, + refreshToken, + inferenceProviderName, + requestId, + payload?.deadline_at_ms, + ); + sendJson(res, 200, { + ok: true, + activation_token: staged.activationToken, + broker_token: staged.brokerToken, + state: staged.state, + }); + return; + } + if (req.url === "/credentials/activate") { + const activationToken = String(payload?.activation_token || "").trim(); + if (!isValidName(sandbox) || !isValidActivationToken(activationToken)) { + sendText(res, 409, "invalid staged destination identity"); + return; + } + const deadlineAtMs = boundedControlDeadline(payload?.deadline_at_ms); + if (deadlineAtMs === null) { + sendText(res, 409, "invalid activation deadline"); + return; + } + activateStagedCloneBinding(sandbox, activationToken, deadlineAtMs); + sendJson(res, 200, { ok: true, state: "activated" }); + return; + } + if (req.url === "/credentials/discard") { + const activationToken = String(payload?.activation_token || "").trim(); + const staged = stagedCloneBinding(activationToken, sandbox); + if (staged) discardStagedCloneBinding(activationToken); + const status = cloneBindingStatus("", activationToken); + sendJson(res, 200, { ok: true, state: status?.state ?? "absent" }); + return; + } + if (req.url === "/credentials/status") { + const status = cloneBindingStatus( + String(payload?.request_id || "").trim(), + String(payload?.activation_token || "").trim(), + ); + if (!status) { + sendText(res, 404, "unknown clone broker request"); + return; + } + sendJson(res, 200, { ok: true, ...status }); + return; + } + if (req.url === "/credentials/register") { + const loaded = loadStateForSandbox(sandbox); + const refreshToken = String(payload?.refresh_token || "").trim(); + if (!loaded || !runtimeRefreshCredentials.register(loaded.state, refreshToken)) { + sendText(res, 409, "credential does not match destination broker state"); + return; + } + try { + await ensureInferenceAgentKey(loaded, refreshToken); + } catch (error) { + runtimeRefreshCredentials.unregister(sandbox); + throw error; + } + sendJson(res, 200, { ok: true }); + return; + } + if (req.url === "/credentials/unregister") { + runtimeRefreshCredentials.unregister(sandbox); + sendJson(res, 200, { ok: true }); + return; + } + sendText(res, 404, "unknown broker control route"); +} + function errorCode(err) { return err && typeof err === "object" && typeof err.code === "string" ? err.code : null; } @@ -635,21 +1028,111 @@ const server = http.createServer((req, res) => { }); }); +let controlServer = null; +let preflightPublicReady = false; +let preflightControlReady = !CONTROL_SOCKET_PATH; +let preflightRunning = false; + +function finishPreflightProbe(status) { + if (!PREFLIGHT_PROBE) return; + if (CONTROL_SOCKET_PATH) { + try { + fs.unlinkSync(CONTROL_SOCKET_PATH); + } catch { + /* ignore */ + } + } + process.exit(status); +} + +function maybeRunPreflightProbe() { + if (!PREFLIGHT_PROBE || preflightRunning || !preflightPublicReady || !preflightControlReady) { + return; + } + preflightRunning = true; + const body = "{}"; + const request = http.request( + { + socketPath: CONTROL_SOCKET_PATH, + path: "/preflight", + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(body), + }, + }, + (response) => { + response.resume(); + response.on("end", () => finishPreflightProbe(response.statusCode === 200 ? 0 : 3)); + }, + ); + request.on("error", () => finishPreflightProbe(3)); + request.end(body); +} + +if (CONTROL_SOCKET_PATH) { + try { + fs.unlinkSync(CONTROL_SOCKET_PATH); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } + fs.mkdirSync(path.dirname(CONTROL_SOCKET_PATH), { recursive: true, mode: 0o700 }); + controlServer = http.createServer((req, res) => { + handleControlRequest(req, res).catch((error) => { + console.error(`Hermes tool gateway control error: ${error?.message || error}`); + if (!res.headersSent) sendText(res, 400, "invalid broker control request"); + else res.end(); + }); + }); + controlServer.listen(CONTROL_SOCKET_PATH, () => { + fs.chmodSync(CONTROL_SOCKET_PATH, 0o600); + preflightControlReady = true; + maybeRunPreflightProbe(); + }); + if (PREFLIGHT_PROBE) controlServer.on("error", () => finishPreflightProbe(2)); +} + server.listen(PORT, "0.0.0.0", () => { + if (PREFLIGHT_PROBE) { + preflightPublicReady = true; + maybeRunPreflightProbe(); + return; + } console.error(`Hermes managed-tool gateway broker listening on :${PORT}`); refreshManagedInferenceForRuntimeCredentials().catch((err) => { const code = errorCode(err) || "agent_key_refresh_failed"; console.error(`Hermes inference provider refresh failed: ${code}`); }); }); +if (PREFLIGHT_PROBE) { + server.on("error", () => finishPreflightProbe(2)); + setTimeout(() => finishPreflightProbe(4), 5000); +} -const refreshTimer = setInterval(() => { - refreshManagedInferenceForRuntimeCredentials().catch((err) => { - const code = errorCode(err) || "agent_key_refresh_failed"; - console.error(`Hermes inference provider refresh failed: ${code}`); - }); -}, AGENT_KEY_REFRESH_INTERVAL_MS); -refreshTimer.unref?.(); +if (!PREFLIGHT_PROBE) { + const refreshTimer = setInterval(() => { + refreshManagedInferenceForRuntimeCredentials().catch((err) => { + const code = errorCode(err) || "agent_key_refresh_failed"; + console.error(`Hermes inference provider refresh failed: ${code}`); + }); + }, AGENT_KEY_REFRESH_INTERVAL_MS); + refreshTimer.unref?.(); +} + +function closeBroker() { + const exit = () => { + if (CONTROL_SOCKET_PATH) { + try { + fs.unlinkSync(CONTROL_SOCKET_PATH); + } catch { + /* ignore */ + } + } + process.exit(0); + }; + if (controlServer) controlServer.close(() => server.close(exit)); + else server.close(exit); +} -process.on("SIGTERM", () => server.close(() => process.exit(0))); -process.on("SIGINT", () => server.close(() => process.exit(0))); +process.on("SIGTERM", closeBroker); +process.on("SIGINT", closeBroker); diff --git a/agents/hermes/host/tool-gateway-control-contract.ts b/agents/hermes/host/tool-gateway-control-contract.ts new file mode 100644 index 00000000000..33dbdb21d75 --- /dev/null +++ b/agents/hermes/host/tool-gateway-control-contract.ts @@ -0,0 +1,59 @@ +// @ts-nocheck +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const path = require("path"); + +const { isValidName, isValidProviderName } = require( + path.join(__dirname, "../../../nemoclaw/dist/shared/sandbox-name.cjs"), +); + +// One end-to-end deadline covers refresh-token exchange, agent-key minting, +// and destination activation. The local client waits slightly longer so it +// cannot abandon a request while the broker still considers it live. +const HERMES_CLONE_CONTROL_DEADLINE_MS = 120_000; +const HERMES_CLONE_CONTROL_CLIENT_MARGIN_MS = 5_000; +const HERMES_CLONE_CONTROL_CLIENT_TIMEOUT_MS = + HERMES_CLONE_CONTROL_DEADLINE_MS + HERMES_CLONE_CONTROL_CLIENT_MARGIN_MS; +const HERMES_CLONE_CONTROL_STATUS_TIMEOUT_MS = 5_000; + +const CONTROL_REQUEST_ID_PATTERN = /^nc_clone_[a-f0-9]{32}$/u; +const ACTIVATION_TOKEN_PATTERN = /^nc_activate_[A-Za-z0-9_-]{32,64}$/u; + +function isValidControlRequestId(value) { + return typeof value === "string" && CONTROL_REQUEST_ID_PATTERN.test(value); +} + +function isValidActivationToken(value) { + return typeof value === "string" && ACTIVATION_TOKEN_PATTERN.test(value); +} + +function newControlDeadline(now = Date.now()) { + return now + HERMES_CLONE_CONTROL_DEADLINE_MS; +} + +function boundedControlDeadline(value, now = Date.now()) { + if (!Number.isSafeInteger(value) || value <= now) return null; + const latest = now + HERMES_CLONE_CONTROL_DEADLINE_MS; + return value <= latest + HERMES_CLONE_CONTROL_CLIENT_MARGIN_MS ? Math.min(value, latest) : null; +} + +function remainingControlTime(deadlineAtMs, capMs, now = Date.now()) { + const remaining = deadlineAtMs - now; + if (!Number.isFinite(remaining) || remaining <= 0) return 0; + return Math.max(1, Math.min(remaining, capMs)); +} + +module.exports = { + HERMES_CLONE_CONTROL_DEADLINE_MS, + HERMES_CLONE_CONTROL_CLIENT_MARGIN_MS, + HERMES_CLONE_CONTROL_CLIENT_TIMEOUT_MS, + HERMES_CLONE_CONTROL_STATUS_TIMEOUT_MS, + boundedControlDeadline, + isValidActivationToken, + isValidControlRequestId, + isValidName, + isValidProviderName, + newControlDeadline, + remainingControlTime, +}; diff --git a/src/lib/actions/sandbox/snapshot-hermes-managed-clone-broker.test.ts b/src/lib/actions/sandbox/snapshot-hermes-managed-clone-broker.test.ts new file mode 100644 index 00000000000..eb9e15f49fc --- /dev/null +++ b/src/lib/actions/sandbox/snapshot-hermes-managed-clone-broker.test.ts @@ -0,0 +1,314 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { describe, expect, it, vi } from "vitest"; +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import type { HermesToolGatewayCloneBroker } from "../../hermes-tool-gateway-clone-broker"; +import { + encodeManagedStartupProfile, + type ManagedStartupProfile, +} from "../../onboard/managed-startup/profile"; +import { captureSandboxRebuildAuthority } from "../../state/registry/rebuild-authority"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; +import type { SnapshotRestoreAuthority } from "../../state/sandbox"; +import { + HERMES_INFERENCE_CREDENTIAL_ENV, + HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV, + HermesManagedCloneBrokerTransactionError, + prepareHermesManagedCloneBrokerTransaction, + provisionHermesManagedCloneBrokerTransaction, + resolveHermesManagedCloneCredentialEnvironment, +} from "./snapshot/hermes-managed-clone-broker"; + +const CONTENT_AUTHORITY = { + schemaVersion: 1, + backupPath: "/tmp/nemoclaw-hermes-clone-source", + contentSha256: "c".repeat(64), +} as const satisfies SnapshotRestoreAuthority; + +function hermesProfile(): ManagedStartupProfile { + const profile = managedStartupE2eProfile("hermes"); + return { + ...profile, + tools: { ...profile.tools, enabledGateways: ["nous-web"] }, + }; +} + +function workload(profile: ManagedStartupProfile) { + const encodedProfile = encodeManagedStartupProfile(profile); + return { + schemaVersion: 1 as const, + kind: "managed-image" as const, + reference: `ghcr.io/nvidia/nemoclaw/hermes-sandbox@sha256:${"a".repeat(64)}`, + platform: "linux/amd64" as const, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile).digest("hex"), + credentialProxyReplayRequired: true, + shared: true, + } satisfies Extract; +} + +function sourceEntry(profile: ManagedStartupProfile): SandboxEntry { + const receipt = workload(profile); + return { + name: "source", + agent: "hermes", + openshellDriver: "docker", + imageTag: receipt.reference, + workload: receipt, + lifecycleGeneration: "generation-source", + lifecycleLiveIdentityFingerprint: createHash("sha256").update("source").digest("hex"), + provider: profile.inference.upstreamProvider, + model: profile.inference.model, + hermesToolGateways: ["nous-web"], + hermesInferenceProvider: "source-hermes-inference", + }; +} + +function handoff(profile = hermesProfile()) { + const source = sourceEntry(profile); + return { + providerId: "docker", + sourceSandboxName: "source", + destinationSandboxName: "destination", + sourceRegistryAuthority: captureSandboxRebuildAuthority(source, "docker"), + snapshotRestoreAuthority: CONTENT_AUTHORITY, + rebound: { + profile, + encodedProfile: + source.workload?.kind === "managed-image" ? source.workload.encodedProfile : "", + startupProfileSha256: + source.workload?.kind === "managed-image" ? source.workload.startupProfileSha256 : "", + }, + }; +} + +function providerMetadata(name: string, type: string, credential: string): string { + return [ + `Name: ${name}`, + `Type: ${type}`, + `Credential keys: ${credential}`, + "Config keys: ", + "", + ].join("\n"); +} + +function providerRunner() { + const live = new Map(); + const createCredentials = new Map(); + const run = vi.fn((args: string[], options: { env?: NodeJS.ProcessEnv } = {}) => { + if (args[0] === "provider" && args[1] === "get") { + const name = args[2] ?? ""; + const binding = live.get(name); + return binding + ? { + status: 0, + stdout: providerMetadata(name, binding.type, binding.credential), + stderr: "", + } + : { status: 1, stdout: "", stderr: `provider '${name}' not found` }; + } + if (args[0] === "provider" && args[1] === "create") { + const name = args[3] ?? ""; + const type = args[5] ?? ""; + const credential = args[7] ?? ""; + live.set(name, { type, credential }); + createCredentials.set(name, options.env?.[credential] ?? ""); + return { status: 0, stdout: "", stderr: "" }; + } + if (args[0] === "provider" && args[1] === "delete") { + live.delete(args[2] ?? ""); + return { status: 0, stdout: "", stderr: "" }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "detach") { + return { status: 0, stdout: "", stderr: "" }; + } + return { status: 1, stdout: "", stderr: "unsupported test command" }; + }); + return { createCredentials, live, run }; +} + +function broker(): HermesToolGatewayCloneBroker & { + activateHermesToolGatewayCloneBinding: ReturnType; + discardHermesToolGatewayCloneBinding: ReturnType; + preflightHermesToolGatewayCloneBinding: ReturnType; + stageHermesToolGatewayCloneBinding: ReturnType; +} { + return { + HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV, + getHermesToolGatewayProviderName: (name) => `${name}-hermes-tool-gateway`, + getHermesInferenceProviderName: (name) => `${name}-hermes-inference`, + preflightHermesToolGatewayCloneBinding: vi.fn(), + stageHermesToolGatewayCloneBinding: vi.fn(() => ({ + activationToken: `nc_activate_${"a".repeat(43)}`, + brokerToken: `nc_broker_${"b".repeat(43)}`, + requestId: `nc_clone_${"1".repeat(32)}`, + })), + activateHermesToolGatewayCloneBinding: vi.fn(() => ({ + file: "/tmp/destination.json", + brokerToken: `nc_broker_${"b".repeat(43)}`, + })), + discardHermesToolGatewayCloneBinding: vi.fn(() => true), + bindHermesToolGatewayCloneProviderState: vi.fn(() => ({ + file: "/tmp/destination.json", + brokerToken: `nc_broker_${"b".repeat(43)}`, + })), + removeHermesToolGatewayProviderState: vi.fn(() => true), + removeHermesToolGatewayProviderStateForSandboxEntry: vi.fn(() => true), + }; +} + +function environment(): NodeJS.ProcessEnv { + return { + [HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV]: "test-only-refresh-token", + [HERMES_INFERENCE_CREDENTIAL_ENV]: "test-only-inference-placeholder", + }; +} + +function authority(source: SandboxEntry) { + return { + readSandbox: (name: string) => (name === "source" ? source : null), + captureSnapshotRestoreAuthority: vi.fn(() => CONTENT_AUTHORITY), + }; +} + +describe("Hermes managed clone broker transaction", () => { + it("never starts a device-code flow unless the caller explicitly opts in", async () => { + const preparedHandoff = handoff(); + const runDeviceCodeFlow = vi.fn(async () => ({ refresh_token: "oauth-refresh" })); + + await expect( + resolveHermesManagedCloneCredentialEnvironment({ + handoff: preparedHandoff, + environment: {}, + runDeviceCodeFlow, + }), + ).rejects.toThrow(`export ${HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV}`); + expect(runDeviceCodeFlow).not.toHaveBeenCalled(); + + await expect( + resolveHermesManagedCloneCredentialEnvironment({ + handoff: preparedHandoff, + environment: {}, + allowDeviceCodeFlow: true, + runDeviceCodeFlow, + }), + ).resolves.toMatchObject({ + [HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV]: "oauth-refresh", + }); + expect(runDeviceCodeFlow).toHaveBeenCalledOnce(); + }); + + it("stages one secret-free provider-neutral transaction and activates after exact creation", () => { + const profile = hermesProfile(); + const source = sourceEntry(profile); + const preparedHandoff = handoff(profile); + const runner = providerRunner(); + const hostBroker = broker(); + const prepared = prepareHermesManagedCloneBrokerTransaction({ + handoff: preparedHandoff, + destination: null, + environment: environment(), + runOpenshell: runner.run, + broker: hostBroker, + transactionId: "1".repeat(32), + }); + + expect(hostBroker.preflightHermesToolGatewayCloneBinding).toHaveBeenCalledWith("destination"); + expect(prepared.providerTransaction.providers.map(({ binding }) => binding.source)).toEqual([ + "hermes-inference", + "hermes-tool-gateway", + ]); + expect(JSON.stringify(prepared)).not.toContain("test-only-refresh-token"); + + const receipt = provisionHermesManagedCloneBrokerTransaction(prepared, { + ...authority(source), + environment: environment(), + runOpenshell: runner.run, + broker: hostBroker, + }); + + expect(hostBroker.stageHermesToolGatewayCloneBinding).toHaveBeenCalledWith( + "destination", + "test-only-refresh-token", + { requestId: `nc_clone_${"1".repeat(32)}` }, + ); + expect(runner.createCredentials.get("destination-hermes-inference")).toBe( + "test-only-inference-placeholder", + ); + expect(runner.createCredentials.get("destination-hermes-tool-gateway")).toBe( + `nc_broker_${"b".repeat(43)}`, + ); + expect(hostBroker.activateHermesToolGatewayCloneBinding).toHaveBeenCalledOnce(); + expect(receipt.phase).toBe("activated"); + }); + + it("preserves exact providers when activation outcome is unknown", () => { + const profile = hermesProfile(); + const source = sourceEntry(profile); + const runner = providerRunner(); + const hostBroker = broker(); + hostBroker.activateHermesToolGatewayCloneBinding.mockImplementation(() => { + throw Object.assign(new Error("lost local response"), { + code: "hermes_clone_activation_outcome_unknown", + }); + }); + const prepared = prepareHermesManagedCloneBrokerTransaction({ + handoff: handoff(profile), + destination: null, + environment: environment(), + runOpenshell: runner.run, + broker: hostBroker, + transactionId: "2".repeat(32), + }); + + expect(() => + provisionHermesManagedCloneBrokerTransaction(prepared, { + ...authority(source), + environment: environment(), + runOpenshell: runner.run, + broker: hostBroker, + }), + ).toThrowError( + expect.objectContaining({ cleanupDeferred: true }), + ); + expect(runner.live.size).toBe(2); + expect(hostBroker.discardHermesToolGatewayCloneBinding).not.toHaveBeenCalled(); + }); + + it("rolls back only its provider receipt when activation fails definitively", () => { + const profile = hermesProfile(); + const source = sourceEntry(profile); + const runner = providerRunner(); + const hostBroker = broker(); + hostBroker.activateHermesToolGatewayCloneBinding.mockImplementation(() => { + throw new Error("activation rejected"); + }); + const prepared = prepareHermesManagedCloneBrokerTransaction({ + handoff: handoff(profile), + destination: null, + environment: environment(), + runOpenshell: runner.run, + broker: hostBroker, + transactionId: "3".repeat(32), + }); + + expect(() => + provisionHermesManagedCloneBrokerTransaction(prepared, { + ...authority(source), + environment: environment(), + runOpenshell: runner.run, + broker: hostBroker, + }), + ).toThrow("activation rejected"); + expect(runner.live.size).toBe(0); + expect(hostBroker.discardHermesToolGatewayCloneBinding).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot/hermes-managed-clone-broker.ts b/src/lib/actions/sandbox/snapshot/hermes-managed-clone-broker.ts new file mode 100644 index 00000000000..5def2c95f9a --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/hermes-managed-clone-broker.ts @@ -0,0 +1,283 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { randomBytes } from "node:crypto"; + +import { deepFreezeOwned } from "../../../core/deep-freeze-owned"; +import { + getHermesToolGatewayCloneBroker, + type HermesToolGatewayCloneBroker, +} from "../../../hermes-tool-gateway-clone-broker"; +import type { PreparedManagedWorkloadCloneHandoff } from "../../../onboard/workload/clone"; +import type { SandboxEntry } from "../../../state/registry/types"; +import * as sandboxState from "../../../state/sandbox"; +import { + cleanupManagedCloneProviderTransaction, + type ManagedCloneProviderBinding, + type ManagedCloneProviderCleanupResult, + type ManagedCloneProviderRunner, + type ManagedCloneProviderTransactionReceipt, + type PreparedManagedCloneProviderTransaction, + prepareManagedCloneProviderTransaction, + provisionManagedCloneProviderTransaction, + revalidateManagedCloneMutationAuthority, +} from "./managed-clone-providers"; + +export const HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV = + "NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN"; +export const HERMES_INFERENCE_CREDENTIAL_ENV = "OPENAI_API_KEY"; + +type ReadSandbox = (sandboxName: string) => SandboxEntry | null; +type CaptureSnapshotRestoreAuthority = typeof sandboxState.captureSnapshotRestoreAuthority; +type HermesCloneHandoff = Pick< + PreparedManagedWorkloadCloneHandoff, + | "destinationSandboxName" + | "messaging" + | "providerId" + | "rebound" + | "snapshotRestoreAuthority" + | "sourceRegistryAuthority" + | "sourceSandboxName" +>; + +export interface PreparedHermesManagedCloneBrokerTransaction { + readonly schemaVersion: 1; + readonly phase: "prepared"; + readonly gatewayProviderName: string; + readonly inferenceProviderName: string; + readonly providerTransaction: PreparedManagedCloneProviderTransaction; +} + +export interface HermesManagedCloneBrokerReceipt { + readonly schemaVersion: 1; + readonly phase: "activated"; + readonly destinationSandboxName: string; + readonly providerReceipt: ManagedCloneProviderTransactionReceipt; +} + +export class HermesManagedCloneBrokerTransactionError extends Error { + readonly providerReceipt?: ManagedCloneProviderTransactionReceipt; + readonly cleanup?: ManagedCloneProviderCleanupResult; + readonly cleanupDeferred: boolean; + + constructor( + message: string, + options: ErrorOptions & { + readonly providerReceipt?: ManagedCloneProviderTransactionReceipt; + readonly cleanup?: ManagedCloneProviderCleanupResult; + readonly cleanupDeferred?: boolean; + } = {}, + ) { + super(`Hermes managed clone broker transaction failed: ${message}`, options); + this.name = "HermesManagedCloneBrokerTransactionError"; + this.providerReceipt = options.providerReceipt; + this.cleanup = options.cleanup; + this.cleanupDeferred = options.cleanupDeferred ?? false; + } +} + +function hermesEnabled(handoff: HermesCloneHandoff): boolean { + const profile = handoff.rebound.profile; + return profile.agent === "hermes" && profile.tools.enabledGateways.length > 0; +} + +/** + * Resolve apply-only Hermes credentials. Device-code OAuth is never selected + * implicitly: a caller must opt in and inject the exact flow it intends to run. + */ +export async function resolveHermesManagedCloneCredentialEnvironment(input: { + readonly handoff: HermesCloneHandoff; + readonly environment?: NodeJS.ProcessEnv; + readonly allowDeviceCodeFlow?: boolean; + readonly runDeviceCodeFlow?: () => Promise<{ readonly refresh_token?: string }>; +}): Promise { + if (!hermesEnabled(input.handoff)) return {}; + const environment = input.environment ?? process.env; + let refreshToken = environment[HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV] + ?.replace(/\r/gu, "") + .trim(); + if (!refreshToken && input.allowDeviceCodeFlow === true) { + if (!input.runDeviceCodeFlow) { + throw new Error("explicit Hermes device-code flow selection requires an injected runner"); + } + const tokens = await input.runDeviceCodeFlow(); + refreshToken = tokens.refresh_token?.replace(/\r/gu, "").trim(); + } + if (!refreshToken) { + throw new Error( + "managed Hermes tool-gateway clone requires an explicit Nous refresh credential; " + + `export ${HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV} before retrying`, + ); + } + return { + [HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV]: refreshToken, + [HERMES_INFERENCE_CREDENTIAL_ENV]: `nc_clone_${randomBytes(32).toString("base64url")}`, + }; +} + +function hermesBindings( + destinationSandboxName: string, + broker: HermesToolGatewayCloneBroker, +): readonly ManagedCloneProviderBinding[] { + if ( + broker.HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV !== HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV + ) { + throw new Error("Hermes managed-tool gateway credential contract changed"); + } + return [ + { + providerName: broker.getHermesInferenceProviderName(destinationSandboxName), + providerType: "openai", + providerEnvKey: HERMES_INFERENCE_CREDENTIAL_ENV, + source: "hermes-inference", + }, + { + providerName: broker.getHermesToolGatewayProviderName(destinationSandboxName), + providerType: "generic", + providerEnvKey: HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV, + source: "hermes-tool-gateway", + }, + ]; +} + +function destinationHermesBindings( + destination: Readonly, + broker: HermesToolGatewayCloneBroker, +): readonly ManagedCloneProviderBinding[] { + if ( + destination.agent !== "hermes" || + !destination.hermesToolGateways?.length || + destination.hermesInferenceProvider !== broker.getHermesInferenceProviderName(destination.name) + ) { + return []; + } + return hermesBindings(destination.name, broker); +} + +export function prepareHermesManagedCloneBrokerTransaction(input: { + readonly handoff: HermesCloneHandoff; + readonly destination: SandboxEntry | null; + readonly environment?: NodeJS.ProcessEnv; + readonly runOpenshell: ManagedCloneProviderRunner; + readonly broker?: HermesToolGatewayCloneBroker; + readonly transactionId?: string; +}): PreparedHermesManagedCloneBrokerTransaction { + if (!hermesEnabled(input.handoff)) { + throw new Error("Hermes managed-tool broker preparation requires an enabled Hermes gateway"); + } + const broker = input.broker ?? getHermesToolGatewayCloneBroker(); + const destinationSandboxName = input.handoff.destinationSandboxName; + broker.preflightHermesToolGatewayCloneBinding(destinationSandboxName); + const bindings = hermesBindings(destinationSandboxName, broker); + const providerTransaction = prepareManagedCloneProviderTransaction({ + handoff: input.handoff, + destination: input.destination, + additionalBindings: bindings, + resolveAdditionalDestinationOwnedBindings: (destination) => + destinationHermesBindings(destination, broker), + environment: input.environment, + runOpenshell: input.runOpenshell, + transactionId: input.transactionId, + }); + return deepFreezeOwned({ + schemaVersion: 1 as const, + phase: "prepared" as const, + gatewayProviderName: bindings[1].providerName, + inferenceProviderName: bindings[0].providerName, + providerTransaction, + }); +} + +function isUnknownActivationOutcome(error: unknown): boolean { + return ( + error !== null && + typeof error === "object" && + "code" in error && + error.code === "hermes_clone_activation_outcome_unknown" + ); +} + +export function provisionHermesManagedCloneBrokerTransaction( + prepared: PreparedHermesManagedCloneBrokerTransaction, + input: { + readonly environment?: NodeJS.ProcessEnv; + readonly runOpenshell: ManagedCloneProviderRunner; + readonly readSandbox: ReadSandbox; + readonly captureSnapshotRestoreAuthority?: CaptureSnapshotRestoreAuthority; + readonly broker?: HermesToolGatewayCloneBroker; + }, +): HermesManagedCloneBrokerReceipt { + const broker = input.broker ?? getHermesToolGatewayCloneBroker(); + const environment = input.environment ?? process.env; + const refreshToken = environment[HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV] + ?.replace(/\r/gu, "") + .trim(); + if (!refreshToken) { + throw new HermesManagedCloneBrokerTransactionError( + `credential ${HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV} disappeared before broker staging`, + ); + } + + revalidateManagedCloneMutationAuthority(prepared.providerTransaction, input); + let staged: ReturnType; + try { + staged = broker.stageHermesToolGatewayCloneBinding( + prepared.providerTransaction.destinationSandboxName, + refreshToken, + { requestId: `nc_clone_${prepared.providerTransaction.transactionId}` }, + ); + } catch (cause) { + const detail = cause instanceof Error ? cause.message : String(cause); + throw new HermesManagedCloneBrokerTransactionError( + `${detail}; no destination provider mutation was attempted`, + { cause }, + ); + } + let providerReceipt: ManagedCloneProviderTransactionReceipt | undefined; + try { + providerReceipt = provisionManagedCloneProviderTransaction(prepared.providerTransaction, { + ...input, + environment, + resolveCredential: (binding, applyEnvironment) => + binding.providerName === prepared.gatewayProviderName + ? staged.brokerToken + : applyEnvironment[binding.providerEnvKey], + }); + revalidateManagedCloneMutationAuthority(prepared.providerTransaction, input); + broker.activateHermesToolGatewayCloneBinding( + prepared.providerTransaction.destinationSandboxName, + refreshToken, + staged, + ); + return deepFreezeOwned({ + schemaVersion: 1 as const, + phase: "activated" as const, + destinationSandboxName: prepared.providerTransaction.destinationSandboxName, + providerReceipt, + }); + } catch (cause) { + if (isUnknownActivationOutcome(cause) && providerReceipt) { + throw new HermesManagedCloneBrokerTransactionError( + "activation outcome is unknown; preserving providers and broker state for reconciliation", + { cause, providerReceipt, cleanupDeferred: true }, + ); + } + const cleanup = providerReceipt + ? cleanupManagedCloneProviderTransaction(providerReceipt, input.runOpenshell) + : undefined; + let discarded = false; + try { + discarded = broker.discardHermesToolGatewayCloneBinding( + prepared.providerTransaction.destinationSandboxName, + staged, + ); + } catch { + discarded = false; + } + const detail = cause instanceof Error ? cause.message : String(cause); + throw new HermesManagedCloneBrokerTransactionError( + `${detail}; broker staging ${discarded ? "discarded" : "could not be proven discarded"}`, + { cause, providerReceipt, cleanup }, + ); + } +} diff --git a/src/lib/actions/sandbox/snapshot/managed-clone-providers.ts b/src/lib/actions/sandbox/snapshot/managed-clone-providers.ts index f87189523da..6b96e5541c1 100644 --- a/src/lib/actions/sandbox/snapshot/managed-clone-providers.ts +++ b/src/lib/actions/sandbox/snapshot/managed-clone-providers.ts @@ -506,6 +506,11 @@ export function provisionManagedCloneProviderTransaction( readonly runOpenshell: ManagedCloneProviderRunner; readonly readSandbox: ReadSandbox; readonly captureSnapshotRestoreAuthority?: CaptureSnapshotRestoreAuthority; + /** Provider-neutral apply-time credential substitution (for host brokers). */ + readonly resolveCredential?: ( + binding: ManagedCloneProviderBinding, + environment: NodeJS.ProcessEnv, + ) => string | null | undefined; }, ): ManagedCloneProviderTransactionReceipt { const environment = input.environment ?? process.env; @@ -531,7 +536,12 @@ export function provisionManagedCloneProviderTransaction( if (current.kind !== "missing") { fail(`provider '${provider.binding.providerName}' appeared after preflight`); } - const credential = environment[provider.binding.providerEnvKey]?.replace(/\r/gu, "").trim(); + const resolved = input.resolveCredential?.(provider.binding, environment); + const credential = ( + resolved === undefined ? environment[provider.binding.providerEnvKey] : resolved + ) + ?.replace(/\r/gu, "") + .trim(); if (!credential) { fail(`credential ${provider.binding.providerEnvKey} disappeared before provider creation`); } diff --git a/src/lib/hermes-tool-gateway-broker.ts b/src/lib/hermes-tool-gateway-broker.ts index b40700d1619..836a88821d0 100644 --- a/src/lib/hermes-tool-gateway-broker.ts +++ b/src/lib/hermes-tool-gateway-broker.ts @@ -6,20 +6,33 @@ const crypto = require("crypto"); const fs = require("fs"); +const os = require("os"); const path = require("path"); -const { spawn } = require("child_process"); +const { spawn, spawnSync } = require("child_process"); const { ROOT, run, runCapture, validateName } = require("./runner"); const { buildSubprocessEnv } = require("./subprocess-env"); const { getCredsDir } = require("./credentials/store"); const oauth = require("./oauth-device-code"); const onboardProviders = require("./onboard/providers"); +const { + HERMES_CLONE_CONTROL_CLIENT_TIMEOUT_MS, + HERMES_CLONE_CONTROL_STATUS_TIMEOUT_MS, + isValidActivationToken, + isValidControlRequestId, + isValidProviderName, + newControlDeadline, +} = require(path.join(ROOT, "agents", "hermes", "host", "tool-gateway-control-contract.ts")); const HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV = "NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN"; const HERMES_TOOL_GATEWAY_PORT = 11436; const HERMES_TOOL_GATEWAY_STATE_DIR = path.join(getCredsDir(), "hermes-tool-gateway"); const HERMES_TOOL_GATEWAY_PID_PATH = path.join(getCredsDir(), "hermes-tool-gateway-broker.pid"); const HERMES_TOOL_GATEWAY_HASH_PATH = path.join(getCredsDir(), "hermes-tool-gateway-broker.hash"); +const HERMES_TOOL_GATEWAY_CONTROL_SOCKET_PATH = path.join( + getCredsDir(), + "hermes-tool-gateway-broker.sock", +); const HERMES_TOOL_GATEWAY_SCRIPT = path.join( ROOT, "agents", @@ -34,6 +47,20 @@ const HERMES_TOOL_GATEWAY_MATRIX_PATH = path.join( "host", "managed-tool-gateway-matrix.json", ); +const HERMES_TOOL_GATEWAY_RUNTIME_CREDENTIALS_PATH = path.join( + ROOT, + "agents", + "hermes", + "host", + "runtime-refresh-credentials.ts", +); +const HERMES_TOOL_GATEWAY_CONTROL_CONTRACT_PATH = path.join( + ROOT, + "agents", + "hermes", + "host", + "tool-gateway-control-contract.ts", +); let brokerStartedThisRun = false; @@ -58,8 +85,23 @@ function generateHermesToolGatewayBrokerToken() { return `nc_broker_${crypto.randomBytes(32).toString("base64url")}`; } +function validateProviderName(value, label) { + if (!isValidProviderName(value)) throw new Error(`${label} is invalid`); + return value; +} + function getHermesToolGatewayProviderName(sandboxName) { - return `${validateName(sandboxName, "sandbox name")}-hermes-tool-gateway`; + return validateProviderName( + `${validateName(sandboxName, "sandbox name")}-hermes-tool-gateway`, + "Hermes tool-gateway provider name", + ); +} + +function getHermesInferenceProviderName(sandboxName) { + return validateProviderName( + `${validateName(sandboxName, "sandbox name")}-hermes-inference`, + "Hermes inference provider name", + ); } function getHermesToolGatewayStatePath(sandboxName) { @@ -100,7 +142,12 @@ function getHermesToolGatewayBrokerToken(sandboxName) { return token || null; } -function persistHermesToolGatewayProviderState(sandboxName, refreshToken, brokerToken = null) { +function persistHermesToolGatewayProviderState( + sandboxName, + refreshToken, + brokerToken = null, + inferenceProviderName = "hermes-provider", +) { const file = getHermesToolGatewayStatePath(sandboxName); const previous = readHermesToolGatewayProviderState(sandboxName); const normalizedBrokerToken = @@ -113,6 +160,11 @@ function persistHermesToolGatewayProviderState(sandboxName, refreshToken, broker version: 1, sandbox: validateName(sandboxName, "sandbox name"), provider_name: getHermesToolGatewayProviderName(sandboxName), + inference_provider_name: validateProviderName( + inferenceProviderName, + "Hermes inference provider name", + ), + inference_credential_env: "OPENAI_API_KEY", credential_env: HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV, broker_token: normalizedBrokerToken, broker_token_sha256: hashRefreshToken(normalizedBrokerToken), @@ -124,6 +176,111 @@ function persistHermesToolGatewayProviderState(sandboxName, refreshToken, broker return { file, brokerToken: normalizedBrokerToken }; } +function brokerControlJsonRequest(route, payload, options = {}) { + if (!fs.existsSync(HERMES_TOOL_GATEWAY_CONTROL_SOCKET_PATH)) return null; + const timeoutMs = options.timeoutMs ?? HERMES_CLONE_CONTROL_CLIENT_TIMEOUT_MS; + const result = spawnSync( + "curl", + [ + "--silent", + "--show-error", + "--fail", + "--unix-socket", + HERMES_TOOL_GATEWAY_CONTROL_SOCKET_PATH, + "--connect-timeout", + "3", + "--max-time", + (timeoutMs / 1000).toFixed(3), + "--request", + "POST", + "--header", + "Content-Type: application/json", + "--data-binary", + "@-", + `http://localhost/${route}`, + ], + { + encoding: "utf8", + input: JSON.stringify(payload), + stdio: ["pipe", "pipe", "ignore"], + timeout: timeoutMs + 1_000, + }, + ); + if (result.status !== 0) return null; + try { + const parsed = JSON.parse(result.stdout || "{}"); + return parsed && typeof parsed === "object" ? parsed : null; + } catch { + return null; + } +} + +function brokerControlStatus(payload) { + return brokerControlJsonRequest("credentials/status", payload, { + timeoutMs: HERMES_CLONE_CONTROL_STATUS_TIMEOUT_MS, + }); +} + +function brokerControlRequest(route, payload) { + return brokerControlJsonRequest(route, payload) !== null; +} + +function registerHermesToolGatewayRuntimeCredential(refreshToken, exactSandboxName = null) { + const digest = hashRefreshToken(refreshToken); + let matched = false; + const stateNames = + exactSandboxName === null + ? fs.readdirSync(HERMES_TOOL_GATEWAY_STATE_DIR) + : [`${validateName(exactSandboxName, "sandbox name")}.json`]; + for (const name of stateNames) { + if (!name.endsWith(".json")) continue; + const sandboxName = name.slice(0, -".json".length); + const state = readHermesToolGatewayProviderState(sandboxName); + if (!state || state.refresh_token_sha256 !== digest) continue; + matched = true; + if ( + !brokerControlRequest("credentials/register", { + sandbox: sandboxName, + refresh_token: refreshToken, + }) + ) { + return false; + } + } + return matched; +} + +function removeHermesToolGatewayProviderState(sandboxName, deps = {}) { + const sandbox = validateName(sandboxName, "sandbox name"); + const file = (deps.getStatePath ?? getHermesToolGatewayStatePath)(sandbox); + const controlSocketExists = + deps.controlSocketExists ?? (() => fs.existsSync(HERMES_TOOL_GATEWAY_CONTROL_SOCKET_PATH)); + const unregister = + deps.unregister ?? + (() => + brokerControlRequest("credentials/unregister", { + sandbox, + })); + // The file is the durable retry identity for an in-memory credential. Keep + // it intact until the live broker confirms unregister; otherwise a later + // cleanup cannot prove which credential remains active. + if (controlSocketExists() && !unregister()) return false; + try { + (deps.unlinkState ?? fs.unlinkSync)(file); + return true; + } catch (error) { + return Boolean(error && error.code === "ENOENT"); + } +} + +function brokerRuntimeFileHash(file) { + try { + return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"); + } catch { + return "missing"; + } +} + function registerHermesToolGatewayRefreshProvider(sandboxName, refreshToken, runOpenshell) { const normalized = String(refreshToken || "").trim(); if (!normalized) { @@ -136,7 +293,7 @@ function registerHermesToolGatewayRefreshProvider(sandboxName, refreshToken, run "generic", HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV, null, - { [HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV]: normalized }, + { [HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV]: state.brokerToken }, runOpenshell, ); if (!result.ok) { @@ -145,6 +302,212 @@ function registerHermesToolGatewayRefreshProvider(sandboxName, refreshToken, run return { providerName, brokerToken: state.brokerToken }; } +/** + * Bind a newly created snapshot destination to its own host-broker identity. + * The refresh credential remains process-local; OpenShell stores only a fresh + * opaque broker token. The durable state file records the refresh digest and + * destination identity without persisting the upstream OAuth secret. + */ +function bindHermesToolGatewayCloneProviderState(sandboxName, refreshToken) { + const normalized = String(refreshToken || "").trim(); + if (!normalized) { + throw new Error("Hermes tool gateway refresh credential is empty"); + } + const state = persistHermesToolGatewayProviderState( + sandboxName, + normalized, + generateHermesToolGatewayBrokerToken(), + getHermesInferenceProviderName(sandboxName), + ); + if ( + ensureHermesToolGatewayBroker({ + refreshToken: normalized, + sandboxName: validateName(sandboxName, "sandbox name"), + }) + ) { + return state; + } + removeHermesToolGatewayProviderState(sandboxName); + throw new Error("Hermes managed-tool gateway broker did not become ready"); +} + +function stageHermesToolGatewayCloneBinding(sandboxName, refreshToken, options = {}) { + const sandbox = validateName(sandboxName, "sandbox name"); + const normalized = String(refreshToken || "").trim(); + const requestId = options.requestId || `nc_clone_${crypto.randomBytes(16).toString("hex")}`; + if (!isValidControlRequestId(requestId)) { + throw new Error("Hermes clone broker request identity is invalid"); + } + if (!normalized) { + throw new Error("Hermes tool gateway refresh credential is empty"); + } + if (!ensureHermesToolGatewayBroker({ startWithoutCredential: true })) { + throw new Error("Hermes managed-tool gateway broker could not start before destination change"); + } + const payload = { + sandbox, + refresh_token: normalized, + inference_provider_name: getHermesInferenceProviderName(sandbox), + request_id: requestId, + deadline_at_ms: newControlDeadline(), + }; + const response = + brokerControlJsonRequest("credentials/stage", payload) ?? + brokerControlStatus({ request_id: requestId }); + const activationToken = + response && typeof response.activation_token === "string" + ? response.activation_token.trim() + : ""; + const brokerToken = + response && typeof response.broker_token === "string" ? response.broker_token.trim() : ""; + if (!isValidActivationToken(activationToken) || !brokerToken.startsWith("nc_broker_")) { + throw new Error("Hermes managed-tool gateway broker could not stage destination credentials"); + } + return Object.freeze({ activationToken, brokerToken, requestId }); +} + +function activateHermesToolGatewayCloneBinding(sandboxName, refreshToken, stagedBinding) { + const sandbox = validateName(sandboxName, "sandbox name"); + const normalized = String(refreshToken || "").trim(); + const activationToken = String(stagedBinding?.activationToken || "").trim(); + const brokerToken = String(stagedBinding?.brokerToken || "").trim(); + if (!normalized || !isValidActivationToken(activationToken) || !brokerToken) { + throw new Error("Hermes staged destination credential binding is incomplete"); + } + const state = persistHermesToolGatewayProviderState( + sandbox, + normalized, + brokerToken, + getHermesInferenceProviderName(sandbox), + ); + const response = brokerControlJsonRequest("credentials/activate", { + sandbox, + activation_token: activationToken, + deadline_at_ms: newControlDeadline(), + }); + const reconciled = response ?? brokerControlStatus({ activation_token: activationToken }); + if (reconciled?.state === "activated") { + return state; + } + if (reconciled?.state === "discarded" || reconciled?.state === "staged") { + removeHermesToolGatewayProviderState(sandbox); + } else { + throw Object.assign( + new Error("Hermes managed-tool gateway broker activation outcome is unknown"), + { code: "hermes_clone_activation_outcome_unknown" }, + ); + } + throw new Error("Hermes managed-tool gateway broker could not activate destination credentials"); +} + +function discardHermesToolGatewayCloneBinding(sandboxName, stagedBinding) { + const activationToken = String(stagedBinding?.activationToken || "").trim(); + if (!activationToken) return true; + const response = brokerControlJsonRequest("credentials/discard", { + sandbox: validateName(sandboxName, "sandbox name"), + activation_token: activationToken, + }); + const reconciled = response ?? brokerControlStatus({ activation_token: activationToken }); + return reconciled?.state === "discarded" || reconciled?.state === "absent"; +} + +function probeHermesToolGatewayBrokerStart(options = {}) { + const spawnProbe = options.spawnSyncImpl || spawnSync; + const probePort = Number.isInteger(options.port) ? options.port : HERMES_TOOL_GATEWAY_PORT; + // AF_UNIX paths are short on macOS; TMPDIR can already consume most of the + // limit before the private control-socket name is appended. + const probeTempRoot = process.platform === "darwin" ? "/tmp" : os.tmpdir(); + const probeRoot = fs.mkdtempSync(path.join(probeTempRoot, "nc-hermes-probe-")); + fs.chmodSync(probeRoot, 0o700); + const stateDir = path.join(probeRoot, "state"); + const controlSocket = path.join(probeRoot, "control.sock"); + ensurePrivateDir(stateDir); + try { + const result = spawnProbe( + process.execPath, + ["--experimental-strip-types", HERMES_TOOL_GATEWAY_SCRIPT], + { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + cwd: ROOT, + env: buildSubprocessEnv({ + HERMES_TOOL_GATEWAY_PORT: String(probePort), + HERMES_TOOL_GATEWAY_STATE_DIR: stateDir, + HERMES_TOOL_GATEWAY_MATRIX_PATH, + HERMES_TOOL_GATEWAY_CONTROL_SOCKET: controlSocket, + HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV, + HERMES_TOOL_GATEWAY_PREFLIGHT_PROBE: "1", + NOUS_PORTAL_BASE_URL: process.env.NOUS_PORTAL_BASE_URL || oauth.DEFAULT_PORTAL_BASE_URL, + NEMOCLAW_OPENSHELL_BIN: process.env.NEMOCLAW_OPENSHELL_BIN || "openshell", + }), + timeout: 10_000, + }, + ); + if (result.error) { + throw new Error( + `Hermes managed-tool broker preflight could not start: ${result.error.message}`, + ); + } + if (result.status === 2) { + throw new Error("Hermes managed-tool broker preflight could not bind its runtime endpoints"); + } + if (result.status === 3) { + throw new Error("Hermes managed-tool broker preflight control registration path failed"); + } + if (result.status !== 0) { + throw new Error( + `Hermes managed-tool broker preflight did not become ready (exit ${String(result.status)})`, + ); + } + } finally { + fs.rmSync(probeRoot, { recursive: true, force: true }); + } +} + +/** + * Prove that a clone can use the current broker runtime before any destination + * is deleted or any OAuth flow begins. The isolated probe creates only + * disposable private runtime files and performs no durable provider, + * credential, or broker-process mutation. + */ +function preflightHermesToolGatewayCloneBinding(sandboxName) { + validateName(sandboxName, "sandbox name"); + const requiredRuntimeFiles = [ + HERMES_TOOL_GATEWAY_SCRIPT, + HERMES_TOOL_GATEWAY_MATRIX_PATH, + HERMES_TOOL_GATEWAY_RUNTIME_CREDENTIALS_PATH, + HERMES_TOOL_GATEWAY_CONTROL_CONTRACT_PATH, + ]; + const missing = requiredRuntimeFiles.filter((file) => brokerRuntimeFileHash(file) === "missing"); + if (missing.length > 0) { + throw new Error( + `Hermes managed-tool broker runtime is incomplete (${missing + .map((file) => path.basename(file)) + .join(", ")})`, + ); + } + + const pid = readPid(); + const currentBrokerOwned = isHermesToolGatewayBrokerProcess(pid) || brokerStartedThisRun; + const currentBrokerHealthy = isHermesToolGatewayBrokerHealthy(); + if (currentBrokerHealthy && !currentBrokerOwned) { + throw new Error("Hermes managed-tool broker health endpoint is not owned by NemoClaw"); + } + if (!currentBrokerOwned || !currentBrokerHealthy) { + probeHermesToolGatewayBrokerStart(); + return; + } + if (readBrokerHash() !== brokerRuntimeHash()) { + throw new Error( + "Hermes managed-tool broker runtime changed while an existing broker is active; " + + "reauthorize every managed-tool Hermes sandbox before retrying", + ); + } + if (!fs.existsSync(HERMES_TOOL_GATEWAY_CONTROL_SOCKET_PATH)) { + throw new Error("Hermes managed-tool broker control socket is unavailable"); + } +} + function readPid() { try { const pid = Number.parseInt(fs.readFileSync(HERMES_TOOL_GATEWAY_PID_PATH, "utf8").trim(), 10); @@ -176,8 +539,17 @@ function brokerRuntimeHash() { JSON.stringify({ port: HERMES_TOOL_GATEWAY_PORT, script: HERMES_TOOL_GATEWAY_SCRIPT, + scriptSha256: brokerRuntimeFileHash(HERMES_TOOL_GATEWAY_SCRIPT), + runtimeCredentials: HERMES_TOOL_GATEWAY_RUNTIME_CREDENTIALS_PATH, + runtimeCredentialsSha256: brokerRuntimeFileHash( + HERMES_TOOL_GATEWAY_RUNTIME_CREDENTIALS_PATH, + ), + controlContract: HERMES_TOOL_GATEWAY_CONTROL_CONTRACT_PATH, + controlContractSha256: brokerRuntimeFileHash(HERMES_TOOL_GATEWAY_CONTROL_CONTRACT_PATH), matrix: HERMES_TOOL_GATEWAY_MATRIX_PATH, + matrixSha256: brokerRuntimeFileHash(HERMES_TOOL_GATEWAY_MATRIX_PATH), stateDir: HERMES_TOOL_GATEWAY_STATE_DIR, + controlSocket: HERMES_TOOL_GATEWAY_CONTROL_SOCKET_PATH, }), ) .digest("hex"); @@ -234,9 +606,14 @@ function killStaleHermesToolGatewayBroker() { } clearPid(); clearBrokerHash(); + try { + fs.unlinkSync(HERMES_TOOL_GATEWAY_CONTROL_SOCKET_PATH); + } catch { + /* ignore */ + } } -function spawnHermesToolGatewayBroker(refreshToken) { +function spawnHermesToolGatewayBroker(refreshToken, initialSandboxName = null) { ensurePrivateDir(HERMES_TOOL_GATEWAY_STATE_DIR); const credentialEnv = {}; if (typeof refreshToken === "string" && refreshToken.trim()) { @@ -253,7 +630,13 @@ function spawnHermesToolGatewayBroker(refreshToken) { HERMES_TOOL_GATEWAY_PORT: String(HERMES_TOOL_GATEWAY_PORT), HERMES_TOOL_GATEWAY_STATE_DIR, HERMES_TOOL_GATEWAY_MATRIX_PATH, + HERMES_TOOL_GATEWAY_CONTROL_SOCKET: HERMES_TOOL_GATEWAY_CONTROL_SOCKET_PATH, HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV, + ...(initialSandboxName === null + ? {} + : { + HERMES_TOOL_GATEWAY_INITIAL_SANDBOX: validateName(initialSandboxName, "sandbox name"), + }), NOUS_PORTAL_BASE_URL: process.env.NOUS_PORTAL_BASE_URL || oauth.DEFAULT_PORTAL_BASE_URL, NEMOCLAW_OPENSHELL_BIN: process.env.NEMOCLAW_OPENSHELL_BIN || "openshell", ...credentialEnv, @@ -266,16 +649,81 @@ function spawnHermesToolGatewayBroker(refreshToken) { return child.pid || null; } +function planHermesToolGatewayBrokerRefresh({ + currentBrokerHealthy, + forceRestart = false, + hashMatches, +}) { + if (!forceRestart && currentBrokerHealthy && !hashMatches) { + return "preserve-runtime-mismatch"; + } + if (!forceRestart && currentBrokerHealthy) { + return "register-with-current"; + } + return "start-or-restart"; +} + function ensureHermesToolGatewayBroker(options = {}) { const refreshToken = typeof options.refreshToken === "string" && options.refreshToken.trim() ? options.refreshToken.trim() : ""; - if (refreshToken) { + const desiredHash = brokerRuntimeHash(); + const hashMatches = readBrokerHash() === desiredHash; + const pid = readPid(); + const currentBrokerOwned = isHermesToolGatewayBrokerProcess(pid) || brokerStartedThisRun; + const currentBrokerHealthy = currentBrokerOwned && isHermesToolGatewayBrokerHealthy(); + if (options.startWithoutCredential) { + if (currentBrokerHealthy) { + return hashMatches && fs.existsSync(HERMES_TOOL_GATEWAY_CONTROL_SOCKET_PATH); + } killStaleHermesToolGatewayBroker(); - const nextPid = spawnHermesToolGatewayBroker(refreshToken); + const nextPid = spawnHermesToolGatewayBroker(""); for (let attempt = 0; attempt < 20; attempt++) { - if (isHermesToolGatewayBrokerProcess(nextPid) && isHermesToolGatewayBrokerHealthy()) { + if ( + isHermesToolGatewayBrokerProcess(nextPid) && + isHermesToolGatewayBrokerHealthy() && + fs.existsSync(HERMES_TOOL_GATEWAY_CONTROL_SOCKET_PATH) + ) { + brokerStartedThisRun = true; + return true; + } + sleep(250); + } + return false; + } + const refreshPlan = refreshToken + ? planHermesToolGatewayBrokerRefresh({ + currentBrokerHealthy, + forceRestart: options.forceRestart, + hashMatches, + }) + : null; + if (refreshPlan === "preserve-runtime-mismatch") { + console.error( + "Hermes managed-tool broker runtime changed while an existing broker is active; " + + "refusing to restart it and discard other in-memory sandbox credentials. " + + "Reauthorize every managed-tool Hermes sandbox using the documented broker recovery flow.", + ); + return false; + } + if (refreshPlan === "register-with-current") { + const registered = registerHermesToolGatewayRuntimeCredential( + refreshToken, + options.sandboxName ?? null, + ); + if (registered) brokerStartedThisRun = true; + return registered; + } + if (refreshPlan === "start-or-restart") { + killStaleHermesToolGatewayBroker(); + const nextPid = spawnHermesToolGatewayBroker(refreshToken, options.sandboxName ?? null); + for (let attempt = 0; attempt < 20; attempt++) { + if ( + isHermesToolGatewayBrokerProcess(nextPid) && + isHermesToolGatewayBrokerHealthy() && + registerHermesToolGatewayRuntimeCredential(refreshToken, options.sandboxName ?? null) + ) { brokerStartedThisRun = true; return true; } @@ -284,8 +732,6 @@ function ensureHermesToolGatewayBroker(options = {}) { return false; } - const desiredHash = brokerRuntimeHash(); - const hashMatches = readBrokerHash() === desiredHash; if ( !options.forceRestart && hashMatches && @@ -294,7 +740,6 @@ function ensureHermesToolGatewayBroker(options = {}) { ) { return true; } - const pid = readPid(); if ( !options.forceRestart && hashMatches && @@ -322,9 +767,60 @@ function isHermesManagedToolGatewayEntry(entry) { return Boolean(enabled); } +function matchesHermesToolGatewayProviderIdentity(entry, state) { + if (entry?.agent !== "hermes" || !state || typeof state !== "object") { + return false; + } + const sandbox = validateName(entry.name, "sandbox name"); + if ( + state.sandbox !== sandbox || + state.provider_name !== getHermesToolGatewayProviderName(sandbox) + ) { + return false; + } + const isolatedProvider = + typeof entry.hermesInferenceProvider === "string" ? entry.hermesInferenceProvider.trim() : ""; + if (!isolatedProvider) { + return ( + state.inference_provider_name === undefined || + state.inference_provider_name === "hermes-provider" + ); + } + return ( + isolatedProvider === getHermesInferenceProviderName(sandbox) && + state.inference_provider_name === isolatedProvider + ); +} + +function matchesHermesToolGatewayProviderState(entry, state) { + return ( + isHermesManagedToolGatewayEntry(entry) && matchesHermesToolGatewayProviderIdentity(entry, state) + ); +} + +function removeHermesToolGatewayProviderStateForSandboxEntry(entry, deps = {}) { + if (entry?.agent !== "hermes") return false; + const sandbox = validateName(entry.name, "sandbox name"); + const isolatedProvider = + typeof entry.hermesInferenceProvider === "string" ? entry.hermesInferenceProvider.trim() : ""; + if (isolatedProvider && isolatedProvider !== getHermesInferenceProviderName(sandbox)) { + return false; + } + const statePath = (deps.getStatePath ?? getHermesToolGatewayStatePath)(sandbox); + if (!(deps.stateExists ?? fs.existsSync)(statePath)) return true; + const state = (deps.readState ?? readHermesToolGatewayProviderState)(sandbox); + if (!matchesHermesToolGatewayProviderIdentity(entry, state)) return false; + return (deps.removeState ?? removeHermesToolGatewayProviderState)(sandbox); +} + function ensureHermesToolGatewayBrokerForSandboxEntry(entry, options = {}) { const enabled = isHermesManagedToolGatewayEntry(entry); if (!enabled) return false; + if ( + !matchesHermesToolGatewayProviderState(entry, readHermesToolGatewayProviderState(entry.name)) + ) { + return false; + } return ensureHermesToolGatewayBroker(options); } @@ -332,16 +828,29 @@ module.exports = { HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV, HERMES_TOOL_GATEWAY_STATE_DIR, HERMES_TOOL_GATEWAY_PORT, + HERMES_TOOL_GATEWAY_CONTROL_SOCKET_PATH, hashRefreshToken, generateHermesToolGatewayBrokerToken, getHermesToolGatewayProviderName, + getHermesInferenceProviderName, getHermesToolGatewayStatePath, getHermesToolGatewayBrokerToken, persistHermesToolGatewayProviderState, + removeHermesToolGatewayProviderState, registerHermesToolGatewayRefreshProvider, + probeHermesToolGatewayBrokerStart, + preflightHermesToolGatewayCloneBinding, + stageHermesToolGatewayCloneBinding, + activateHermesToolGatewayCloneBinding, + discardHermesToolGatewayCloneBinding, + bindHermesToolGatewayCloneProviderState, + planHermesToolGatewayBrokerRefresh, isHermesToolGatewayBrokerHealthy, killStaleHermesToolGatewayBroker, ensureHermesToolGatewayBroker, isHermesManagedToolGatewayEntry, + matchesHermesToolGatewayProviderIdentity, + matchesHermesToolGatewayProviderState, + removeHermesToolGatewayProviderStateForSandboxEntry, ensureHermesToolGatewayBrokerForSandboxEntry, }; diff --git a/src/lib/hermes-tool-gateway-clone-broker.ts b/src/lib/hermes-tool-gateway-clone-broker.ts new file mode 100644 index 00000000000..ccc49ea26ac --- /dev/null +++ b/src/lib/hermes-tool-gateway-clone-broker.ts @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type HermesToolGatewayProviderOwner = { + readonly name: string; + readonly agent?: string | null; + readonly hermesInferenceProvider?: string; + readonly hermesToolGateways?: readonly string[]; +}; + +export type HermesToolGatewayCloneBroker = { + readonly HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV: string; + getHermesToolGatewayProviderName(sandboxName: string): string; + getHermesInferenceProviderName(sandboxName: string): string; + preflightHermesToolGatewayCloneBinding(sandboxName: string): void; + stageHermesToolGatewayCloneBinding( + sandboxName: string, + refreshToken: string, + options?: { readonly requestId?: string }, + ): { + readonly activationToken: string; + readonly brokerToken: string; + readonly requestId: string; + }; + activateHermesToolGatewayCloneBinding( + sandboxName: string, + refreshToken: string, + stagedBinding: { + readonly activationToken: string; + readonly brokerToken: string; + readonly requestId?: string; + }, + ): { readonly file: string; readonly brokerToken: string }; + discardHermesToolGatewayCloneBinding( + sandboxName: string, + stagedBinding: { + readonly activationToken: string; + readonly brokerToken: string; + readonly requestId?: string; + }, + ): boolean; + bindHermesToolGatewayCloneProviderState( + sandboxName: string, + refreshToken: string, + ): { readonly file: string; readonly brokerToken: string }; + removeHermesToolGatewayProviderState(sandboxName: string): boolean; + removeHermesToolGatewayProviderStateForSandboxEntry( + entry: HermesToolGatewayProviderOwner, + ): boolean; +}; + +/** Lazy CommonJS bridge, kept injectable so tests never start a host broker. */ +export function getHermesToolGatewayCloneBroker(): HermesToolGatewayCloneBroker { + return require("./hermes-tool-gateway-broker") as HermesToolGatewayCloneBroker; +} diff --git a/test/helpers/hermes-tool-gateway-broker-fixture.ts b/test/helpers/hermes-tool-gateway-broker-fixture.ts new file mode 100644 index 00000000000..70fdf189e66 --- /dev/null +++ b/test/helpers/hermes-tool-gateway-broker-fixture.ts @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { IncomingMessage, ServerResponse } from "node:http"; + +export function handleHermesBrokerCoexistencePortal( + refreshHeaders: string[], + req: IncomingMessage, + res: ServerResponse, +): void { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => { + const refreshToken = String(req.headers["x-nous-refresh-token"] || ""); + const respond = () => { + res.writeHead(200, { "Content-Type": "application/json" }); + if (req.url === "/api/oauth/agent-key") { + res.end( + JSON.stringify({ + api_key: "local-agent-key", + expires_in: 1800, + inference_base_url: "https://inference-api.nousresearch.com/v1", + }), + ); + return; + } + refreshHeaders.push(refreshToken); + const identity = refreshToken.split("-")[0] || "source"; + res.end( + JSON.stringify({ + access_token: `access-${identity}`, + refresh_token: refreshToken, + expires_in: 900, + token_type: "Bearer", + }), + ); + }; + if (refreshToken === "deadline-refresh-token") setTimeout(respond, 250); + else respond(); + }); +} diff --git a/test/hermes-tool-gateway-broker.test.ts b/test/hermes-tool-gateway-broker.test.ts index e17b974c0ed..ea517d23190 100644 --- a/test/hermes-tool-gateway-broker.test.ts +++ b/test/hermes-tool-gateway-broker.test.ts @@ -10,6 +10,8 @@ import { createRequire } from "node:module"; import net from "node:net"; import path from "node:path"; import zlib from "node:zlib"; +import { vi } from "vitest"; +import { handleHermesBrokerCoexistencePortal } from "./helpers/hermes-tool-gateway-broker-fixture"; import { describe, expect, test as it } from "./helpers/owned-test-resources"; import { testTimeout } from "./helpers/timeouts"; @@ -29,6 +31,14 @@ const BROKER_WRAPPER = path.join( "lib", "hermes-tool-gateway-broker.ts", ); +const CONTROL_CONTRACT = path.join( + import.meta.dirname, + "..", + "agents", + "hermes", + "host", + "tool-gateway-control-contract.ts", +); const BROKER_READINESS_TIMEOUT_MS = 15_000; const BROKER_TEST_TIMEOUT_MS = testTimeout(45_000); @@ -70,6 +80,45 @@ function brokerDiagnostics(child: ChildProcess, output: () => string): string { ].join("; "); } +function controlRequest( + socketPath: string, + route: + | "/credentials/activate" + | "/credentials/discard" + | "/credentials/register" + | "/credentials/stage" + | "/credentials/status" + | "/credentials/unregister", + payload: Record, +): Promise<{ readonly body: string; readonly status: number }> { + const body = JSON.stringify(payload); + return new Promise((resolve, reject) => { + const request = http.request( + { + socketPath, + path: route, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(body), + }, + }, + (response) => { + const chunks: Buffer[] = []; + response.on("data", (chunk) => chunks.push(chunk)); + response.on("end", () => + resolve({ + body: Buffer.concat(chunks).toString("utf8"), + status: response.statusCode ?? 0, + }), + ); + }, + ); + request.on("error", reject); + request.end(body); + }); +} + async function waitForBrokerCondition( description: string, child: ChildProcess, @@ -97,6 +146,14 @@ async function waitForBrokerCondition( } describe("Hermes managed-tool gateway broker", () => { + it("keeps the local client deadline beyond the broker's end-to-end deadline", () => { + const contract = require(CONTROL_CONTRACT); + expect(contract.HERMES_CLONE_CONTROL_CLIENT_TIMEOUT_MS).toBe( + contract.HERMES_CLONE_CONTROL_DEADLINE_MS + contract.HERMES_CLONE_CONTROL_CLIENT_MARGIN_MS, + ); + expect(contract.HERMES_CLONE_CONTROL_CLIENT_MARGIN_MS).toBeGreaterThan(0); + }); + it("only auto-recovers for Hermes sandboxes with selected managed tools", () => { delete require.cache[require.resolve(BROKER_WRAPPER)]; const broker = require(BROKER_WRAPPER); @@ -125,6 +182,132 @@ describe("Hermes managed-tool gateway broker", () => { hermesToolGateways: ["nous-web"], }), ).toBe(true); + expect( + broker.matchesHermesToolGatewayProviderState( + { + name: "clone", + agent: "hermes", + hermesToolGateways: ["nous-web"], + hermesInferenceProvider: "clone-hermes-inference", + }, + { + sandbox: "clone", + provider_name: "clone-hermes-tool-gateway", + inference_provider_name: "clone-hermes-inference", + }, + ), + ).toBe(true); + expect( + broker.matchesHermesToolGatewayProviderState( + { + name: "clone", + agent: "hermes", + hermesToolGateways: ["nous-web"], + hermesInferenceProvider: "clone-hermes-inference", + }, + { + sandbox: "clone", + provider_name: "clone-hermes-tool-gateway", + inference_provider_name: "hermes-provider", + }, + ), + ).toBe(false); + expect( + broker.planHermesToolGatewayBrokerRefresh({ + currentBrokerHealthy: true, + hashMatches: false, + }), + ).toBe("preserve-runtime-mismatch"); + expect( + broker.planHermesToolGatewayBrokerRefresh({ + currentBrokerHealthy: true, + hashMatches: true, + }), + ).toBe("register-with-current"); + expect( + broker.planHermesToolGatewayBrokerRefresh({ + currentBrokerHealthy: false, + hashMatches: false, + }), + ).toBe("start-or-restart"); + }); + + it("preserves durable state when live credential unregister fails", () => { + delete require.cache[require.resolve(BROKER_WRAPPER)]; + const broker = require(BROKER_WRAPPER); + const unlinkState = vi.fn(); + + expect( + broker.removeHermesToolGatewayProviderState("clone", { + getStatePath: () => "/test-only/clone.json", + controlSocketExists: () => true, + unregister: () => false, + unlinkState, + }), + ).toBe(false); + expect(unlinkState).not.toHaveBeenCalled(); + }); + + it("removes broker state only for the exact registry identity", () => { + delete require.cache[require.resolve(BROKER_WRAPPER)]; + const broker = require(BROKER_WRAPPER); + const removeState = vi.fn(() => true); + const entry = { + name: "clone", + agent: "hermes", + hermesToolGateways: [], + hermesInferenceProvider: "clone-hermes-inference", + }; + const exactState = { + sandbox: "clone", + provider_name: "clone-hermes-tool-gateway", + inference_provider_name: "clone-hermes-inference", + }; + const deps = { + getStatePath: () => "/test-only/clone.json", + stateExists: () => true, + removeState, + }; + + expect( + broker.removeHermesToolGatewayProviderStateForSandboxEntry( + { ...entry, hermesInferenceProvider: "other-hermes-inference" }, + { ...deps, stateExists: () => false }, + ), + ).toBe(false); + expect( + broker.removeHermesToolGatewayProviderStateForSandboxEntry(entry, { + ...deps, + readState: () => ({ ...exactState, sandbox: "other" }), + }), + ).toBe(false); + expect(removeState).not.toHaveBeenCalled(); + + expect( + broker.removeHermesToolGatewayProviderStateForSandboxEntry(entry, { + ...deps, + readState: () => exactState, + }), + ).toBe(true); + expect(removeState).toHaveBeenCalledExactlyOnceWith("clone"); + }); + + it("probes private broker boot/control and classifies failures before clone mutation", async () => { + delete require.cache[require.resolve(BROKER_WRAPPER)]; + const broker = require(BROKER_WRAPPER); + + expect(() => + broker.probeHermesToolGatewayBrokerStart({ + spawnSyncImpl: () => ({ status: 2 }), + }), + ).toThrow("could not bind its runtime endpoints"); + expect(() => + broker.probeHermesToolGatewayBrokerStart({ + spawnSyncImpl: () => ({ status: 3 }), + }), + ).toThrow("control registration path failed"); + const probePort = await freePort(); + expect(() => broker.probeHermesToolGatewayBrokerStart({ port: probePort })).not.toThrow(); }); it("refreshes via header, replaces upstream auth, normalizes responses, and rotates OpenShell storage", { @@ -343,7 +526,8 @@ describe("Hermes managed-tool gateway broker", () => { expect(openshellOutput).toContain( "provider update sandbox-hermes-tool-gateway --credential NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN", ); - expect(openshellOutput).toContain("refresh=refresh-2"); + expect(openshellOutput).toContain("refresh=broker-1"); + expect(openshellOutput).not.toContain("refresh=refresh-2"); expect(openshellOutput).toContain( "provider update hermes-provider --credential OPENAI_API_KEY --config OPENAI_BASE_URL=https://inference-api.nousresearch.com/v1", ); @@ -391,4 +575,263 @@ describe("Hermes managed-tool gateway broker", () => { expect(output).not.toContain("sandbox-secret"); expect(output).not.toContain("agent-key-2"); }); + + it("keeps source and destination credentials live in one broker process and unregisters only the destination", { + timeout: BROKER_TEST_TIMEOUT_MS, + }, async ({ resources }) => { + const tmp = resources.temporaryDirectory("nemoclaw-hermes-tool-broker-coexistence-"); + const stateDir = path.join(tmp, "state"); + const binDir = path.join(tmp, "bin"); + // AF_UNIX paths are short on macOS; the Vitest-owned TMPDIR itself can + // exceed that limit before the socket name is appended. + const socketDir = resources.ownDirectory(fs.mkdtempSync("/tmp/nc-hermes-broker-")); + const controlSocket = path.join(socketDir, "control.sock"); + fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); + fs.mkdirSync(binDir, { recursive: true }); + const openshellLog = path.join(tmp, "openshell.log"); + const openshellBin = path.join(binDir, "openshell"); + fs.writeFileSync( + openshellBin, + `#!/bin/sh\nprintf '%s\\n' "$*" >> "${openshellLog}"\nexit 0\n`, + { mode: 0o755 }, + ); + + const writeState = ( + sandbox: string, + brokerToken: string, + refreshToken: string, + inferenceProviderName: string, + ): void => { + fs.writeFileSync( + path.join(stateDir, `${sandbox}.json`), + JSON.stringify( + { + version: 1, + sandbox, + provider_name: `${sandbox}-hermes-tool-gateway`, + inference_provider_name: inferenceProviderName, + inference_credential_env: "OPENAI_API_KEY", + credential_env: "NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN", + broker_token: brokerToken, + broker_token_sha256: sha256(brokerToken), + refresh_token_sha256: sha256(refreshToken), + client_id: "hermes-cli", + }, + null, + 2, + ), + { mode: 0o600 }, + ); + }; + writeState("source", "source-broker-token", "source-refresh-token", "hermes-provider"); + writeState( + "destination", + "destination-broker-token", + "destination-refresh-token", + "destination-hermes-inference", + ); + + const refreshHeaders: string[] = []; + const portal = resources.ownServer( + http.createServer((req, res) => + handleHermesBrokerCoexistencePortal(refreshHeaders, req, res), + ), + ); + const portalPort = await listen(portal); + + const upstreamAuthorizations: string[] = []; + const upstream = resources.ownServer( + http.createServer((req, res) => { + upstreamAuthorizations.push(String(req.headers.authorization || "")); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true })); + }), + ); + const upstreamPort = await listen(upstream); + const matrixPath = path.join(tmp, "matrix.json"); + fs.writeFileSync( + matrixPath, + JSON.stringify({ + "nous-web": { + service: "firecrawl", + upstream: `http://127.0.0.1:${upstreamPort}`, + }, + }), + ); + const brokerPort = await freePort(); + const child = resources.ownChild( + spawn(process.execPath, ["--experimental-strip-types", SCRIPT], { + env: { + ...process.env, + HERMES_TOOL_GATEWAY_PORT: String(brokerPort), + HERMES_TOOL_GATEWAY_STATE_DIR: stateDir, + HERMES_TOOL_GATEWAY_MATRIX_PATH: matrixPath, + HERMES_TOOL_GATEWAY_CONTROL_SOCKET: controlSocket, + HERMES_INFERENCE_AGENT_KEY_REFRESH_INTERVAL_MS: "3600000", + NOUS_PORTAL_BASE_URL: `http://127.0.0.1:${portalPort}`, + NEMOCLAW_OPENSHELL_BIN: openshellBin, + NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN: "source-refresh-token", + }, + stdio: ["ignore", "pipe", "pipe"], + }), + ); + + let output = ""; + child.stdout.on("data", (chunk) => { + output += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + output += chunk.toString(); + }); + await waitForBrokerCondition( + "broker and private control socket", + child, + () => output, + async () => { + const response = await fetch(`http://127.0.0.1:${brokerPort}/health`, { + signal: AbortSignal.timeout(1_000), + }); + return ( + response.status === 200 && + fs.existsSync(controlSocket) && + (fs.statSync(controlSocket).mode & 0o777) === 0o600 + ); + }, + ); + + const proxy = (brokerToken: string) => + fetch(`http://127.0.0.1:${brokerPort}/firecrawl/v1/scrape`, { + method: "POST", + headers: { Authorization: `Bearer ${brokerToken}` }, + body: "{}", + }); + + expect((await proxy("source-broker-token")).status).toBe(200); + await expect( + controlRequest(controlSocket, "/credentials/register", { + sandbox: "destination", + refresh_token: "destination-refresh-token", + }), + ).resolves.toMatchObject({ status: 200 }); + expect((await proxy("destination-broker-token")).status).toBe(200); + + const stagedPayload = { + sandbox: "staged", + refresh_token: "staged-refresh-token", + inference_provider_name: "staged-hermes-inference", + request_id: `nc_clone_${"3".repeat(32)}`, + deadline_at_ms: Date.now() + 120_000, + }; + const stagedResponse = await controlRequest(controlSocket, "/credentials/stage", stagedPayload); + expect(stagedResponse.status, `${stagedResponse.body}\n${output}`).toBe(200); + const staged = JSON.parse(stagedResponse.body) as { + activation_token: string; + broker_token: string; + }; + expect(staged.activation_token).toMatch(/^nc_activate_/u); + expect(staged.broker_token).toMatch(/^nc_broker_/u); + const repeatedStage = await controlRequest(controlSocket, "/credentials/stage", stagedPayload); + expect(repeatedStage.status).toBe(200); + expect(JSON.parse(repeatedStage.body)).toMatchObject(staged); + writeState("staged", staged.broker_token, "staged-refresh-token", "staged-hermes-inference"); + await expect( + controlRequest(controlSocket, "/credentials/activate", { + sandbox: "staged", + activation_token: staged.activation_token, + deadline_at_ms: Date.now() + 120_000, + }), + ).resolves.toMatchObject({ status: 200 }); + await expect( + controlRequest(controlSocket, "/credentials/activate", { + sandbox: "staged", + activation_token: staged.activation_token, + deadline_at_ms: Date.now() + 120_000, + }), + ).resolves.toMatchObject({ status: 200 }); + await expect( + controlRequest(controlSocket, "/credentials/status", { + activation_token: staged.activation_token, + }), + ).resolves.toMatchObject({ status: 200 }); + + const discardedPayload = { + sandbox: "discarded", + refresh_token: "discarded-refresh-token", + inference_provider_name: "discarded-hermes-inference", + request_id: `nc_clone_${"4".repeat(32)}`, + deadline_at_ms: Date.now() + 120_000, + }; + const discardedStage = await controlRequest( + controlSocket, + "/credentials/stage", + discardedPayload, + ); + const discarded = JSON.parse(discardedStage.body) as { activation_token: string }; + await expect( + controlRequest(controlSocket, "/credentials/discard", { + sandbox: "discarded", + activation_token: discarded.activation_token, + }), + ).resolves.toMatchObject({ status: 200 }); + await expect( + controlRequest(controlSocket, "/credentials/discard", { + sandbox: "discarded", + activation_token: discarded.activation_token, + }), + ).resolves.toMatchObject({ status: 200 }); + await expect( + controlRequest(controlSocket, "/credentials/stage", discardedPayload), + ).resolves.toMatchObject({ status: 400 }); + + await expect( + controlRequest(controlSocket, "/credentials/stage", { + sandbox: "deadline", + refresh_token: "deadline-refresh-token", + inference_provider_name: "deadline-hermes-inference", + request_id: `nc_clone_${"5".repeat(32)}`, + deadline_at_ms: Date.now() + 50, + }), + ).resolves.toMatchObject({ status: 400 }); + await expect( + controlRequest(controlSocket, "/credentials/stage", { + sandbox: "Invalid_Sandbox", + refresh_token: "must-not-reach-portal", + inference_provider_name: "valid-hermes-inference", + request_id: `nc_clone_${"6".repeat(32)}`, + deadline_at_ms: Date.now() + 120_000, + }), + ).resolves.toMatchObject({ status: 400 }); + expect(refreshHeaders).not.toContain("must-not-reach-portal"); + expect((await proxy(staged.broker_token)).status).toBe(200); + + await expect( + controlRequest(controlSocket, "/credentials/unregister", { + sandbox: "destination", + }), + ).resolves.toMatchObject({ status: 200 }); + expect((await proxy("destination-broker-token")).status).toBe(401); + expect((await proxy("source-broker-token")).status).toBe(200); + + expect(upstreamAuthorizations).toEqual([ + "Bearer access-source", + "Bearer access-destination", + "Bearer access-staged", + "Bearer access-source", + ]); + expect(refreshHeaders).toEqual( + expect.arrayContaining(["source-refresh-token", "destination-refresh-token"]), + ); + const openshellUpdates = fs.readFileSync(openshellLog, "utf8"); + expect(openshellUpdates).toContain( + "provider update hermes-provider --credential OPENAI_API_KEY", + ); + expect(openshellUpdates).toContain( + "provider update destination-hermes-inference --credential OPENAI_API_KEY", + ); + expect(openshellUpdates).toContain( + "provider update staged-hermes-inference --credential OPENAI_API_KEY", + ); + expect(output).not.toContain("source-refresh-token"); + expect(output).not.toContain("destination-refresh-token"); + }); }); diff --git a/test/hermes-tool-gateway-runtime-credentials.test.ts b/test/hermes-tool-gateway-runtime-credentials.test.ts new file mode 100644 index 00000000000..fe0ca7a44e2 --- /dev/null +++ b/test/hermes-tool-gateway-runtime-credentials.test.ts @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; + +const { RuntimeRefreshCredentialStore } = + require("../agents/hermes/host/runtime-refresh-credentials.ts") as { + RuntimeRefreshCredentialStore: new ( + hashCredential: (value: string) => string, + ) => { + register(state: Record, refreshToken: string): boolean; + resolve(state: Record): string | null; + unregister(sandboxName: string): boolean; + }; + }; + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +describe("Hermes tool-gateway runtime credentials", () => { + it("keeps source and destination credentials live and cleans only the destination", () => { + const sourceToken = "test-only-source-refresh"; + const destinationToken = "test-only-destination-refresh"; + const source = { + sandbox: "source", + refresh_token_sha256: sha256(sourceToken), + }; + const destination = { + sandbox: "destination", + refresh_token_sha256: sha256(destinationToken), + }; + const store = new RuntimeRefreshCredentialStore(sha256); + + expect(store.register(source, sourceToken)).toBe(true); + expect(store.register(destination, destinationToken)).toBe(true); + expect(store.resolve(source)).toBe(sourceToken); + expect(store.resolve(destination)).toBe(destinationToken); + + expect(store.unregister("destination")).toBe(true); + expect(store.resolve(destination)).toBeNull(); + expect(store.resolve(source)).toBe(sourceToken); + }); + + it("rejects a credential that does not match the destination state hash", () => { + const store = new RuntimeRefreshCredentialStore(sha256); + const destination = { + sandbox: "destination", + refresh_token_sha256: sha256("expected-refresh"), + }; + + expect(store.register(destination, "wrong-refresh")).toBe(false); + expect(store.resolve(destination)).toBeNull(); + }); +}); From 888e2f98f2a106306a325bd9821efc1cf7b60454 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 04:01:37 -0700 Subject: [PATCH 054/117] test(snapshot): satisfy Hermes broker CLI typings Signed-off-by: Aaron Erickson --- ...apshot-hermes-managed-clone-broker.test.ts | 58 ++++++++++++++----- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/src/lib/actions/sandbox/snapshot-hermes-managed-clone-broker.test.ts b/src/lib/actions/sandbox/snapshot-hermes-managed-clone-broker.test.ts index eb9e15f49fc..1076d616665 100644 --- a/src/lib/actions/sandbox/snapshot-hermes-managed-clone-broker.test.ts +++ b/src/lib/actions/sandbox/snapshot-hermes-managed-clone-broker.test.ts @@ -3,7 +3,7 @@ import { createHash } from "node:crypto"; -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, it, type Mock, vi } from "vitest"; import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; import type { HermesToolGatewayCloneBroker } from "../../hermes-tool-gateway-clone-broker"; import { @@ -135,27 +135,50 @@ function providerRunner() { return { createCredentials, live, run }; } -function broker(): HermesToolGatewayCloneBroker & { - activateHermesToolGatewayCloneBinding: ReturnType; - discardHermesToolGatewayCloneBinding: ReturnType; - preflightHermesToolGatewayCloneBinding: ReturnType; - stageHermesToolGatewayCloneBinding: ReturnType; -} { +type HermesBrokerMock = Omit< + HermesToolGatewayCloneBroker, + | "activateHermesToolGatewayCloneBinding" + | "discardHermesToolGatewayCloneBinding" + | "preflightHermesToolGatewayCloneBinding" + | "stageHermesToolGatewayCloneBinding" +> & { + activateHermesToolGatewayCloneBinding: Mock< + HermesToolGatewayCloneBroker["activateHermesToolGatewayCloneBinding"] + >; + discardHermesToolGatewayCloneBinding: Mock< + HermesToolGatewayCloneBroker["discardHermesToolGatewayCloneBinding"] + >; + preflightHermesToolGatewayCloneBinding: Mock< + HermesToolGatewayCloneBroker["preflightHermesToolGatewayCloneBinding"] + >; + stageHermesToolGatewayCloneBinding: Mock< + HermesToolGatewayCloneBroker["stageHermesToolGatewayCloneBinding"] + >; +}; + +function broker(): HermesBrokerMock { return { HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV, getHermesToolGatewayProviderName: (name) => `${name}-hermes-tool-gateway`, getHermesInferenceProviderName: (name) => `${name}-hermes-inference`, - preflightHermesToolGatewayCloneBinding: vi.fn(), - stageHermesToolGatewayCloneBinding: vi.fn(() => ({ + preflightHermesToolGatewayCloneBinding: + vi.fn(), + stageHermesToolGatewayCloneBinding: vi.fn< + HermesToolGatewayCloneBroker["stageHermesToolGatewayCloneBinding"] + >(() => ({ activationToken: `nc_activate_${"a".repeat(43)}`, brokerToken: `nc_broker_${"b".repeat(43)}`, requestId: `nc_clone_${"1".repeat(32)}`, })), - activateHermesToolGatewayCloneBinding: vi.fn(() => ({ + activateHermesToolGatewayCloneBinding: vi.fn< + HermesToolGatewayCloneBroker["activateHermesToolGatewayCloneBinding"] + >(() => ({ file: "/tmp/destination.json", brokerToken: `nc_broker_${"b".repeat(43)}`, })), - discardHermesToolGatewayCloneBinding: vi.fn(() => true), + discardHermesToolGatewayCloneBinding: vi.fn< + HermesToolGatewayCloneBroker["discardHermesToolGatewayCloneBinding"] + >(() => true), bindHermesToolGatewayCloneProviderState: vi.fn(() => ({ file: "/tmp/destination.json", brokerToken: `nc_broker_${"b".repeat(43)}`, @@ -269,16 +292,19 @@ describe("Hermes managed clone broker transaction", () => { transactionId: "2".repeat(32), }); - expect(() => + let thrown: unknown; + try { provisionHermesManagedCloneBrokerTransaction(prepared, { ...authority(source), environment: environment(), runOpenshell: runner.run, broker: hostBroker, - }), - ).toThrowError( - expect.objectContaining({ cleanupDeferred: true }), - ); + }); + } catch (error) { + thrown = error; + } + expect(thrown).toBeInstanceOf(HermesManagedCloneBrokerTransactionError); + expect((thrown as HermesManagedCloneBrokerTransactionError).cleanupDeferred).toBe(true); expect(runner.live.size).toBe(2); expect(hostBroker.discardHermesToolGatewayCloneBinding).not.toHaveBeenCalled(); }); From c176a4f109be406dd96d391190e0b04716550a41 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 19:36:16 -0700 Subject: [PATCH 055/117] fix(snapshot): use shared immutable boundary Signed-off-by: Aaron Erickson --- .../actions/sandbox/snapshot/hermes-managed-clone-broker.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/actions/sandbox/snapshot/hermes-managed-clone-broker.ts b/src/lib/actions/sandbox/snapshot/hermes-managed-clone-broker.ts index 5def2c95f9a..32d984a7e74 100644 --- a/src/lib/actions/sandbox/snapshot/hermes-managed-clone-broker.ts +++ b/src/lib/actions/sandbox/snapshot/hermes-managed-clone-broker.ts @@ -3,7 +3,7 @@ import { randomBytes } from "node:crypto"; -import { deepFreezeOwned } from "../../../core/deep-freeze-owned"; +import { cloneAndDeepFreeze } from "../../../core/immutable"; import { getHermesToolGatewayCloneBroker, type HermesToolGatewayCloneBroker, @@ -179,7 +179,7 @@ export function prepareHermesManagedCloneBrokerTransaction(input: { runOpenshell: input.runOpenshell, transactionId: input.transactionId, }); - return deepFreezeOwned({ + return cloneAndDeepFreeze({ schemaVersion: 1 as const, phase: "prepared" as const, gatewayProviderName: bindings[1].providerName, @@ -249,7 +249,7 @@ export function provisionHermesManagedCloneBrokerTransaction( refreshToken, staged, ); - return deepFreezeOwned({ + return cloneAndDeepFreeze({ schemaVersion: 1 as const, phase: "activated" as const, destinationSandboxName: prepared.providerTransaction.destinationSandboxName, From 8bc9ea13c379e15ca892382272cc01ceeb564b6c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 19:49:42 -0700 Subject: [PATCH 056/117] feat(onboard): define dormant transactional managed bootstrap Signed-off-by: Aaron Erickson --- ci/source-shape-test-budget.json | 5 + scripts/managed-bootstrap-trampoline.sh | 110 ++ src/lib/onboard/managed-bootstrap/README.md | 39 + .../onboard/managed-bootstrap/adapter.test.ts | 669 ++++++++ src/lib/onboard/managed-bootstrap/adapter.ts | 1444 +++++++++++++++++ .../managed-bootstrap/envelope.test.ts | 87 + src/lib/onboard/managed-bootstrap/envelope.ts | 172 ++ src/lib/onboard/managed-bootstrap/index.ts | 22 + test/managed-bootstrap-trampoline.test.ts | 174 ++ test/runtime-provider-source-shape.test.ts | 50 + 10 files changed, 2772 insertions(+) create mode 100755 scripts/managed-bootstrap-trampoline.sh create mode 100644 src/lib/onboard/managed-bootstrap/README.md create mode 100644 src/lib/onboard/managed-bootstrap/adapter.test.ts create mode 100644 src/lib/onboard/managed-bootstrap/adapter.ts create mode 100644 src/lib/onboard/managed-bootstrap/envelope.test.ts create mode 100644 src/lib/onboard/managed-bootstrap/envelope.ts create mode 100644 src/lib/onboard/managed-bootstrap/index.ts create mode 100644 test/managed-bootstrap-trampoline.test.ts diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index c5f842289eb..a4d7cfe4b74 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -511,6 +511,11 @@ "test": "keeps migrated provider identities and implementations behind the one bundle composition", "category": "compatibility" }, + { + "file": "test/runtime-provider-source-shape.test.ts", + "test": "keeps managed bootstrap provider-neutral, image-owned, and dormant", + "category": "security" + }, { "file": "test/pr-limit-policy.test.ts", "test": "keeps contributor guidance aligned with the enforced maintainer exemption", diff --git a/scripts/managed-bootstrap-trampoline.sh b/scripts/managed-bootstrap-trampoline.sh new file mode 100755 index 00000000000..7c621bc1b80 --- /dev/null +++ b/scripts/managed-bootstrap-trampoline.sh @@ -0,0 +1,110 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Image-owned bootstrap boundary. A runtime provider creates a replacement +# without starting it, writes one bounded root-owned request into its writable +# layer, and starts this trampoline as PID 1. The exact captured supervisor argv +# cannot run until the request has been authenticated and applied. + +set -euo pipefail + +fail() { + printf '[SECURITY] Managed bootstrap trampoline: %s\n' "$*" >&2 + exit 1 +} + +[ "$(/usr/bin/id -u)" -eq 0 ] && [ "$(/usr/bin/id -g)" -eq 0 ] \ + || fail "must run as root" +[ "$#" -ge 16 ] \ + || fail "managed bootstrap arguments are incomplete" +[ "$1" = "--agent" ] || fail "agent argument is missing" +_nemoclaw_agent="$2" +[ "$3" = "--profile-fingerprint" ] || fail "profile fingerprint argument is missing" +_nemoclaw_fingerprint="$4" +[ "$5" = "--bootstrap-identity" ] || fail "bootstrap identity argument is missing" +_nemoclaw_bootstrap_identity="$6" +[ "$7" = "--agent-uid" ] || fail "agent uid argument is missing" +_nemoclaw_agent_uid="$8" +[ "$9" = "--agent-gid" ] || fail "agent gid argument is missing" +_nemoclaw_agent_gid="${10}" +[ "${11}" = "--agent-workdir" ] || fail "agent workdir argument is missing" +_nemoclaw_agent_workdir="${12}" +[ "${13}" = "--request-file" ] || fail "request-file argument is missing" +_nemoclaw_request="${14:-}" +[ "${15:-}" = "--" ] || fail "supervisor delimiter is missing" +shift 15 +[ "$#" -gt 0 ] || fail "supervisor argv is empty" + +case "$_nemoclaw_agent" in + openclaw | hermes | langchain-deepagents-code) ;; + *) fail "agent is unsupported" ;; +esac +case "$_nemoclaw_fingerprint" in + *[!0-9a-f]* | "") fail "profile fingerprint must be lowercase SHA-256" ;; +esac +[ "${#_nemoclaw_fingerprint}" -eq 64 ] \ + || fail "profile fingerprint must be lowercase SHA-256" +case "$_nemoclaw_bootstrap_identity" in + *[!0-9a-f]* | "") fail "bootstrap identity must be lowercase hex" ;; +esac +[ "${#_nemoclaw_bootstrap_identity}" -eq 64 ] \ + || fail "bootstrap identity must encode 32 bytes" +case "$_nemoclaw_agent_uid:$_nemoclaw_agent_gid" in + *[!0-9:]* | :* | *:) fail "agent uid/gid must be numeric" ;; +esac +[ "$(/usr/bin/id -u sandbox)" = "$_nemoclaw_agent_uid" ] \ + && [ "$(/usr/bin/id -g sandbox)" = "$_nemoclaw_agent_gid" ] \ + || fail "agent identity does not match the image sandbox account" +[ "$_nemoclaw_agent_workdir" = "/sandbox" ] \ + && [ -d "$_nemoclaw_agent_workdir" ] \ + && [ ! -L "$_nemoclaw_agent_workdir" ] \ + || fail "agent workdir does not match the image sandbox workspace" +[ "$_nemoclaw_request" = "/var/lib/nemoclaw-managed-bootstrap-request.json" ] \ + || fail "request file path is not the fixed bootstrap path" + +_nemoclaw_runtime="/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs" +[ -f "$_nemoclaw_runtime" ] && [ ! -L "$_nemoclaw_runtime" ] \ + || fail "managed startup runtime is missing" +if [ -L "$_nemoclaw_request" ]; then + fail "bootstrap request path is a symbolic link" +fi +if [ -e "$_nemoclaw_request" ]; then + if [ ! -f "$_nemoclaw_request" ] \ + || [ "$(/usr/bin/stat -c '%u:%g:%a:%h' "$_nemoclaw_request")" != "0:0:400:1" ]; then + fail "bootstrap request failed root ownership validation" + fi + /usr/bin/env -i \ + HOME="/root" \ + LANG="C.UTF-8" \ + LC_ALL="C.UTF-8" \ + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION="1" \ + PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + /usr/local/bin/node "$_nemoclaw_runtime" \ + --apply-bootstrap-file \ + --agent "$_nemoclaw_agent" \ + --profile-fingerprint "$_nemoclaw_fingerprint" \ + --bootstrap-identity "$_nemoclaw_bootstrap_identity" + /usr/bin/rm -f -- "$_nemoclaw_request" + [ ! -e "$_nemoclaw_request" ] && [ ! -L "$_nemoclaw_request" ] \ + || fail "bootstrap runtime did not consume its request" +fi +/usr/bin/env -i \ + HOME="/root" \ + LANG="C.UTF-8" \ + LC_ALL="C.UTF-8" \ + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION="1" \ + PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + /usr/local/bin/node "$_nemoclaw_runtime" \ + --verify-bootstrap-completion \ + --agent "$_nemoclaw_agent" \ + --profile-fingerprint "$_nemoclaw_fingerprint" \ + --bootstrap-identity "$_nemoclaw_bootstrap_identity" + +unset _nemoclaw_agent _nemoclaw_fingerprint _nemoclaw_bootstrap_identity +unset _nemoclaw_agent_uid _nemoclaw_agent_gid _nemoclaw_agent_workdir +unset _nemoclaw_request _nemoclaw_runtime +unset -f fail +exec 3<&- 4<&- 5<&- 6<&- 7<&- 8<&- 9<&- +exec 3>&- 4>&- 5>&- 6>&- 7>&- 8>&- 9>&- +exec "$@" diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md new file mode 100644 index 00000000000..79b17729eac --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -0,0 +1,39 @@ +# Managed bootstrap protocol + +This directory defines a dormant, driver-neutral transaction contract. It does +not register a runtime provider or change sandbox creation, onboarding, +snapshot, clone, or restore behavior. + +The protocol binds one random bootstrap identity to: + +- the expected managed-image manifest digest and startup-profile fingerprint; +- the Ready sandbox and immutable runtime receipts; +- the exact captured supervisor `argv`; +- the replacement runtime, spec hashes, and image-owned completion receipt; and +- an identity-bound rollback receipt when any stage fails. + +The coordinator deliberately exposes two phases. Preparation may create and +inspect a stopped replacement, but it cannot alter the Ready held workload. +Activation first records a complete, fingerprinted authority receipt through +the injected durable store; only then may the provider quiesce or replace the +original runtime. Provider results are copied into deeply frozen coordinator +authority, and finalization receipts must either prove exact snapshot restore +or exact workload absence. + +`scripts/managed-bootstrap-trampoline.sh` defines the image-owned executable +that the later all-agent packaging slice will install as +`/usr/local/bin/nemoclaw-managed-bootstrap`. It authenticates a fixed, +root-owned request, verifies an identity-bound completion, clears its private +bootstrap variables and file descriptors, and then uses `exec "$@"` to preserve +the captured supervisor argument boundaries. + +The trampoline is intentionally not packaged or selected yet, and no production +TypeScript module imports this protocol. The current image definitions also do +not package `nemoclaw-managed-startup-hold` or +`managed-startup-image-runtime.cjs`. A later provider integration must add those +prerequisites together with their image-runtime bootstrap modes, implement +driver-specific prepare, durable-record, activate, exact cleanup, and rollback, +and only then wire the coordinator into create. The same contract is exercised +for OpenClaw, Hermes, and DCode without a provider-specific central switch. +Until that complete boundary lands, every registered runtime provider keeps +its bootstrap surface unsupported. diff --git a/src/lib/onboard/managed-bootstrap/adapter.test.ts b/src/lib/onboard/managed-bootstrap/adapter.test.ts new file mode 100644 index 00000000000..40ec9d744b0 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/adapter.test.ts @@ -0,0 +1,669 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + encodeManagedStartupProfile, + MANAGED_STARTUP_AGENTS, + type ManagedStartupAgent, +} from "../managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import { + activateManagedBootstrapSequence, + finalizeManagedBootstrapSequence, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapAdapter, + type ManagedBootstrapAuthorityStore, + type ManagedBootstrapCompletionReceipt, + type ManagedBootstrapCreateReceipt, + type ManagedBootstrapFinalizationReceipt, + type ManagedBootstrapHeldWorkloadHandle, + type ManagedBootstrapObservedSnapshot, + type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapReplacementHandle, + prepareManagedBootstrapSequence, + renderManagedBootstrapHeldCommand, +} from "./adapter"; + +const IDENTITY = "1".repeat(64); +const CONFIG_ID = `sha256:${"2".repeat(64)}`; +const MANIFEST_DIGEST = `sha256:${"3".repeat(64)}` as const; +const SPEC_JSON = "{}\n"; +const SPEC_HASH = createHash("sha256").update(SPEC_JSON, "utf8").digest("hex"); +const PREPARED_SPEC_JSON = '{"name":"prepared"}\n'; +const PREPARED_HASH = createHash("sha256").update(PREPARED_SPEC_JSON, "utf8").digest("hex"); +const ACTIVATED_SPEC_JSON = '{"name":"active"}\n'; +const ACTIVATED_HASH = createHash("sha256").update(ACTIVATED_SPEC_JSON, "utf8").digest("hex"); +const RUNTIME_ID = "7".repeat(64); +const PREPARED_ID = "8".repeat(64); + +function requestFor(agent: ManagedStartupAgent) { + return createManagedStartupRootApplyRequest({ + agent, + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agent, false, false)), + }); +} + +function planFor(request: ReturnType) { + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandboxName: "alpha", + driverId: "mxc-fixture", + image: { + repository: `registry.example/nemoclaw/${request.agent}`, + manifestDigest: MANIFEST_DIGEST, + }, + profile: { agent: request.agent, fingerprint: request.profileFingerprint }, + agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + intendedWorkloadArgv: ["env", "A=1", "nemoclaw-start"], + expectedSupervisorArgv: ["/runtime/sandbox-supervisor", "supervise", "--foreground"], + metadata: { "nemoclaw.ai/managed-profile": request.profileFingerprint }, + } as const; +} + +function sandbox() { + return { + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + driverId: "mxc-fixture", + } as const; +} + +function createReceipt() { + return { + sandbox: sandbox(), + ready: true as const, + readyAt: "2026-07-29T12:00:00.000Z", + }; +} + +function handleFor( + request: ReturnType, + receipt: ManagedBootstrapCreateReceipt = createReceipt(), +): ManagedBootstrapHeldWorkloadHandle { + const plan = planFor(request); + const heldWorkloadArgv = renderManagedBootstrapHeldCommand( + request, + IDENTITY, + plan.intendedWorkloadArgv, + ); + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: receipt.sandbox, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv, + intendedWorkloadArgv: plan.intendedWorkloadArgv, + plan, + createReceipt: receipt, + }; +} + +function snapshotFor( + request: ReturnType, + handle = handleFor(request), +): ManagedBootstrapObservedSnapshot { + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: RUNTIME_ID, + bootstrapIdentity: IDENTITY, + image: handle.plan.image, + runtimeImageContentId: CONFIG_ID, + specHash: SPEC_HASH, + specCanonicalJson: SPEC_JSON, + agentIdentity: handle.plan.agentIdentity, + supervisorArgv: handle.plan.expectedSupervisorArgv, + heldWorkloadArgv: handle.heldWorkloadArgv, + metadata: handle.plan.metadata, + }; +} + +function preparedFor( + request: ReturnType, + handle = handleFor(request), +): ManagedBootstrapPreparedReplacementHandle { + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: IDENTITY, + originalRuntimeId: RUNTIME_ID, + preparedRuntimeId: PREPARED_ID, + image: handle.plan.image, + runtimeImageContentId: CONFIG_ID, + originalSpecHash: SPEC_HASH, + preparedSpecHash: PREPARED_HASH, + preparedSpecCanonicalJson: PREPARED_SPEC_JSON, + expectedActivatedSpecHash: ACTIVATED_HASH, + expectedActivatedSpecCanonicalJson: ACTIVATED_SPEC_JSON, + profileFingerprint: request.profileFingerprint, + rollbackAuthority: "mxc-opaque-rollback-authority", + }; +} + +function replacementFor( + request: ReturnType, + handle = handleFor(request), +): ManagedBootstrapReplacementHandle { + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: IDENTITY, + originalRuntimeId: RUNTIME_ID, + replacementRuntimeId: PREPARED_ID, + image: handle.plan.image, + runtimeImageContentId: CONFIG_ID, + originalSpecHash: SPEC_HASH, + replacementSpecHash: ACTIVATED_HASH, + replacementSpecCanonicalJson: ACTIVATED_SPEC_JSON, + profileFingerprint: request.profileFingerprint, + }; +} + +function completionFor( + request: ReturnType, + handle = handleFor(request), +): ManagedBootstrapCompletionReceipt { + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: PREPARED_ID, + image: handle.plan.image, + runtimeImageContentId: CONFIG_ID, + originalSpecHash: SPEC_HASH, + replacementSpecHash: ACTIVATED_HASH, + profileFingerprint: request.profileFingerprint, + bootstrapIdentity: IDENTITY, + transactionPending: true, + completedAt: "2026-07-29T12:01:00.000Z", + }; +} + +function rolledBackReceipt( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot | null, +): ManagedBootstrapFinalizationReceipt { + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: IDENTITY, + outcome: "rolled-back", + restoredRuntimeId: snapshot?.runtimeId ?? null, + restoredSpecHash: snapshot?.specHash ?? null, + heldWorkloadRemoved: snapshot === null, + alreadyRolledBack: false, + finalizedAt: "2026-07-29T12:02:00.000Z", + }; +} + +function cleanupReceipt(): ManagedBootstrapFinalizationReceipt { + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: sandbox(), + bootstrapIdentity: IDENTITY, + outcome: "rolled-back", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: true, + alreadyRolledBack: false, + finalizedAt: "2026-07-29T12:02:00.000Z", + }; +} + +interface Fixture { + readonly adapter: ManagedBootstrapAdapter; + readonly order: string[]; + readonly raw: { + handle: ManagedBootstrapHeldWorkloadHandle | null; + snapshot: ManagedBootstrapObservedSnapshot | null; + prepared: ManagedBootstrapPreparedReplacementHandle | null; + }; +} + +function adapterFor(agent: ManagedStartupAgent): Fixture { + const request = requestFor(agent); + const order: string[] = []; + const raw: Fixture["raw"] = { handle: null, snapshot: null, prepared: null }; + const adapter: ManagedBootstrapAdapter = { + createHeldWorkload: vi.fn(async (input) => { + order.push("create"); + const receipt = await input.launch({ + heldWorkloadArgv: renderManagedBootstrapHeldCommand( + input.request, + input.bootstrapIdentity as string, + input.plan.intendedWorkloadArgv, + ), + bootstrapIdentity: input.bootstrapIdentity as string, + }); + raw.handle = handleFor(request, receipt); + return raw.handle; + }), + cleanupIncompleteCreate: vi.fn(async () => { + order.push("cleanup-incomplete"); + return cleanupReceipt(); + }), + discoverHeldWorkload: vi.fn(async (input) => { + order.push("discover"); + return { + sandbox: input.sandbox, + runtimeId: RUNTIME_ID, + bootstrapIdentity: IDENTITY, + }; + }), + inspectHeldWorkload: vi.fn(async ({ handle }) => { + order.push("inspect"); + const observed = snapshotFor(request, handle); + raw.snapshot = { + ...observed, + sandbox: { ...observed.sandbox }, + image: { ...observed.image }, + agentIdentity: { ...observed.agentIdentity }, + supervisorArgv: [...observed.supervisorArgv], + heldWorkloadArgv: [...observed.heldWorkloadArgv], + metadata: { ...observed.metadata }, + }; + return raw.snapshot; + }), + prepareBootstrapReplacement: vi.fn(async ({ handle }) => { + order.push("prepare-replacement"); + raw.prepared = preparedFor(request, handle); + return raw.prepared; + }), + activateBootstrapReplacement: vi.fn(async ({ handle }) => { + order.push("activate"); + return replacementFor(request, handle); + }), + awaitBootstrap: vi.fn(async ({ handle }) => { + order.push("await"); + return completionFor(request, handle); + }), + finalizeBootstrap: vi.fn(async (input) => { + order.push(input.outcome); + if (input.outcome === "rollback") return rolledBackReceipt(input.handle, input.snapshot); + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: input.handle.sandbox, + bootstrapIdentity: IDENTITY, + outcome: "committed" as const, + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: "2026-07-29T12:03:00.000Z", + }; + }), + }; + return { adapter, order, raw }; +} + +function authorityStore(order: string[]): ManagedBootstrapAuthorityStore { + return { + recordPreparedAuthority: vi.fn(async (authority) => { + order.push("record"); + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: authority.sandbox, + bootstrapIdentity: authority.bootstrapIdentity, + authorityFingerprint: authority.authorityFingerprint, + recordId: "durable-record-alpha", + recordedAt: "2026-07-29T12:00:30.000Z", + }; + }), + }; +} + +function preparationInput(agent: ManagedStartupAgent) { + const request = requestFor(agent); + return { + create: { + plan: planFor(request), + request, + bootstrapIdentity: IDENTITY, + launch: vi.fn(async () => createReceipt()), + }, + request, + replacementOptions: { values: { mode: "native", groups: ["44", "109"] } }, + } as const; +} + +async function prepareAndActivate(agent: ManagedStartupAgent, fixture = adapterFor(agent)) { + const prepared = await prepareManagedBootstrapSequence(fixture.adapter, preparationInput(agent)); + const activated = await activateManagedBootstrapSequence(fixture.adapter, { + transaction: prepared, + authorityStore: authorityStore(fixture.order), + timeoutSecs: 30, + }); + return { ...fixture, prepared, activated }; +} + +async function captureFailure(promise: Promise) { + try { + await promise; + } catch (error) { + return error as Error & { + managedBootstrapRollback?: ManagedBootstrapFinalizationReceipt; + managedBootstrapRollbackError?: unknown; + }; + } + throw new Error("Expected managed bootstrap operation to fail."); +} + +describe("managed bootstrap adapter contract", () => { + it.each( + MANAGED_STARTUP_AGENTS, + )("prepares, durably records, and only then activates %s through a provider-neutral adapter", async (agent) => { + const result = await prepareAndActivate(agent); + + expect(result.order).toEqual([ + "create", + "discover", + "inspect", + "prepare-replacement", + "record", + "activate", + "await", + ]); + expect(result.activated.completion).toMatchObject({ + bootstrapIdentity: IDENTITY, + runtimeId: PREPARED_ID, + profileFingerprint: requestFor(agent).profileFingerprint, + }); + expect(Object.isFrozen(result.prepared)).toBe(true); + expect(Object.isFrozen(result.prepared.handle.plan.metadata)).toBe(true); + expect(Object.isFrozen(result.prepared.prepared)).toBe(true); + expect(Object.isFrozen(result.activated.durablePreparation)).toBe(true); + const prepareInput = vi.mocked(result.adapter.prepareBootstrapReplacement).mock.calls[0]?.[0]; + expect(Object.isFrozen(prepareInput?.replacementOptions.values)).toBe(true); + expect(Object.isFrozen(prepareInput?.replacementOptions.values.groups)).toBe(true); + }); + + it("stops after non-destructive preparation until durable activation is requested", async () => { + const fixture = adapterFor("openclaw"); + const prepared = await prepareManagedBootstrapSequence( + fixture.adapter, + preparationInput("openclaw"), + ); + + expect(fixture.order).toEqual(["create", "discover", "inspect", "prepare-replacement"]); + expect(fixture.adapter.activateBootstrapReplacement).not.toHaveBeenCalled(); + expect(prepared.prepared.originalRuntimeId).toBe(RUNTIME_ID); + expect(prepared.prepared.preparedRuntimeId).toBe(PREPARED_ID); + }); + + it("consumes prepared authority exactly once and rejects a second activation", async () => { + const fixture = adapterFor("openclaw"); + const prepared = await prepareManagedBootstrapSequence( + fixture.adapter, + preparationInput("openclaw"), + ); + await activateManagedBootstrapSequence(fixture.adapter, { + transaction: prepared, + authorityStore: authorityStore(fixture.order), + timeoutSecs: 30, + }); + + await expect( + activateManagedBootstrapSequence(fixture.adapter, { + transaction: prepared, + authorityStore: authorityStore(fixture.order), + timeoutSecs: 30, + }), + ).rejects.toThrow("exact prepared transaction"); + expect(fixture.order.filter((event) => event === "activate")).toHaveLength(1); + }); + + it("deeply clones authority so later provider mutation cannot change the transaction", async () => { + const fixture = adapterFor("hermes"); + const prepared = await prepareManagedBootstrapSequence( + fixture.adapter, + preparationInput("hermes"), + ); + const rawHandle = fixture.raw.handle as ManagedBootstrapHeldWorkloadHandle; + const rawSnapshot = fixture.raw.snapshot as ManagedBootstrapObservedSnapshot; + const rawPrepared = fixture.raw.prepared as ManagedBootstrapPreparedReplacementHandle; + + (rawHandle.plan.metadata as Record)["nemoclaw.ai/managed-profile"] = "changed"; + (rawSnapshot.supervisorArgv as string[])[0] = "/attacker"; + (rawPrepared as { preparedRuntimeId: string }).preparedRuntimeId = "future-runtime"; + + expect(prepared.handle.plan.metadata["nemoclaw.ai/managed-profile"]).toBe( + requestFor("hermes").profileFingerprint, + ); + expect(prepared.snapshot.supervisorArgv[0]).toBe("/runtime/sandbox-supervisor"); + expect(prepared.prepared.preparedRuntimeId).toBe(PREPARED_ID); + expect(() => { + (prepared.handle.plan.intendedWorkloadArgv as string[]).push("attacker"); + }).toThrow(TypeError); + }); + + it.each([ + "throws", + "returns without launch", + "returns an invalid handle", + ] as const)("runs exact incomplete-create cleanup when create %s", async (failureMode) => { + const fixture = adapterFor("langchain-deepagents-code"); + const original = fixture.adapter.createHeldWorkload; + if (failureMode === "throws") { + vi.mocked(original).mockImplementationOnce(async (input) => { + await input.launch({ + heldWorkloadArgv: renderManagedBootstrapHeldCommand( + input.request, + input.bootstrapIdentity as string, + input.plan.intendedWorkloadArgv, + ), + bootstrapIdentity: input.bootstrapIdentity as string, + }); + throw new Error("create failed after materialization"); + }); + } else if (failureMode === "returns without launch") { + vi.mocked(original).mockResolvedValueOnce(handleFor(requestFor("langchain-deepagents-code"))); + } else { + vi.mocked(original).mockImplementationOnce(async (input) => { + const receipt = await input.launch({ + heldWorkloadArgv: renderManagedBootstrapHeldCommand( + input.request, + input.bootstrapIdentity as string, + input.plan.intendedWorkloadArgv, + ), + bootstrapIdentity: input.bootstrapIdentity as string, + }); + return { + ...handleFor(requestFor("langchain-deepagents-code"), receipt), + sandbox: { ...receipt.sandbox, sandboxId: "wrong-owner" }, + }; + }); + } + + const failure = await captureFailure( + prepareManagedBootstrapSequence( + fixture.adapter, + preparationInput("langchain-deepagents-code"), + ), + ); + + expect(fixture.adapter.cleanupIncompleteCreate).toHaveBeenCalledWith({ + plan: expect.objectContaining({ sandboxName: "alpha", driverId: "mxc-fixture" }), + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: expect.arrayContaining([IDENTITY]), + }); + expect(failure.managedBootstrapRollback).toMatchObject({ + outcome: "rolled-back", + heldWorkloadRemoved: true, + bootstrapIdentity: IDENTITY, + }); + }); + + it("rolls back a prepared replacement when durable recording fails, before activation", async () => { + const fixture = adapterFor("openclaw"); + const prepared = await prepareManagedBootstrapSequence( + fixture.adapter, + preparationInput("openclaw"), + ); + const failure = await captureFailure( + activateManagedBootstrapSequence(fixture.adapter, { + transaction: prepared, + authorityStore: { + recordPreparedAuthority: vi.fn(async () => { + throw new Error("durable store unavailable"); + }), + }, + timeoutSecs: 30, + }), + ); + + expect(failure.message).toContain("durable store unavailable"); + expect(fixture.adapter.activateBootstrapReplacement).not.toHaveBeenCalled(); + expect(fixture.adapter.finalizeBootstrap).toHaveBeenCalledWith({ + outcome: "rollback", + handle: prepared.handle, + snapshot: prepared.snapshot, + prepared: prepared.prepared, + durablePreparation: null, + replacement: null, + completion: null, + }); + }); + + it("rejects a durable receipt that is not bound to the complete prepared authority", async () => { + const fixture = adapterFor("hermes"); + const prepared = await prepareManagedBootstrapSequence( + fixture.adapter, + preparationInput("hermes"), + ); + const failure = await captureFailure( + activateManagedBootstrapSequence(fixture.adapter, { + transaction: prepared, + authorityStore: { + recordPreparedAuthority: vi.fn(async (authority) => ({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: authority.sandbox, + bootstrapIdentity: authority.bootstrapIdentity, + authorityFingerprint: "f".repeat(64), + recordId: "wrong-record", + recordedAt: "2026-07-29T12:00:30.000Z", + })), + }, + timeoutSecs: 30, + }), + ); + + expect(failure.message).toContain("changed prepared authority"); + expect(fixture.adapter.activateBootstrapReplacement).not.toHaveBeenCalled(); + expect(failure.managedBootstrapRollback).toMatchObject({ outcome: "rolled-back" }); + }); + + it("rejects activated-runtime drift and rolls back from the prepared authority", async () => { + const fixture = adapterFor("openclaw"); + vi.mocked(fixture.adapter.activateBootstrapReplacement).mockImplementationOnce( + async ({ handle }) => ({ + ...replacementFor(requestFor("openclaw"), handle), + replacementRuntimeId: "9".repeat(64), + }), + ); + const prepared = await prepareManagedBootstrapSequence( + fixture.adapter, + preparationInput("openclaw"), + ); + const failure = await captureFailure( + activateManagedBootstrapSequence(fixture.adapter, { + transaction: prepared, + authorityStore: authorityStore(fixture.order), + timeoutSecs: 30, + }), + ); + + expect(failure.message).toContain("changed immutable prepared authority"); + expect(fixture.adapter.finalizeBootstrap).toHaveBeenLastCalledWith({ + outcome: "rollback", + handle: prepared.handle, + snapshot: prepared.snapshot, + prepared: prepared.prepared, + durablePreparation: expect.objectContaining({ bootstrapIdentity: IDENTITY }), + replacement: null, + completion: null, + }); + }); + + it("rejects completion drift and retains the primary failure when rollback also fails", async () => { + const fixture = adapterFor("hermes"); + const rollbackFailure = new Error("rollback unavailable"); + vi.mocked(fixture.adapter.awaitBootstrap).mockImplementationOnce(async ({ handle }) => ({ + ...completionFor(requestFor("hermes"), handle), + bootstrapIdentity: "a".repeat(64), + })); + vi.mocked(fixture.adapter.finalizeBootstrap).mockRejectedValueOnce(rollbackFailure); + const prepared = await prepareManagedBootstrapSequence( + fixture.adapter, + preparationInput("hermes"), + ); + const failure = await captureFailure( + activateManagedBootstrapSequence(fixture.adapter, { + transaction: prepared, + authorityStore: authorityStore(fixture.order), + timeoutSecs: 30, + }), + ); + + expect(failure.message).toContain("completion receipt changed"); + expect(failure.message).toContain("rollback unavailable"); + expect(failure.managedBootstrapRollbackError).toBe(rollbackFailure); + }); + + it("binds rollback and commit receipts to the exact captured snapshot", async () => { + const result = await prepareAndActivate("langchain-deepagents-code"); + vi.mocked(result.adapter.finalizeBootstrap).mockResolvedValueOnce({ + ...rolledBackReceipt(result.activated.handle, result.activated.snapshot), + restoredRuntimeId: "b".repeat(64), + }); + await expect( + finalizeManagedBootstrapSequence(result.adapter, { + outcome: "rollback", + transaction: result.activated, + }), + ).rejects.toThrow("does not restore the exact captured runtime and spec"); + + const commitResult = await prepareAndActivate("langchain-deepagents-code"); + const committed = await finalizeManagedBootstrapSequence(commitResult.adapter, { + outcome: "commit", + transaction: commitResult.activated, + }); + expect(committed).toMatchObject({ + outcome: "committed", + bootstrapIdentity: IDENTITY, + restoredRuntimeId: null, + }); + }); + + it("refuses commit before activation has produced an exact completion receipt", async () => { + const fixture = adapterFor("openclaw"); + const prepared = await prepareManagedBootstrapSequence( + fixture.adapter, + preparationInput("openclaw"), + ); + await expect( + finalizeManagedBootstrapSequence(fixture.adapter, { + outcome: "commit", + transaction: prepared, + }), + ).rejects.toThrow("commit requires a completed activated transaction"); + expect(fixture.adapter.finalizeBootstrap).not.toHaveBeenCalled(); + }); + + it.each([ + "BASH_ENV=/sandbox/attacker", + "ENV=/sandbox/attacker", + "LD_PRELOAD=/sandbox/attacker.so", + "LD_AUDIT=/sandbox/attacker.so", + "LD_LIBRARY_PATH=/sandbox/lib", + "SHELLOPTS=xtrace", + "PS4=$(touch /sandbox/bypass)", + "BASH_FUNC_attacker%%=() { touch /sandbox/bypass; }", + ])("rejects a process-control assignment before rendering the held command: %s", (assignment) => { + const request = requestFor("hermes"); + expect(() => + renderManagedBootstrapHeldCommand(request, IDENTITY, ["env", assignment, "nemoclaw-start"]), + ).toThrow("process-control environment assignment"); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts new file mode 100644 index 00000000000..e346b7b4e32 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -0,0 +1,1444 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, randomBytes as defaultRandomBytes } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; + +import { MANAGED_STARTUP_HOLD_EXECUTABLE } from "../managed-startup/hold"; +import type { ManagedStartupAgent } from "../managed-startup/profile"; +import { + type ManagedStartupRootApplyRequest, + parseManagedStartupRootApplyRequest, + serializeManagedStartupRootApplyRequest, +} from "../managed-startup/root-apply"; + +export const MANAGED_BOOTSTRAP_SCHEMA_VERSION = 1 as const; +export const MANAGED_BOOTSTRAP_IDENTITY_BYTES = 32; + +const SHA256_RE = /^[a-f0-9]{64}$/u; +const MANIFEST_DIGEST_RE = /^sha256:[a-f0-9]{64}$/u; +const ENV_ASSIGNMENT_RE = /^[A-Za-z_][A-Za-z0-9_]*=/u; +const PROCESS_INJECTION_ENV_KEYS = new Set([ + "BASHOPTS", + "BASH_ENV", + "ENV", + "LD_AUDIT", + "LD_LIBRARY_PATH", + "LD_PRELOAD", + "PS4", + "SHELLOPTS", +]); +const PROCESS_INJECTION_ENV_PREFIXES = ["BASH_FUNC_"] as const; + +export interface ManagedBootstrapImageIdentity { + readonly repository: string; + /** Registry/platform manifest digest, not a runtime-specific image config ID. */ + readonly manifestDigest: `sha256:${string}`; +} + +export interface ManagedBootstrapSandboxIdentity { + readonly sandboxName: string; + readonly sandboxId: string; + readonly driverId: string; +} + +export interface ManagedBootstrapAgentIdentity { + readonly uid: number; + readonly gid: number; + readonly workdir: string; +} + +export interface ManagedBootstrapExpectedPlan { + readonly schemaVersion: typeof MANAGED_BOOTSTRAP_SCHEMA_VERSION; + readonly sandboxName: string; + readonly driverId: string; + readonly image: ManagedBootstrapImageIdentity; + readonly profile: { + readonly agent: ManagedStartupAgent; + readonly fingerprint: string; + }; + readonly agentIdentity: ManagedBootstrapAgentIdentity; + readonly intendedWorkloadArgv: readonly string[]; + readonly expectedSupervisorArgv: readonly string[]; + readonly metadata: Readonly>; +} + +export interface ManagedBootstrapCreateReceipt { + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly ready: true; + readonly readyAt: string; +} + +export interface ManagedBootstrapCreateInput { + readonly plan: ManagedBootstrapExpectedPlan; + readonly request: ManagedStartupRootApplyRequest; + /** + * A caller that already rendered the create argv supplies the same one-time + * identity here. Providers generate it when rendering is deferred. + */ + readonly bootstrapIdentity?: string; + readonly launch: (input: { + readonly heldWorkloadArgv: readonly string[]; + readonly bootstrapIdentity: string; + }) => Promise; +} + +export interface ManagedBootstrapHeldWorkloadHandle { + readonly schemaVersion: typeof MANAGED_BOOTSTRAP_SCHEMA_VERSION; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly bootstrapIdentity: string; + readonly heldWorkloadArgv: readonly string[]; + readonly intendedWorkloadArgv: readonly string[]; + readonly plan: ManagedBootstrapExpectedPlan; + readonly createReceipt: ManagedBootstrapCreateReceipt; +} + +export interface ManagedBootstrapIncompleteCreateCleanupInput { + readonly plan: ManagedBootstrapExpectedPlan; + readonly bootstrapIdentity: string; + readonly heldWorkloadArgv: readonly string[]; +} + +export interface ManagedBootstrapDiscoveryInput { + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly bootstrapIdentity: string; + readonly expectedImage: ManagedBootstrapImageIdentity; + readonly metadata: Readonly>; +} + +export interface ManagedBootstrapDiscoveredWorkload { + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly runtimeId: string; + readonly bootstrapIdentity: string; +} + +export interface ManagedBootstrapObservedSnapshot { + readonly schemaVersion: typeof MANAGED_BOOTSTRAP_SCHEMA_VERSION; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly runtimeId: string; + readonly bootstrapIdentity: string; + readonly image: ManagedBootstrapImageIdentity; + /** Driver-local immutable content/config identity, distinct from manifestDigest. */ + readonly runtimeImageContentId: string; + readonly specHash: string; + readonly specCanonicalJson: string; + readonly agentIdentity: ManagedBootstrapAgentIdentity; + readonly supervisorArgv: readonly string[]; + readonly heldWorkloadArgv: readonly string[]; + readonly metadata: Readonly>; +} + +export interface ManagedBootstrapReplacementOptions { + /** + * Driver-neutral options contributed by startup compatibility. A provider + * rejects keys it does not explicitly support. + */ + readonly values: Readonly>; +} + +/** + * Exact stopped replacement authority returned before the owning provider may + * quiesce, rename, or otherwise mutate the Ready held workload. + */ +export interface ManagedBootstrapPreparedReplacementHandle { + readonly schemaVersion: typeof MANAGED_BOOTSTRAP_SCHEMA_VERSION; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly bootstrapIdentity: string; + readonly originalRuntimeId: string; + readonly preparedRuntimeId: string; + readonly image: ManagedBootstrapImageIdentity; + readonly runtimeImageContentId: string; + readonly originalSpecHash: string; + readonly preparedSpecHash: string; + readonly preparedSpecCanonicalJson: string; + readonly expectedActivatedSpecHash: string; + readonly expectedActivatedSpecCanonicalJson: string; + readonly profileFingerprint: string; + /** Provider-opaque, bounded authority needed to clean up or restore exactly this transaction. */ + readonly rollbackAuthority: string; +} + +export interface ManagedBootstrapPreparedAuthority { + readonly schemaVersion: typeof MANAGED_BOOTSTRAP_SCHEMA_VERSION; + readonly phase: "prepared"; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly bootstrapIdentity: string; + readonly authorityFingerprint: string; + readonly planFingerprint: string; + readonly image: ManagedBootstrapImageIdentity; + readonly runtimeImageContentId: string; + readonly profileFingerprint: string; + readonly originalRuntimeId: string; + readonly preparedRuntimeId: string; + readonly originalSpecHash: string; + readonly preparedSpecHash: string; + readonly expectedActivatedSpecHash: string; + readonly rollbackTargetRuntimeId: string; + readonly rollbackTargetSpecHash: string; + readonly rollbackAuthority: string; +} + +export interface ManagedBootstrapDurablePreparationReceipt { + readonly schemaVersion: typeof MANAGED_BOOTSTRAP_SCHEMA_VERSION; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly bootstrapIdentity: string; + readonly authorityFingerprint: string; + readonly recordId: string; + readonly recordedAt: string; +} + +export interface ManagedBootstrapAuthorityStore { + /** Return only after the complete prepared authority is durably recoverable. */ + recordPreparedAuthority( + authority: ManagedBootstrapPreparedAuthority, + ): Promise; +} + +export interface ManagedBootstrapReplacementHandle { + readonly schemaVersion: typeof MANAGED_BOOTSTRAP_SCHEMA_VERSION; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly bootstrapIdentity: string; + readonly originalRuntimeId: string; + readonly replacementRuntimeId: string; + readonly image: ManagedBootstrapImageIdentity; + readonly runtimeImageContentId: string; + readonly originalSpecHash: string; + readonly replacementSpecHash: string; + readonly replacementSpecCanonicalJson: string; + readonly profileFingerprint: string; +} + +export interface ManagedBootstrapCompletionReceipt { + readonly schemaVersion: typeof MANAGED_BOOTSTRAP_SCHEMA_VERSION; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly runtimeId: string; + readonly image: ManagedBootstrapImageIdentity; + readonly runtimeImageContentId: string; + readonly originalSpecHash: string; + readonly replacementSpecHash: string; + readonly profileFingerprint: string; + readonly bootstrapIdentity: string; + /** True when image-owned bootstrap left a protected shared-state transaction pending. */ + readonly transactionPending: boolean; + readonly completedAt: string; +} + +export interface ManagedBootstrapFinalizationReceipt { + readonly schemaVersion: typeof MANAGED_BOOTSTRAP_SCHEMA_VERSION; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly bootstrapIdentity: string; + readonly outcome: "committed" | "rolled-back"; + readonly restoredRuntimeId: string | null; + readonly restoredSpecHash: string | null; + readonly heldWorkloadRemoved: boolean; + readonly alreadyRolledBack: boolean; + readonly finalizedAt: string; +} + +export class ManagedBootstrapDurableCommitCleanupPendingError extends Error { + readonly bootstrapIdentity: string; + readonly cleanupRuntimeId: string; + + constructor(input: { + readonly bootstrapIdentity: string; + readonly cleanupRuntimeId: string; + readonly detail: string; + }) { + super( + `Managed bootstrap shared state is durably committed, but finalization cleanup is pending for runtime ${input.cleanupRuntimeId}: ${input.detail}`, + ); + this.name = "ManagedBootstrapDurableCommitCleanupPendingError"; + this.bootstrapIdentity = input.bootstrapIdentity; + this.cleanupRuntimeId = input.cleanupRuntimeId; + } +} + +export class ManagedBootstrapCommitStateIndeterminateError extends Error { + readonly bootstrapIdentity: string; + readonly runtimeId: string; + + constructor(input: { + readonly bootstrapIdentity: string; + readonly runtimeId: string; + readonly detail: string; + }) { + super( + `Managed bootstrap commit state is indeterminate for runtime ${input.runtimeId}; rollback is unsafe until immutable status is recovered: ${input.detail}`, + ); + this.name = "ManagedBootstrapCommitStateIndeterminateError"; + this.bootstrapIdentity = input.bootstrapIdentity; + this.runtimeId = input.runtimeId; + } +} + +/** A provider without exact-ID deletion retains the bounded held workload. */ +export class ManagedBootstrapOwnerCleanupRequiredError extends Error { + readonly sandboxName: string; + readonly sandboxId: string; + readonly runtimeId: string; + + constructor(input: { + readonly sandboxName: string; + readonly sandboxId: string; + readonly runtimeId: string; + readonly detail?: string; + }) { + super( + `Managed bootstrap quiesced and retained sandbox '${input.sandboxName}' (ID ${input.sandboxId}, runtime ${input.runtimeId}) because deletion cannot atomically require this durable ID.${input.detail ? ` ${input.detail}` : ""}`, + ); + this.name = "ManagedBootstrapOwnerCleanupRequiredError"; + this.sandboxName = input.sandboxName; + this.sandboxId = input.sandboxId; + this.runtimeId = input.runtimeId; + } +} + +export function attachManagedBootstrapRollbackError(failure: Error, rollbackError: unknown): void { + ( + failure as Error & { + managedBootstrapRollbackError?: unknown; + } + ).managedBootstrapRollbackError = rollbackError; + const detail = rollbackError instanceof Error ? rollbackError.message : String(rollbackError); + if (!failure.message.includes(detail)) { + failure.message = `${failure.message}\nManaged bootstrap rollback requires attention: ${detail}`; + } +} + +export interface ManagedBootstrapAdapter { + /** Return only after one durable sandbox/driver identity reports Ready. */ + createHeldWorkload( + input: ManagedBootstrapCreateInput, + ): Promise; + + /** + * Clean up a materialized create that failed before returning a Ready + * identity-bound handle. + */ + cleanupIncompleteCreate( + input: ManagedBootstrapIncompleteCreateCleanupInput, + ): Promise; + + /** Resolve exactly one runtime from the complete durable identity. */ + discoverHeldWorkload( + input: ManagedBootstrapDiscoveryInput, + ): Promise; + + /** Capture one immutable normalized runtime snapshot before mutation. */ + inspectHeldWorkload(input: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly discovered: ManagedBootstrapDiscoveredWorkload; + }): Promise; + + /** + * Create and fully inspect a stopped replacement without changing the held + * workload. The returned authority must be sufficient for exact cleanup. + */ + prepareBootstrapReplacement(input: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot; + readonly request: ManagedStartupRootApplyRequest; + readonly replacementOptions: ManagedBootstrapReplacementOptions; + }): Promise; + + /** + * Perform the destructive cutover only after the coordinator supplies the + * exact receipt proving that prepared authority was durably recorded. + */ + activateBootstrapReplacement(input: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot; + readonly prepared: ManagedBootstrapPreparedReplacementHandle; + readonly durablePreparation: ManagedBootstrapDurablePreparationReceipt; + }): Promise; + + /** Return an identity-bound completion receipt, never an unqualified boolean. */ + awaitBootstrap(input: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot; + readonly replacement: ManagedBootstrapReplacementHandle; + readonly timeoutSecs: number; + }): Promise; + + /** Commit or roll back using exact handles captured by this transaction. */ + finalizeBootstrap(input: { + readonly outcome: "commit" | "rollback"; + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot | null; + readonly prepared: ManagedBootstrapPreparedReplacementHandle | null; + readonly durablePreparation: ManagedBootstrapDurablePreparationReceipt | null; + readonly replacement: ManagedBootstrapReplacementHandle | null; + readonly completion: ManagedBootstrapCompletionReceipt | null; + }): Promise; +} + +export interface ManagedBootstrapPreparationInput { + readonly create: ManagedBootstrapCreateInput; + readonly request: ManagedStartupRootApplyRequest; + readonly replacementOptions: ManagedBootstrapReplacementOptions; +} + +export interface ManagedBootstrapActivationInput { + readonly transaction: ManagedBootstrapPreparedTransaction; + readonly authorityStore: ManagedBootstrapAuthorityStore; + readonly timeoutSecs: number; +} + +export interface ManagedBootstrapPreparedTransaction { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot; + readonly prepared: ManagedBootstrapPreparedReplacementHandle; +} + +export interface ManagedBootstrapActivatedTransaction extends ManagedBootstrapPreparedTransaction { + readonly durablePreparation: ManagedBootstrapDurablePreparationReceipt; + readonly replacement: ManagedBootstrapReplacementHandle; + readonly completion: ManagedBootstrapCompletionReceipt; +} + +function protocolFail(message: string): never { + throw new Error(`Managed bootstrap protocol violation: ${message}`); +} + +function assertOpaqueString(value: unknown, label: string): asserts value is string { + if ( + typeof value !== "string" || + value.length === 0 || + value.includes("\0") || + Buffer.byteLength(value, "utf8") > 64 * 1024 + ) { + protocolFail(`${label} must be one bounded non-empty string`); + } +} + +function assertExact(actual: unknown, expected: unknown, label: string): void { + if (!isDeepStrictEqual(actual, expected)) { + protocolFail(`${label} does not match the transaction authority`); + } +} + +function assertTimestamp(value: unknown, label: string): void { + assertOpaqueString(value, label); + const timestamp = new Date(value); + if (!Number.isFinite(timestamp.getTime()) || timestamp.toISOString() !== value) { + protocolFail(`${label} must be a canonical ISO timestamp`); + } +} + +function assertSandboxIdentity( + sandbox: ManagedBootstrapSandboxIdentity, + expected?: Pick, +): void { + if (typeof sandbox !== "object" || sandbox === null || Array.isArray(sandbox)) { + protocolFail("sandbox identity must be an object"); + } + assertOpaqueString(sandbox.sandboxName, "sandbox name"); + assertOpaqueString(sandbox.sandboxId, "sandbox ID"); + assertOpaqueString(sandbox.driverId, "driver ID"); + if ( + expected && + (sandbox.sandboxName !== expected.sandboxName || sandbox.driverId !== expected.driverId) + ) { + protocolFail("sandbox identity does not match the expected plan"); + } +} + +function assertImageIdentity(image: ManagedBootstrapImageIdentity): void { + if (typeof image !== "object" || image === null || Array.isArray(image)) { + protocolFail("image identity must be an object"); + } + assertOpaqueString(image.repository, "image repository"); + if (!MANIFEST_DIGEST_RE.test(image.manifestDigest)) { + protocolFail("image manifest digest must be canonical sha256"); + } +} + +function assertAgentIdentity(identity: ManagedBootstrapAgentIdentity): void { + if (typeof identity !== "object" || identity === null || Array.isArray(identity)) { + protocolFail("agent identity must be an object"); + } + if ( + !Number.isSafeInteger(identity.uid) || + identity.uid < 0 || + !Number.isSafeInteger(identity.gid) || + identity.gid < 0 + ) { + protocolFail("agent uid and gid must be non-negative safe integers"); + } + assertOpaqueString(identity.workdir, "agent workdir"); + if (!identity.workdir.startsWith("/")) { + protocolFail("agent workdir must be absolute"); + } +} + +function assertMetadata(metadata: Readonly>): void { + if (typeof metadata !== "object" || metadata === null || Array.isArray(metadata)) { + protocolFail("metadata must be a string record"); + } + const prototype = Object.getPrototypeOf(metadata); + if (prototype !== Object.prototype && prototype !== null) { + protocolFail("metadata must be a plain string record"); + } + for (const [key, value] of Object.entries(metadata)) { + assertOpaqueString(key, "metadata key"); + assertOpaqueString(value, `metadata value '${key}'`); + } +} + +function assertArgv(argv: readonly string[], label: string): void { + if ( + !Array.isArray(argv) || + argv.length === 0 || + argv.some( + (value) => + typeof value !== "string" || + value.length === 0 || + value.includes("\0") || + Buffer.byteLength(value, "utf8") > 64 * 1024, + ) || + Buffer.byteLength(JSON.stringify(argv), "utf8") > 128 * 1024 + ) { + protocolFail(`${label} must be one bounded exact argv`); + } +} + +function assertExpectedPlan( + plan: ManagedBootstrapExpectedPlan, + request: ManagedStartupRootApplyRequest, +): void { + if ( + typeof plan !== "object" || + plan === null || + Array.isArray(plan) || + plan.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION + ) { + protocolFail("expected plan schema version is unsupported"); + } + assertOpaqueString(plan.sandboxName, "planned sandbox name"); + assertOpaqueString(plan.driverId, "planned driver ID"); + assertImageIdentity(plan.image); + if ( + typeof plan.profile !== "object" || + plan.profile === null || + Array.isArray(plan.profile) || + plan.profile.agent !== request.agent || + plan.profile.fingerprint !== request.profileFingerprint || + !SHA256_RE.test(plan.profile.fingerprint) + ) { + protocolFail("planned profile does not match the root application request"); + } + assertAgentIdentity(plan.agentIdentity); + assertArgv(plan.intendedWorkloadArgv, "intended workload"); + assertArgv(plan.expectedSupervisorArgv, "expected supervisor"); + assertMetadata(plan.metadata); +} + +function freezeArgv(argv: readonly string[], label: string): readonly string[] { + assertArgv(argv, label); + return Object.freeze([...argv]); +} + +function freezeMetadata( + metadata: Readonly>, +): Readonly> { + assertMetadata(metadata); + const copy: Record = {}; + for (const key of Object.keys(metadata).sort()) { + Object.defineProperty(copy, key, { + configurable: false, + enumerable: true, + writable: false, + value: metadata[key] as string, + }); + } + return Object.freeze(copy); +} + +function normalizeRootApplyRequest( + request: ManagedStartupRootApplyRequest, +): ManagedStartupRootApplyRequest { + return parseManagedStartupRootApplyRequest(serializeManagedStartupRootApplyRequest(request)); +} + +function normalizeExpectedPlan( + plan: ManagedBootstrapExpectedPlan, + request: ManagedStartupRootApplyRequest, +): ManagedBootstrapExpectedPlan { + assertExpectedPlan(plan, request); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandboxName: plan.sandboxName, + driverId: plan.driverId, + image: Object.freeze({ + repository: plan.image.repository, + manifestDigest: plan.image.manifestDigest, + }), + profile: Object.freeze({ + agent: plan.profile.agent, + fingerprint: plan.profile.fingerprint, + }), + agentIdentity: Object.freeze({ + uid: plan.agentIdentity.uid, + gid: plan.agentIdentity.gid, + workdir: plan.agentIdentity.workdir, + }), + intendedWorkloadArgv: freezeArgv(plan.intendedWorkloadArgv, "intended workload"), + expectedSupervisorArgv: freezeArgv(plan.expectedSupervisorArgv, "expected supervisor"), + metadata: freezeMetadata(plan.metadata), + }); +} + +function normalizeReplacementOptions( + options: ManagedBootstrapReplacementOptions, +): ManagedBootstrapReplacementOptions { + if ( + typeof options !== "object" || + options === null || + Array.isArray(options) || + typeof options.values !== "object" || + options.values === null || + Array.isArray(options.values) + ) { + protocolFail("replacement options must be a plain value record"); + } + const prototype = Object.getPrototypeOf(options.values); + if (prototype !== Object.prototype && prototype !== null) { + protocolFail("replacement options must be a plain value record"); + } + const values: Record = Object.create( + null, + ) as Record; + for (const key of Object.keys(options.values).sort()) { + assertOpaqueString(key, "replacement option key"); + const value = options.values[key]; + if (Array.isArray(value)) { + if (value.length > 1024) protocolFail(`replacement option '${key}' has too many values`); + const entries = value.map((entry) => { + if ( + typeof entry !== "string" || + entry.includes("\0") || + Buffer.byteLength(entry, "utf8") > 64 * 1024 + ) { + protocolFail(`replacement option '${key}' has an invalid list value`); + } + return entry; + }); + values[key] = Object.freeze(entries); + continue; + } + if (typeof value === "number") { + if (!Number.isFinite(value)) protocolFail(`replacement option '${key}' must be finite`); + values[key] = value; + continue; + } + if (typeof value === "boolean") { + values[key] = value; + continue; + } + if ( + typeof value !== "string" || + value.includes("\0") || + Buffer.byteLength(value, "utf8") > 64 * 1024 + ) { + protocolFail(`replacement option '${key}' has an invalid value`); + } + values[key] = value; + } + return Object.freeze({ values: Object.freeze(values) }); +} + +function canonicalJson(value: unknown): string { + if (value === null || typeof value === "string" || typeof value === "boolean") { + return JSON.stringify(value); + } + if (typeof value === "number") { + if (!Number.isFinite(value)) protocolFail("authority contains a non-finite number"); + return JSON.stringify(value); + } + if (Array.isArray(value)) { + return `[${value.map((entry) => canonicalJson(entry)).join(",")}]`; + } + if (typeof value !== "object") protocolFail("authority contains an unsupported value"); + const record = value as Readonly>; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; +} + +export function assertManagedBootstrapIdentity(value: string): void { + if (!SHA256_RE.test(value)) { + protocolFail("identity must be 32 random bytes encoded as lowercase hex"); + } +} + +export function createManagedBootstrapIdentity( + randomBytes: (size: number) => Buffer = defaultRandomBytes, +): string { + const identity = randomBytes(MANAGED_BOOTSTRAP_IDENTITY_BYTES).toString("hex"); + assertManagedBootstrapIdentity(identity); + return identity; +} + +export function assertManagedBootstrapSafeProcessEnvironmentKey(key: string): void { + if ( + PROCESS_INJECTION_ENV_KEYS.has(key) || + PROCESS_INJECTION_ENV_PREFIXES.some((prefix) => key.startsWith(prefix)) + ) { + throw new Error(`Managed bootstrap refuses process-control environment assignment '${key}'.`); + } +} + +export function renderManagedBootstrapHeldCommand( + request: ManagedStartupRootApplyRequest, + bootstrapIdentity: string, + intendedWorkloadArgv: readonly string[], +): readonly string[] { + assertManagedBootstrapIdentity(bootstrapIdentity); + assertArgv(intendedWorkloadArgv, "intended workload"); + if (intendedWorkloadArgv[0] !== "env") { + protocolFail("intended workload must begin with env"); + } + let executableIndex = 1; + while (executableIndex < intendedWorkloadArgv.length) { + const assignment = intendedWorkloadArgv[executableIndex] as string; + const separator = assignment.indexOf("="); + if (separator > 0 && assignment.startsWith("BASH_FUNC_")) { + assertManagedBootstrapSafeProcessEnvironmentKey(assignment.slice(0, separator)); + } + if (!ENV_ASSIGNMENT_RE.test(assignment)) break; + assertManagedBootstrapSafeProcessEnvironmentKey(assignment.slice(0, separator)); + executableIndex += 1; + } + if (executableIndex >= intendedWorkloadArgv.length) { + protocolFail("intended workload executable is missing"); + } + return Object.freeze([ + ...intendedWorkloadArgv.slice(0, executableIndex), + MANAGED_STARTUP_HOLD_EXECUTABLE, + "--agent", + request.agent, + "--profile-fingerprint", + request.profileFingerprint, + "--bootstrap-identity", + bootstrapIdentity, + ]); +} + +function freezeSandboxIdentity( + sandbox: ManagedBootstrapSandboxIdentity, + expected?: Pick, +): ManagedBootstrapSandboxIdentity { + assertSandboxIdentity(sandbox, expected); + return Object.freeze({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + driverId: sandbox.driverId, + }); +} + +function normalizeCreateReceipt( + receipt: ManagedBootstrapCreateReceipt, + plan: ManagedBootstrapExpectedPlan, +): ManagedBootstrapCreateReceipt { + if (typeof receipt !== "object" || receipt === null || Array.isArray(receipt)) { + protocolFail("create receipt must be an object"); + } + if (receipt.ready !== true) protocolFail("create receipt is not Ready"); + const sandbox = freezeSandboxIdentity(receipt.sandbox, plan); + assertTimestamp(receipt.readyAt, "create receipt timestamp"); + return Object.freeze({ sandbox, ready: true, readyAt: receipt.readyAt }); +} + +function normalizeHeldHandle( + candidate: ManagedBootstrapHeldWorkloadHandle, + input: ManagedBootstrapCreateInput, + launchReceipt: ManagedBootstrapCreateReceipt, +): ManagedBootstrapHeldWorkloadHandle { + if ( + typeof candidate !== "object" || + candidate === null || + Array.isArray(candidate) || + candidate.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION + ) { + protocolFail("held workload schema version is unsupported"); + } + assertManagedBootstrapIdentity(candidate.bootstrapIdentity); + if (candidate.bootstrapIdentity !== input.bootstrapIdentity) { + protocolFail("held workload changed the caller-supplied bootstrap identity"); + } + assertExact(candidate.plan, input.plan, "held workload plan"); + assertExact(candidate.sandbox, launchReceipt.sandbox, "held workload sandbox"); + assertExact(candidate.createReceipt, launchReceipt, "held workload create receipt"); + assertExact( + candidate.intendedWorkloadArgv, + input.plan.intendedWorkloadArgv, + "intended workload argv", + ); + const heldWorkloadArgv = renderManagedBootstrapHeldCommand( + input.request, + candidate.bootstrapIdentity, + input.plan.intendedWorkloadArgv, + ); + assertExact(candidate.heldWorkloadArgv, heldWorkloadArgv, "held workload argv"); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: launchReceipt.sandbox, + bootstrapIdentity: candidate.bootstrapIdentity, + heldWorkloadArgv, + intendedWorkloadArgv: input.plan.intendedWorkloadArgv, + plan: input.plan, + createReceipt: launchReceipt, + }); +} + +function normalizeDiscoveredWorkload( + candidate: ManagedBootstrapDiscoveredWorkload, + handle: ManagedBootstrapHeldWorkloadHandle, +): ManagedBootstrapDiscoveredWorkload { + if (typeof candidate !== "object" || candidate === null || Array.isArray(candidate)) { + protocolFail("discovered workload must be an object"); + } + assertExact(candidate.sandbox, handle.sandbox, "discovered sandbox"); + if (candidate.bootstrapIdentity !== handle.bootstrapIdentity) { + protocolFail("discovered workload bootstrap identity changed"); + } + assertOpaqueString(candidate.runtimeId, "discovered runtime ID"); + return Object.freeze({ + sandbox: handle.sandbox, + runtimeId: candidate.runtimeId, + bootstrapIdentity: handle.bootstrapIdentity, + }); +} + +function normalizeObservedSnapshot( + candidate: ManagedBootstrapObservedSnapshot, + handle: ManagedBootstrapHeldWorkloadHandle, + discovered: ManagedBootstrapDiscoveredWorkload, +): ManagedBootstrapObservedSnapshot { + if ( + typeof candidate !== "object" || + candidate === null || + Array.isArray(candidate) || + candidate.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION + ) { + protocolFail("observed snapshot schema version is unsupported"); + } + assertExact(candidate.sandbox, handle.sandbox, "observed sandbox"); + assertExact(candidate.image, handle.plan.image, "observed image"); + assertExact(candidate.agentIdentity, handle.plan.agentIdentity, "observed agent identity"); + assertExact(candidate.supervisorArgv, handle.plan.expectedSupervisorArgv, "supervisor argv"); + assertExact(candidate.heldWorkloadArgv, handle.heldWorkloadArgv, "observed held workload argv"); + assertExact(candidate.metadata, handle.plan.metadata, "observed metadata"); + if ( + candidate.runtimeId !== discovered.runtimeId || + candidate.bootstrapIdentity !== handle.bootstrapIdentity + ) { + protocolFail("observed runtime identity changed after discovery"); + } + assertOpaqueString(candidate.runtimeImageContentId, "runtime image content ID"); + if (!SHA256_RE.test(candidate.specHash)) { + protocolFail("observed spec hash must be canonical sha256"); + } + assertOpaqueString(candidate.specCanonicalJson, "canonical runtime spec"); + if ( + createHash("sha256").update(candidate.specCanonicalJson, "utf8").digest("hex") !== + candidate.specHash + ) { + protocolFail("observed spec hash does not match its canonical runtime spec"); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: candidate.runtimeId, + bootstrapIdentity: handle.bootstrapIdentity, + image: handle.plan.image, + runtimeImageContentId: candidate.runtimeImageContentId, + specHash: candidate.specHash, + specCanonicalJson: candidate.specCanonicalJson, + agentIdentity: handle.plan.agentIdentity, + supervisorArgv: handle.plan.expectedSupervisorArgv, + heldWorkloadArgv: handle.heldWorkloadArgv, + metadata: handle.plan.metadata, + }); +} + +function normalizePreparedReplacement( + candidate: ManagedBootstrapPreparedReplacementHandle, + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, +): ManagedBootstrapPreparedReplacementHandle { + if ( + typeof candidate !== "object" || + candidate === null || + Array.isArray(candidate) || + candidate.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION + ) { + protocolFail("prepared replacement schema version is unsupported"); + } + assertExact(candidate.sandbox, handle.sandbox, "prepared replacement sandbox"); + assertExact(candidate.image, snapshot.image, "prepared replacement image"); + if ( + candidate.bootstrapIdentity !== handle.bootstrapIdentity || + candidate.originalRuntimeId !== snapshot.runtimeId || + candidate.runtimeImageContentId !== snapshot.runtimeImageContentId || + candidate.originalSpecHash !== snapshot.specHash || + candidate.profileFingerprint !== handle.plan.profile.fingerprint + ) { + protocolFail("prepared replacement changed immutable transaction authority"); + } + assertOpaqueString(candidate.preparedRuntimeId, "prepared runtime ID"); + if (candidate.preparedRuntimeId === candidate.originalRuntimeId) { + protocolFail("prepared runtime ID must differ from the captured runtime"); + } + for (const [label, hash] of [ + ["prepared spec hash", candidate.preparedSpecHash], + ["expected activated spec hash", candidate.expectedActivatedSpecHash], + ] as const) { + if (!SHA256_RE.test(hash)) protocolFail(`${label} must be canonical sha256`); + } + for (const [label, text, hash] of [ + ["prepared spec", candidate.preparedSpecCanonicalJson, candidate.preparedSpecHash], + [ + "expected activated spec", + candidate.expectedActivatedSpecCanonicalJson, + candidate.expectedActivatedSpecHash, + ], + ] as const) { + assertOpaqueString(text, `${label} canonical JSON`); + if (createHash("sha256").update(text, "utf8").digest("hex") !== hash) { + protocolFail(`${label} hash does not match its canonical runtime spec`); + } + } + assertOpaqueString(candidate.rollbackAuthority, "prepared rollback authority"); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + originalRuntimeId: snapshot.runtimeId, + preparedRuntimeId: candidate.preparedRuntimeId, + image: snapshot.image, + runtimeImageContentId: snapshot.runtimeImageContentId, + originalSpecHash: snapshot.specHash, + preparedSpecHash: candidate.preparedSpecHash, + preparedSpecCanonicalJson: candidate.preparedSpecCanonicalJson, + expectedActivatedSpecHash: candidate.expectedActivatedSpecHash, + expectedActivatedSpecCanonicalJson: candidate.expectedActivatedSpecCanonicalJson, + profileFingerprint: handle.plan.profile.fingerprint, + rollbackAuthority: candidate.rollbackAuthority, + }); +} + +function createPreparedAuthority( + transaction: ManagedBootstrapPreparedTransaction, +): ManagedBootstrapPreparedAuthority { + const { handle, snapshot, prepared } = transaction; + const planFingerprint = createHash("sha256") + .update(canonicalJson(handle.plan), "utf8") + .digest("hex"); + const bound = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + phase: "prepared" as const, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + planFingerprint, + image: handle.plan.image, + runtimeImageContentId: snapshot.runtimeImageContentId, + profileFingerprint: handle.plan.profile.fingerprint, + originalRuntimeId: snapshot.runtimeId, + preparedRuntimeId: prepared.preparedRuntimeId, + originalSpecHash: snapshot.specHash, + preparedSpecHash: prepared.preparedSpecHash, + expectedActivatedSpecHash: prepared.expectedActivatedSpecHash, + rollbackTargetRuntimeId: snapshot.runtimeId, + rollbackTargetSpecHash: snapshot.specHash, + rollbackAuthority: prepared.rollbackAuthority, + }); + const authorityFingerprint = createHash("sha256") + .update(canonicalJson(bound), "utf8") + .digest("hex"); + return Object.freeze({ ...bound, authorityFingerprint }); +} + +function normalizeDurablePreparationReceipt( + candidate: ManagedBootstrapDurablePreparationReceipt, + authority: ManagedBootstrapPreparedAuthority, +): ManagedBootstrapDurablePreparationReceipt { + if ( + typeof candidate !== "object" || + candidate === null || + Array.isArray(candidate) || + candidate.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION + ) { + protocolFail("durable preparation receipt schema version is unsupported"); + } + assertExact(candidate.sandbox, authority.sandbox, "durable preparation sandbox"); + if ( + candidate.bootstrapIdentity !== authority.bootstrapIdentity || + candidate.authorityFingerprint !== authority.authorityFingerprint + ) { + protocolFail("durable preparation receipt changed prepared authority"); + } + assertOpaqueString(candidate.recordId, "durable preparation record ID"); + assertTimestamp(candidate.recordedAt, "durable preparation timestamp"); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: authority.sandbox, + bootstrapIdentity: authority.bootstrapIdentity, + authorityFingerprint: authority.authorityFingerprint, + recordId: candidate.recordId, + recordedAt: candidate.recordedAt, + }); +} + +function normalizeReplacementHandle( + candidate: ManagedBootstrapReplacementHandle, + transaction: ManagedBootstrapPreparedTransaction, +): ManagedBootstrapReplacementHandle { + const { handle, snapshot, prepared } = transaction; + if ( + typeof candidate !== "object" || + candidate === null || + Array.isArray(candidate) || + candidate.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION + ) { + protocolFail("replacement schema version is unsupported"); + } + assertExact(candidate.sandbox, handle.sandbox, "replacement sandbox"); + assertExact(candidate.image, snapshot.image, "replacement image"); + if ( + candidate.bootstrapIdentity !== handle.bootstrapIdentity || + candidate.originalRuntimeId !== snapshot.runtimeId || + candidate.replacementRuntimeId !== prepared.preparedRuntimeId || + candidate.runtimeImageContentId !== snapshot.runtimeImageContentId || + candidate.originalSpecHash !== snapshot.specHash || + candidate.replacementSpecHash !== prepared.expectedActivatedSpecHash || + candidate.replacementSpecCanonicalJson !== prepared.expectedActivatedSpecCanonicalJson || + candidate.profileFingerprint !== handle.plan.profile.fingerprint + ) { + protocolFail("replacement receipt changed immutable prepared authority"); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + originalRuntimeId: snapshot.runtimeId, + replacementRuntimeId: prepared.preparedRuntimeId, + image: snapshot.image, + runtimeImageContentId: snapshot.runtimeImageContentId, + originalSpecHash: snapshot.specHash, + replacementSpecHash: prepared.expectedActivatedSpecHash, + replacementSpecCanonicalJson: prepared.expectedActivatedSpecCanonicalJson, + profileFingerprint: handle.plan.profile.fingerprint, + }); +} + +function normalizeCompletionReceipt( + candidate: ManagedBootstrapCompletionReceipt, + handle: ManagedBootstrapHeldWorkloadHandle, + replacement: ManagedBootstrapReplacementHandle, +): ManagedBootstrapCompletionReceipt { + if ( + typeof candidate !== "object" || + candidate === null || + Array.isArray(candidate) || + candidate.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION + ) { + protocolFail("completion schema version is unsupported"); + } + assertExact(candidate.sandbox, handle.sandbox, "completion sandbox"); + assertExact(candidate.image, replacement.image, "completion image"); + if ( + candidate.bootstrapIdentity !== replacement.bootstrapIdentity || + candidate.runtimeId !== replacement.replacementRuntimeId || + candidate.runtimeImageContentId !== replacement.runtimeImageContentId || + candidate.originalSpecHash !== replacement.originalSpecHash || + candidate.replacementSpecHash !== replacement.replacementSpecHash || + candidate.profileFingerprint !== replacement.profileFingerprint + ) { + protocolFail("completion receipt changed immutable transaction authority"); + } + if (typeof candidate.transactionPending !== "boolean") { + protocolFail("completion receipt transaction state is invalid"); + } + assertTimestamp(candidate.completedAt, "completion timestamp"); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: replacement.replacementRuntimeId, + image: replacement.image, + runtimeImageContentId: replacement.runtimeImageContentId, + originalSpecHash: replacement.originalSpecHash, + replacementSpecHash: replacement.replacementSpecHash, + profileFingerprint: replacement.profileFingerprint, + bootstrapIdentity: replacement.bootstrapIdentity, + transactionPending: candidate.transactionPending, + completedAt: candidate.completedAt, + }); +} + +function normalizeFinalizationReceipt( + candidate: ManagedBootstrapFinalizationReceipt, + input: { + readonly outcome: "commit" | "rollback"; + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot | null; + }, +): ManagedBootstrapFinalizationReceipt { + if ( + typeof candidate !== "object" || + candidate === null || + Array.isArray(candidate) || + candidate.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + candidate.outcome !== (input.outcome === "commit" ? "committed" : "rolled-back") + ) { + protocolFail("finalization receipt has an invalid schema or outcome"); + } + assertExact(candidate.sandbox, input.handle.sandbox, "finalization sandbox"); + if (candidate.bootstrapIdentity !== input.handle.bootstrapIdentity) { + protocolFail("finalization receipt bootstrap identity changed"); + } + if ( + typeof candidate.heldWorkloadRemoved !== "boolean" || + typeof candidate.alreadyRolledBack !== "boolean" + ) { + protocolFail("finalization receipt state is invalid"); + } + if (input.outcome === "commit") { + if ( + candidate.restoredRuntimeId !== null || + candidate.restoredSpecHash !== null || + candidate.alreadyRolledBack + ) { + protocolFail("commit receipt cannot report rollback state"); + } + } else if (candidate.heldWorkloadRemoved) { + if (candidate.restoredRuntimeId !== null || candidate.restoredSpecHash !== null) { + protocolFail("removed workload receipt cannot also report a restored runtime"); + } + } else if ( + input.snapshot === null || + candidate.restoredRuntimeId !== input.snapshot.runtimeId || + candidate.restoredSpecHash !== input.snapshot.specHash + ) { + protocolFail("rollback receipt does not restore the exact captured runtime and spec"); + } + assertTimestamp(candidate.finalizedAt, "finalization timestamp"); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: input.handle.sandbox, + bootstrapIdentity: input.handle.bootstrapIdentity, + outcome: candidate.outcome, + restoredRuntimeId: candidate.restoredRuntimeId, + restoredSpecHash: candidate.restoredSpecHash, + heldWorkloadRemoved: candidate.heldWorkloadRemoved, + alreadyRolledBack: candidate.alreadyRolledBack, + finalizedAt: candidate.finalizedAt, + }); +} + +function normalizeIncompleteCreateCleanupReceipt( + candidate: ManagedBootstrapFinalizationReceipt, + plan: ManagedBootstrapExpectedPlan, + bootstrapIdentity: string, +): ManagedBootstrapFinalizationReceipt { + if ( + typeof candidate !== "object" || + candidate === null || + Array.isArray(candidate) || + candidate.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + candidate.outcome !== "rolled-back" || + candidate.bootstrapIdentity !== bootstrapIdentity || + candidate.heldWorkloadRemoved !== true || + candidate.restoredRuntimeId !== null || + candidate.restoredSpecHash !== null || + typeof candidate.alreadyRolledBack !== "boolean" + ) { + protocolFail("incomplete-create cleanup receipt does not prove exact absence"); + } + const sandbox = freezeSandboxIdentity(candidate.sandbox, plan); + assertTimestamp(candidate.finalizedAt, "incomplete-create cleanup timestamp"); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: true, + alreadyRolledBack: candidate.alreadyRolledBack, + finalizedAt: candidate.finalizedAt, + }); +} + +function discoveryInput( + handle: ManagedBootstrapHeldWorkloadHandle, +): ManagedBootstrapDiscoveryInput { + return Object.freeze({ + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + expectedImage: handle.plan.image, + metadata: handle.plan.metadata, + }); +} + +async function rollbackAfterFailure( + adapter: ManagedBootstrapAdapter, + error: unknown, + input: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot | null; + readonly prepared: ManagedBootstrapPreparedReplacementHandle | null; + readonly durablePreparation: ManagedBootstrapDurablePreparationReceipt | null; + readonly replacement: ManagedBootstrapReplacementHandle | null; + }, +): Promise { + const failure = error instanceof Error ? error : new Error(String(error)); + try { + const rollback = normalizeFinalizationReceipt( + await adapter.finalizeBootstrap({ + outcome: "rollback", + ...input, + completion: null, + }), + { outcome: "rollback", handle: input.handle, snapshot: input.snapshot }, + ); + ( + failure as Error & { managedBootstrapRollback?: ManagedBootstrapFinalizationReceipt } + ).managedBootstrapRollback = rollback; + } catch (rollbackError) { + attachManagedBootstrapRollbackError(failure, rollbackError); + } + throw failure; +} + +type ManagedBootstrapCoordinatorState = + | "prepared" + | "activating" + | "activation-consumed" + | "activated" + | "finalizing" + | "finalized" + | "rollback-attempted" + | "finalization-indeterminate"; + +const TRANSACTION_STATES = new WeakMap(); + +/** + * Create the held workload and stopped replacement without a destructive + * cutover. Production wiring is intentionally outside this dormant module. + */ +export async function prepareManagedBootstrapSequence( + adapter: ManagedBootstrapAdapter, + input: ManagedBootstrapPreparationInput, +): Promise { + const request = normalizeRootApplyRequest(input.request); + const createRequest = normalizeRootApplyRequest(input.create.request); + assertExact(createRequest, request, "create root application request"); + const plan = normalizeExpectedPlan(input.create.plan, request); + const replacementOptions = normalizeReplacementOptions(input.replacementOptions); + const bootstrapIdentity = input.create.bootstrapIdentity ?? createManagedBootstrapIdentity(); + assertManagedBootstrapIdentity(bootstrapIdentity); + const heldWorkloadArgv = renderManagedBootstrapHeldCommand( + request, + bootstrapIdentity, + plan.intendedWorkloadArgv, + ); + + let launchCalls = 0; + let launchProtocolViolation = false; + let launchReceipt: ManagedBootstrapCreateReceipt | null = null; + const create: ManagedBootstrapCreateInput = Object.freeze({ + plan, + request, + bootstrapIdentity, + launch: async (candidate: Parameters[0]) => { + launchCalls += 1; + if (launchCalls !== 1) { + launchProtocolViolation = true; + protocolFail("provider attempted more than one held workload launch"); + } + if ( + candidate.bootstrapIdentity !== bootstrapIdentity || + !isDeepStrictEqual(candidate.heldWorkloadArgv, heldWorkloadArgv) + ) { + launchProtocolViolation = true; + protocolFail("provider changed the identity-bound held workload launch"); + } + launchReceipt = normalizeCreateReceipt( + await input.create.launch({ heldWorkloadArgv, bootstrapIdentity }), + plan, + ); + return launchReceipt; + }, + }); + + let handle: ManagedBootstrapHeldWorkloadHandle; + try { + const candidate = await adapter.createHeldWorkload(create); + if (launchProtocolViolation || launchCalls !== 1 || !launchReceipt) { + protocolFail("provider returned a held workload without one exact authorized launch"); + } + handle = normalizeHeldHandle(candidate, create, launchReceipt); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + try { + const rollback = normalizeIncompleteCreateCleanupReceipt( + await adapter.cleanupIncompleteCreate({ plan, bootstrapIdentity, heldWorkloadArgv }), + plan, + bootstrapIdentity, + ); + ( + failure as Error & { managedBootstrapRollback?: ManagedBootstrapFinalizationReceipt } + ).managedBootstrapRollback = rollback; + } catch (cleanupError) { + attachManagedBootstrapRollbackError(failure, cleanupError); + } + throw failure; + } + + let snapshot: ManagedBootstrapObservedSnapshot | null = null; + let prepared: ManagedBootstrapPreparedReplacementHandle | null = null; + try { + const discovered = normalizeDiscoveredWorkload( + await adapter.discoverHeldWorkload(discoveryInput(handle)), + handle, + ); + snapshot = normalizeObservedSnapshot( + await adapter.inspectHeldWorkload({ handle, discovered }), + handle, + discovered, + ); + prepared = normalizePreparedReplacement( + await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request, + replacementOptions, + }), + handle, + snapshot, + ); + const transaction = Object.freeze({ handle, snapshot, prepared }); + TRANSACTION_STATES.set(transaction, "prepared"); + return transaction; + } catch (error) { + return rollbackAfterFailure(adapter, error, { + handle, + snapshot, + prepared, + durablePreparation: null, + replacement: null, + }); + } +} + +/** Durably record prepared authority before allowing provider cutover. */ +export async function activateManagedBootstrapSequence( + adapter: ManagedBootstrapAdapter, + input: ManagedBootstrapActivationInput, +): Promise { + if (TRANSACTION_STATES.get(input.transaction) !== "prepared") { + protocolFail("activation requires the exact prepared transaction returned by this coordinator"); + } + if (!Number.isFinite(input.timeoutSecs) || input.timeoutSecs <= 0) { + protocolFail("bootstrap timeout must be positive and finite"); + } + const { handle, snapshot, prepared } = input.transaction; + TRANSACTION_STATES.set(input.transaction, "activating"); + let durablePreparation: ManagedBootstrapDurablePreparationReceipt | null = null; + let replacement: ManagedBootstrapReplacementHandle | null = null; + try { + const authority = createPreparedAuthority(input.transaction); + durablePreparation = normalizeDurablePreparationReceipt( + await input.authorityStore.recordPreparedAuthority(authority), + authority, + ); + replacement = normalizeReplacementHandle( + await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation, + }), + input.transaction, + ); + const completion = normalizeCompletionReceipt( + await adapter.awaitBootstrap({ + handle, + snapshot, + replacement, + timeoutSecs: input.timeoutSecs, + }), + handle, + replacement, + ); + const transaction = Object.freeze({ + ...input.transaction, + durablePreparation, + replacement, + completion, + }); + TRANSACTION_STATES.set(input.transaction, "activation-consumed"); + TRANSACTION_STATES.set(transaction, "activated"); + return transaction; + } catch (error) { + TRANSACTION_STATES.set(input.transaction, "rollback-attempted"); + return rollbackAfterFailure(adapter, error, { + handle, + snapshot, + prepared, + durablePreparation, + replacement, + }); + } +} + +/** Validate provider finalization against the exact prepared or activated authority. */ +export async function finalizeManagedBootstrapSequence( + adapter: ManagedBootstrapAdapter, + input: { + readonly outcome: "commit" | "rollback"; + readonly transaction: + | ManagedBootstrapPreparedTransaction + | ManagedBootstrapActivatedTransaction; + }, +): Promise { + const state = TRANSACTION_STATES.get(input.transaction); + const activated = state === "activated"; + if (!activated && state !== "prepared") { + protocolFail("finalization requires an exact coordinator-owned transaction"); + } + if (input.outcome === "commit" && !activated) { + protocolFail("commit requires a completed activated transaction"); + } + const transaction = input.transaction; + const replacement = activated + ? (transaction as ManagedBootstrapActivatedTransaction).replacement + : null; + const completion = activated + ? (transaction as ManagedBootstrapActivatedTransaction).completion + : null; + TRANSACTION_STATES.set(transaction, "finalizing"); + try { + const receipt = normalizeFinalizationReceipt( + await adapter.finalizeBootstrap({ + outcome: input.outcome, + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: activated + ? (transaction as ManagedBootstrapActivatedTransaction).durablePreparation + : null, + replacement, + completion, + }), + { outcome: input.outcome, handle: transaction.handle, snapshot: transaction.snapshot }, + ); + TRANSACTION_STATES.set(transaction, "finalized"); + return receipt; + } catch (error) { + TRANSACTION_STATES.set(transaction, "finalization-indeterminate"); + throw error; + } +} diff --git a/src/lib/onboard/managed-bootstrap/envelope.test.ts b/src/lib/onboard/managed-bootstrap/envelope.test.ts new file mode 100644 index 00000000000..6364bb0fbb5 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/envelope.test.ts @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + encodeManagedStartupProfile, + MANAGED_STARTUP_AGENTS, + type ManagedStartupAgent, +} from "../managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import { + MANAGED_BOOTSTRAP_COMPLETION_MAX_BYTES, + parseManagedBootstrapEnvelope, + parseManagedBootstrapImageCompletion, + serializeManagedBootstrapEnvelope, + serializeManagedBootstrapImageCompletion, +} from "./envelope"; + +function requestFor(agent: ManagedStartupAgent) { + return createManagedStartupRootApplyRequest({ + agent, + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agent, false, false)), + }); +} + +describe("managed bootstrap envelope", () => { + it.each( + MANAGED_STARTUP_AGENTS, + )("round-trips one canonical identity-bound %s root request", (agent) => { + const request = requestFor(agent); + const identity = "a".repeat(64); + const serialized = serializeManagedBootstrapEnvelope({ + bootstrapIdentity: identity, + rootApplyRequest: request, + }); + + expect(parseManagedBootstrapEnvelope(serialized)).toEqual({ + schemaVersion: 1, + bootstrapIdentity: identity, + rootApplyRequest: request, + }); + }); + + it("rejects malformed identities and non-canonical transport", () => { + const request = requestFor("openclaw"); + const identity = "a".repeat(64); + const serialized = serializeManagedBootstrapEnvelope({ + bootstrapIdentity: identity, + rootApplyRequest: request, + }); + + expect(parseManagedBootstrapEnvelope(serialized)).toEqual({ + schemaVersion: 1, + bootstrapIdentity: identity, + rootApplyRequest: request, + }); + expect(() => parseManagedBootstrapEnvelope(` ${serialized}`)).toThrow(/canonical/u); + expect(() => + serializeManagedBootstrapEnvelope({ + bootstrapIdentity: "not-an-identity", + rootApplyRequest: request, + }), + ).toThrow(/bootstrap identity/u); + }); + + it("round-trips a canonical identity-bound image completion receipt", () => { + const request = requestFor("hermes"); + const completion = { + agent: request.agent, + bootstrapIdentity: "b".repeat(64), + profileFingerprint: request.profileFingerprint, + transactionPending: true, + } as const; + expect( + parseManagedBootstrapImageCompletion(serializeManagedBootstrapImageCompletion(completion)), + ).toEqual({ schemaVersion: 1, ...completion }); + }); + + it("bounds image-owned completion input before parsing", () => { + expect(() => + parseManagedBootstrapImageCompletion(" ".repeat(MANAGED_BOOTSTRAP_COMPLETION_MAX_BYTES + 1)), + ).toThrow(/too large/u); + expect(() => parseManagedBootstrapImageCompletion("{}\0")).toThrow(/empty or too large/u); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/envelope.ts b/src/lib/onboard/managed-bootstrap/envelope.ts new file mode 100644 index 00000000000..817f7f22fd7 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/envelope.ts @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES, + type ManagedStartupRootApplyRequest, + parseManagedStartupRootApplyRequest, + serializeManagedStartupRootApplyRequest, +} from "../managed-startup/root-apply"; + +export const MANAGED_BOOTSTRAP_ENVELOPE_SCHEMA_VERSION = 1 as const; +export const MANAGED_BOOTSTRAP_REQUEST_FILE = "/var/lib/nemoclaw-managed-bootstrap-request.json"; +export const MANAGED_BOOTSTRAP_COMPLETION_FILE = "/run/nemoclaw/managed-bootstrap-completion.json"; +export const MANAGED_BOOTSTRAP_ENVELOPE_MAX_BYTES = MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES + 1024; +export const MANAGED_BOOTSTRAP_COMPLETION_MAX_BYTES = 1024; + +const BOOTSTRAP_IDENTITY_RE = /^[a-f0-9]{64}$/u; +const STANDARD_BASE64_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u; + +export interface ManagedBootstrapEnvelope { + readonly schemaVersion: typeof MANAGED_BOOTSTRAP_ENVELOPE_SCHEMA_VERSION; + readonly bootstrapIdentity: string; + readonly rootApplyRequest: ManagedStartupRootApplyRequest; +} + +export interface ManagedBootstrapImageCompletion { + readonly schemaVersion: typeof MANAGED_BOOTSTRAP_ENVELOPE_SCHEMA_VERSION; + readonly bootstrapIdentity: string; + readonly agent: ManagedStartupRootApplyRequest["agent"]; + readonly profileFingerprint: string; + readonly transactionPending: boolean; +} + +function fail(message: string): never { + throw new Error(`Managed bootstrap envelope is invalid: ${message}`); +} + +export function serializeManagedBootstrapEnvelope(input: { + readonly bootstrapIdentity: string; + readonly rootApplyRequest: ManagedStartupRootApplyRequest; +}): string { + if (!BOOTSTRAP_IDENTITY_RE.test(input.bootstrapIdentity)) { + fail("bootstrap identity must be 32 random bytes encoded as lowercase hex"); + } + const request = Buffer.from( + serializeManagedStartupRootApplyRequest(input.rootApplyRequest), + "utf8", + ).toString("base64"); + const serialized = `${JSON.stringify({ + bootstrapIdentity: input.bootstrapIdentity, + rootApplyRequestB64: request, + schemaVersion: MANAGED_BOOTSTRAP_ENVELOPE_SCHEMA_VERSION, + })}\n`; + if (Buffer.byteLength(serialized, "utf8") > MANAGED_BOOTSTRAP_ENVELOPE_MAX_BYTES) { + fail("serialized envelope exceeds its bounded transport"); + } + return serialized; +} + +export function parseManagedBootstrapEnvelope(text: string): ManagedBootstrapEnvelope { + if ( + text.length === 0 || + Buffer.byteLength(text, "utf8") > MANAGED_BOOTSTRAP_ENVELOPE_MAX_BYTES || + text.includes("\0") + ) { + fail("serialized envelope is empty or too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("serialized envelope is not valid JSON"); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + fail("serialized envelope must be an object"); + } + const record = parsed as Record; + if ( + Object.keys(record).sort().join(",") !== + ["bootstrapIdentity", "rootApplyRequestB64", "schemaVersion"].sort().join(",") || + record.schemaVersion !== MANAGED_BOOTSTRAP_ENVELOPE_SCHEMA_VERSION || + typeof record.bootstrapIdentity !== "string" || + !BOOTSTRAP_IDENTITY_RE.test(record.bootstrapIdentity) || + typeof record.rootApplyRequestB64 !== "string" || + !STANDARD_BASE64_RE.test(record.rootApplyRequestB64) + ) { + fail("serialized envelope has an invalid schema"); + } + const requestBytes = Buffer.from(record.rootApplyRequestB64, "base64"); + if (requestBytes.toString("base64") !== record.rootApplyRequestB64) { + fail("root application request transport is non-canonical"); + } + const rootApplyRequest = parseManagedStartupRootApplyRequest(requestBytes.toString("utf8")); + const envelope = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_ENVELOPE_SCHEMA_VERSION, + bootstrapIdentity: record.bootstrapIdentity, + rootApplyRequest, + }); + if (serializeManagedBootstrapEnvelope(envelope) !== text) { + fail("serialized envelope is not canonical"); + } + return envelope; +} + +export function serializeManagedBootstrapImageCompletion( + completion: Omit, +): string { + if ( + !BOOTSTRAP_IDENTITY_RE.test(completion.bootstrapIdentity) || + !BOOTSTRAP_IDENTITY_RE.test(completion.profileFingerprint) || + !["openclaw", "hermes", "langchain-deepagents-code"].includes(completion.agent) || + typeof completion.transactionPending !== "boolean" + ) { + fail("image completion identity is invalid"); + } + return `${JSON.stringify({ + agent: completion.agent, + bootstrapIdentity: completion.bootstrapIdentity, + profileFingerprint: completion.profileFingerprint, + schemaVersion: MANAGED_BOOTSTRAP_ENVELOPE_SCHEMA_VERSION, + transactionPending: completion.transactionPending, + })}\n`; +} + +export function parseManagedBootstrapImageCompletion( + text: string, +): ManagedBootstrapImageCompletion { + if ( + text.length === 0 || + Buffer.byteLength(text, "utf8") > MANAGED_BOOTSTRAP_COMPLETION_MAX_BYTES || + text.includes("\0") + ) { + fail("image completion is empty or too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + return fail("image completion is not valid JSON"); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + return fail("image completion must be an object"); + } + const completion = parsed as Record; + if ( + Object.keys(completion).sort().join(",") !== + ["agent", "bootstrapIdentity", "profileFingerprint", "schemaVersion", "transactionPending"] + .sort() + .join(",") || + completion.schemaVersion !== MANAGED_BOOTSTRAP_ENVELOPE_SCHEMA_VERSION || + typeof completion.agent !== "string" || + !["openclaw", "hermes", "langchain-deepagents-code"].includes(completion.agent) || + typeof completion.bootstrapIdentity !== "string" || + !BOOTSTRAP_IDENTITY_RE.test(completion.bootstrapIdentity) || + typeof completion.profileFingerprint !== "string" || + !BOOTSTRAP_IDENTITY_RE.test(completion.profileFingerprint) || + typeof completion.transactionPending !== "boolean" + ) { + return fail("image completion schema is invalid"); + } + const normalized = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_ENVELOPE_SCHEMA_VERSION, + bootstrapIdentity: completion.bootstrapIdentity, + agent: completion.agent as ManagedStartupRootApplyRequest["agent"], + profileFingerprint: completion.profileFingerprint, + transactionPending: completion.transactionPending, + }); + if (serializeManagedBootstrapImageCompletion(normalized) !== text) { + return fail("image completion is not canonical"); + } + return normalized; +} diff --git a/src/lib/onboard/managed-bootstrap/index.ts b/src/lib/onboard/managed-bootstrap/index.ts new file mode 100644 index 00000000000..17099572608 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/index.ts @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { + activateManagedBootstrapSequence, + finalizeManagedBootstrapSequence, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapActivatedTransaction, + type ManagedBootstrapAdapter, + type ManagedBootstrapAuthorityStore, + type ManagedBootstrapExpectedPlan, + type ManagedBootstrapPreparedTransaction, + prepareManagedBootstrapSequence, +} from "./adapter"; +export { + MANAGED_BOOTSTRAP_COMPLETION_FILE, + MANAGED_BOOTSTRAP_REQUEST_FILE, + parseManagedBootstrapEnvelope, + parseManagedBootstrapImageCompletion, + serializeManagedBootstrapEnvelope, + serializeManagedBootstrapImageCompletion, +} from "./envelope"; diff --git a/test/managed-bootstrap-trampoline.test.ts b/test/managed-bootstrap-trampoline.test.ts new file mode 100644 index 00000000000..92e173da6c6 --- /dev/null +++ b/test/managed-bootstrap-trampoline.test.ts @@ -0,0 +1,174 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { MANAGED_STARTUP_AGENTS } from "../src/lib/onboard/managed-startup/profile"; + +const ROOT = path.resolve(import.meta.dirname, ".."); +const TRAMPOLINE = path.join(ROOT, "scripts", "managed-bootstrap-trampoline.sh"); + +function executable(target: string, contents: string): void { + fs.writeFileSync(target, contents, { mode: 0o755 }); + fs.chmodSync(target, 0o755); +} + +describe("managed bootstrap image trampoline", () => { + it("uses the absolute image-owned Bash interpreter even with an attacker PATH", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-bootstrap-path-")); + try { + const attackerTrace = path.join(directory, "attacker-trace"); + executable( + path.join(directory, "bash"), + `#!/bin/sh\nprintf 'attacker bash ran\\n' >${JSON.stringify(attackerTrace)}\n`, + ); + const result = spawnSync(TRAMPOLINE, [], { + encoding: "utf8", + env: { ...process.env, PATH: directory }, + }); + + expect(result.status).not.toBe(0); + expect(fs.existsSync(attackerTrace)).toBe(false); + expect(fs.readFileSync(TRAMPOLINE, "utf8").startsWith("#!/bin/bash\n")).toBe(true); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + + it.each( + MANAGED_STARTUP_AGENTS, + )("consumes the protected %s request before exact supervisor exec and drops bootstrap variables", (agent) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-bootstrap-trampoline-")); + try { + const request = path.join(directory, "request.json"); + const completion = path.join(directory, "completion"); + const runtime = path.join(directory, "runtime.cjs"); + const sandbox = path.join(directory, "sandbox"); + const trace = path.join(directory, "trace"); + const script = path.join(directory, "trampoline.sh"); + const supervisor = path.join(directory, "supervisor"); + const injection = path.join(directory, "injection"); + fs.mkdirSync(sandbox); + fs.writeFileSync(runtime, ""); + fs.writeFileSync(request, "{}\n", { mode: 0o400 }); + executable( + path.join(directory, "id"), + `#!/bin/sh +case "$*" in + "-u") printf '0\\n' ;; + "-g") printf '0\\n' ;; + "-u sandbox") printf '1000\\n' ;; + "-g sandbox") printf '1000\\n' ;; + *) exit 1 ;; +esac +`, + ); + executable(path.join(directory, "stat"), "#!/bin/sh\nprintf '0:0:400:1\\n'\n"); + executable(path.join(directory, "rm"), '#!/bin/sh\nexec /bin/rm "$@"\n'); + executable( + path.join(directory, "node"), + `#!/bin/sh +printf 'node:%s:home=%s:path=%s:lang=%s:capability=%s\\n' "$*" "$HOME" "$PATH" "$LANG" "$NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION" >>${JSON.stringify(trace)} +case "$*" in + *--apply-bootstrap-file*) + /bin/rm -f ${JSON.stringify(request)} + printf '%s\\n' '${agent}:${"a".repeat(64)}:${"b".repeat(64)}' >${JSON.stringify(completion)} + ;; + *--verify-bootstrap-completion*) + test "$(/bin/cat ${JSON.stringify(completion)})" = '${agent}:${"a".repeat(64)}:${"b".repeat(64)}' + ;; +esac +`, + ); + executable( + supervisor, + `#!/bin/sh +test ! -e "$REQUEST" +test "$#" -eq 3 +test "$1" = "supervise" +test "$2" = "two words" +test "$3" = "\\$(touch ${injection})" +test ! -e ${JSON.stringify(injection)} +printf 'supervisor:%s|%s|%s:identity=%s:request=%s:home=%s:path=%s:lang=%s:capability=%s\\n' "$1" "$2" "$3" "\${_nemoclaw_bootstrap_identity-unset}" "\${_nemoclaw_request-unset}" "$HOME" "$PATH" "$LANG" "\${NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION-unset}" >>"$TRACE" +`, + ); + const source = fs + .readFileSync(TRAMPOLINE, "utf8") + .replaceAll("/usr/bin/id", path.join(directory, "id")) + .replaceAll("/usr/bin/stat", path.join(directory, "stat")) + .replaceAll("/usr/bin/rm", path.join(directory, "rm")) + .replace( + '_nemoclaw_runtime="/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs"', + `_nemoclaw_runtime=${JSON.stringify(runtime)}`, + ) + .replaceAll("/var/lib/nemoclaw-managed-bootstrap-request.json", request) + .replaceAll("/sandbox", sandbox) + .replaceAll("/usr/local/bin/node", path.join(directory, "node")); + fs.writeFileSync(script, source, { mode: 0o755 }); + fs.chmodSync(script, 0o755); + const fingerprint = "a".repeat(64); + const identity = "b".repeat(64); + const argv = [ + "--agent", + agent, + "--profile-fingerprint", + fingerprint, + "--bootstrap-identity", + identity, + "--agent-uid", + "1000", + "--agent-gid", + "1000", + "--agent-workdir", + sandbox, + "--request-file", + request, + "--", + supervisor, + "supervise", + "two words", + `$(touch ${injection})`, + ]; + const environment = { + REQUEST: request, + TRACE: trace, + HOME: "/preserved-home", + PATH: "/preserved-path", + LANG: "zz_TEST", + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "preserved-capability", + }; + + execFileSync(script, argv, { env: environment }); + + expect(fs.existsSync(request)).toBe(false); + expect(fs.existsSync(injection)).toBe(false); + expect(fs.readFileSync(trace, "utf8").trim().split("\n")).toEqual([ + `node:${runtime} --apply-bootstrap-file --agent ${agent} --profile-fingerprint ${fingerprint} --bootstrap-identity ${identity}:home=/root:path=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:lang=C.UTF-8:capability=1`, + `node:${runtime} --verify-bootstrap-completion --agent ${agent} --profile-fingerprint ${fingerprint} --bootstrap-identity ${identity}:home=/root:path=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:lang=C.UTF-8:capability=1`, + `supervisor:supervise|two words|$(touch ${injection}):identity=unset:request=unset:home=/preserved-home:path=/preserved-path:lang=zz_TEST:capability=preserved-capability`, + ]); + + execFileSync(script, argv, { env: environment }); + let lines = fs.readFileSync(trace, "utf8").trim().split("\n"); + expect(lines.filter((line) => line.includes("--apply-bootstrap-file"))).toHaveLength(1); + expect(lines.filter((line) => line.startsWith("supervisor:"))).toHaveLength(2); + + fs.writeFileSync(completion, `${agent}:${fingerprint}:${"c".repeat(64)}\n`); + const tamperedRestart = spawnSync(script, argv, { + encoding: "utf8", + env: environment, + }); + expect(tamperedRestart.status).not.toBe(0); + lines = fs.readFileSync(trace, "utf8").trim().split("\n"); + expect(lines.filter((line) => line.includes("--apply-bootstrap-file"))).toHaveLength(1); + expect(lines.filter((line) => line.startsWith("supervisor:"))).toHaveLength(2); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); +}); diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 816fd9f903e..9992230a0df 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -56,4 +56,54 @@ describe("runtime provider central source boundary", () => { expect(Object.values(providerContract).join("\n")).not.toMatch(/managed-bootstrap/u); expect(providerContract.current).not.toMatch(/\b(?:podman|mxc)\b/iu); }); + + // source-shape-contract: security -- The bootstrap protocol and image-owned trampoline must remain dormant until a later provider slice supplies runtime packaging and exact activation + it("keeps managed bootstrap provider-neutral, image-owned, and dormant", () => { + const bootstrapProtocol = [ + readFileSync(join(repoRoot, "src/lib/onboard/managed-bootstrap/adapter.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/managed-bootstrap/envelope.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/managed-bootstrap/index.ts"), "utf8"), + ]; + const activationSources = [ + readFileSync(join(repoRoot, "src/lib/onboard.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/docker-gpu-sandbox-create.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/sandbox-create-launch.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/sandbox-create-step.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/sandbox-gpu-create-flow.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/sandbox-gpu-create-run-attempt.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/runtime-provider/contract.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/runtime-provider/current.ts"), "utf8"), + readFileSync(join(repoRoot, "src/lib/onboard/runtime-provider/registry.ts"), "utf8"), + ]; + const dockerProvider = readFileSync( + join(repoRoot, "src/lib/onboard/runtime-provider/docker.ts"), + "utf8", + ); + const managedDockerfiles = [ + readFileSync(join(repoRoot, "Dockerfile"), "utf8"), + readFileSync(join(repoRoot, "agents/hermes/Dockerfile"), "utf8"), + readFileSync(join(repoRoot, "agents/langchain-deepagents-code/Dockerfile"), "utf8"), + ]; + + expect(bootstrapProtocol.join("\n")).not.toMatch( + /from\s+["'][^"']*(?:docker|podman)[^"']*["']/iu, + ); + expect(bootstrapProtocol.join("\n")).not.toMatch( + /(?:driverId|providerId)\s*(?:===|!==)\s*["'](?:docker|podman)["']/iu, + ); + expect(bootstrapProtocol.join("\n")).not.toMatch(/\b(?:docker|podman|openshell|mxc)\b/iu); + expect(activationSources.join("\n")).not.toMatch( + /(?:from\s+["'][^"']*managed-bootstrap|require\([^)]*managed-bootstrap)/u, + ); + expect(dockerProvider).not.toMatch( + /(?:from\s+["'][^"']*managed-bootstrap|require\([^)]*managed-bootstrap)/u, + ); + expect(dockerProvider.match(/bootstrap:\s*unsupported\(/gu)).toHaveLength(2); + + for (const dockerfile of managedDockerfiles) { + expect(dockerfile).not.toContain("nemoclaw-managed-bootstrap"); + expect(dockerfile).not.toContain("managed-startup-image-runtime.cjs"); + expect(dockerfile).not.toContain("nemoclaw-managed-startup-hold"); + } + }); }); From 69bcdc40721995a85f84cb911ac54698a7e254f8 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 04:45:21 -0700 Subject: [PATCH 057/117] feat(onboard): add transactional Docker bootstrap adapter Signed-off-by: Aaron Erickson --- .../openshell/sandbox-identity.test.ts | 21 + .../adapters/openshell/sandbox-identity.ts | 16 + src/lib/onboard/docker-gpu-patch-clone.ts | 23 +- src/lib/onboard/docker-gpu-patch-types.ts | 17 + src/lib/onboard/managed-bootstrap/README.md | 41 +- src/lib/onboard/managed-bootstrap/adapter.ts | 4 +- .../managed-bootstrap/docker-journal.test.ts | 99 + .../managed-bootstrap/docker-journal.ts | 389 +++ .../managed-bootstrap/docker-shared-state.ts | 629 ++++ .../managed-bootstrap/docker-spec.test.ts | 66 + .../onboard/managed-bootstrap/docker-spec.ts | 312 ++ .../onboard/managed-bootstrap/docker.test.ts | 651 ++++ src/lib/onboard/managed-bootstrap/docker.ts | 2957 +++++++++++++++++ .../openshell-docker-sandbox-containers.ts | 1 + test/runtime-provider-source-shape.test.ts | 7 + 15 files changed, 5215 insertions(+), 18 deletions(-) create mode 100644 src/lib/adapters/openshell/sandbox-identity.test.ts create mode 100644 src/lib/adapters/openshell/sandbox-identity.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-journal.test.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-journal.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-shared-state.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-spec.test.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-spec.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker.test.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker.ts diff --git a/src/lib/adapters/openshell/sandbox-identity.test.ts b/src/lib/adapters/openshell/sandbox-identity.test.ts new file mode 100644 index 00000000000..422bf71ccd8 --- /dev/null +++ b/src/lib/adapters/openshell/sandbox-identity.test.ts @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { parseOpenShellSandboxId } from "./sandbox-identity"; + +describe("OpenShell sandbox identity parsing", () => { + it("accepts one exact durable ID with optional terminal color", () => { + expect(parseOpenShellSandboxId("Name: alpha\nID: sandbox-alpha\n")).toBe("sandbox-alpha"); + expect(parseOpenShellSandboxId("\u001b[32mId: sandbox.alpha_2\u001b[0m\n")).toBe( + "sandbox.alpha_2", + ); + }); + + it("rejects ambiguous or non-canonical IDs", () => { + expect(parseOpenShellSandboxId("ID: first\nID: second\n")).toBeNull(); + expect(parseOpenShellSandboxId("ID: sandbox/alpha\n")).toBeNull(); + expect(parseOpenShellSandboxId("id: sandbox-alpha\n")).toBeNull(); + }); +}); diff --git a/src/lib/adapters/openshell/sandbox-identity.ts b/src/lib/adapters/openshell/sandbox-identity.ts new file mode 100644 index 00000000000..1820a8f8f7d --- /dev/null +++ b/src/lib/adapters/openshell/sandbox-identity.ts @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const ANSI_RE = /\x1b\[[0-9;]*m/gu; +const SANDBOX_ID_RE = /^[A-Za-z0-9._-]+$/u; + +export function parseOpenShellSandboxId(output: string): string | null { + const matches = [ + ...String(output) + .replace(ANSI_RE, "") + .matchAll(/^\s*(?:Id|ID):\s*(\S+)\s*$/gm), + ].map((match) => match[1] ?? ""); + return matches.length === 1 && SANDBOX_ID_RE.test(matches[0] as string) + ? (matches[0] as string) + : null; +} diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts index c92767adad0..828ce0c540b 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -329,7 +329,15 @@ export function buildDockerGpuCloneRunArgs( const image = String(options.image || config.Image || "").trim(); if (!image) throw new Error("Docker inspect output did not include Config.Image."); - const args: string[] = ["--name", dockerContainerName(inspect), ...mode.args]; + const containerName = String(options.containerName ?? dockerContainerName(inspect)).trim(); + if ( + containerName.length === 0 || + containerName.length > 253 || + !/^[A-Za-z0-9][A-Za-z0-9_.-]*$/u.test(containerName) + ) { + throw new Error("Docker clone container name is invalid."); + } + const args: string[] = ["--name", containerName, ...mode.args]; const gpuAugment = mode.kind !== "startup-command"; // Startup-command recreation must retain OpenShell's native CDI attachment. @@ -435,8 +443,17 @@ export function buildDockerGpuCloneRunArgs( if (host.Init) args.push("--init"); const entrypoint = stringArray(config.Entrypoint); - if (entrypoint.length > 0) args.push("--entrypoint", entrypoint[0]); - const commandArgs = sandboxCommand ? [] : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; + const replacementEntrypoint = String(options.containerEntrypoint ?? "").trim(); + if (replacementEntrypoint) { + args.push("--entrypoint", replacementEntrypoint); + } else if (entrypoint.length > 0) { + args.push("--entrypoint", entrypoint[0]); + } + const commandArgs = options.containerCommand + ? [...options.containerCommand] + : sandboxCommand + ? [] + : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; args.push(image, ...commandArgs); return args; } diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 32be0afe46f..d72046bd320 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -112,6 +112,14 @@ export type DockerGpuCloneRunOptions = { sandboxFallbackDns?: string | null; openshellSandboxCommand?: readonly string[] | null; requiredUlimits?: readonly DockerUlimit[] | null; + /** + * Exact replacement process boundary used only by dormant managed bootstrap. + * Ordinary recreation leaves both fields unset. + */ + containerEntrypoint?: string | null; + containerCommand?: readonly string[] | null; + /** Stopped staging name used before exact-name cutover. */ + containerName?: string | null; /** * Extra supplementary group IDs to add to the recreated container via * `--group-add`. On Jetson these are the host group(s) owning the Tegra GPU @@ -190,6 +198,14 @@ export type DockerContainerInspect = { Hostname?: string; Tty?: boolean; OpenStdin?: boolean; + StopTimeout?: number | null; + Volumes?: Record | null; + } | null; + State?: { + Running?: boolean; + Paused?: boolean; + Restarting?: boolean; + Dead?: boolean; } | null; HostConfig?: { Binds?: string[] | null; @@ -244,6 +260,7 @@ export type DockerContainerInspect = { DeviceIDs?: string[] | null; }> | null; ShmSize?: number; + ReadonlyRootfs?: boolean; ReadonlyPaths?: string[] | null; MaskedPaths?: string[] | null; } | null; diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 79b17729eac..61a2efbf8f8 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -1,8 +1,8 @@ # Managed bootstrap protocol -This directory defines a dormant, driver-neutral transaction contract. It does -not register a runtime provider or change sandbox creation, onboarding, -snapshot, clone, or restore behavior. +This directory defines a dormant, driver-neutral transaction contract and its +first driver adapter. It does not register a runtime provider or change sandbox +creation, onboarding, snapshot, clone, or restore behavior. The protocol binds one random bootstrap identity to: @@ -27,13 +27,28 @@ root-owned request, verifies an identity-bound completion, clears its private bootstrap variables and file descriptors, and then uses `exec "$@"` to preserve the captured supervisor argument boundaries. -The trampoline is intentionally not packaged or selected yet, and no production -TypeScript module imports this protocol. The current image definitions also do -not package `nemoclaw-managed-startup-hold` or -`managed-startup-image-runtime.cjs`. A later provider integration must add those -prerequisites together with their image-runtime bootstrap modes, implement -driver-specific prepare, durable-record, activate, exact cleanup, and rollback, -and only then wire the coordinator into create. The same contract is exercised -for OpenClaw, Hermes, and DCode without a provider-specific central switch. -Until that complete boundary lands, every registered runtime provider keeps -its bootstrap surface unsupported. +The trampoline is intentionally not an entrypoint, and no production TypeScript +module imports this protocol or the Docker adapter. + +The Docker adapter creates and validates a stopped replacement under an +identity-derived staging name while the original remains running. It stages the +0400 envelope and returns exact cleanup authority without quiescing, renaming, +or otherwise mutating the original. Only after the coordinator durably records +that complete prepared authority may activation journal both full runtime IDs, +all three names, both launch-spec hashes, image identity, profile fingerprint, +and sandbox ID and then enter the destructive cutover. Rollback publishes +`rollback-authorized` before exact replacement deletion; commit publishes +`shared-state-committed` before exact backup deletion. Cleanup is bound to full +runtime IDs. Mutable OpenShell names are read only to detect ownership reuse, +and unsafe name-only deletion returns a typed retention error. The dormant +adapter assumes the protocol's single coordinator; multi-process +lease/arbitration remains an explicit production-activation gate. Activation +must also inject the selected gateway's canonical state root. + +The current image definitions still do not package +`nemoclaw-managed-startup-hold`, `managed-startup-image-runtime.cjs`, or the +shared-state bootstrap modes consumed by this adapter. A later activation slice +must add those prerequisites and wire the coordinator into Docker create as one +boundary. The same contract is exercised for OpenClaw, Hermes, and DCode without +a provider-specific central switch. Until that complete boundary lands, every +registered runtime provider keeps bootstrap unsupported. diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index e346b7b4e32..22b6fc27650 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -929,7 +929,7 @@ function normalizePreparedReplacement( }); } -function createPreparedAuthority( +export function createManagedBootstrapPreparedAuthority( transaction: ManagedBootstrapPreparedTransaction, ): ManagedBootstrapPreparedAuthority { const { handle, snapshot, prepared } = transaction; @@ -1349,7 +1349,7 @@ export async function activateManagedBootstrapSequence( let durablePreparation: ManagedBootstrapDurablePreparationReceipt | null = null; let replacement: ManagedBootstrapReplacementHandle | null = null; try { - const authority = createPreparedAuthority(input.transaction); + const authority = createManagedBootstrapPreparedAuthority(input.transaction); durablePreparation = normalizeDurablePreparationReceipt( await input.authorityStore.recordPreparedAuthority(authority), authority, diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts new file mode 100644 index 00000000000..91a398efa51 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapJournal, + parseDockerManagedBootstrapJournal, + serializeDockerManagedBootstrapJournal, +} from "./docker-journal"; + +const roots: string[] = []; +const IDENTITY = "1".repeat(64); +const journal = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + phase: "staged", + bootstrapIdentity: IDENTITY, + sandbox: { + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + driverId: "docker", + }, + profileFingerprint: "2".repeat(64), + imageReference: `registry.example/image@sha256:${"3".repeat(64)}`, + runtimeImageContentId: `sha256:${"4".repeat(64)}`, + originalRuntimeId: "5".repeat(64), + replacementRuntimeId: "6".repeat(64), + originalName: "openshell-alpha", + replacementStagingName: "openshell-alpha-staged", + backupName: "openshell-alpha-backup", + originalSpecHash: "7".repeat(64), + replacementSpecHash: "8".repeat(64), +} satisfies DockerManagedBootstrapJournal); + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +describe("Docker managed bootstrap journal", () => { + it("publishes private canonical state through only monotonic phases", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + const directory = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); + const file = path.join(directory, `${IDENTITY}.json`); + expect(fs.statSync(directory).mode & 0o777).toBe(0o700); + expect(fs.statSync(file).mode & 0o777).toBe(0o600); + expect(parseDockerManagedBootstrapJournal(fs.readFileSync(file, "utf8"))).toEqual(journal); + expect(() => store.create(journal)).toThrow("already exists"); + expect(() => store.transition(IDENTITY, "staged", "shared-state-committed")).toThrow( + "unsupported", + ); + + expect(store.transition(IDENTITY, "staged", "cutover").phase).toBe("cutover"); + expect(store.transition(IDENTITY, "cutover", "shared-state-committed").phase).toBe( + "shared-state-committed", + ); + store.remove(IDENTITY, ["shared-state-committed"]); + expect(store.load(IDENTITY)).toBeNull(); + }); + + it("recovers one durable cutover decision before journal replacement", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + store.transition(IDENTITY, "staged", "cutover"); + const file = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, `${IDENTITY}.json`); + fs.writeFileSync(`${file}.decision`, "rollback-authorized\n", { mode: 0o600 }); + + expect(store.load(IDENTITY)?.phase).toBe("rollback-authorized"); + expect(parseDockerManagedBootstrapJournal(fs.readFileSync(file, "utf8")).phase).toBe( + "rollback-authorized", + ); + fs.unlinkSync(`${file}.decision`); + expect(store.load(IDENTITY)?.phase).toBe("rollback-authorized"); + expect(() => store.transition(IDENTITY, "cutover", "shared-state-committed")).toThrow( + "expected phase cutover", + ); + store.remove(IDENTITY, ["rollback-authorized"]); + }); + + it("rejects non-canonical authority", () => { + expect(() => + parseDockerManagedBootstrapJournal(`${JSON.stringify({ ...journal, phase: "unknown" })}\n`), + ).toThrow("phase is unsupported"); + expect( + serializeDockerManagedBootstrapJournal(Object.freeze({ ...journal, phase: "staged" })), + ).toBe(`${JSON.stringify(journal)}\n`); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts new file mode 100644 index 00000000000..e3a7e72b3a2 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -0,0 +1,389 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import type { ManagedBootstrapSandboxIdentity } from "./adapter"; + +export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 1 as const; +export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY = "managed-bootstrap"; + +const SHA256_RE = /^[a-f0-9]{64}$/u; +const MAX_JOURNAL_BYTES = 32 * 1024; +const JOURNAL_DIRECTORY_MODE = 0o700; +const JOURNAL_FILE_MODE = 0o600; +const DECISION_PHASES = new Set([ + "rollback-authorized", + "shared-state-committed", +]); + +export type DockerManagedBootstrapJournalPhase = + | "staged" + | "cutover" + | "rollback-authorized" + | "shared-state-committed"; + +export interface DockerManagedBootstrapJournal { + readonly schemaVersion: typeof DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION; + readonly phase: DockerManagedBootstrapJournalPhase; + readonly bootstrapIdentity: string; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly profileFingerprint: string; + readonly imageReference: string; + readonly runtimeImageContentId: string; + readonly originalRuntimeId: string; + readonly replacementRuntimeId: string; + readonly originalName: string; + readonly replacementStagingName: string; + readonly backupName: string; + readonly originalSpecHash: string; + readonly replacementSpecHash: string; +} + +export interface DockerManagedBootstrapJournalStore { + create(journal: DockerManagedBootstrapJournal): void; + load(bootstrapIdentity: string): DockerManagedBootstrapJournal | null; + transition( + bootstrapIdentity: string, + expected: DockerManagedBootstrapJournalPhase, + next: DockerManagedBootstrapJournalPhase, + ): DockerManagedBootstrapJournal; + remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]): void; +} + +/** + * Alternate stores may use this only when the durable mutation completed and + * the caller lost its acknowledgement. Ordinary I/O and fsync failures must + * retain their original error type and are never reconciled as success. + */ +export class DockerManagedBootstrapJournalAcknowledgementLostError extends Error { + constructor(message: string) { + super(message); + this.name = "DockerManagedBootstrapJournalAcknowledgementLostError"; + } +} + +const ALLOWED_TRANSITIONS = new Set([ + "staged->cutover", + "cutover->rollback-authorized", + "cutover->shared-state-committed", +]); + +function fail(message: string): never { + throw new Error(`Managed bootstrap Docker journal is invalid: ${message}`); +} + +function exactString(value: unknown, label: string, maxBytes = 4096): string { + if ( + typeof value !== "string" || + value.length === 0 || + value !== value.trim() || + value.includes("\0") || + Buffer.byteLength(value, "utf8") > maxBytes + ) { + fail(`${label} must be one bounded exact string`); + } + return value; +} + +function exactSha256(value: unknown, label: string): string { + if (typeof value !== "string" || !SHA256_RE.test(value)) { + fail(`${label} must be lowercase SHA-256`); + } + return value; +} + +function exactPhase(value: unknown): DockerManagedBootstrapJournalPhase { + if ( + !["staged", "cutover", "rollback-authorized", "shared-state-committed"].includes(String(value)) + ) { + fail("phase is unsupported"); + } + return value as DockerManagedBootstrapJournalPhase; +} + +function exactSandbox(value: unknown): ManagedBootstrapSandboxIdentity { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("sandbox identity must be an object"); + } + const sandbox = value as Record; + if (Object.keys(sandbox).sort().join(",") !== "driverId,sandboxId,sandboxName") { + fail("sandbox identity schema is invalid"); + } + return Object.freeze({ + sandboxName: exactString(sandbox.sandboxName, "sandbox name"), + sandboxId: exactString(sandbox.sandboxId, "sandbox ID"), + driverId: exactString(sandbox.driverId, "driver ID"), + }); +} + +export function normalizeDockerManagedBootstrapJournal( + value: unknown, +): DockerManagedBootstrapJournal { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("journal must be an object"); + } + const journal = value as Record; + const expectedKeys = [ + "backupName", + "bootstrapIdentity", + "imageReference", + "originalName", + "originalRuntimeId", + "originalSpecHash", + "phase", + "profileFingerprint", + "replacementRuntimeId", + "replacementSpecHash", + "replacementStagingName", + "runtimeImageContentId", + "sandbox", + "schemaVersion", + ]; + if ( + Object.keys(journal).sort().join(",") !== expectedKeys.sort().join(",") || + journal.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION + ) { + fail("journal schema is invalid"); + } + const normalized = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + phase: exactPhase(journal.phase), + bootstrapIdentity: exactSha256(journal.bootstrapIdentity, "bootstrap identity"), + sandbox: exactSandbox(journal.sandbox), + profileFingerprint: exactSha256(journal.profileFingerprint, "profile fingerprint"), + imageReference: exactString(journal.imageReference, "image reference"), + runtimeImageContentId: exactString(journal.runtimeImageContentId, "runtime image content ID"), + originalRuntimeId: exactSha256(journal.originalRuntimeId, "original runtime ID"), + replacementRuntimeId: exactSha256(journal.replacementRuntimeId, "replacement runtime ID"), + originalName: exactString(journal.originalName, "original name", 253), + replacementStagingName: exactString( + journal.replacementStagingName, + "replacement staging name", + 253, + ), + backupName: exactString(journal.backupName, "backup name", 253), + originalSpecHash: exactSha256(journal.originalSpecHash, "original spec hash"), + replacementSpecHash: exactSha256(journal.replacementSpecHash, "replacement spec hash"), + } satisfies DockerManagedBootstrapJournal); + if (normalized.originalRuntimeId === normalized.replacementRuntimeId) { + fail("original and replacement runtime IDs must differ"); + } + if ( + new Set([normalized.originalName, normalized.replacementStagingName, normalized.backupName]) + .size !== 3 + ) { + fail("original, staging, and backup names must be distinct"); + } + return normalized; +} + +export function serializeDockerManagedBootstrapJournal( + journal: DockerManagedBootstrapJournal, +): string { + const normalized = normalizeDockerManagedBootstrapJournal(journal); + const serialized = `${JSON.stringify(normalized)}\n`; + if (Buffer.byteLength(serialized, "utf8") > MAX_JOURNAL_BYTES) { + fail("serialized journal exceeds its bounded transport"); + } + return serialized; +} + +export function parseDockerManagedBootstrapJournal(text: string): DockerManagedBootstrapJournal { + if ( + text.length === 0 || + text.includes("\0") || + Buffer.byteLength(text, "utf8") > MAX_JOURNAL_BYTES + ) { + fail("serialized journal is empty or too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("serialized journal is not valid JSON"); + } + const journal = normalizeDockerManagedBootstrapJournal(parsed); + if (serializeDockerManagedBootstrapJournal(journal) !== text) { + fail("serialized journal is not canonical"); + } + return journal; +} + +function assertDirectory(directory: string): void { + fs.mkdirSync(directory, { recursive: true, mode: JOURNAL_DIRECTORY_MODE }); + const stat = fs.lstatSync(directory); + if (!stat.isDirectory() || stat.isSymbolicLink() || (stat.mode & 0o077) !== 0) { + fail("journal directory must be a private real directory"); + } +} + +function journalPath(directory: string, bootstrapIdentity: string): string { + exactSha256(bootstrapIdentity, "bootstrap identity"); + return path.join(directory, `${bootstrapIdentity}.json`); +} + +function decisionPath(target: string): string { + return `${target}.decision`; +} + +function readPrivateFile(target: string, label: string): string | null { + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.nlink !== 1 || + (stat.mode & 0o077) !== 0 || + stat.size <= 0 || + stat.size > MAX_JOURNAL_BYTES + ) { + fail(`${label} file ownership boundary is invalid`); + } + return fs.readFileSync(target, "utf8"); +} + +function fsyncDirectory(directory: string): void { + const descriptor = fs.openSync(directory, "r"); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +function atomicWrite( + directory: string, + target: string, + contents: string, + exclusive: boolean, +): void { + const temporary = path.join( + directory, + `.${path.basename(target)}.${process.pid}.${Date.now().toString(16)}.tmp`, + ); + let descriptor: number | null = null; + try { + descriptor = fs.openSync(temporary, "wx", JOURNAL_FILE_MODE); + fs.writeFileSync(descriptor, contents, "utf8"); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = null; + if (exclusive) { + try { + fs.linkSync(temporary, target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + fail("journal already exists for this bootstrap identity"); + } + throw error; + } + fs.unlinkSync(temporary); + } else { + fs.renameSync(temporary, target); + } + fs.chmodSync(target, JOURNAL_FILE_MODE); + fsyncDirectory(directory); + } finally { + if (descriptor !== null) fs.closeSync(descriptor); + try { + fs.unlinkSync(temporary); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } +} + +export function createFileDockerManagedBootstrapJournalStore( + stateRoot: string, +): DockerManagedBootstrapJournalStore { + const directory = path.join(stateRoot, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); + const load = (bootstrapIdentity: string): DockerManagedBootstrapJournal | null => { + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const contents = readPrivateFile(target, "journal"); + if (contents === null) return null; + const journal = parseDockerManagedBootstrapJournal(contents); + const decision = readPrivateFile(decisionPath(target), "decision"); + if (decision === null) return journal; + const phase = decision.endsWith("\n") ? decision.slice(0, -1) : ""; + if ( + !DECISION_PHASES.has(phase as DockerManagedBootstrapJournalPhase) || + (journal.phase !== "cutover" && journal.phase !== phase) + ) { + fail("decision does not match its cutover journal"); + } + const decided = normalizeDockerManagedBootstrapJournal({ ...journal, phase }); + if (journal.phase === "cutover") { + atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(decided), false); + } + return decided; + }; + return Object.freeze({ + create(journal: DockerManagedBootstrapJournal) { + const normalized = normalizeDockerManagedBootstrapJournal(journal); + assertDirectory(directory); + const target = journalPath(directory, normalized.bootstrapIdentity); + if (readPrivateFile(decisionPath(target), "decision") !== null) { + fail("stale decision exists for this bootstrap identity"); + } + atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(normalized), true); + }, + load, + transition( + bootstrapIdentity: string, + expected: DockerManagedBootstrapJournalPhase, + next: DockerManagedBootstrapJournalPhase, + ) { + if (!ALLOWED_TRANSITIONS.has(`${expected}->${next}`)) { + fail(`transition ${expected} to ${next} is unsupported`); + } + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const current = load(bootstrapIdentity); + if (current?.phase === next) return current; + if (!current || current.phase !== expected) { + fail(`expected phase ${expected} before transition to ${next}`); + } + const updated = normalizeDockerManagedBootstrapJournal({ ...current, phase: next }); + if (expected === "cutover") { + const decision = decisionPath(target); + try { + atomicWrite(directory, decision, `${next}\n`, true); + } catch (error) { + if ( + !(error instanceof Error) || + error.message !== + "Managed bootstrap Docker journal is invalid: journal already exists for this bootstrap identity" || + readPrivateFile(decision, "decision") !== `${next}\n` + ) { + throw error; + } + } + } + atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(updated), false); + return updated; + }, + remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]) { + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const current = load(bootstrapIdentity); + if (!current || !expected.includes(current.phase)) { + fail(`journal removal is not authorized from phase ${current?.phase ?? "absent"}`); + } + const decision = decisionPath(target); + if (readPrivateFile(decision, "decision") !== null) { + fs.unlinkSync(decision); + fsyncDirectory(directory); + } + fs.unlinkSync(target); + fsyncDirectory(directory); + }, + }); +} diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts new file mode 100644 index 00000000000..5953fc2b356 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts @@ -0,0 +1,629 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { + dockerRm as defaultDockerRm, + dockerStop as defaultDockerStop, +} from "../../adapters/docker/container"; +import { dockerRun as defaultDockerRun } from "../../adapters/docker/run"; +import { hasZeroDockerExitStatus } from "../docker-command-result"; +import { + DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + DOCKER_GPU_PATCH_TIMEOUT_MS, +} from "../docker-gpu-patch-constants"; +import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "../docker-gpu-patch-types"; +import { MANAGED_STARTUP_RUNTIME_EXECUTABLE } from "../managed-startup/image-runtime"; +import { MANAGED_STARTUP_AGENTS, type ManagedStartupAgent } from "../managed-startup/profile"; +import { + MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, +} from "../managed-startup/shared-state-transaction"; +import { isImmutableDockerImageId } from "../openshell-docker-sandbox-containers"; +import { cleanupTempDir, secureTempFile } from "../temp-files"; + +const RECEIPT_TEMP_PREFIX = "nemoclaw-managed-startup-receipt"; +const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY = + "/var/lib/nemoclaw/managed-startup-shared-state-commit-v1"; +const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; +const DURABLE_IDENTITY_RE = /^[a-f0-9]{64}$/u; +const NEUTRALIZED_PROCESS_INJECTION_ENV = [ + "--env", + "NODE_OPTIONS=", + "--env", + "NODE_PATH=", + "--env", + "BASH_ENV=", + "--env", + "ENV=", + "--env", + "LD_PRELOAD=", + "--env", + "LD_AUDIT=", + "--env", + "LD_LIBRARY_PATH=", + "--env", + "SHELLOPTS=", + "--env", + "PS4=", +] as const; + +export interface DockerManagedBootstrapSharedStateTransaction { + readonly agent: ManagedStartupAgent; + readonly bootstrapIdentity: string; + readonly containerId: string; + readonly image: string; + readonly profileFingerprint: string; +} + +export interface DockerManagedStartupSharedStateOutcome { + /** + * True only when the new supervisor is still eligible for successful + * container cutover. A commit failure forces shared-state rollback first. + */ + readonly supervisorReady: boolean; + /** Original commit failure after a successful shared-state rollback. */ + readonly failure: Error | null; +} + +export class DockerManagedStartupSharedStateCommitIndeterminateError extends Error { + constructor(detail: string, options?: ErrorOptions) { + super( + `Managed-startup shared-state commit may have completed, but immutable status is unavailable: ${detail}`, + options, + ); + this.name = "DockerManagedStartupSharedStateCommitIndeterminateError"; + } +} + +export function probeDockerManagedStartupSharedState( + input: { + readonly transaction: DockerManagedBootstrapSharedStateTransaction; + readonly profileFingerprint: string; + }, + deps: DockerGpuPatchDeps = {}, +): "committed" | "none" | "pending" { + const transaction = input.transaction; + assertValidManagedStartupTransaction(transaction); + if (input.profileFingerprint !== transaction.profileFingerprint) { + throw new Error("Managed bootstrap shared-state status fingerprint does not match."); + } + const committedReceiptPath = copyManagedStartupReceiptAt( + transaction, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, + deps, + true, + ); + if (committedReceiptPath) { + let verified = false; + try { + verifyCopiedManagedStartupReceipt( + transaction, + input.profileFingerprint, + committedReceiptPath, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, + "committed", + deps, + ); + verified = true; + return "committed"; + } finally { + if (verified) cleanupReceiptBestEffort(committedReceiptPath); + } + } + const receiptPath = copyManagedStartupReceipt(transaction, deps, true); + if (!receiptPath) return "none"; + let verified = false; + try { + verifyCopiedManagedStartupReceipt( + transaction, + input.profileFingerprint, + receiptPath, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + "pending", + deps, + ); + verified = true; + return "pending"; + } finally { + if (verified) cleanupReceiptBestEffort(receiptPath); + } +} + +function verifyCopiedManagedStartupReceipt( + transaction: DockerManagedBootstrapSharedStateTransaction, + profileFingerprint: string, + receiptPath: string, + receiptDirectory: string, + expectedStatus: "committed" | "pending", + deps: DockerGpuPatchDeps, +): void { + if (!transaction.bootstrapIdentity || !/^[a-f0-9]{64}$/u.test(profileFingerprint)) { + throw new Error("Managed bootstrap copied-receipt identity is incomplete."); + } + const dockerRun = deps.dockerRun ?? defaultDockerRun; + const result = dockerRun( + [ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + "--mount", + transactionReceiptMount(receiptPath, receiptDirectory), + "--entrypoint", + "/usr/local/bin/node", + transaction.image, + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + "--shared-state-transaction-status", + "--agent", + transaction.agent, + "--profile-fingerprint", + profileFingerprint, + "--bootstrap-identity", + transaction.bootstrapIdentity, + ], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(result)) { + throw new Error( + `Immutable managed-startup helper could not verify shared-state status: ${commandDetail(result)}. ` + + `Protected receipt retained at ${receiptPath}`, + ); + } + if (String(result.stdout ?? "").trim() !== expectedStatus) { + throw new Error( + `Immutable managed-startup helper returned an invalid copied transaction status. Protected receipt retained at ${receiptPath}`, + ); + } +} + +function commandDetail(result: { + readonly stderr?: string | Buffer | null; + readonly stdout?: string | Buffer | null; + readonly error?: Error | null; +}): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}` + .trim() + .slice(-800); +} + +function cleanupReceiptBestEffort(receiptPath: string): void { + try { + cleanupTempDir(receiptPath, RECEIPT_TEMP_PREFIX); + } catch (error) { + console.warn( + ` ⚠ Managed-startup shared state is finalized, but its protected host receipt could not be removed (${receiptPath}): ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +function assertValidManagedStartupTransaction( + transaction: DockerManagedBootstrapSharedStateTransaction, +): asserts transaction is DockerManagedBootstrapSharedStateTransaction & { + readonly bootstrapIdentity: string; + readonly profileFingerprint: string; +} { + if (!(MANAGED_STARTUP_AGENTS as readonly string[]).includes(transaction.agent)) { + throw new Error("Managed bootstrap shared-state transaction agent is invalid."); + } + if (!FULL_CONTAINER_ID_RE.test(transaction.containerId)) { + throw new Error("Managed bootstrap shared-state transaction container identity is invalid."); + } + if (!isImmutableDockerImageId(transaction.image)) { + throw new Error("Managed bootstrap shared-state transaction image identity is not immutable."); + } + if (!transaction.bootstrapIdentity || !DURABLE_IDENTITY_RE.test(transaction.bootstrapIdentity)) { + throw new Error("Managed bootstrap shared-state transaction identity is missing or invalid."); + } + if ( + !transaction.profileFingerprint || + !DURABLE_IDENTITY_RE.test(transaction.profileFingerprint) + ) { + throw new Error( + "Managed bootstrap shared-state transaction profile fingerprint is missing or invalid.", + ); + } +} + +function transactionCommand( + action: "clear-shared-state-commit-receipt" | "commit" | "rollback", + transaction: DockerManagedBootstrapSharedStateTransaction, +): string[] { + assertValidManagedStartupTransaction(transaction); + return [ + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + action === "clear-shared-state-commit-receipt" + ? "--clear-shared-state-commit-receipt" + : `--${action}-shared-state-transaction`, + "--agent", + transaction.agent, + "--bootstrap-identity", + transaction.bootstrapIdentity, + ]; +} + +export function clearDockerManagedStartupSharedStateCommitReceipt( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps = {}, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + assertValidManagedStartupTransaction(transaction); + const command = transactionCommand("clear-shared-state-commit-receipt", transaction); + const cleared = dockerRun( + [ + "exec", + "--user", + "0:0", + "--workdir", + "/", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + transaction.containerId, + "/usr/bin/env", + "-i", + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + ...command, + ], + DOCKER_MUTATION_OPTIONS, + ); + // Accept a lost Docker acknowledgement only when both exact image-owned + // receipt paths are independently proven absent by the immutable helper. + let status: "committed" | "none" | "pending"; + try { + status = probeDockerManagedStartupSharedState( + { + transaction, + profileFingerprint: transaction.profileFingerprint, + }, + deps, + ); + } catch (error) { + throw new DockerManagedStartupSharedStateCommitIndeterminateError( + error instanceof Error ? error.message : String(error), + { cause: error }, + ); + } + if (status === "none") return; + if (!hasZeroDockerExitStatus(cleared)) { + throw new Error( + `Managed-startup durable commit receipt cleanup failed and exact absence was not proven (status=${status}): ${commandDetail(cleared)}`, + ); + } + throw new Error( + `Managed-startup durable commit receipt cleanup returned success, but exact absence was not proven (status=${status}).`, + ); +} + +function commitManagedStartupSharedState( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + assertValidManagedStartupTransaction(transaction); + const command = transactionCommand("commit", transaction); + const commit = dockerRun( + [ + "exec", + "--user", + "0:0", + "--workdir", + "/", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + transaction.containerId, + "/usr/bin/env", + "-i", + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + ...command, + ], + DOCKER_MUTATION_OPTIONS, + ); + // The commit helper atomically renames the rollback receipt into a compact + // identity-bound commit receipt before Docker returns. Always probe it + // afterward so a lost daemon acknowledgement is accepted only when durable + // commit state is independently proven. + let status: "committed" | "none" | "pending"; + try { + status = probeDockerManagedStartupSharedState( + { + transaction, + profileFingerprint: transaction.profileFingerprint, + }, + deps, + ); + } catch (error) { + throw new DockerManagedStartupSharedStateCommitIndeterminateError( + error instanceof Error ? error.message : String(error), + { cause: error }, + ); + } + if (status === "committed") return; + if (!hasZeroDockerExitStatus(commit)) { + throw new Error( + `Managed-startup shared-state commit helper failed and durable commit was not proven (status=${status}): ${commandDetail(commit)}`, + ); + } + throw new Error( + `Managed-startup shared-state commit helper returned success, but durable commit was not proven (status=${status}).`, + ); +} + +const DOCKER_MUTATION_OPTIONS = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, +} as const; + +function quiesceManagedStartupContainer( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerStop = deps.dockerStop ?? defaultDockerStop; + const stopped = dockerStop(transaction.containerId, { + ...DOCKER_MUTATION_OPTIONS, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopped)) { + throw new Error( + `Could not quiesce the failed managed-startup container before shared-state rollback: ${commandDetail(stopped)}`, + ); + } +} + +function isExactMissingReceiptCopy( + transaction: DockerManagedBootstrapSharedStateTransaction, + sourcePath: string, + result: { + readonly stderr?: string | Buffer | null; + readonly stdout?: string | Buffer | null; + readonly error?: Error | null; + }, +): boolean { + const detail = commandDetail(result); + const escapedPath = sourcePath.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const escapedContainer = transaction.containerId.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + return [ + new RegExp( + `^(?:Error response from daemon: )?Could not find the file ${escapedPath} in container ${escapedContainer}$`, + "u", + ), + new RegExp(`^(?:lstat|stat) ${escapedPath}: no such file or directory$`, "u"), + ].some((pattern) => pattern.test(detail)); +} + +function transactionReceiptMount(receiptPath: string, receiptDirectory: string): string { + return `type=bind,src=${receiptPath},dst=${receiptDirectory},readonly`; +} + +function copyManagedStartupReceiptAt( + transaction: DockerManagedBootstrapSharedStateTransaction, + sourcePath: string, + deps: DockerGpuPatchDeps, + allowAbsent = false, +): string | null { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + const tempSeed = secureTempFile(RECEIPT_TEMP_PREFIX); + const receiptPath = path.join(path.dirname(tempSeed), path.basename(sourcePath)); + try { + const copy = dockerRun( + ["cp", "-a", `${transaction.containerId}:${sourcePath}`, receiptPath], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(copy)) { + if (allowAbsent && isExactMissingReceiptCopy(transaction, sourcePath, copy)) { + cleanupReceiptBestEffort(receiptPath); + return null; + } + throw new Error( + `Could not copy the managed-startup rollback receipt from the failed container: ${commandDetail(copy)}`, + ); + } + if (receiptPath.includes(",") || /[\r\n\0]/u.test(receiptPath)) { + throw new Error("Managed-startup rollback receipt path is unsafe for a Docker bind mount"); + } + return receiptPath; + } catch (error) { + cleanupReceiptBestEffort(receiptPath); + throw error; + } +} + +function copyManagedStartupReceipt( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, + allowAbsent = false, +): string | null { + return copyManagedStartupReceiptAt( + transaction, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + deps, + allowAbsent, + ); +} + +function rollbackManagedStartupSharedState( + transaction: DockerManagedBootstrapSharedStateTransaction, + receiptPath: string, + deps: DockerGpuPatchDeps, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + let restored = false; + try { + const helper = dockerRun( + [ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--cap-add", + "CHOWN", + "--cap-add", + "DAC_OVERRIDE", + "--cap-add", + "FOWNER", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + "--volumes-from", + transaction.containerId, + "--mount", + `type=bind,src=${receiptPath},dst=${MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY},readonly`, + "--entrypoint", + "/usr/local/bin/node", + transaction.image, + ...transactionCommand("rollback", transaction), + "--read-only-receipt", + ], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(helper)) { + throw new Error( + `Immutable managed-startup helper could not restore and verify shared state: ${commandDetail(helper)}. ` + + `Protected receipt retained at ${receiptPath}`, + ); + } + restored = true; + } finally { + if (restored) { + cleanupReceiptBestEffort(receiptPath); + } + } +} + +function removeFailedUnbackedContainer( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerRm = deps.dockerRm ?? defaultDockerRm; + const removed = dockerRm(transaction.containerId, DOCKER_MUTATION_OPTIONS); + if (!hasZeroDockerExitStatus(removed)) { + throw new Error( + `Could not remove the failed managed-startup container after shared-state rollback: ${commandDetail(removed)}`, + ); + } +} + +/** + * Finalize the shared-state half of managed container cutover before generic + * backup removal or rollback. A shared-state rollback failure deliberately + * throws so callers cannot remove the new container or restart the old one + * while `/sandbox` remains partially applied. + */ +export function finalizeDockerManagedStartupSharedState( + input: { + readonly transaction: DockerManagedBootstrapSharedStateTransaction | null; + readonly patchResult?: DockerGpuPatchResult | null; + /** + * The managed-bootstrap journal owns exact replacement removal. Retaining + * it lets the caller publish rollback authorization after shared-state + * restoration and before the first runtime deletion. + */ + readonly retainContainerAfterRollback?: boolean; + readonly supervisorReady: boolean; + }, + deps: DockerGpuPatchDeps = {}, +): DockerManagedStartupSharedStateOutcome { + const transaction = input.transaction; + if (!transaction) { + return { supervisorReady: input.supervisorReady, failure: null }; + } + assertValidManagedStartupTransaction(transaction); + if (input.supervisorReady) { + // Preserve and validate an explicit writable-layer receipt before logical + // commit. The helper receives the copy read-only and does not delete it; + // this keeps rollback possible when Docker loses the helper acknowledgement. + // --volumes-from exposes shared mounts only; it cannot expose this + // container-local transaction directory to an immutable helper. + let receiptPath: string; + try { + const copiedReceipt = copyManagedStartupReceipt(transaction, deps); + if (!copiedReceipt) { + throw new Error("Managed-startup pending receipt disappeared before commit."); + } + receiptPath = copiedReceipt; + } catch (error) { + try { + quiesceManagedStartupContainer(transaction, deps); + } catch (stopError) { + throw new Error( + `Managed-startup receipt preservation failed and the new workload could not be quiesced: ${ + error instanceof Error ? error.message : String(error) + }; ${stopError instanceof Error ? stopError.message : String(stopError)}`, + ); + } + throw error; + } + let commitFailure: Error | null = null; + try { + verifyCopiedManagedStartupReceipt( + transaction, + transaction.profileFingerprint, + receiptPath, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + "pending", + deps, + ); + commitManagedStartupSharedState(transaction, deps); + cleanupReceiptBestEffort(receiptPath); + return { supervisorReady: true, failure: null }; + } catch (error) { + if (error instanceof DockerManagedStartupSharedStateCommitIndeterminateError) { + throw error; + } + commitFailure = error instanceof Error ? error : new Error(String(error)); + } + const failure = new Error( + `OpenShell supervisor reconnected, but managed shared-state logical commit validation failed: ${commitFailure.message}`, + ); + quiesceManagedStartupContainer(transaction, deps); + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!input.patchResult && !input.retainContainerAfterRollback) { + removeFailedUnbackedContainer(transaction, deps); + } + return { supervisorReady: false, failure }; + } + + quiesceManagedStartupContainer(transaction, deps); + const receiptPath = copyManagedStartupReceipt(transaction, deps, true); + if (!receiptPath) { + if (!input.patchResult && !input.retainContainerAfterRollback) { + removeFailedUnbackedContainer(transaction, deps); + } + return { supervisorReady: false, failure: null }; + } + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!input.patchResult && !input.retainContainerAfterRollback) { + removeFailedUnbackedContainer(transaction, deps); + } + return { supervisorReady: false, failure: null }; +} diff --git a/src/lib/onboard/managed-bootstrap/docker-spec.test.ts b/src/lib/onboard/managed-bootstrap/docker-spec.test.ts new file mode 100644 index 00000000000..57bcc8633ff --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-spec.test.ts @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { createDockerGpuInspectFixture } from "../__test-helpers__/docker-gpu-patch-fixtures"; +import { + normalizeDockerManagedBootstrapLaunchSpec, + parseDockerManagedBootstrapLaunchSpec, +} from "./docker-spec"; + +describe("managed bootstrap Docker launch spec", () => { + it("hashes reproducible launch state while excluding runtime ID, phase, IP, and gateway", () => { + const first = createDockerGpuInspectFixture(); + const second = structuredClone(first); + second.Id = "another-runtime-id"; + second.State = { Running: false, Dead: true }; + second.NetworkSettings!.Networks!["openshell-docker"]!.IPAddress = "172.18.0.99"; + second.NetworkSettings!.Networks!["openshell-docker"]!.Gateway = "172.18.0.254"; + + const expected = normalizeDockerManagedBootstrapLaunchSpec(first); + const observed = normalizeDockerManagedBootstrapLaunchSpec(second); + + expect(observed.hash).toBe(expected.hash); + expect(observed.canonicalJson).toBe(expected.canonicalJson); + expect(parseDockerManagedBootstrapLaunchSpec(expected.canonicalJson)).toEqual(expected.spec); + }); + + it("changes the hash when a reproducible launch field changes", () => { + const first = createDockerGpuInspectFixture(); + const second = structuredClone(first); + second.Config!.StopTimeout = 45; + + expect(normalizeDockerManagedBootstrapLaunchSpec(second).hash).not.toBe( + normalizeDockerManagedBootstrapLaunchSpec(first).hash, + ); + }); + + it.each([ + { + name: "anonymous Config.Volumes whose data source cannot be proven", + mutate: (inspect: ReturnType) => { + inspect.Config!.Volumes = { "/var/lib/state": {} }; + }, + error: /config fields it cannot reproduce exactly: Volumes\./u, + }, + { + name: "multiple attached networks", + mutate: (inspect: ReturnType) => { + inspect.NetworkSettings!.Networks!.secondary = { Aliases: ["alpha-secondary"] }; + }, + error: /multiple attached networks/u, + }, + { + name: "an unknown HostConfig field", + mutate: (inspect: ReturnType) => { + (inspect.HostConfig as Record).FutureRuntimeField = true; + }, + error: /unsupported fields: FutureRuntimeField/u, + }, + ])("fails closed for $name", ({ mutate, error }) => { + const inspect = createDockerGpuInspectFixture(); + mutate(inspect); + expect(() => normalizeDockerManagedBootstrapLaunchSpec(inspect)).toThrow(error); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker-spec.ts b/src/lib/onboard/managed-bootstrap/docker-spec.ts new file mode 100644 index 00000000000..bf96451f99a --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-spec.ts @@ -0,0 +1,312 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import type { DockerContainerInspect } from "../docker-gpu-patch-types"; + +const CONFIG_KEYS = new Set([ + "ArgsEscaped", + "AttachStderr", + "AttachStdin", + "AttachStdout", + "Cmd", + "Domainname", + "Entrypoint", + "Env", + "ExposedPorts", + "Healthcheck", + "Hostname", + "Image", + "Labels", + "MacAddress", + "NetworkDisabled", + "OnBuild", + "OpenStdin", + "Shell", + "StdinOnce", + "StopSignal", + "StopTimeout", + "Tty", + "User", + "Volumes", + "WorkingDir", +]); + +const HOST_CONFIG_KEYS = new Set([ + "AutoRemove", + "Binds", + "BlkioDeviceReadBps", + "BlkioDeviceReadIOps", + "BlkioDeviceWriteBps", + "BlkioDeviceWriteIOps", + "BlkioWeight", + "BlkioWeightDevice", + "CapAdd", + "CapDrop", + "Cgroup", + "CgroupParent", + "CgroupnsMode", + "ConsoleSize", + "ContainerIDFile", + "CpuCount", + "CpuPercent", + "CpuPeriod", + "CpuQuota", + "CpuRealtimePeriod", + "CpuRealtimeRuntime", + "CpuShares", + "CpusetCpus", + "CpusetMems", + "DeviceCgroupRules", + "DeviceRequests", + "Devices", + "Dns", + "DnsOptions", + "DnsSearch", + "ExtraHosts", + "GroupAdd", + "IOMaximumBandwidth", + "IOMaximumIOps", + "Init", + "IpcMode", + "Isolation", + "Links", + "LogConfig", + "MaskedPaths", + "Memory", + "MemoryReservation", + "MemorySwap", + "MemorySwappiness", + "Mounts", + "NanoCpus", + "NetworkMode", + "OomKillDisable", + "OomScoreAdj", + "PidMode", + "PidsLimit", + "PortBindings", + "Privileged", + "PublishAllPorts", + "ReadonlyPaths", + "ReadonlyRootfs", + "RestartPolicy", + "Runtime", + "SecurityOpt", + "ShmSize", + "StorageOpt", + "Sysctls", + "Tmpfs", + "UTSMode", + "Ulimits", + "UsernsMode", + "VolumeDriver", + "VolumesFrom", +]); + +const UNSUPPORTED_CONFIG_KEYS = new Set([ + "ArgsEscaped", + "AttachStderr", + "AttachStdin", + "AttachStdout", + "MacAddress", + "OnBuild", + "Shell", + "Volumes", +]); + +const UNSUPPORTED_HOST_CONFIG_KEYS = new Set([ + "BlkioDeviceReadBps", + "BlkioDeviceReadIOps", + "BlkioDeviceWriteBps", + "BlkioDeviceWriteIOps", + "BlkioWeight", + "BlkioWeightDevice", + "Cgroup", + "ConsoleSize", + "ContainerIDFile", + "CpuCount", + "CpuPercent", + "CpuRealtimePeriod", + "CpuRealtimeRuntime", + "IOMaximumBandwidth", + "IOMaximumIOps", + "Isolation", + "Links", + "MaskedPaths", + "MemorySwappiness", + "ReadonlyPaths", + "StorageOpt", + "VolumeDriver", + "VolumesFrom", +]); + +export interface DockerManagedBootstrapLaunchSpec { + readonly schemaVersion: 1; + readonly inspect: Pick< + DockerContainerInspect, + "Name" | "Config" | "HostConfig" | "NetworkSettings" + > & { readonly Platform?: string }; +} + +function isEmptyDefault(value: unknown): boolean { + if (value === undefined || value === null || value === false || value === "" || value === 0) { + return true; + } + if (Array.isArray(value)) return value.length === 0; + if (typeof value === "object") return Object.keys(value as object).length === 0; + return false; +} + +function exactObject(value: unknown, label: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`Managed bootstrap Docker ${label} must be an object.`); + } + return value as Record; +} + +function assertKnownKeys( + record: Record, + allowed: ReadonlySet, + label: string, +): void { + const unknown = Object.keys(record).filter((key) => !allowed.has(key)); + if (unknown.length > 0) { + throw new Error( + `Managed bootstrap Docker ${label} contains unsupported fields: ${unknown.sort().join(", ")}.`, + ); + } +} + +function assertUnsupportedDefaults(host: Record): void { + const active = [...UNSUPPORTED_HOST_CONFIG_KEYS].filter((key) => !isEmptyDefault(host[key])); + if (active.length > 0) { + throw new Error( + `Managed bootstrap refuses Docker launch fields it cannot reproduce exactly: ${active + .sort() + .join(", ")}.`, + ); + } +} + +function normalizedNetworkSettings( + value: DockerContainerInspect["NetworkSettings"], +): DockerContainerInspect["NetworkSettings"] { + const networks = value?.Networks ?? {}; + return { + Networks: Object.fromEntries( + Object.entries(networks) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([name, network]) => [ + name, + { + Aliases: [...(network.Aliases ?? [])].sort(), + }, + ]), + ), + }; +} + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalize); + if (typeof value !== "object" || value === null) return value; + return Object.fromEntries( + Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, nested]) => [key, canonicalize(nested)]), + ); +} + +export function parseExactDockerContainerInspect(output: string): DockerContainerInspect { + let parsed: unknown; + try { + parsed = JSON.parse(output); + } catch { + throw new Error("Managed bootstrap Docker inspect output is malformed."); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("Managed bootstrap Docker inspect must return exactly one workload."); + } + return exactObject(parsed[0], "inspect") as DockerContainerInspect; +} + +export function normalizeDockerManagedBootstrapLaunchSpec(inspect: DockerContainerInspect): { + readonly canonicalJson: string; + readonly hash: string; + readonly spec: DockerManagedBootstrapLaunchSpec; +} { + const raw = inspect as DockerContainerInspect & Record; + const config = exactObject(raw.Config, "Config"); + const hostConfig = exactObject(raw.HostConfig, "HostConfig"); + assertKnownKeys(config, CONFIG_KEYS, "Config"); + assertKnownKeys(hostConfig, HOST_CONFIG_KEYS, "HostConfig"); + const unsupportedConfig = [...UNSUPPORTED_CONFIG_KEYS].filter( + (key) => !isEmptyDefault(config[key]), + ); + if (unsupportedConfig.length > 0) { + throw new Error( + `Managed bootstrap refuses Docker config fields it cannot reproduce exactly: ${unsupportedConfig + .sort() + .join(", ")}.`, + ); + } + assertUnsupportedDefaults(hostConfig); + + if (config.NetworkDisabled === true) { + throw new Error("Managed bootstrap does not support Config.NetworkDisabled."); + } + if (config.StdinOnce === true) { + throw new Error("Managed bootstrap does not support Config.StdinOnce."); + } + if (hostConfig.AutoRemove === true) { + throw new Error("Managed bootstrap cannot preserve an auto-remove held workload."); + } + if (hostConfig.PublishAllPorts === true) { + throw new Error("Managed bootstrap requires explicit Docker port bindings."); + } + if (Object.keys(inspect.NetworkSettings?.Networks ?? {}).length > 1) { + throw new Error("Managed bootstrap refuses a Docker workload with multiple attached networks."); + } + + const spec: DockerManagedBootstrapLaunchSpec = { + schemaVersion: 1, + inspect: { + Name: inspect.Name, + Config: config as DockerContainerInspect["Config"], + HostConfig: hostConfig as DockerContainerInspect["HostConfig"], + NetworkSettings: normalizedNetworkSettings(inspect.NetworkSettings), + ...("Platform" in raw && typeof raw.Platform === "string" ? { Platform: raw.Platform } : {}), + }, + }; + const canonicalJson = `${JSON.stringify(canonicalize(spec))}\n`; + return Object.freeze({ + canonicalJson, + hash: createHash("sha256").update(canonicalJson, "utf8").digest("hex"), + spec: Object.freeze(spec), + }); +} + +export function parseDockerManagedBootstrapLaunchSpec( + canonicalJson: string, +): DockerManagedBootstrapLaunchSpec { + let parsed: unknown; + try { + parsed = JSON.parse(canonicalJson); + } catch { + throw new Error("Managed bootstrap Docker launch snapshot is malformed."); + } + const record = exactObject(parsed, "launch snapshot"); + if ( + Object.keys(record).sort().join(",") !== ["inspect", "schemaVersion"].join(",") || + record.schemaVersion !== 1 + ) { + throw new Error("Managed bootstrap Docker launch snapshot schema is invalid."); + } + const normalized = normalizeDockerManagedBootstrapLaunchSpec( + exactObject(record.inspect, "launch snapshot inspect") as DockerContainerInspect, + ); + if (normalized.canonicalJson !== canonicalJson) { + throw new Error("Managed bootstrap Docker launch snapshot is not canonical."); + } + return normalized.spec; +} diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts new file mode 100644 index 00000000000..14c65a9e926 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -0,0 +1,651 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import type { DockerContainerInspect } from "../docker-gpu-patch-types"; +import { encodeManagedStartupProfile, type ManagedStartupAgent } from "../managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import { + createManagedBootstrapPreparedAuthority, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapCompletionReceipt, + type ManagedBootstrapDurablePreparationReceipt, + type ManagedBootstrapHeldWorkloadHandle, + type ManagedBootstrapObservedSnapshot, + ManagedBootstrapOwnerCleanupRequiredError, + type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapReplacementHandle, +} from "./adapter"; +import { createDockerManagedBootstrapAdapter, type DockerManagedBootstrapDeps } from "./docker"; +import type { + DockerManagedBootstrapJournal, + DockerManagedBootstrapJournalStore, +} from "./docker-journal"; +import { DockerManagedBootstrapJournalAcknowledgementLostError } from "./docker-journal"; +import { normalizeDockerManagedBootstrapLaunchSpec } from "./docker-spec"; +import { parseManagedBootstrapEnvelope } from "./envelope"; + +const IDENTITY = "1".repeat(64); +const OLD_ID = "2".repeat(64); +const NEW_ID = "3".repeat(64); +const CONFIG_ID = `sha256:${"4".repeat(64)}`; +const MANIFEST = `sha256:${"5".repeat(64)}` as const; +const REPOSITORY = "registry.example/nemoclaw/hermes"; +const IMAGE = `${REPOSITORY}@${MANIFEST}`; +const SUPERVISOR = ["/opt/openshell/bin/openshell-sandbox", "supervise"] as const; +const SUPPORTED_AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; + +function agentInputs(agent: ManagedStartupAgent = "hermes") { + const request = createManagedStartupRootApplyRequest({ + agent, + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agent, false, false)), + }); + const heldArgv = [ + "env", + "A=1", + "/usr/local/bin/nemoclaw-managed-startup-hold", + "--agent", + agent, + "--profile-fingerprint", + request.profileFingerprint, + "--bootstrap-identity", + IDENTITY, + ] as const; + return { + request, + heldArgv, + metadata: { "nemoclaw.ai/managed-profile": request.profileFingerprint }, + }; +} + +const { heldArgv } = agentInputs(); +const sandbox = { sandboxName: "alpha", sandboxId: "sandbox-alpha", driverId: "docker" }; + +function shellArgv(argv: readonly string[]): string { + return argv.join(" "); +} + +function originalInspect(inputs = agentInputs()): DockerContainerInspect { + return { + Id: OLD_ID, + Image: CONFIG_ID, + Name: "/openshell-alpha", + Config: { + Image: IMAGE, + Env: ["A=1", `OPENSHELL_SANDBOX_COMMAND=${shellArgv(inputs.heldArgv)}`], + Labels: { + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-name": "alpha", + "openshell.ai/sandbox-id": "sandbox-alpha", + ...inputs.metadata, + }, + Entrypoint: [SUPERVISOR[0]], + Cmd: SUPERVISOR.slice(1), + User: "root", + WorkingDir: "/sandbox", + Hostname: "alpha", + }, + State: { Running: true, Paused: false, Restarting: false, Dead: false }, + HostConfig: { + Binds: ["/host/workspace:/sandbox:rw"], + NetworkMode: "openshell", + RestartPolicy: { Name: "unless-stopped" }, + CapDrop: ["NET_RAW"], + SecurityOpt: ["no-new-privileges"], + Ulimits: [{ Name: "nofile", Soft: 65_536, Hard: 65_536 }], + }, + NetworkSettings: { Networks: { openshell: { Aliases: ["openshell-alpha"] } } }, + }; +} + +function authority(agent: ManagedStartupAgent = "hermes") { + const inputs = agentInputs(agent); + const inspect = originalInspect(inputs); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(inspect); + const plan = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandboxName: "alpha", + driverId: "docker", + image: { repository: REPOSITORY, manifestDigest: MANIFEST }, + profile: { agent, fingerprint: inputs.request.profileFingerprint }, + agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + intendedWorkloadArgv: ["env", "A=1", "nemoclaw-start"], + expectedSupervisorArgv: SUPERVISOR, + metadata: inputs.metadata, + }; + const handle: ManagedBootstrapHeldWorkloadHandle = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: inputs.heldArgv, + intendedWorkloadArgv: plan.intendedWorkloadArgv, + plan, + createReceipt: { sandbox, ready: true, readyAt: "2026-07-31T12:00:00.000Z" }, + }; + const snapshot: ManagedBootstrapObservedSnapshot = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + runtimeId: OLD_ID, + bootstrapIdentity: IDENTITY, + image: plan.image, + runtimeImageContentId: CONFIG_ID, + specHash: normalized.hash, + specCanonicalJson: normalized.canonicalJson, + agentIdentity: plan.agentIdentity, + supervisorArgv: SUPERVISOR, + heldWorkloadArgv: inputs.heldArgv, + metadata: inputs.metadata, + }; + return { handle, plan, request: inputs.request, snapshot }; +} + +type FixtureOptions = { + agent?: ManagedStartupAgent; + failAfterCutoverFence?: boolean; + failStart?: boolean; + lostAcks?: boolean; + ownerId?: string; + sharedState?: "committed" | "none" | "pending"; +}; + +function fixture(options: FixtureOptions = {}) { + let original = originalInspect(agentInputs(options.agent)); + let replacement: DockerContainerInspect | null = null; + let journal: DockerManagedBootstrapJournal | null = null; + let sharedState: "committed" | "none" | "pending" = options.sharedState ?? "none"; + const events: string[] = []; + const lostTransitions = new Set(["cutover", "shared-state-committed"]); + let loseCreateAck = options.lostAcks === true; + let loseRemoveAck = options.lostAcks === true; + const ok = (stdout = "") => ({ status: 0, stdout, stderr: "" }); + const copyJournal = () => (journal ? structuredClone(journal) : null); + const store: DockerManagedBootstrapJournalStore = { + create(value) { + journal = structuredClone(value); + events.push("journal:staged"); + if (loseCreateAck) { + loseCreateAck = false; + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal create acknowledgement", + ); + } + }, + load: () => copyJournal(), + transition(_identity, expected, next) { + if (!journal || journal.phase !== expected) throw new Error("stale journal transition"); + journal = { ...journal, phase: next }; + events.push(`journal:${next}`); + if (next === "cutover" && options.failAfterCutoverFence) { + throw new Error("injected crash after durable cutover fence"); + } + if (options.lostAcks && lostTransitions.delete(next)) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal transition acknowledgement", + ); + } + return structuredClone(journal); + }, + remove(_identity, expected) { + if (!journal || !expected.includes(journal.phase)) throw new Error("stale journal remove"); + journal = null; + events.push("journal:removed"); + if (loseRemoveAck) { + loseRemoveAck = false; + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal remove acknowledgement", + ); + } + }, + }; + const inspect = (reference: string): DockerContainerInspect => { + const candidates = [original, replacement].filter( + (value): value is DockerContainerInspect => value !== null, + ); + const found = candidates.find( + (value) => + value.Id === reference || String(value.Name ?? "").replace(/^\/+/u, "") === reference, + ); + if (!found) throw new Error(`No such container: ${reference}`); + return structuredClone(found); + }; + const dockerCapture: NonNullable = vi.fn((args) => { + if (args[0] === "image") { + return JSON.stringify([{ Id: CONFIG_ID, RepoDigests: [IMAGE] }]); + } + return JSON.stringify([inspect(String(args[3] ?? ""))]); + }); + const dockerRun: NonNullable = vi.fn( + (args: readonly string[]) => { + if (args[0] === "create") { + events.push("create:replacement"); + const name = String(args[args.indexOf("--name") + 1] ?? ""); + const entrypoint = String(args[args.indexOf("--entrypoint") + 1] ?? ""); + const imageIndex = args.indexOf(IMAGE); + const env: string[] = []; + args.forEach((value, index) => { + if (value === "--env") env.push(String(args[index + 1] ?? "")); + }); + replacement = { + ...structuredClone(original), + Id: NEW_ID, + Name: `/${name}`, + Config: { + ...structuredClone(original.Config), + Image: IMAGE, + Env: env, + Entrypoint: [entrypoint], + Cmd: args.slice(imageIndex + 1), + }, + State: { Running: false, Paused: false, Restarting: false, Dead: false }, + }; + return options.lostAcks + ? { status: 1, stdout: "", stderr: "lost create acknowledgement" } + : ok(NEW_ID); + } + if (args[0] === "ps") return ok(original ? OLD_ID : ""); + if (args[0] === "inspect") { + const id = String(args[3] ?? ""); + try { + inspect(id); + return ok(`[{"Id":"${id}"}]`); + } catch { + return { status: 1, stderr: `Error response from daemon: No such container: ${id}` }; + } + } + if (args[0] === "cp") { + const sourceIndex = args[1] === "-a" ? 2 : 1; + const source = String(args[sourceIndex] ?? ""); + const destination = String(args[sourceIndex + 1] ?? ""); + if (!source.includes(":")) { + events.push("stage:envelope"); + expect(fs.statSync(source).mode & 0o777).toBe(0o400); + expect( + parseManagedBootstrapEnvelope(fs.readFileSync(source, "utf8")).bootstrapIdentity, + ).toBe(IDENTITY); + return ok(); + } + const receipt = source.split(":")[1]; + const expected = receipt?.includes("shared-state-commit") ? "committed" : "pending"; + if (sharedState === expected) { + fs.mkdirSync(destination, { recursive: true }); + return ok(); + } + return { + status: 1, + stderr: `Error response from daemon: Could not find the file ${receipt} in container ${NEW_ID}`, + }; + } + if (args[0] === "run" && args.includes("--shared-state-transaction-status")) { + return ok(`${sharedState}\n`); + } + if (args[0] === "run" && args.includes("--rollback-shared-state-transaction")) { + sharedState = "none"; + events.push("shared:rollback"); + return ok(); + } + if (args[0] === "exec" && args.includes("--commit-shared-state-transaction")) { + sharedState = "committed"; + events.push("shared:commit"); + return ok(); + } + if (args[0] === "exec" && args.includes("--clear-shared-state-commit-receipt")) { + sharedState = "none"; + events.push("shared:clear"); + return ok(); + } + throw new Error(`unexpected Docker command: ${args.join(" ")}`); + }, + ); + const deps: DockerManagedBootstrapDeps = { + journalStore: store, + dockerCapture, + dockerRun, + dockerStop: vi.fn((id) => { + events.push(`stop:${id}`); + const target = id === OLD_ID ? original : replacement; + if (target?.State) target.State = { ...target.State, Running: false }; + return options.lostAcks ? { status: 1, stderr: "lost stop acknowledgement" } : ok(); + }), + dockerRename: vi.fn((id, name) => { + events.push(`rename:${id}:${name}`); + const target = id === OLD_ID ? original : replacement; + if (target) target.Name = `/${name}`; + return options.lostAcks ? { status: 1, stderr: "lost rename acknowledgement" } : ok(); + }), + dockerStart: vi.fn((id) => { + events.push(`start:${id}`); + const target = id === OLD_ID ? original : replacement; + if (target?.State && !(id === NEW_ID && options.failStart)) { + target.State = { ...target.State, Running: true }; + } + return id === NEW_ID && options.failStart + ? { status: 1, stderr: "injected start failure" } + : options.lostAcks + ? { status: 1, stderr: "lost start acknowledgement" } + : ok(); + }), + dockerRm: vi.fn((id) => { + events.push(`rm:${id}`); + if (id === OLD_ID) original = null as unknown as DockerContainerInspect; + if (id === NEW_ID) replacement = null; + return options.lostAcks ? { status: 1, stderr: "lost rm acknowledgement" } : ok(); + }), + runCaptureOpenshell: vi.fn(() => `Name: alpha\nID: ${options.ownerId ?? "sandbox-alpha"}\n`), + runOpenshell: vi.fn(() => ok()), + now: () => new Date("2026-07-31T12:30:00.000Z"), + }; + return { + deps, + events, + get journal() { + return journal; + }, + get original() { + return original; + }, + get replacement() { + return replacement; + }, + get sharedState() { + return sharedState; + }, + }; +} + +function completion( + replacement: ManagedBootstrapReplacementHandle, +): ManagedBootstrapCompletionReceipt { + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + runtimeId: replacement.replacementRuntimeId, + image: replacement.image, + runtimeImageContentId: replacement.runtimeImageContentId, + originalSpecHash: replacement.originalSpecHash, + replacementSpecHash: replacement.replacementSpecHash, + profileFingerprint: replacement.profileFingerprint, + bootstrapIdentity: replacement.bootstrapIdentity, + transactionPending: true, + completedAt: "2026-07-31T12:15:00.000Z", + }; +} + +function durablePreparation( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, +): ManagedBootstrapDurablePreparationReceipt { + const authority = createManagedBootstrapPreparedAuthority({ handle, snapshot, prepared }); + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + authorityFingerprint: authority.authorityFingerprint, + recordId: `test-authority-${handle.plan.profile.agent}`, + recordedAt: "2026-07-31T12:10:00.000Z", + }; +} + +describe("Docker managed bootstrap adapter", () => { + it("journals both exact identities before cutover and reconciles lost acknowledgements", async () => { + const fake = fixture({ lostAcks: true, sharedState: "pending" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + expect(fake.journal).toBeNull(); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + fake.events.push("authority:recorded"); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + const order = fake.events; + expect(order.indexOf("journal:staged")).toBeGreaterThan(order.indexOf("authority:recorded")); + expect(order.indexOf("journal:cutover")).toBeLessThan(order.indexOf(`stop:${OLD_ID}`)); + expect(fake.journal).toMatchObject({ + phase: "cutover", + originalRuntimeId: OLD_ID, + replacementRuntimeId: NEW_ID, + }); + + await expect( + adapter.finalizeBootstrap({ + outcome: "commit", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: completion(replacement), + }), + ).resolves.toMatchObject({ outcome: "committed" }); + expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( + fake.events.indexOf(`rm:${OLD_ID}`), + ); + expect(fake.journal).toBeNull(); + expect(fake.sharedState).toBe("none"); + expect(fake.replacement?.Id).toBe(NEW_ID); + }); + + it("recovers a failed cutover after adapter restart from exact journal authority", async () => { + const fake = fixture({ failStart: true }); + const first = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await first.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + await expect( + first.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }), + ).rejects.toThrow("could not prove its exact replacement running"); + expect(fake.journal?.phase).toBe("cutover"); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect( + restarted.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement: null, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( + fake.events.indexOf(`rm:${NEW_ID}`), + ); + expect(fake.journal).toBeNull(); + expect(fake.replacement).toBeNull(); + expect(fake.original.Name).toBe("/openshell-alpha"); + expect(fake.original.State?.Running).toBe(false); + }); + + it("recovers the pre-stop cutover crash state after adapter restart", async () => { + const fake = fixture({ failAfterCutoverFence: true }); + const { handle, request: rootRequest, snapshot } = authority(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + await expect( + adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }), + ).rejects.toThrow("crash after durable cutover fence"); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + await expect( + createDockerManagedBootstrapAdapter(fake.deps).finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement: null, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( + fake.events.indexOf(`rm:${NEW_ID}`), + ); + expect(fake.journal).toBeNull(); + }); + + it("fences rollback when image-owned shared state is already committed", async () => { + const fake = fixture({ sharedState: "committed" }); + const { handle, request: rootRequest, snapshot } = authority(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + const eventCount = fake.events.length; + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: null, + }), + ).rejects.toMatchObject({ name: "ManagedBootstrapDurableCommitCleanupPendingError" }); + expect(fake.journal?.phase).toBe("shared-state-committed"); + expect(fake.events.slice(eventCount)).toEqual(["journal:shared-state-committed"]); + }); + + it("rejects cutover before the exact durable authority receipt", async () => { + const fake = fixture(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const invalid = { + ...durablePreparation(handle, snapshot, prepared), + authorityFingerprint: "f".repeat(64), + }; + await expect( + adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: invalid, + }), + ).rejects.toThrow("exact durable prepared-authority receipt"); + expect(fake.journal).toBeNull(); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: null, + replacement: null, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.replacement).toBeNull(); + }); + + it.each( + SUPPORTED_AGENTS, + )("prepares, activates, and exactly rolls back the %s agent without a central switch", async (agent) => { + const fake = fixture({ agent, sharedState: "pending" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(agent); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.journal).toBeNull(); + expect(fake.replacement).toBeNull(); + expect( + vi.mocked(fake.deps.dockerRun).mock.calls.some(([args]) => { + const agentIndex = args.indexOf("--agent"); + return args.includes("--shared-state-transaction-status") && args[agentIndex + 1] === agent; + }), + ).toBe(true); + }); + + it("quiesces and retains an exact incomplete create when its mutable name is reused", async () => { + const fake = fixture({ ownerId: "sandbox-alpha-recreated" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { plan } = authority(); + await expect( + adapter.cleanupIncompleteCreate({ + plan, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: heldArgv, + }), + ).rejects.toMatchObject({ + name: "ManagedBootstrapOwnerCleanupRequiredError", + sandboxId: "sandbox-alpha", + runtimeId: OLD_ID, + }); + expect(fake.original.State?.Running).toBe(false); + expect(fake.events).not.toContain(`rm:${OLD_ID}`); + expect(vi.mocked(fake.deps.runOpenshell!)).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts new file mode 100644 index 00000000000..ab99a258731 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -0,0 +1,2957 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { + dockerRename as defaultDockerRename, + dockerRm as defaultDockerRm, + dockerStart as defaultDockerStart, + dockerStop as defaultDockerStop, +} from "../../adapters/docker/container"; +import { + dockerCapture as defaultDockerCapture, + dockerRun as defaultDockerRun, +} from "../../adapters/docker/run"; +import { parseOpenShellSandboxId } from "../../adapters/openshell/sandbox-identity"; +import { hasZeroDockerExitStatus } from "../docker-command-result"; +import { buildDockerGpuCloneRunArgs, dockerContainerName } from "../docker-gpu-patch-clone"; +import { + DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + DOCKER_GPU_PATCH_TIMEOUT_MS, +} from "../docker-gpu-patch-constants"; +import type { + DockerContainerInspect, + DockerGpuPatchDeps, + DockerGpuPatchMode, + DockerGpuPatchModeKind, + DockerUlimit, +} from "../docker-gpu-patch-types"; +import { waitForOpenShellSupervisorReconnect } from "../docker-gpu-supervisor-reconnect"; +import { openshellSandboxCommandEnvValue } from "../docker-startup-command-env"; +import { + isImmutableDockerImageId, + OPENSHELL_MANAGED_BY_LABEL, + OPENSHELL_MANAGED_BY_VALUE, + OPENSHELL_SANDBOX_ID_LABEL, + OPENSHELL_SANDBOX_NAME_LABEL, + queryOpenShellDockerSandboxContainers, +} from "../openshell-docker-sandbox-containers"; +import { cleanupTempDir, secureTempFile } from "../temp-files"; +import { + assertManagedBootstrapIdentity, + assertManagedBootstrapSafeProcessEnvironmentKey, + attachManagedBootstrapRollbackError, + createManagedBootstrapIdentity, + createManagedBootstrapPreparedAuthority, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapAdapter, + ManagedBootstrapCommitStateIndeterminateError, + type ManagedBootstrapCompletionReceipt, + type ManagedBootstrapDiscoveredWorkload, + type ManagedBootstrapDiscoveryInput, + ManagedBootstrapDurableCommitCleanupPendingError, + type ManagedBootstrapDurablePreparationReceipt, + type ManagedBootstrapFinalizationReceipt, + type ManagedBootstrapHeldWorkloadHandle, + type ManagedBootstrapIncompleteCreateCleanupInput, + type ManagedBootstrapObservedSnapshot, + ManagedBootstrapOwnerCleanupRequiredError, + type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapReplacementHandle, + type ManagedBootstrapReplacementOptions, + type ManagedBootstrapSandboxIdentity, + renderManagedBootstrapHeldCommand, +} from "./adapter"; +import { + createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapJournal, + DockerManagedBootstrapJournalAcknowledgementLostError, + type DockerManagedBootstrapJournalStore, + parseDockerManagedBootstrapJournal, + serializeDockerManagedBootstrapJournal, +} from "./docker-journal"; +import { + clearDockerManagedStartupSharedStateCommitReceipt, + DockerManagedStartupSharedStateCommitIndeterminateError, + finalizeDockerManagedStartupSharedState, + probeDockerManagedStartupSharedState, +} from "./docker-shared-state"; +import { + normalizeDockerManagedBootstrapLaunchSpec, + parseDockerManagedBootstrapLaunchSpec, + parseExactDockerContainerInspect, +} from "./docker-spec"; +import { + MANAGED_BOOTSTRAP_COMPLETION_FILE, + MANAGED_BOOTSTRAP_REQUEST_FILE, + parseManagedBootstrapImageCompletion, + serializeManagedBootstrapEnvelope, +} from "./envelope"; + +const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; +const FULL_SHA256_RE = /^sha256:[a-f0-9]{64}$/u; +const MAX_ARGV_BYTES = 128 * 1024; +const MAX_CONTAINER_NAME_LENGTH = 253; +const REQUEST_TEMP_PREFIX = "nemoclaw-managed-bootstrap-request"; +const COMPLETION_TEMP_PREFIX = "nemoclaw-managed-bootstrap-completion"; +const COMPLETION_MAX_BYTES = 4096; +const DOCKER_DRIVER_ID = "docker"; + +export const MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE = "/usr/local/bin/nemoclaw-managed-bootstrap"; + +type DockerCommandResult = { + readonly status?: number | null; + readonly stdout?: string | Buffer | null; + readonly stderr?: string | Buffer | null; + readonly error?: Error | null; +}; + +export type DockerManagedBootstrapDeps = Pick< + DockerGpuPatchDeps, + | "dockerCapture" + | "dockerRename" + | "dockerRm" + | "dockerRun" + | "dockerStart" + | "dockerStop" + | "runCaptureOpenshell" + | "runOpenshell" + | "sleep" + | "now" +> & { + readonly createBootstrapIdentity?: () => string; + readonly journalStore?: DockerManagedBootstrapJournalStore; + /** Canonical gateway-scoped state root; required when no store is injected. */ + readonly stateRoot?: string; +}; + +type ResolvedDeps = Required< + Pick< + DockerManagedBootstrapDeps, + | "dockerCapture" + | "dockerRename" + | "dockerRm" + | "dockerRun" + | "dockerStart" + | "dockerStop" + | "journalStore" + | "now" + | "createBootstrapIdentity" + > +> & + DockerManagedBootstrapDeps; + +type DockerBootstrapTransaction = DockerManagedBootstrapJournal; + +interface DockerBootstrapRollbackTombstone { + readonly profileFingerprint: string; + readonly imageReference: string; + readonly receipt: ManagedBootstrapFinalizationReceipt; +} + +export interface DockerManagedBootstrapAdapter extends ManagedBootstrapAdapter {} + +function resolveDeps(deps: DockerManagedBootstrapDeps): ResolvedDeps { + const journalStore = + deps.journalStore ?? + (deps.stateRoot ? createFileDockerManagedBootstrapJournalStore(deps.stateRoot) : null); + if (!journalStore) { + throw new Error( + "Managed bootstrap Docker requires its canonical state root or an injected journal store.", + ); + } + return { + dockerCapture: defaultDockerCapture, + dockerRename: defaultDockerRename, + dockerRm: defaultDockerRm, + dockerRun: defaultDockerRun, + dockerStart: defaultDockerStart, + dockerStop: defaultDockerStop, + journalStore, + now: () => new Date(), + createBootstrapIdentity: createManagedBootstrapIdentity, + ...deps, + }; +} + +function commandDetail(result: DockerCommandResult): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}` + .trim() + .slice(-1200); +} + +function isExactMissingDockerContainer(containerId: string, result: DockerCommandResult): boolean { + const escapedContainerId = containerId.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const patterns = [ + new RegExp( + `^(?:Error response from daemon: )?No such (?:container|object): ${escapedContainerId}$`, + "u", + ), + new RegExp(`^Error: No such (?:container|object): ${escapedContainerId}$`, "u"), + ]; + return [result.stderr, result.stdout, result.error?.message] + .map((value) => String(value ?? "").trim()) + .filter(Boolean) + .some((detail) => patterns.some((pattern) => pattern.test(detail))); +} + +function probeExactDockerContainerAbsence( + containerId: string, + deps: ResolvedDeps, +): "absent" | "present" | "unknown" { + let result: DockerCommandResult; + try { + result = deps.dockerRun(["inspect", "--type", "container", containerId], { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + } catch { + return "unknown"; + } + if (hasZeroDockerExitStatus(result)) return "present"; + return isExactMissingDockerContainer(containerId, result) ? "absent" : "unknown"; +} + +function assertZero(result: DockerCommandResult, message: string): void { + if (!hasZeroDockerExitStatus(result)) { + throw new Error(`${message}: ${commandDetail(result) || "Docker command failed"}`); + } +} + +function exactStringArray(value: unknown, label: string): string[] { + if (value === null || value === undefined) return []; + const values = typeof value === "string" ? [value] : value; + if ( + !Array.isArray(values) || + values.some( + (item) => + typeof item !== "string" || + item.length === 0 || + item.includes("\0") || + Buffer.byteLength(item, "utf8") > 64 * 1024, + ) + ) { + throw new Error(`Managed bootstrap Docker ${label} is not an exact bounded argv.`); + } + const result = [...values]; + if (Buffer.byteLength(JSON.stringify(result), "utf8") > MAX_ARGV_BYTES) { + throw new Error(`Managed bootstrap Docker ${label} exceeds its bounded argv transport.`); + } + return result; +} + +function exactSupervisorArgv(inspect: DockerContainerInspect): readonly string[] { + const argv = [ + ...exactStringArray(inspect.Config?.Entrypoint, "entrypoint"), + ...exactStringArray(inspect.Config?.Cmd, "command"), + ]; + if (argv.length === 0 || !argv[0]?.startsWith("/")) { + throw new Error( + "Managed bootstrap requires one bounded absolute supervisor argv from Docker inspect.", + ); + } + return Object.freeze(argv); +} + +function exactArrayEqual(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + +function envValue(env: readonly string[] | null | undefined, key: string): string | null { + const prefix = `${key}=`; + const matches = (env ?? []).filter((value) => value.startsWith(prefix)); + return matches.length === 1 ? (matches[0]?.slice(prefix.length) ?? null) : null; +} + +function assertNoRootProcessInjectionEnvironment(env: readonly string[] | null | undefined): void { + for (const entry of env ?? []) { + const separator = entry.indexOf("="); + const key = separator < 0 ? entry : entry.slice(0, separator); + try { + assertManagedBootstrapSafeProcessEnvironmentKey(key); + } catch { + throw new Error(`Managed bootstrap refuses root-process injection environment '${key}'.`); + } + } +} + +function assertRootSupervisor(inspect: DockerContainerInspect): void { + const user = String(inspect.Config?.User ?? "") + .trim() + .toLowerCase(); + if (!["", "0", "0:0", "root", "root:root"].includes(user)) { + throw new Error("Managed bootstrap Docker workload must retain a root supervisor user."); + } +} + +function isStableRunning(inspect: DockerContainerInspect): boolean { + return inspect.State?.Running !== true || + inspect.State.Paused === true || + inspect.State.Restarting === true || + inspect.State.Dead === true + ? false + : true; +} + +function assertStableRunning(inspect: DockerContainerInspect, label: string): void { + if (!isStableRunning(inspect)) { + throw new Error(`Managed bootstrap Docker ${label} is not stably running.`); + } +} + +function isExplicitlyStopped(inspect: DockerContainerInspect): boolean { + return ( + inspect.State?.Running === false && + inspect.State.Paused === false && + inspect.State.Restarting === false && + inspect.State.Dead === false + ); +} + +function assertExplicitlyStopped(inspect: DockerContainerInspect, label: string): void { + if (!isExplicitlyStopped(inspect)) { + throw new Error(`Managed bootstrap Docker ${label} is not explicitly stopped.`); + } +} + +function expectedImageReference(repository: string, manifestDigest: string): string { + if ( + repository.length === 0 || + repository !== repository.trim() || + repository.includes("@") || + repository.includes("\0") || + !FULL_SHA256_RE.test(manifestDigest) + ) { + throw new Error("Managed bootstrap image repository/manifest identity is invalid."); + } + return `${repository}@${manifestDigest}`; +} + +function assertImage( + inspect: DockerContainerInspect, + image: ManagedBootstrapHeldWorkloadHandle["plan"]["image"], + deps: ResolvedDeps, +): string { + const runtimeContentId = String(inspect.Image ?? "").toLowerCase(); + if (!FULL_SHA256_RE.test(runtimeContentId)) { + throw new Error("Managed bootstrap Docker image does not have an immutable local content ID."); + } + const expectedReference = expectedImageReference(image.repository, image.manifestDigest); + const configuredImage = String(inspect.Config?.Image ?? "").trim(); + if (configuredImage !== expectedReference) { + throw new Error( + "Managed bootstrap Docker configured image is not the exact repository@manifestDigest.", + ); + } + const imageOutput = deps.dockerCapture(["image", "inspect", expectedReference], { + ignoreError: false, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + let parsed: unknown; + try { + parsed = JSON.parse(imageOutput); + } catch { + throw new Error("Managed bootstrap Docker image evidence is malformed."); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("Managed bootstrap Docker image evidence is not exact."); + } + const evidence = parsed[0] as { + readonly Id?: unknown; + readonly RepoDigests?: unknown; + }; + const evidenceId = String(evidence.Id ?? "").toLowerCase(); + const repoDigests = Array.isArray(evidence.RepoDigests) + ? evidence.RepoDigests.filter((value): value is string => typeof value === "string") + : []; + if (evidenceId !== runtimeContentId || !repoDigests.includes(expectedReference)) { + throw new Error( + "Managed bootstrap Docker image manifest evidence does not match its local content ID.", + ); + } + return runtimeContentId; +} + +function assertMetadata( + inspect: DockerContainerInspect, + sandbox: ManagedBootstrapHeldWorkloadHandle["sandbox"], + metadata: Readonly>, +): void { + const labels = inspect.Config?.Labels ?? {}; + if ( + labels[OPENSHELL_MANAGED_BY_LABEL] !== OPENSHELL_MANAGED_BY_VALUE || + labels[OPENSHELL_SANDBOX_NAME_LABEL] !== sandbox.sandboxName || + labels[OPENSHELL_SANDBOX_ID_LABEL] !== sandbox.sandboxId + ) { + throw new Error( + "Managed bootstrap Docker workload does not match the durable OpenShell sandbox identity.", + ); + } + for (const [key, value] of Object.entries(metadata)) { + if (labels[key] !== value) { + throw new Error(`Managed bootstrap Docker metadata label '${key}' changed.`); + } + } +} + +function assertHeldCommand( + inspect: DockerContainerInspect, + heldWorkloadArgv: readonly string[], + bootstrapIdentity: string, +): void { + assertManagedBootstrapIdentity(bootstrapIdentity); + const expected = openshellSandboxCommandEnvValue(heldWorkloadArgv); + const observed = envValue(inspect.Config?.Env, "OPENSHELL_SANDBOX_COMMAND"); + if (!expected || observed !== expected) { + throw new Error( + "Managed bootstrap Docker workload does not contain the exact identity-bound hold.", + ); + } + const identityIndexes = heldWorkloadArgv + .map((value, index) => (value === bootstrapIdentity ? index : -1)) + .filter((index) => index >= 0); + if (identityIndexes.length !== 1) { + throw new Error("Managed bootstrap hold does not contain exactly one bootstrap identity."); + } +} + +function assertBootstrapIdentityInObservedHold( + inspect: DockerContainerInspect, + bootstrapIdentity: string, +): void { + assertManagedBootstrapIdentity(bootstrapIdentity); + const observed = envValue(inspect.Config?.Env, "OPENSHELL_SANDBOX_COMMAND"); + if (!observed) { + throw new Error("Managed bootstrap Docker workload is missing its held command."); + } + const occurrences = observed.split(bootstrapIdentity).length - 1; + if (occurrences !== 1) { + throw new Error( + "Managed bootstrap Docker held command does not contain one exact bootstrap identity.", + ); + } +} + +function inspectExact(containerId: string, deps: ResolvedDeps): DockerContainerInspect { + if (!FULL_CONTAINER_ID_RE.test(containerId)) { + throw new Error("Managed bootstrap requires one full lowercase Docker container ID."); + } + const output = deps.dockerCapture(["inspect", "--type", "container", containerId], { + ignoreError: false, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + const inspect = parseExactDockerContainerInspect(output); + if (String(inspect.Id ?? "").toLowerCase() !== containerId) { + throw new Error("Managed bootstrap Docker workload identity changed during inspection."); + } + return inspect; +} + +function inspectDockerContainerReference( + reference: string, + deps: ResolvedDeps, +): DockerContainerInspect { + if ( + reference.length === 0 || + reference !== reference.trim() || + reference.includes("\0") || + Buffer.byteLength(reference, "utf8") > MAX_CONTAINER_NAME_LENGTH + ) { + throw new Error("Managed bootstrap Docker lookup reference is invalid."); + } + const output = deps.dockerCapture(["inspect", "--type", "container", reference], { + ignoreError: false, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + const inspect = parseExactDockerContainerInspect(output); + const runtimeId = String(inspect.Id ?? "").toLowerCase(); + if (!FULL_CONTAINER_ID_RE.test(runtimeId)) { + throw new Error("Managed bootstrap Docker lookup did not resolve one full runtime ID."); + } + return inspect; +} + +function tryInspectExact(containerId: string, deps: ResolvedDeps): DockerContainerInspect | null { + try { + return inspectExact(containerId, deps); + } catch { + return null; + } +} + +function backupName(originalName: string, bootstrapIdentity: string): string { + const suffix = `-nemoclaw-bootstrap-${bootstrapIdentity.slice(0, 20)}`; + return `${originalName.slice(0, Math.max(1, MAX_CONTAINER_NAME_LENGTH - suffix.length))}${suffix}`; +} + +function replacementStagingName(originalName: string, bootstrapIdentity: string): string { + const suffix = `-nemoclaw-staged-${bootstrapIdentity.slice(0, 20)}`; + return `${originalName.slice(0, Math.max(1, MAX_CONTAINER_NAME_LENGTH - suffix.length))}${suffix}`; +} + +function writeProtectedEnvelope( + bootstrapIdentity: string, + request: Parameters[0]["rootApplyRequest"], +): string { + const file = secureTempFile(REQUEST_TEMP_PREFIX, ".json"); + try { + fs.writeFileSync( + file, + serializeManagedBootstrapEnvelope({ bootstrapIdentity, rootApplyRequest: request }), + { encoding: "utf8", flag: "wx", mode: 0o400 }, + ); + fs.chmodSync(file, 0o400); + const stat = fs.lstatSync(file); + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.nlink !== 1 || + (stat.mode & 0o777) !== 0o400 + ) { + throw new Error("Managed bootstrap request source is not one protected 0400 file."); + } + return file; + } catch (error) { + cleanupTempDir(file, REQUEST_TEMP_PREFIX); + throw error; + } +} + +function readProtectedImageCompletion( + replacementRuntimeId: string, + deps: ResolvedDeps, +): ReturnType { + const file = secureTempFile(COMPLETION_TEMP_PREFIX, ".json"); + let descriptor: number | undefined; + try { + const copied = deps.dockerRun( + ["cp", `${replacementRuntimeId}:${MANAGED_BOOTSTRAP_COMPLETION_FILE}`, file], + { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }, + ); + assertZero(copied, "Managed bootstrap could not retrieve its image completion receipt"); + descriptor = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + const before = fs.fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.isSymbolicLink() || + before.nlink !== 1n || + Number(before.mode & 0o777n) !== 0o444 || + before.size < 1n || + before.size > BigInt(COMPLETION_MAX_BYTES) + ) { + throw new Error("Managed bootstrap image completion is not one protected bounded 0444 file."); + } + const bytes = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < bytes.length) { + const count = fs.readSync(descriptor, bytes, offset, bytes.length - offset, offset); + if (count === 0) break; + offset += count; + } + const after = fs.fstatSync(descriptor, { bigint: true }); + if ( + offset !== bytes.length || + after.dev !== before.dev || + after.ino !== before.ino || + after.size !== before.size || + after.mtimeNs !== before.mtimeNs || + after.ctimeNs !== before.ctimeNs || + after.mode !== before.mode || + after.nlink !== before.nlink + ) { + throw new Error("Managed bootstrap image completion changed during stable read."); + } + return parseManagedBootstrapImageCompletion(bytes.toString("utf8")); + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + cleanupTempDir(file, COMPLETION_TEMP_PREFIX); + } +} + +function parseRequiredUlimits(value: unknown): DockerUlimit[] { + if (value === undefined) return []; + if ( + !Array.isArray(value) || + value.some((entry) => typeof entry !== "string" || entry.includes("\0")) + ) { + throw new Error("Managed bootstrap Docker requiredUlimits must be string entries."); + } + return value.map((entry) => { + const match = /^([a-z][a-z0-9_]*)=(\d+):(\d+)$/u.exec(entry); + if (!match) { + throw new Error(`Managed bootstrap Docker ulimit '${entry}' is invalid.`); + } + const soft = Number(match[2]); + const hard = Number(match[3]); + if (!Number.isSafeInteger(soft) || !Number.isSafeInteger(hard) || hard < soft) { + throw new Error(`Managed bootstrap Docker ulimit '${entry}' is invalid.`); + } + return { name: match[1] as string, soft, hard }; + }); +} + +function replacementPlan(options: ManagedBootstrapReplacementOptions): { + readonly mode: DockerGpuPatchMode; + readonly requiredUlimits: readonly DockerUlimit[]; + readonly extraGroupGids: readonly string[]; +} { + const allowed = new Set([ + "gpuModeArgs", + "gpuModeDevice", + "gpuModeKind", + "gpuModeLabel", + "extraGroupGids", + "requiredUlimits", + ]); + const unknown = Object.keys(options.values).filter((key) => !allowed.has(key)); + if (unknown.length > 0) { + throw new Error( + `Managed bootstrap Docker replacement options are unsupported: ${unknown.sort().join(", ")}.`, + ); + } + const kind = String(options.values.gpuModeKind ?? "startup-command") as DockerGpuPatchModeKind; + if (!["gpus", "nvidia-runtime", "cdi", "startup-command"].includes(kind)) { + throw new Error(`Managed bootstrap Docker GPU mode '${kind}' is invalid.`); + } + const args = exactStringArray(options.values.gpuModeArgs ?? [], "GPU mode arguments"); + return { + mode: { + kind, + label: String(options.values.gpuModeLabel ?? "managed bootstrap"), + device: String(options.values.gpuModeDevice ?? ""), + args, + }, + extraGroupGids: exactStringArray(options.values.extraGroupGids ?? [], "extra group GIDs").map( + (value) => { + if (!/^\d+$/u.test(value)) { + throw new Error(`Managed bootstrap Docker supplementary group '${value}' is invalid.`); + } + return value; + }, + ), + requiredUlimits: parseRequiredUlimits(options.values.requiredUlimits), + }; +} + +function replacementCommand( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, +): readonly string[] { + return Object.freeze([ + "--agent", + handle.plan.profile.agent, + "--profile-fingerprint", + handle.plan.profile.fingerprint, + "--bootstrap-identity", + handle.bootstrapIdentity, + "--agent-uid", + String(snapshot.agentIdentity.uid), + "--agent-gid", + String(snapshot.agentIdentity.gid), + "--agent-workdir", + snapshot.agentIdentity.workdir, + "--request-file", + MANAGED_BOOTSTRAP_REQUEST_FILE, + "--", + ...snapshot.supervisorArgv, + ]); +} + +function assertReplacementBoundary( + inspect: DockerContainerInspect, + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, +): void { + const entrypoint = exactStringArray(inspect.Config?.Entrypoint, "replacement entrypoint"); + const command = exactStringArray(inspect.Config?.Cmd, "replacement command"); + if ( + !exactArrayEqual(entrypoint, [MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE]) || + !exactArrayEqual(command, replacementCommand(handle, snapshot)) + ) { + throw new Error("Managed bootstrap Docker replacement process boundary changed."); + } + const intended = openshellSandboxCommandEnvValue(handle.intendedWorkloadArgv); + if (envValue(inspect.Config?.Env, "OPENSHELL_SANDBOX_COMMAND") !== intended) { + throw new Error( + "Managed bootstrap Docker replacement did not restore the intended sandbox command.", + ); + } +} + +const REPLACED_GPU_ENV_KEYS = new Set([ + "NVIDIA_DISABLE_REQUIRE", + "NVIDIA_DRIVER_CAPABILITIES", + "NVIDIA_REQUIRE_CUDA", + "NVIDIA_VISIBLE_DEVICES", +]); + +function canonicalObject(text: string): Record { + const value = JSON.parse(text) as unknown; + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("Managed bootstrap normalized Docker spec is not an object."); + } + return value as Record; +} + +function objectField(record: Record, key: string): Record { + const value = record[key]; + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`Managed bootstrap normalized Docker spec is missing ${key}.`); + } + return value as Record; +} + +function exactJson(value: unknown): string { + return JSON.stringify(value ?? null); +} + +function stringSet(value: unknown, label: string): string[] { + const values = exactStringArray(value ?? [], label); + if (new Set(values).size !== values.length) { + throw new Error(`Managed bootstrap Docker ${label} contains duplicate entries.`); + } + return values.sort(); +} + +function assertExactStringSet(observed: unknown, expected: readonly string[], label: string): void { + if (!exactArrayEqual(stringSet(observed, label), [...expected].sort())) { + throw new Error(`Managed bootstrap Docker ${label} changed outside declared deltas.`); + } +} + +function modeEnvironment(mode: DockerGpuPatchMode): string[] { + const values: string[] = []; + for (let index = 0; index < mode.args.length; index += 1) { + if (mode.args[index] === "--env") { + const value = mode.args[index + 1]; + if (!value || !value.includes("=")) { + throw new Error("Managed bootstrap Docker GPU mode has an invalid environment delta."); + } + values.push(value); + index += 1; + } + } + return values; +} + +function assertExactEnvironmentDelta( + original: Record, + replacement: Record, + mode: DockerGpuPatchMode, + intendedSandboxCommand: string, +): void { + const gpuAugment = mode.kind !== "startup-command"; + const originalEnv = exactStringArray(original.Env ?? [], "original environment"); + const expected = [ + ...modeEnvironment(mode), + ...originalEnv + .filter((entry) => !gpuAugment || !REPLACED_GPU_ENV_KEYS.has(entry.split("=", 1)[0] ?? "")) + .map((entry) => + entry.startsWith("OPENSHELL_SANDBOX_COMMAND=") + ? `OPENSHELL_SANDBOX_COMMAND=${intendedSandboxCommand}` + : entry, + ), + ]; + const observed = exactStringArray(replacement.Env ?? [], "replacement environment"); + if (!exactArrayEqual(observed, expected)) { + throw new Error( + "Managed bootstrap Docker replacement environment changed outside declared deltas.", + ); + } +} + +function canonicalUlimits(value: unknown, label: string): string { + if (!Array.isArray(value)) { + if (value === undefined || value === null) return "[]"; + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + const normalized = value.map((entry) => { + if (typeof entry !== "object" || entry === null || Array.isArray(entry)) { + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + const record = entry as Record; + const name = String(record.Name ?? ""); + const soft = record.Soft; + const hard = record.Hard; + if (!name || !Number.isSafeInteger(soft) || !Number.isSafeInteger(hard)) { + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + return { Hard: hard, Name: name, Soft: soft }; + }); + if (new Set(normalized.map((entry) => entry.Name)).size !== normalized.length) { + throw new Error(`Managed bootstrap Docker ${label} contains duplicate entries.`); + } + return JSON.stringify(normalized.sort((left, right) => left.Name.localeCompare(right.Name))); +} + +function expectedUlimits(original: unknown, required: readonly DockerUlimit[]): string { + const existing = JSON.parse(canonicalUlimits(original, "original ulimits")) as Array<{ + Hard: number; + Name: string; + Soft: number; + }>; + const merged = new Map(existing.map((entry) => [entry.Name, entry])); + for (const requiredEntry of required) { + merged.set(requiredEntry.name, { + Name: requiredEntry.name, + Soft: requiredEntry.soft, + Hard: requiredEntry.hard, + }); + } + return JSON.stringify( + [...merged.values()].sort((left, right) => left.Name.localeCompare(right.Name)), + ); +} + +function assertExactDeviceRequests( + original: unknown, + observed: unknown, + mode: DockerGpuPatchMode, +): void { + if (mode.kind === "startup-command") { + if (exactJson(observed) !== exactJson(original)) { + throw new Error("Managed bootstrap Docker device requests were not preserved exactly."); + } + return; + } + if (Array.isArray(original) && original.length > 0) { + throw new Error( + "Managed bootstrap Docker GPU augmentation cannot replace an existing device request.", + ); + } + const requests = Array.isArray(observed) ? observed : []; + if (mode.kind === "nvidia-runtime") { + if (requests.length !== 0) { + throw new Error( + "Managed bootstrap Docker NVIDIA runtime added an undeclared device request.", + ); + } + return; + } + if (requests.length !== 1 || typeof requests[0] !== "object" || requests[0] === null) { + throw new Error("Managed bootstrap Docker GPU mode did not add one exact device request."); + } + const request = requests[0] as Record; + if (mode.kind === "gpus") { + const all = mode.device === "all"; + const expectedIds = all ? [] : [mode.device]; + const ids = Array.isArray(request.DeviceIDs) ? request.DeviceIDs : []; + if ( + String(request.Driver ?? "") !== "" || + Number(request.Count) !== (all ? -1 : 0) || + !exactArrayEqual(ids.map(String), expectedIds) || + exactJson(request.Capabilities) !== JSON.stringify([["gpu"]]) || + exactJson(request.Options ?? {}) !== "{}" + ) { + throw new Error("Managed bootstrap Docker --gpus request changed outside its exact delta."); + } + return; + } + const ids = Array.isArray(request.DeviceIDs) ? request.DeviceIDs.map(String) : []; + if ( + request.Driver !== "cdi" || + ![-1, 0].includes(Number(request.Count ?? 0)) || + !exactArrayEqual(ids, [mode.device]) || + (request.Capabilities != null && + (!Array.isArray(request.Capabilities) || request.Capabilities.length > 0)) || + exactJson(request.Options ?? {}) !== "{}" + ) { + throw new Error("Managed bootstrap Docker CDI request changed outside its exact delta."); + } +} + +function scrubVerifiedReplacementDeltas(canonicalJson: string): string { + const root = canonicalObject(canonicalJson); + const inspect = objectField(root, "inspect"); + const config = objectField(inspect, "Config"); + const host = objectField(inspect, "HostConfig"); + config.Image = ""; + config.Entrypoint = [""]; + config.Cmd = [""]; + config.Env = ""; + for (const key of [ + "CapAdd", + "DeviceRequests", + "Devices", + "GroupAdd", + "Runtime", + "SecurityOpt", + "Ulimits", + ]) { + host[key] = ``; + } + return JSON.stringify(root); +} + +function assertReplacementMatchesIntent( + originalCanonicalJson: string, + replacement: DockerContainerInspect, + authoritativeName: string, + plan: { + readonly mode: DockerGpuPatchMode; + readonly requiredUlimits: readonly DockerUlimit[]; + readonly extraGroupGids: readonly string[]; + }, + intendedSandboxCommand: string, +): string { + const original = canonicalObject(originalCanonicalJson); + const originalInspect = objectField(original, "inspect"); + const originalConfig = objectField(originalInspect, "Config"); + const originalHost = objectField(originalInspect, "HostConfig"); + const replacementSpec = normalizeDockerManagedBootstrapLaunchSpec({ + ...replacement, + Name: `/${authoritativeName}`, + }); + const observed = canonicalObject(replacementSpec.canonicalJson); + const observedInspect = objectField(observed, "inspect"); + const observedConfig = objectField(observedInspect, "Config"); + const observedHost = objectField(observedInspect, "HostConfig"); + const gpuAugment = plan.mode.kind !== "startup-command"; + assertExactEnvironmentDelta(originalConfig, observedConfig, plan.mode, intendedSandboxCommand); + assertExactStringSet( + observedHost.CapAdd, + [ + ...stringSet(originalHost.CapAdd, "original capability additions"), + ...(gpuAugment ? ["SYS_PTRACE"] : []), + ].filter((value, index, values) => values.indexOf(value) === index), + "capability additions", + ); + const originalSecurity = stringSet(originalHost.SecurityOpt, "original security options"); + assertExactStringSet( + observedHost.SecurityOpt, + [ + ...originalSecurity, + ...(gpuAugment && !originalSecurity.some((value) => value.startsWith("apparmor")) + ? ["apparmor=unconfined"] + : []), + ], + "security options", + ); + if (exactJson(observedHost.Devices) !== exactJson(originalHost.Devices)) { + throw new Error("Managed bootstrap Docker non-GPU devices were not preserved exactly."); + } + assertExactDeviceRequests(originalHost.DeviceRequests, observedHost.DeviceRequests, plan.mode); + const expectedRuntime = plan.mode.kind === "nvidia-runtime" ? "nvidia" : originalHost.Runtime; + if (exactJson(observedHost.Runtime) !== exactJson(expectedRuntime)) { + throw new Error("Managed bootstrap Docker runtime changed outside its selected GPU delta."); + } + assertExactStringSet( + observedHost.GroupAdd, + [ + ...stringSet(originalHost.GroupAdd, "original supplementary groups"), + ...plan.extraGroupGids, + ].filter((value, index, values) => values.indexOf(value) === index), + "supplementary groups", + ); + if ( + canonicalUlimits(observedHost.Ulimits, "replacement ulimits") !== + expectedUlimits(originalHost.Ulimits, plan.requiredUlimits) + ) { + throw new Error("Managed bootstrap Docker ulimits changed outside declared requirements."); + } + const expectedPreserved = scrubVerifiedReplacementDeltas(originalCanonicalJson); + const observedPreserved = scrubVerifiedReplacementDeltas(replacementSpec.canonicalJson); + if (observedPreserved !== expectedPreserved) { + throw new Error( + "Managed bootstrap Docker replacement normalized spec changed outside declared deltas.", + ); + } + return replacementSpec.hash; +} + +function inspectTransactionRuntime( + transaction: DockerBootstrapTransaction, + runtimeId: string, + deps: ResolvedDeps, +): DockerContainerInspect | null { + const presence = probeExactDockerContainerAbsence(runtimeId, deps); + if (presence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId, + detail: "exact Docker runtime presence could not be proven before mutation", + }); + } + if (presence === "absent") return null; + try { + return inspectExact(runtimeId, deps); + } catch (error) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId, + detail: `exact Docker runtime inspection became unavailable: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + } +} + +function assertTransactionOriginal( + transaction: DockerBootstrapTransaction, + inspect: DockerContainerInspect, +): void { + const name = dockerContainerName(inspect); + if (name !== transaction.originalName && name !== transaction.backupName) { + throw new Error("Managed bootstrap original container has an unexpected transaction name."); + } + const normalized = normalizeDockerManagedBootstrapLaunchSpec({ + ...inspect, + Name: `/${transaction.originalName}`, + }); + if (normalized.hash !== transaction.originalSpecHash) { + throw new Error( + "Managed bootstrap refused mutation because the exact original launch spec changed.", + ); + } +} + +function assertTransactionReplacement( + transaction: DockerBootstrapTransaction, + inspect: DockerContainerInspect, +): void { + const name = dockerContainerName(inspect); + if (name !== transaction.replacementStagingName && name !== transaction.originalName) { + throw new Error("Managed bootstrap replacement container has an unexpected transaction name."); + } + const normalized = normalizeDockerManagedBootstrapLaunchSpec({ + ...inspect, + Name: `/${transaction.originalName}`, + }); + if (normalized.hash !== transaction.replacementSpecHash) { + throw new Error( + "Managed bootstrap refused mutation because the exact replacement launch spec changed.", + ); + } +} + +function assertCompletedCutoverRuntimeState( + transaction: DockerBootstrapTransaction, + deps: ResolvedDeps, +): void { + const original = inspectTransactionRuntime(transaction, transaction.originalRuntimeId, deps); + const replacement = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (!original || !replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: original ? transaction.replacementRuntimeId : transaction.originalRuntimeId, + detail: "completed cutover requires both exact transaction runtimes", + }); + } + assertTransactionOriginal(transaction, original); + assertTransactionReplacement(transaction, replacement); + assertExplicitlyStopped(original, "rollback backup"); + assertStableRunning(replacement, "replacement"); + if ( + dockerContainerName(original) !== transaction.backupName || + dockerContainerName(replacement) !== transaction.originalName + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.replacementRuntimeId, + detail: "completed cutover runtime names do not match durable authority", + }); + } +} + +function removeExactReplacement( + transaction: DockerBootstrapTransaction, + replacement: DockerContainerInspect, + deps: ResolvedDeps, +): void { + assertTransactionReplacement(transaction, replacement); + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + if (replacement.State?.Running === true) { + const stopped = deps.dockerStop(transaction.replacementRuntimeId, { + ...options, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopped)) { + const afterStop = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (!afterStop || afterStop.State?.Running === true) { + throw new Error( + `Managed bootstrap could not quiesce its exact replacement: ${ + commandDetail(stopped) || "Docker stop failed" + }`, + ); + } + assertTransactionReplacement(transaction, afterStop); + } + } + const removed = deps.dockerRm(transaction.replacementRuntimeId, options); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(transaction.replacementRuntimeId, deps) !== "absent" + ) { + throw new Error( + `Managed bootstrap could not remove its exact replacement: ${ + commandDetail(removed) || "Docker removal failed" + }`, + ); + } +} + +function restoreOriginal(transaction: DockerBootstrapTransaction, deps: ResolvedDeps): void { + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + const originalBeforeReplacementRemoval = inspectTransactionRuntime( + transaction, + transaction.originalRuntimeId, + deps, + ); + if (!originalBeforeReplacementRemoval) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "the exact rollback original is absent", + }); + } + assertTransactionOriginal(transaction, originalBeforeReplacementRemoval); + const replacement = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (replacement) { + removeExactReplacement(transaction, replacement, deps); + } + const original = inspectExact(transaction.originalRuntimeId, deps); + assertTransactionOriginal(transaction, original); + const currentName = dockerContainerName(original); + if (currentName !== transaction.originalName) { + if (currentName !== transaction.backupName) { + throw new Error("Managed bootstrap original container has an unexpected rollback name."); + } + const renamed = deps.dockerRename( + transaction.originalRuntimeId, + transaction.originalName, + options, + ); + if (!hasZeroDockerExitStatus(renamed)) { + const afterRename = inspectTransactionRuntime( + transaction, + transaction.originalRuntimeId, + deps, + ); + if (!afterRename || dockerContainerName(afterRename) !== transaction.originalName) { + throw new Error( + `Managed bootstrap could not restore the original container name: ${ + commandDetail(renamed) || "Docker rename failed" + }`, + ); + } + assertTransactionOriginal(transaction, afterRename); + } + } + const restoredBeforeStart = inspectExact(transaction.originalRuntimeId, deps); + if (restoredBeforeStart.State?.Running !== true) { + const started = deps.dockerStart(transaction.originalRuntimeId, options); + if (!hasZeroDockerExitStatus(started)) { + const afterStart = inspectTransactionRuntime( + transaction, + transaction.originalRuntimeId, + deps, + ); + if (!afterStart || afterStart.State?.Running !== true) { + throw new Error( + `Managed bootstrap could not restart the original container: ${ + commandDetail(started) || "Docker start failed" + }`, + ); + } + assertTransactionOriginal(transaction, afterStart); + } + } + const restored = inspectExact(transaction.originalRuntimeId, deps); + assertStableRunning(restored, "restored workload"); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(restored); + if (normalized.hash !== transaction.originalSpecHash) { + throw new Error("Managed bootstrap rollback did not restore the exact launch spec."); + } +} + +function removeOwnedWorkload( + sandbox: ManagedBootstrapSandboxIdentity, + deps: ResolvedDeps, + expectedRuntimeId?: string, +): never { + const expectedIdentity = + expectedRuntimeId === undefined + ? `sandbox ${sandbox.sandboxId} with no previously resolved runtime ID` + : `sandbox ${sandbox.sandboxId} expected runtime ${expectedRuntimeId}`; + let containers: DockerCommandResult; + try { + containers = deps.dockerRun( + [ + "ps", + "-a", + "--no-trunc", + "--filter", + `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, + "--filter", + `label=${OPENSHELL_SANDBOX_ID_LABEL}=${sandbox.sandboxId}`, + "--format", + "{{.ID}}", + ], + { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }, + ); + } catch (error) { + throw new Error( + `Managed bootstrap owner cleanup could not enumerate the exact held runtime for ${expectedIdentity}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + if (Number(containers.status ?? 1) !== 0) { + throw new Error( + `Managed bootstrap owner cleanup could not verify the exact held runtime for ${expectedIdentity}: ${ + commandDetail(containers) || "Docker enumeration failed" + }`, + ); + } + const runtimeIds = String(containers.stdout ?? "") + .trim() + .split(/\r?\n/u) + .map((value) => value.trim().toLowerCase()) + .filter(Boolean); + if ( + runtimeIds.length !== 1 || + !FULL_CONTAINER_ID_RE.test(runtimeIds[0] ?? "") || + (expectedRuntimeId !== undefined && runtimeIds[0] !== expectedRuntimeId) + ) { + throw new Error( + `Managed bootstrap owner cleanup could not bind retention for ${expectedIdentity}; resolved runtime IDs: ${ + runtimeIds.length === 0 ? "none" : runtimeIds.join(", ") + }.`, + ); + } + const runtimeId = runtimeIds[0] as string; + let inspect: DockerContainerInspect; + try { + inspect = inspectExact(runtimeId, deps); + } catch (error) { + throw new Error( + `Managed bootstrap could not inspect retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + const labels = inspect.Config?.Labels ?? {}; + if ( + labels[OPENSHELL_MANAGED_BY_LABEL] !== OPENSHELL_MANAGED_BY_VALUE || + labels[OPENSHELL_SANDBOX_NAME_LABEL] !== sandbox.sandboxName || + labels[OPENSHELL_SANDBOX_ID_LABEL] !== sandbox.sandboxId + ) { + throw new Error( + `Managed bootstrap owner cleanup refused retention after exact runtime ${runtimeId} ownership changed for sandbox ${sandbox.sandboxId}.`, + ); + } + let stopped: DockerCommandResult; + try { + stopped = deps.dockerStop(runtimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + } catch (error) { + throw new Error( + `Managed bootstrap could not quiesce retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + assertZero( + stopped, + `Managed bootstrap could not quiesce retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}`, + ); + let retained: DockerContainerInspect; + try { + retained = inspectExact(runtimeId, deps); + } catch (error) { + throw new Error( + `Managed bootstrap could not re-inspect quiesced sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + if ( + retained.State?.Running !== false || + retained.State.Paused !== false || + retained.State.Restarting !== false + ) { + throw new Error( + `Managed bootstrap retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId} did not prove an explicitly quiescent state.`, + ); + } + if (!deps.runCaptureOpenshell) { + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + }); + } + let getBeforeDelete: string; + try { + getBeforeDelete = deps.runCaptureOpenshell(["sandbox", "get", sandbox.sandboxName], { + ignoreError: false, + }); + } catch (error) { + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + detail: `OpenShell owner lookup also failed: ${ + error instanceof Error ? error.message : String(error) + }.`, + }); + } + const sandboxIdBeforeDelete = parseOpenShellSandboxId(getBeforeDelete); + if (sandboxIdBeforeDelete !== sandbox.sandboxId) { + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + detail: `The same mutable name now resolves to durable sandbox ID ${ + sandboxIdBeforeDelete ?? "unknown" + } instead of ${sandbox.sandboxId}.`, + }); + } + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + }); +} + +function resolveIncompleteCreateSandbox( + input: ManagedBootstrapIncompleteCreateCleanupInput, + deps: ResolvedDeps, +): { + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly runtimeId: string; +} { + if ( + input.plan.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + input.plan.driverId !== DOCKER_DRIVER_ID + ) { + throw new Error("Managed bootstrap Docker incomplete-create cleanup received another driver."); + } + assertManagedBootstrapIdentity(input.bootstrapIdentity); + const query = queryOpenShellDockerSandboxContainers(input.plan.sandboxName, deps); + if (!query.ok) { + throw new Error(`Managed bootstrap Docker incomplete-create discovery failed: ${query.error}`); + } + if (query.ids.length !== 1) { + throw new Error( + `Managed bootstrap incomplete-create cleanup requires exactly one labeled Docker workload; found ${String( + query.ids.length, + )}.`, + ); + } + const runtimeId = String(query.ids[0] ?? "").toLowerCase(); + const inspect = inspectExact(runtimeId, deps); + const sandboxId = String(inspect.Config?.Labels?.[OPENSHELL_SANDBOX_ID_LABEL] ?? ""); + if (parseOpenShellSandboxId(`ID: ${sandboxId}\n`) !== sandboxId) { + throw new Error( + "Managed bootstrap Docker incomplete-create workload has no exact durable sandbox ID.", + ); + } + const sandbox = Object.freeze({ + sandboxName: input.plan.sandboxName, + sandboxId, + driverId: input.plan.driverId, + }); + assertImage(inspect, input.plan.image, deps); + assertMetadata(inspect, sandbox, input.plan.metadata); + assertHeldCommand(inspect, input.heldWorkloadArgv, input.bootstrapIdentity); + return { sandbox, runtimeId }; +} + +function managedSharedStateTransaction( + handle: ManagedBootstrapHeldWorkloadHandle, + containerId: string, + image: string, +) { + return { + agent: handle.plan.profile.agent, + bootstrapIdentity: handle.bootstrapIdentity, + containerId, + image, + profileFingerprint: handle.plan.profile.fingerprint, + } as const; +} + +function sameDockerBootstrapJournal( + left: DockerBootstrapTransaction, + right: DockerBootstrapTransaction, +): boolean { + return ( + serializeDockerManagedBootstrapJournal(left) === serializeDockerManagedBootstrapJournal(right) + ); +} + +function createDockerBootstrapJournalDurably( + journal: DockerBootstrapTransaction, + deps: ResolvedDeps, +): DockerBootstrapTransaction { + try { + deps.journalStore.create(journal); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + if (!recovered || !sameDockerBootstrapJournal(recovered, journal)) throw error; + return recovered; + } + const persisted = deps.journalStore.load(journal.bootstrapIdentity); + if (!persisted || !sameDockerBootstrapJournal(persisted, journal)) { + throw new Error("Managed bootstrap Docker staged journal was not durably re-readable."); + } + return persisted; +} + +function transitionDockerBootstrapJournalDurably( + journal: DockerBootstrapTransaction, + next: "cutover" | "rollback-authorized" | "shared-state-committed", + deps: ResolvedDeps, +): DockerBootstrapTransaction { + try { + deps.journalStore.transition(journal.bootstrapIdentity, journal.phase, next); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + const expected = Object.freeze({ ...journal, phase: next }); + if (!recovered || !sameDockerBootstrapJournal(recovered, expected)) throw error; + return recovered; + } + const persisted = deps.journalStore.load(journal.bootstrapIdentity); + const expected = Object.freeze({ ...journal, phase: next }); + if (!persisted || !sameDockerBootstrapJournal(persisted, expected)) { + throw new Error(`Managed bootstrap Docker journal transition to ${next} was not durable.`); + } + return persisted; +} + +function removeDockerBootstrapJournalDurably( + journal: DockerBootstrapTransaction, + deps: ResolvedDeps, +): void { + try { + deps.journalStore.remove(journal.bootstrapIdentity, [journal.phase]); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + if (recovered !== null) throw error; + return; + } + if (deps.journalStore.load(journal.bootstrapIdentity) !== null) { + throw new Error("Managed bootstrap Docker journal removal was not durable."); + } +} + +function assertDockerBootstrapTransactionAuthority( + transaction: DockerBootstrapTransaction, + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared?: ManagedBootstrapPreparedReplacementHandle | null, + replacement?: ManagedBootstrapReplacementHandle | null, +): void { + const originalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, + ); + const expectedSandbox = handle.sandbox; + if ( + transaction.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION || + transaction.bootstrapIdentity !== handle.bootstrapIdentity || + transaction.sandbox.sandboxName !== expectedSandbox.sandboxName || + transaction.sandbox.sandboxId !== expectedSandbox.sandboxId || + transaction.sandbox.driverId !== expectedSandbox.driverId || + transaction.profileFingerprint !== handle.plan.profile.fingerprint || + transaction.imageReference !== + expectedImageReference(snapshot.image.repository, snapshot.image.manifestDigest) || + transaction.runtimeImageContentId !== snapshot.runtimeImageContentId || + transaction.originalRuntimeId !== snapshot.runtimeId || + transaction.originalName !== originalName || + transaction.replacementStagingName !== + replacementStagingName(originalName, handle.bootstrapIdentity) || + transaction.backupName !== backupName(originalName, handle.bootstrapIdentity) || + transaction.originalSpecHash !== snapshot.specHash || + (prepared !== undefined && + prepared !== null && + (transaction.originalRuntimeId !== prepared.originalRuntimeId || + transaction.replacementRuntimeId !== prepared.preparedRuntimeId || + transaction.replacementSpecHash !== prepared.expectedActivatedSpecHash)) || + (replacement !== undefined && + replacement !== null && + (transaction.originalRuntimeId !== replacement.originalRuntimeId || + transaction.replacementRuntimeId !== replacement.replacementRuntimeId || + transaction.replacementSpecHash !== replacement.replacementSpecHash)) + ) { + throw new Error( + "Managed bootstrap receipts do not match the durable Docker transaction authority.", + ); + } +} + +function transactionFromPreparedAuthority( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, +): DockerBootstrapTransaction { + const transaction = parseDockerManagedBootstrapJournal(prepared.rollbackAuthority); + if (transaction.phase !== "staged") { + throw new Error("Managed bootstrap Docker prepared authority must describe a staged runtime."); + } + assertDockerBootstrapTransactionAuthority(transaction, handle, snapshot, prepared); + return transaction; +} + +function assertDurablePreparationAuthority( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, + receipt: ManagedBootstrapDurablePreparationReceipt, +): void { + const authority = createManagedBootstrapPreparedAuthority({ handle, snapshot, prepared }); + const recordedAt = new Date(receipt.recordedAt); + if ( + receipt.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + receipt.sandbox.sandboxName !== authority.sandbox.sandboxName || + receipt.sandbox.sandboxId !== authority.sandbox.sandboxId || + receipt.sandbox.driverId !== authority.sandbox.driverId || + receipt.bootstrapIdentity !== authority.bootstrapIdentity || + receipt.authorityFingerprint !== authority.authorityFingerprint || + typeof receipt.recordId !== "string" || + receipt.recordId.length === 0 || + receipt.recordId.includes("\0") || + typeof receipt.recordedAt !== "string" || + !Number.isFinite(recordedAt.getTime()) || + recordedAt.toISOString() !== receipt.recordedAt + ) { + throw new Error( + "Managed bootstrap Docker activation requires the exact durable prepared-authority receipt.", + ); + } +} + +function reconstructDockerBootstrapTransaction( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + replacement: ManagedBootstrapReplacementHandle, + deps: ResolvedDeps, +): DockerBootstrapTransaction { + if ( + replacement.bootstrapIdentity !== handle.bootstrapIdentity || + replacement.originalRuntimeId !== snapshot.runtimeId || + replacement.originalSpecHash !== snapshot.specHash || + replacement.replacementRuntimeId === replacement.originalRuntimeId + ) { + throw new Error( + "Managed bootstrap finalization receipts do not reconstruct one exact Docker transaction.", + ); + } + const transaction = deps.journalStore.load(handle.bootstrapIdentity); + if (!transaction) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: "the durable Docker cutover journal is absent", + }); + } + assertDockerBootstrapTransactionAuthority(transaction, handle, snapshot, null, replacement); + return transaction; +} + +function rollbackReplacementSharedStateIfPending( + input: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly replacementRuntimeId: string; + readonly runtimeImageContentId: string; + }, + deps: ResolvedDeps, +): void { + if (!tryInspectExact(input.replacementRuntimeId, deps)) { + throw new Error( + "Managed bootstrap replacement disappeared before shared-state rollback could be proven; the preserved original remains stopped.", + ); + } + const transaction = managedSharedStateTransaction( + input.handle, + input.replacementRuntimeId, + input.runtimeImageContentId, + ); + finalizeDockerManagedStartupSharedState( + { transaction, supervisorReady: false, retainContainerAfterRollback: true }, + deps, + ); +} + +function cleanupUnjournaledPreparedContainer( + input: { + readonly snapshot: ManagedBootstrapObservedSnapshot; + readonly preparedRuntimeId: string; + readonly stagingName: string; + }, + deps: ResolvedDeps, +): void { + if (!FULL_CONTAINER_ID_RE.test(input.preparedRuntimeId)) return; + const original = inspectExact(input.snapshot.runtimeId, deps); + const originalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(input.snapshot.specCanonicalJson).inspect, + ); + if ( + !isStableRunning(original) || + dockerContainerName(original) !== originalName || + normalizeDockerManagedBootstrapLaunchSpec(original).hash !== input.snapshot.specHash + ) { + throw new Error( + "Managed bootstrap cannot clean an unjournaled replacement after original drift.", + ); + } + const prepared = tryInspectExact(input.preparedRuntimeId, deps); + if (!prepared) return; + if ( + String(prepared.Id ?? "").toLowerCase() !== input.preparedRuntimeId || + dockerContainerName(prepared) !== input.stagingName || + !isExplicitlyStopped(prepared) + ) { + throw new Error( + "Managed bootstrap refused cleanup because the unjournaled prepared runtime changed.", + ); + } + const removed = deps.dockerRm(input.preparedRuntimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(input.preparedRuntimeId, deps) !== "absent" + ) { + throw new Error( + `Managed bootstrap could not remove its unjournaled prepared runtime: ${ + commandDetail(removed) || "Docker removal failed" + }`, + ); + } +} + +function resolvePreparedRollbackAuthority(input: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot; + readonly prepared: ManagedBootstrapPreparedReplacementHandle | null; + readonly durablePreparation: ManagedBootstrapDurablePreparationReceipt | null; +}): DockerBootstrapTransaction | null { + if (input.durablePreparation && !input.prepared) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: input.handle.bootstrapIdentity, + runtimeId: input.snapshot.runtimeId, + detail: "durable prepared authority is present without its exact prepared handle", + }); + } + if (!input.prepared) return null; + const authority = transactionFromPreparedAuthority(input.handle, input.snapshot, input.prepared); + if (input.durablePreparation) { + assertDurablePreparationAuthority( + input.handle, + input.snapshot, + input.prepared, + input.durablePreparation, + ); + } + return authority; +} + +export function createDockerManagedBootstrapAdapter( + dependencies: DockerManagedBootstrapDeps = {}, +): DockerManagedBootstrapAdapter { + const deps = resolveDeps(dependencies); + const committedTransactions = new Set(); + const rollbackTombstones = new Map(); + const completedRollback = ( + handle: ManagedBootstrapHeldWorkloadHandle, + alreadyRolledBack: boolean, + ): ManagedBootstrapFinalizationReceipt => { + const receipt = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: true, + alreadyRolledBack, + finalizedAt: deps.now().toISOString(), + } satisfies ManagedBootstrapFinalizationReceipt); + rollbackTombstones.set(handle.bootstrapIdentity, { + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + handle.plan.image.repository, + handle.plan.image.manifestDigest, + ), + receipt, + }); + return receipt; + }; + const priorRollback = ( + handle: ManagedBootstrapHeldWorkloadHandle, + ): ManagedBootstrapFinalizationReceipt | null => { + const tombstone = rollbackTombstones.get(handle.bootstrapIdentity); + if (!tombstone) return null; + const receipt = tombstone.receipt; + if ( + receipt.sandbox.sandboxName !== handle.sandbox.sandboxName || + receipt.sandbox.sandboxId !== handle.sandbox.sandboxId || + receipt.sandbox.driverId !== handle.sandbox.driverId || + tombstone.profileFingerprint !== handle.plan.profile.fingerprint || + tombstone.imageReference !== + expectedImageReference(handle.plan.image.repository, handle.plan.image.manifestDigest) + ) { + throw new Error("Managed bootstrap rollback tombstone does not match its durable identity."); + } + return Object.freeze({ + ...receipt, + alreadyRolledBack: true, + }); + }; + const rollbackBootstrapNow = ({ + handle, + snapshot, + prepared, + durablePreparation, + replacement, + sharedStateAlreadyRolledBack = false, + }: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot | null; + readonly prepared: ManagedBootstrapPreparedReplacementHandle | null; + readonly durablePreparation: ManagedBootstrapDurablePreparationReceipt | null; + readonly replacement: ManagedBootstrapReplacementHandle | null; + readonly sharedStateAlreadyRolledBack?: boolean; + }): ManagedBootstrapFinalizationReceipt => { + const finalized = priorRollback(handle); + if (finalized) return finalized; + const journal = deps.journalStore.load(handle.bootstrapIdentity); + if ( + committedTransactions.has(handle.bootstrapIdentity) || + journal?.phase === "shared-state-committed" + ) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: handle.bootstrapIdentity, + cleanupRuntimeId: journal?.originalRuntimeId ?? snapshot?.runtimeId ?? "unknown", + detail: + "rollback is no longer legal after the durable Docker commit fence; retry commit finalization", + }); + } + if (!snapshot) { + if (journal || prepared || durablePreparation || replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal?.originalRuntimeId ?? prepared?.originalRuntimeId ?? "unknown", + detail: "Docker replacement authority exists without its observed snapshot", + }); + } + removeOwnedWorkload(handle.sandbox, deps); + return completedRollback(handle, false); + } + + const preparedAuthority = resolvePreparedRollbackAuthority({ + handle, + snapshot, + prepared, + durablePreparation, + }); + + if (!journal) { + const originalPresence = probeExactDockerContainerAbsence(snapshot.runtimeId, deps); + if (originalPresence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: snapshot.runtimeId, + detail: "the original runtime presence is unknown and no durable journal is available", + }); + } + if (originalPresence === "absent") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: handle.bootstrapIdentity, + cleanupRuntimeId: snapshot.runtimeId, + detail: + "rollback is forbidden because the exact original is absent after journal retirement", + }); + } + const original = inspectExact(snapshot.runtimeId, deps); + const expectedOriginalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, + ); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(original); + if ( + dockerContainerName(original) !== expectedOriginalName || + original.State?.Running !== true || + normalized.hash !== snapshot.specHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: snapshot.runtimeId, + detail: "the journal is absent and the exact original is not a proven restored workload", + }); + } + if (preparedAuthority) { + const observedPrepared = inspectTransactionRuntime( + preparedAuthority, + preparedAuthority.replacementRuntimeId, + deps, + ); + if (observedPrepared) { + assertExplicitlyStopped(observedPrepared, "prepared replacement"); + if ( + dockerContainerName(observedPrepared) !== preparedAuthority.replacementStagingName || + normalizeDockerManagedBootstrapLaunchSpec(observedPrepared).canonicalJson !== + prepared?.preparedSpecCanonicalJson + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: preparedAuthority.replacementRuntimeId, + detail: "the unjournaled prepared runtime changed before exact cleanup", + }); + } + removeExactReplacement(preparedAuthority, observedPrepared, deps); + } + } else if (replacement) { + const replacementPresence = probeExactDockerContainerAbsence( + replacement.replacementRuntimeId, + deps, + ); + if (replacementPresence !== "absent") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: + replacementPresence === "present" + ? "the replacement still exists without durable journal authority" + : "replacement absence is unknown without durable journal authority", + }); + } + } + removeOwnedWorkload(handle.sandbox, deps, snapshot.runtimeId); + return completedRollback(handle, true); + } + + if (!preparedAuthority || !durablePreparation) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker cutover lacks its coordinator-recorded prepared authority", + }); + } + const stagedJournal = Object.freeze({ ...journal, phase: "staged" as const }); + if (!sameDockerBootstrapJournal(stagedJournal, preparedAuthority)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker cutover changed its prepared rollback authority", + }); + } + assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); + const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); + if (!original) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "the exact rollback original is absent", + }); + } + assertTransactionOriginal(journal, original); + const observedReplacement = inspectTransactionRuntime( + journal, + journal.replacementRuntimeId, + deps, + ); + + if (journal.phase === "staged") { + assertStableRunning(original, "staged original"); + if (observedReplacement) { + assertExplicitlyStopped(observedReplacement, "staged replacement"); + } + if ( + dockerContainerName(original) !== journal.originalName || + (observedReplacement !== null && + dockerContainerName(observedReplacement) !== journal.replacementStagingName) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "staged transaction runtime state does not match its pre-cutover fence", + }); + } + if (observedReplacement) { + removeExactReplacement(journal, observedReplacement, deps); + } + removeDockerBootstrapJournalDurably(journal, deps); + removeOwnedWorkload(handle.sandbox, deps, journal.originalRuntimeId); + return completedRollback(handle, false); + } + + if (journal.phase !== "cutover" && journal.phase !== "rollback-authorized") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: journal.bootstrapIdentity, + cleanupRuntimeId: journal.originalRuntimeId, + detail: "rollback is forbidden by the durable Docker commit phase", + }); + } + + const originalNameNow = dockerContainerName(original); + const replacementNameNow = observedReplacement + ? dockerContainerName(observedReplacement) + : null; + const originalAtTargetRecoverable = + originalNameNow === journal.originalName && + (isStableRunning(original) || isExplicitlyStopped(original)); + const originalAtBackupRecoverable = + originalNameNow === journal.backupName && isExplicitlyStopped(original); + const replacementAtStagingRecoverable = + replacementNameNow === journal.replacementStagingName && + observedReplacement !== null && + isExplicitlyStopped(observedReplacement); + const replacementAtTargetRecoverable = + replacementNameNow === journal.originalName && + observedReplacement !== null && + (isStableRunning(observedReplacement) || isExplicitlyStopped(observedReplacement)); + const validCutoverState = + (originalAtTargetRecoverable && replacementAtStagingRecoverable) || + (originalAtBackupRecoverable && replacementAtStagingRecoverable) || + (originalAtBackupRecoverable && replacementAtTargetRecoverable); + let activeJournal = journal; + + if (journal.phase === "cutover") { + if ( + (!sharedStateAlreadyRolledBack && (!observedReplacement || !validCutoverState)) || + (sharedStateAlreadyRolledBack && + (observedReplacement !== null || + !( + (originalNameNow === journal.backupName && isExplicitlyStopped(original)) || + originalAtTargetRecoverable + ))) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: + observedReplacement === null && !sharedStateAlreadyRolledBack + ? "the exact replacement disappeared before rollback authorization was durable" + : "cutover runtime names or states do not match a recoverable phase", + }); + } + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable transaction authority changed before rollback authorization", + }); + } + + let sharedStatus: "committed" | "none" | "pending" = "none"; + const sharedTransaction = managedSharedStateTransaction( + handle, + journal.replacementRuntimeId, + journal.runtimeImageContentId, + ); + if (!sharedStateAlreadyRolledBack) { + sharedStatus = probeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + profileFingerprint: journal.profileFingerprint, + }, + deps, + ); + if (sharedStatus === "committed") { + const committedJournal = transitionDockerBootstrapJournalDurably( + journal, + "shared-state-committed", + deps, + ); + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: committedJournal.bootstrapIdentity, + cleanupRuntimeId: committedJournal.originalRuntimeId, + detail: "image-owned shared state is durably committed; rollback is no longer legal", + }); + } + } + activeJournal = transitionDockerBootstrapJournalDurably(journal, "rollback-authorized", deps); + if (!sharedStateAlreadyRolledBack && sharedStatus === "pending") { + finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: false, + retainContainerAfterRollback: true, + }, + deps, + ); + } + } else { + if (!originalAtTargetRecoverable && !originalAtBackupRecoverable) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "rollback-authorized original runtime state is not recoverable", + }); + } + if ( + observedReplacement && + originalNameNow === journal.originalName && + replacementNameNow === journal.originalName + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "both transaction runtimes claim the authoritative workload name", + }); + } + if (observedReplacement) { + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "rollback authorization changed before replacement cleanup", + }); + } + const sharedTransaction = managedSharedStateTransaction( + handle, + journal.replacementRuntimeId, + journal.runtimeImageContentId, + ); + const sharedStatus = probeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + profileFingerprint: journal.profileFingerprint, + }, + deps, + ); + if (sharedStatus === "committed") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: + "shared state became committed after rollback authorization; no mutation was attempted", + }); + } + if (sharedStatus === "pending") { + finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: false, + retainContainerAfterRollback: true, + }, + deps, + ); + } + } + } + + const beforeRestore = deps.journalStore.load(activeJournal.bootstrapIdentity); + if (!beforeRestore || !sameDockerBootstrapJournal(beforeRestore, activeJournal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "durable transaction authority changed before original restoration", + }); + } + restoreOriginal(activeJournal, deps); + const restored = inspectExact(activeJournal.originalRuntimeId, deps); + assertStableRunning(restored, "restored workload"); + if ( + dockerContainerName(restored) !== activeJournal.originalName || + normalizeDockerManagedBootstrapLaunchSpec(restored).hash !== activeJournal.originalSpecHash + ) { + throw new Error("Managed bootstrap Docker rollback did not restore its exact original."); + } + removeDockerBootstrapJournalDurably(activeJournal, deps); + removeOwnedWorkload(handle.sandbox, deps, activeJournal.originalRuntimeId); + return completedRollback(handle, false); + }; + const commitBootstrapNow = ( + receipt: ManagedBootstrapCompletionReceipt, + transaction: DockerBootstrapTransaction, + input: { + readonly sharedStateStatus: "committed" | "none"; + readonly sharedStateTransaction: ReturnType; + }, + ): void => { + if (committedTransactions.has(receipt.bootstrapIdentity)) return; + if ( + transaction.phase !== "shared-state-committed" || + transaction.replacementRuntimeId !== receipt.runtimeId || + transaction.originalSpecHash !== receipt.originalSpecHash || + transaction.replacementSpecHash !== receipt.replacementSpecHash + ) { + throw new Error("Managed bootstrap Docker commit receipt does not match its commit fence."); + } + const current = deps.journalStore.load(transaction.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, transaction)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "durable commit authority changed before exact cleanup", + }); + } + + const replacement = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (!replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.replacementRuntimeId, + detail: "the exact committed replacement is absent", + }); + } + assertTransactionReplacement(transaction, replacement); + if ( + dockerContainerName(replacement) !== transaction.originalName || + replacement.State?.Running !== true + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.replacementRuntimeId, + detail: "the exact replacement is not running under the authoritative workload name", + }); + } + + const original = inspectTransactionRuntime(transaction, transaction.originalRuntimeId, deps); + if (original) { + assertTransactionOriginal(transaction, original); + if (dockerContainerName(original) !== transaction.backupName) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "the exact rollback backup is not quiescent under its durable backup name", + }); + } + assertExplicitlyStopped(original, "commit rollback backup"); + const beforeRemove = deps.journalStore.load(transaction.bootstrapIdentity); + if (!beforeRemove || !sameDockerBootstrapJournal(beforeRemove, transaction)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "durable commit authority changed before exact rollback-backup removal", + }); + } + const removed = deps.dockerRm(transaction.originalRuntimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(transaction.originalRuntimeId, deps) !== "absent" + ) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: receipt.bootstrapIdentity, + cleanupRuntimeId: transaction.originalRuntimeId, + detail: `${commandDetail(removed) || "Docker removal failed"}; exact backup absence was not proven`, + }); + } + } + + if (input.sharedStateStatus === "committed") { + try { + clearDockerManagedStartupSharedStateCommitReceipt(input.sharedStateTransaction, deps); + } catch (error) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: receipt.bootstrapIdentity, + cleanupRuntimeId: transaction.replacementRuntimeId, + detail: `exact rollback backup is absent, but its image-owned commit receipt could not be retired: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + } + } + removeDockerBootstrapJournalDurably(transaction, deps); + committedTransactions.add(receipt.bootstrapIdentity); + }; + const finalizeBootstrap = async ( + input: Parameters[0], + ): Promise => { + if (input.outcome === "rollback") { + return rollbackBootstrapNow(input); + } + const { completion, durablePreparation, handle, prepared, replacement, snapshot } = input; + if (!completion || !snapshot || !prepared || !durablePreparation || !replacement) { + throw new Error("Managed bootstrap commit requires one complete cutover receipt."); + } + const preparedAuthority = transactionFromPreparedAuthority(handle, snapshot, prepared); + assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); + const sharedTransaction = managedSharedStateTransaction( + handle, + replacement.replacementRuntimeId, + replacement.runtimeImageContentId, + ); + let sharedStatus = probeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + profileFingerprint: completion.profileFingerprint, + }, + deps, + ); + let journal = deps.journalStore.load(handle.bootstrapIdentity); + + if (!journal) { + if (committedTransactions.has(completion.bootstrapIdentity)) { + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + } + const originalPresence = probeExactDockerContainerAbsence(snapshot.runtimeId, deps); + if (originalPresence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: snapshot.runtimeId, + detail: "the retired-journal commit cannot prove exact backup absence", + }); + } + if (originalPresence !== "absent" || sharedStatus !== "none") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: + originalPresence === "absent" ? replacement.replacementRuntimeId : snapshot.runtimeId, + detail: + "the durable journal is absent before both exact backup and shared commit receipt retirement were proven", + }); + } + const committedReplacement = inspectExact(replacement.replacementRuntimeId, deps); + assertStableRunning(committedReplacement, "committed replacement"); + const originalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, + ); + if ( + dockerContainerName(committedReplacement) !== originalName || + normalizeDockerManagedBootstrapLaunchSpec(committedReplacement).hash !== + replacement.replacementSpecHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: "the retired-journal replacement does not match the exact completion receipt", + }); + } + committedTransactions.add(completion.bootstrapIdentity); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + } + + if ( + !sameDockerBootstrapJournal( + Object.freeze({ ...journal, phase: "staged" as const }), + preparedAuthority, + ) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker commit changed its prepared rollback authority", + }); + } + assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); + if (journal.phase === "staged" || journal.phase === "rollback-authorized") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: `commit is forbidden from durable journal phase ${journal.phase}`, + }); + } + if (!completion.transactionPending && sharedStatus !== "none") { + throw new Error( + "Managed bootstrap image completion disagrees with shared-state transaction status.", + ); + } + + if (journal.phase === "cutover") { + if (completion.transactionPending && sharedStatus === "none") { + throw new Error( + "Managed bootstrap image completion lost its shared-state receipt before the durable commit fence.", + ); + } + if (sharedStatus === "pending") { + let outcome; + try { + outcome = finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: true, + retainContainerAfterRollback: true, + }, + deps, + ); + } catch (error) { + if (error instanceof DockerManagedStartupSharedStateCommitIndeterminateError) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: error.message, + }); + } + throw error; + } + if (!outcome.supervisorReady) { + const failure = + outcome.failure ?? new Error("Managed bootstrap shared-state commit did not complete."); + try { + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: + "durable authority changed after shared-state rollback and before restoration", + }); + } + journal = transitionDockerBootstrapJournalDurably(journal, "rollback-authorized", deps); + await rollbackBootstrapNow({ + handle, + snapshot, + prepared, + durablePreparation, + replacement, + sharedStateAlreadyRolledBack: true, + }); + } catch (rollbackError) { + attachManagedBootstrapRollbackError(failure, rollbackError); + } + throw failure; + } + sharedStatus = "committed"; + } + journal = transitionDockerBootstrapJournalDurably(journal, "shared-state-committed", deps); + } else if (completion.transactionPending && sharedStatus === "pending") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "shared state is pending after the durable Docker commit fence", + }); + } + + commitBootstrapNow(completion, journal, { + sharedStateStatus: sharedStatus === "committed" ? "committed" : "none", + sharedStateTransaction: sharedTransaction, + }); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + }; + return { + async createHeldWorkload(input) { + if ( + input.plan.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + input.plan.driverId !== DOCKER_DRIVER_ID || + input.request.agent !== input.plan.profile.agent || + input.request.profileFingerprint !== input.plan.profile.fingerprint + ) { + throw new Error("Managed bootstrap Docker create plan does not match its root request."); + } + const bootstrapIdentity = input.bootstrapIdentity ?? deps.createBootstrapIdentity(); + assertManagedBootstrapIdentity(bootstrapIdentity); + const heldWorkloadArgv = renderManagedBootstrapHeldCommand( + input.request, + bootstrapIdentity, + input.plan.intendedWorkloadArgv, + ); + const createReceipt = await input.launch({ heldWorkloadArgv, bootstrapIdentity }); + if ( + createReceipt.ready !== true || + createReceipt.sandbox.sandboxName !== input.plan.sandboxName || + createReceipt.sandbox.driverId !== input.plan.driverId || + !createReceipt.sandbox.sandboxId + ) { + throw new Error( + "Managed bootstrap Docker create did not return one Ready durable sandbox identity.", + ); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: Object.freeze({ ...createReceipt.sandbox }), + bootstrapIdentity, + heldWorkloadArgv, + intendedWorkloadArgv: Object.freeze([...input.plan.intendedWorkloadArgv]), + plan: input.plan, + createReceipt, + }); + }, + + async cleanupIncompleteCreate(input) { + const { sandbox, runtimeId } = resolveIncompleteCreateSandbox(input, deps); + removeOwnedWorkload(sandbox, deps, runtimeId); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + bootstrapIdentity: input.bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: true, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + }, + + async discoverHeldWorkload( + input: ManagedBootstrapDiscoveryInput, + ): Promise { + if (input.sandbox.driverId !== DOCKER_DRIVER_ID) { + throw new Error("Managed bootstrap Docker adapter received another runtime driver."); + } + const query = queryOpenShellDockerSandboxContainers(input.sandbox.sandboxName, deps); + if (!query.ok) { + throw new Error(`Managed bootstrap Docker discovery failed: ${query.error}`); + } + if (query.ids.length !== 1) { + throw new Error( + `Managed bootstrap requires exactly one labeled Docker workload after Ready; found ${String( + query.ids.length, + )}.`, + ); + } + const runtimeId = String(query.ids[0] ?? "").toLowerCase(); + const inspect = inspectExact(runtimeId, deps); + assertStableRunning(inspect, "held workload"); + assertRootSupervisor(inspect); + assertImage(inspect, input.expectedImage, deps); + assertMetadata(inspect, input.sandbox, input.metadata); + assertBootstrapIdentityInObservedHold(inspect, input.bootstrapIdentity); + return Object.freeze({ + sandbox: input.sandbox, + runtimeId, + bootstrapIdentity: input.bootstrapIdentity, + }); + }, + + async inspectHeldWorkload({ handle, discovered }) { + if ( + discovered.bootstrapIdentity !== handle.bootstrapIdentity || + discovered.sandbox.sandboxId !== handle.sandbox.sandboxId || + discovered.sandbox.driverId !== handle.sandbox.driverId + ) { + throw new Error("Managed bootstrap Docker identity changed before inspection."); + } + const first = inspectExact(discovered.runtimeId, deps); + assertStableRunning(first, "held workload"); + assertRootSupervisor(first); + assertNoRootProcessInjectionEnvironment(first.Config?.Env); + const runtimeImageContentId = assertImage(first, handle.plan.image, deps); + assertMetadata(first, handle.sandbox, handle.plan.metadata); + assertHeldCommand(first, handle.heldWorkloadArgv, handle.bootstrapIdentity); + const firstNormalized = normalizeDockerManagedBootstrapLaunchSpec(first); + const inspect = inspectExact(discovered.runtimeId, deps); + assertStableRunning(inspect, "held workload"); + assertRootSupervisor(inspect); + assertNoRootProcessInjectionEnvironment(inspect.Config?.Env); + if (assertImage(inspect, handle.plan.image, deps) !== runtimeImageContentId) { + throw new Error("Managed bootstrap Docker image content changed during stable capture."); + } + assertMetadata(inspect, handle.sandbox, handle.plan.metadata); + assertHeldCommand(inspect, handle.heldWorkloadArgv, handle.bootstrapIdentity); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(inspect); + if ( + normalized.hash !== firstNormalized.hash || + normalized.canonicalJson !== firstNormalized.canonicalJson + ) { + throw new Error("Managed bootstrap Docker launch spec changed during stable capture."); + } + const supervisorArgv = exactSupervisorArgv(inspect); + if (!exactArrayEqual(supervisorArgv, handle.plan.expectedSupervisorArgv)) { + throw new Error("Managed bootstrap Docker supervisor argv changed before replacement."); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: discovered.runtimeId, + bootstrapIdentity: handle.bootstrapIdentity, + image: handle.plan.image, + runtimeImageContentId, + specHash: normalized.hash, + specCanonicalJson: normalized.canonicalJson, + agentIdentity: Object.freeze({ ...handle.plan.agentIdentity }), + supervisorArgv, + heldWorkloadArgv: handle.heldWorkloadArgv, + metadata: handle.plan.metadata, + }); + }, + + async prepareBootstrapReplacement({ handle, snapshot, request, replacementOptions }) { + if ( + snapshot.bootstrapIdentity !== handle.bootstrapIdentity || + !FULL_CONTAINER_ID_RE.test(snapshot.runtimeId) || + request.agent !== handle.plan.profile.agent || + request.profileFingerprint !== handle.plan.profile.fingerprint + ) { + throw new Error("Managed bootstrap Docker replacement identities do not match."); + } + const parsed = parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson); + const normalizedOriginal = normalizeDockerManagedBootstrapLaunchSpec(parsed.inspect); + if (normalizedOriginal.hash !== snapshot.specHash) { + throw new Error("Managed bootstrap Docker replacement snapshot is not exact."); + } + if (parsed.inspect.HostConfig?.ReadonlyRootfs === true) { + throw new Error( + "Managed bootstrap cannot stage its root-owned request in a read-only root filesystem.", + ); + } + const plan = replacementPlan(replacementOptions); + const originalName = dockerContainerName(parsed.inspect); + const backupContainerName = backupName(originalName, handle.bootstrapIdentity); + const stagingName = replacementStagingName(originalName, handle.bootstrapIdentity); + const existingJournal = deps.journalStore.load(handle.bootstrapIdentity); + if (existingJournal) { + assertDockerBootstrapTransactionAuthority(existingJournal, handle, snapshot); + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: existingJournal.bootstrapIdentity, + runtimeId: existingJournal.replacementRuntimeId, + detail: `preparation requires rollback or commit from durable phase ${existingJournal.phase}`, + }); + } + const trampolineCommand = replacementCommand(handle, snapshot); + const cloneArgs = buildDockerGpuCloneRunArgs(parsed.inspect, plan.mode, { + image: expectedImageReference(snapshot.image.repository, snapshot.image.manifestDigest), + openshellSandboxCommand: handle.intendedWorkloadArgv, + requiredUlimits: plan.requiredUlimits, + extraGroupGids: plan.extraGroupGids, + containerEntrypoint: MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, + containerCommand: trampolineCommand, + containerName: stagingName, + }); + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + + let requestFile = ""; + let replacementRuntimeId = ""; + let stagedAuthority: DockerBootstrapTransaction | null = null; + try { + const created = deps.dockerRun(["create", ...cloneArgs], options); + const returnedRuntimeId = String(created.stdout ?? "") + .trim() + .toLowerCase(); + let createdInspect: DockerContainerInspect; + if (FULL_CONTAINER_ID_RE.test(returnedRuntimeId)) { + replacementRuntimeId = returnedRuntimeId; + createdInspect = inspectExact(replacementRuntimeId, deps); + } else { + try { + createdInspect = inspectDockerContainerReference(stagingName, deps); + } catch (lookupError) { + throw new Error( + "Managed bootstrap could not prove a stopped Docker replacement after create: " + + (commandDetail(created) || + (lookupError instanceof Error ? lookupError.message : String(lookupError))), + ); + } + replacementRuntimeId = String(createdInspect.Id ?? "").toLowerCase(); + } + if ( + !FULL_CONTAINER_ID_RE.test(replacementRuntimeId) || + dockerContainerName(createdInspect) !== stagingName + ) { + throw new Error( + "Managed bootstrap Docker create did not resolve one stopped identity-bound staging container.", + ); + } + assertExplicitlyStopped(createdInspect, "created replacement"); + const createdImageContentId = assertImage(createdInspect, snapshot.image, deps); + if (createdImageContentId !== snapshot.runtimeImageContentId) { + throw new Error( + "Managed bootstrap Docker replacement resolved a different image content ID.", + ); + } + assertMetadata(createdInspect, handle.sandbox, snapshot.metadata); + assertRootSupervisor(createdInspect); + assertReplacementBoundary(createdInspect, handle, snapshot); + const expectedActivatedSpecHash = assertReplacementMatchesIntent( + snapshot.specCanonicalJson, + createdInspect, + originalName, + plan, + openshellSandboxCommandEnvValue(handle.intendedWorkloadArgv) as string, + ); + const preparedSpec = normalizeDockerManagedBootstrapLaunchSpec(createdInspect); + const expectedActivatedSpec = normalizeDockerManagedBootstrapLaunchSpec({ + ...createdInspect, + Name: `/${originalName}`, + }); + if (expectedActivatedSpec.hash !== expectedActivatedSpecHash) { + throw new Error("Managed bootstrap Docker expected activation spec is inconsistent."); + } + stagedAuthority = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + phase: "staged", + bootstrapIdentity: handle.bootstrapIdentity, + sandbox: Object.freeze({ ...handle.sandbox }), + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + snapshot.image.repository, + snapshot.image.manifestDigest, + ), + runtimeImageContentId: snapshot.runtimeImageContentId, + originalRuntimeId: snapshot.runtimeId, + replacementRuntimeId, + originalName, + replacementStagingName: stagingName, + backupName: backupContainerName, + originalSpecHash: snapshot.specHash, + replacementSpecHash: expectedActivatedSpecHash, + }); + + requestFile = writeProtectedEnvelope(handle.bootstrapIdentity, request); + const copied = deps.dockerRun( + ["cp", requestFile, replacementRuntimeId + ":" + MANAGED_BOOTSTRAP_REQUEST_FILE], + options, + ); + assertZero( + copied, + "Managed bootstrap could not stage its protected root-owned 0400 envelope", + ); + + const originalBeforeJournal = inspectExact(snapshot.runtimeId, deps); + assertStableRunning(originalBeforeJournal, "pre-journal original"); + if ( + dockerContainerName(originalBeforeJournal) !== originalName || + normalizeDockerManagedBootstrapLaunchSpec(originalBeforeJournal).hash !== + snapshot.specHash + ) { + throw new Error( + "Managed bootstrap Docker original changed while the replacement was staged.", + ); + } + const replacementBeforeJournal = inspectExact(replacementRuntimeId, deps); + assertTransactionReplacement(stagedAuthority, replacementBeforeJournal); + const observedPreparedSpec = + normalizeDockerManagedBootstrapLaunchSpec(replacementBeforeJournal); + const observedActivatedSpec = normalizeDockerManagedBootstrapLaunchSpec({ + ...replacementBeforeJournal, + Name: `/${originalName}`, + }); + if ( + dockerContainerName(replacementBeforeJournal) !== stagingName || + observedPreparedSpec.canonicalJson !== preparedSpec.canonicalJson || + observedActivatedSpec.canonicalJson !== expectedActivatedSpec.canonicalJson + ) { + throw new Error("Managed bootstrap Docker replacement changed before durable staging."); + } + assertExplicitlyStopped(replacementBeforeJournal, "pre-journal replacement"); + + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + originalRuntimeId: snapshot.runtimeId, + preparedRuntimeId: replacementRuntimeId, + image: snapshot.image, + runtimeImageContentId: snapshot.runtimeImageContentId, + originalSpecHash: snapshot.specHash, + preparedSpecHash: preparedSpec.hash, + preparedSpecCanonicalJson: preparedSpec.canonicalJson, + expectedActivatedSpecHash, + expectedActivatedSpecCanonicalJson: expectedActivatedSpec.canonicalJson, + profileFingerprint: handle.plan.profile.fingerprint, + rollbackAuthority: serializeDockerManagedBootstrapJournal(stagedAuthority), + }); + } catch (error) { + let rollbackError: unknown = null; + try { + const durable = deps.journalStore.load(handle.bootstrapIdentity); + if (!durable) { + cleanupUnjournaledPreparedContainer( + { snapshot, preparedRuntimeId: replacementRuntimeId, stagingName }, + deps, + ); + } + } catch (cleanupError) { + rollbackError = cleanupError; + } + const failure = error instanceof Error ? error : new Error(String(error)); + if (rollbackError) attachManagedBootstrapRollbackError(failure, rollbackError); + throw failure; + } finally { + if (requestFile) cleanupTempDir(requestFile, REQUEST_TEMP_PREFIX); + } + }, + async activateBootstrapReplacement({ handle, snapshot, prepared, durablePreparation }) { + const authority = transactionFromPreparedAuthority(handle, snapshot, prepared); + assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); + const existingJournal = deps.journalStore.load(handle.bootstrapIdentity); + if (existingJournal) { + assertDockerBootstrapTransactionAuthority(existingJournal, handle, snapshot, prepared); + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: existingJournal.bootstrapIdentity, + runtimeId: existingJournal.replacementRuntimeId, + detail: `activation requires rollback or commit from durable phase ${existingJournal.phase}`, + }); + } + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + try { + const originalBeforeJournal = inspectExact(snapshot.runtimeId, deps); + const preparedBeforeJournal = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionOriginal(authority, originalBeforeJournal); + assertTransactionReplacement(authority, preparedBeforeJournal); + assertStableRunning(originalBeforeJournal, "pre-activation original"); + assertExplicitlyStopped(preparedBeforeJournal, "pre-activation replacement"); + if ( + dockerContainerName(originalBeforeJournal) !== authority.originalName || + dockerContainerName(preparedBeforeJournal) !== authority.replacementStagingName || + normalizeDockerManagedBootstrapLaunchSpec(preparedBeforeJournal).canonicalJson !== + prepared.preparedSpecCanonicalJson + ) { + throw new Error( + "Managed bootstrap Docker prepared runtimes changed before durable activation.", + ); + } + + let journal = createDockerBootstrapJournalDurably(authority, deps); + const originalAtFence = inspectExact(snapshot.runtimeId, deps); + const replacementAtFence = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionOriginal(journal, originalAtFence); + assertTransactionReplacement(journal, replacementAtFence); + if ( + dockerContainerName(originalAtFence) !== journal.originalName || + originalAtFence.State?.Running !== true || + dockerContainerName(replacementAtFence) !== journal.replacementStagingName || + normalizeDockerManagedBootstrapLaunchSpec(replacementAtFence).canonicalJson !== + prepared.preparedSpecCanonicalJson + ) { + throw new Error( + "Managed bootstrap Docker staged runtimes changed before the cutover fence.", + ); + } + assertExplicitlyStopped(replacementAtFence, "staged replacement"); + journal = transitionDockerBootstrapJournalDurably(journal, "cutover", deps); + + const stopped = deps.dockerStop(snapshot.runtimeId, { + ...options, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + const afterStop = inspectExact(snapshot.runtimeId, deps); + assertTransactionOriginal(journal, afterStop); + if (dockerContainerName(afterStop) !== journal.originalName) { + throw new Error( + "Managed bootstrap could not prove its exact original stopped after Docker stop: " + + (commandDetail(stopped) || "state did not reach stopped"), + ); + } + assertExplicitlyStopped(afterStop, "stopped original"); + + const renamedOriginal = deps.dockerRename(snapshot.runtimeId, journal.backupName, options); + const afterOriginalRename = inspectExact(snapshot.runtimeId, deps); + assertTransactionOriginal(journal, afterOriginalRename); + if (dockerContainerName(afterOriginalRename) !== journal.backupName) { + throw new Error( + "Managed bootstrap could not prove its exact original backup rename: " + + (commandDetail(renamedOriginal) || "name did not reach backup"), + ); + } + assertExplicitlyStopped(afterOriginalRename, "renamed rollback backup"); + + const renamedReplacement = deps.dockerRename( + prepared.preparedRuntimeId, + journal.originalName, + options, + ); + const afterReplacementRename = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionReplacement(journal, afterReplacementRename); + if (dockerContainerName(afterReplacementRename) !== journal.originalName) { + throw new Error( + "Managed bootstrap could not prove its exact replacement cutover rename: " + + (commandDetail(renamedReplacement) || "name did not reach target"), + ); + } + + const started = deps.dockerStart(prepared.preparedRuntimeId, options); + const running = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionReplacement(journal, running); + const runningSpec = normalizeDockerManagedBootstrapLaunchSpec(running); + if ( + dockerContainerName(running) !== journal.originalName || + running.State?.Running !== true || + running.State.Paused === true || + running.State.Restarting === true || + running.State.Dead === true || + runningSpec.canonicalJson !== prepared.expectedActivatedSpecCanonicalJson + ) { + throw new Error( + "Managed bootstrap could not prove its exact replacement running after Docker start: " + + (commandDetail(started) || "state did not reach running"), + ); + } + assertReplacementBoundary(running, handle, snapshot); + const preservedOriginal = inspectExact(snapshot.runtimeId, deps); + assertTransactionOriginal(journal, preservedOriginal); + if (dockerContainerName(preservedOriginal) !== journal.backupName) { + throw new Error("Managed bootstrap Docker rollback backup changed during cutover."); + } + assertExplicitlyStopped(preservedOriginal, "preserved rollback backup"); + + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + originalRuntimeId: snapshot.runtimeId, + replacementRuntimeId: prepared.preparedRuntimeId, + image: snapshot.image, + runtimeImageContentId: snapshot.runtimeImageContentId, + originalSpecHash: snapshot.specHash, + replacementSpecHash: prepared.expectedActivatedSpecHash, + replacementSpecCanonicalJson: prepared.expectedActivatedSpecCanonicalJson, + profileFingerprint: handle.plan.profile.fingerprint, + }); + } catch (error) { + let rollbackError: unknown = null; + try { + if (!deps.journalStore.load(handle.bootstrapIdentity)) { + cleanupUnjournaledPreparedContainer( + { + snapshot, + preparedRuntimeId: prepared.preparedRuntimeId, + stagingName: authority.replacementStagingName, + }, + deps, + ); + } + } catch (cleanupError) { + rollbackError = cleanupError; + } + const failure = error instanceof Error ? error : new Error(String(error)); + if (rollbackError) attachManagedBootstrapRollbackError(failure, rollbackError); + throw failure; + } + }, + async awaitBootstrap({ handle, snapshot, replacement, timeoutSecs }) { + if ( + replacement.bootstrapIdentity !== handle.bootstrapIdentity || + replacement.originalRuntimeId !== snapshot.runtimeId || + replacement.profileFingerprint !== handle.plan.profile.fingerprint + ) { + throw new Error("Managed bootstrap Docker completion identities do not match."); + } + const journal = reconstructDockerBootstrapTransaction(handle, snapshot, replacement, deps); + if (journal.phase !== "cutover") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: `bootstrap completion is invalid from durable journal phase ${journal.phase}`, + }); + } + assertCompletedCutoverRuntimeState(journal, deps); + const before = inspectExact(replacement.replacementRuntimeId, deps); + assertStableRunning(before, "replacement"); + const beforeImageContentId = assertImage(before, replacement.image, deps); + if (beforeImageContentId !== replacement.runtimeImageContentId) { + throw new Error("Managed bootstrap Docker replacement image content changed."); + } + assertReplacementBoundary(before, handle, snapshot); + if (!waitForOpenShellSupervisorReconnect(handle.sandbox.sandboxName, timeoutSecs, deps)) { + throw new Error("Managed bootstrap Docker supervisor did not reconnect."); + } + const afterWaitJournal = deps.journalStore.load(journal.bootstrapIdentity); + if (!afterWaitJournal || !sameDockerBootstrapJournal(afterWaitJournal, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable transaction authority changed while awaiting bootstrap", + }); + } + assertCompletedCutoverRuntimeState(afterWaitJournal, deps); + const after = inspectExact(replacement.replacementRuntimeId, deps); + assertStableRunning(after, "completed replacement"); + if (assertImage(after, replacement.image, deps) !== replacement.runtimeImageContentId) { + throw new Error("Managed bootstrap Docker completed image content changed."); + } + assertReplacementBoundary(after, handle, snapshot); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(after); + if (normalized.hash !== replacement.replacementSpecHash) { + throw new Error("Managed bootstrap Docker replacement changed during bootstrap."); + } + const imageCompletion = readProtectedImageCompletion(replacement.replacementRuntimeId, deps); + if ( + imageCompletion.bootstrapIdentity !== replacement.bootstrapIdentity || + imageCompletion.agent !== handle.plan.profile.agent || + imageCompletion.profileFingerprint !== replacement.profileFingerprint + ) { + throw new Error( + "Managed bootstrap Docker image completion identities do not match the transaction.", + ); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: replacement.replacementRuntimeId, + image: replacement.image, + runtimeImageContentId: replacement.runtimeImageContentId, + originalSpecHash: replacement.originalSpecHash, + replacementSpecHash: replacement.replacementSpecHash, + profileFingerprint: replacement.profileFingerprint, + bootstrapIdentity: replacement.bootstrapIdentity, + transactionPending: imageCompletion.transactionPending, + completedAt: deps.now().toISOString(), + }); + }, + + finalizeBootstrap, + }; +} diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.ts b/src/lib/onboard/openshell-docker-sandbox-containers.ts index a8934027fd6..94c067f027c 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.ts @@ -7,6 +7,7 @@ import type { DockerGpuPatchDeps } from "./docker-gpu-patch-types"; export const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; export const OPENSHELL_MANAGED_BY_VALUE = "openshell"; export const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; +export const OPENSHELL_SANDBOX_ID_LABEL = "openshell.ai/sandbox-id"; const DOCKER_SANDBOX_QUERY_TIMEOUT_MS = 30_000; diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 9992230a0df..ece9d79adaf 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -79,6 +79,10 @@ describe("runtime provider central source boundary", () => { join(repoRoot, "src/lib/onboard/runtime-provider/docker.ts"), "utf8", ); + const dockerBootstrapAdapter = readFileSync( + join(repoRoot, "src/lib/onboard/managed-bootstrap/docker.ts"), + "utf8", + ); const managedDockerfiles = [ readFileSync(join(repoRoot, "Dockerfile"), "utf8"), readFileSync(join(repoRoot, "agents/hermes/Dockerfile"), "utf8"), @@ -99,6 +103,9 @@ describe("runtime provider central source boundary", () => { /(?:from\s+["'][^"']*managed-bootstrap|require\([^)]*managed-bootstrap)/u, ); expect(dockerProvider.match(/bootstrap:\s*unsupported\(/gu)).toHaveLength(2); + expect(dockerBootstrapAdapter).toContain('"rollback-authorized"'); + expect(dockerBootstrapAdapter).toContain('"shared-state-committed"'); + expect(bootstrapProtocol[2]).not.toMatch(/from\s+["'][^"']*docker/u); for (const dockerfile of managedDockerfiles) { expect(dockerfile).not.toContain("nemoclaw-managed-bootstrap"); From 22c198a7fcecd154d5836e7ccf63aa4ae4fba10b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 04:48:08 -0700 Subject: [PATCH 058/117] test(onboard): satisfy strict Docker bootstrap fixture typing Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/docker.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 14c65a9e926..4716da993ed 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -622,7 +622,7 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.journal).toBeNull(); expect(fake.replacement).toBeNull(); expect( - vi.mocked(fake.deps.dockerRun).mock.calls.some(([args]) => { + vi.mocked(fake.deps.dockerRun!).mock.calls.some(([args]) => { const agentIndex = args.indexOf("--agent"); return args.includes("--shared-state-transaction-status") && args[agentIndex + 1] === agent; }), From ff836adc7db965275fceb79e097568a3d05c9f3d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 05:21:28 -0700 Subject: [PATCH 059/117] test(snapshot): complete dependency adapter mocks Signed-off-by: Aaron Erickson --- .../snapshot-auto-create-failure.test.ts | 18 ++++++++++++++++-- .../sandbox/snapshot-restore-test-fixture.ts | 5 +++++ src/lib/actions/sandbox/snapshot.test.ts | 10 +++++----- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts b/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts index 65612e0ae3d..9dc200a96e1 100644 --- a/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts +++ b/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts @@ -28,13 +28,22 @@ const streamSandboxCreateMock = vi.fn(async () forcedReady: false, })); -vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => "") })); +vi.mock("../../adapters/docker", () => ({ + dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(), + dockerRunDetached: vi.fn(), +})); vi.mock("../../adapters/openshell/runtime", () => ({ captureOpenshell: captureOpenshellMock, getOpenshellBinary: vi.fn(() => "openshell"), runOpenshell: vi.fn(() => ({ status: 0, output: "" })), })); -vi.mock("../../credentials/store", () => ({ prompt: vi.fn() })); +vi.mock("../../credentials/store", () => ({ + deleteCredential: vi.fn(), + getCredential: vi.fn(() => null), + prompt: vi.fn(), + saveCredential: vi.fn(), +})); vi.mock("../../domain/sandbox/destroy", () => ({ getSandboxDeleteOutcome: vi.fn(() => ({ alreadyGone: false, gatewayUnreachable: false })), })); @@ -50,6 +59,11 @@ vi.mock("../../inference/nim", () => ({ stopNimContainerByName: vi.fn(), })); vi.mock("../../messaging/channels", () => ({ + BUILT_IN_CHANNEL_MANIFESTS: [], + getMessagingConfigEnvAliases: vi.fn(() => ({})), + getMessagingCredentialEnvKeysByChannel: vi.fn(() => ({})), + getMessagingProviderSuffixesByChannel: vi.fn(() => ({})), + listBuiltInMessagingChannelManifests: vi.fn(() => []), listMessagingProviderSuffixes: vi.fn(() => []), listMessagingCredentialMetadata: vi.fn(() => []), })); diff --git a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts index 66819eb0951..0d99c458141 100644 --- a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts +++ b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts @@ -213,7 +213,9 @@ export { lifecycleMock, shieldsMock }; vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(), dockerInspect: dockerInspectMock, + dockerRunDetached: vi.fn(), })); vi.mock("../../agent/defs", () => ({ @@ -227,7 +229,10 @@ vi.mock("../../adapters/openshell/runtime", () => ({ })); vi.mock("../../credentials/store", () => ({ + deleteCredential: vi.fn(), + getCredential: vi.fn(() => null), prompt: vi.fn(), + saveCredential: vi.fn(), })); vi.mock("../../domain/sandbox/destroy", () => ({ diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 977a2f49822..19cf6428de0 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -144,19 +144,21 @@ const latestBackupFixture = { vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(), dockerInspect: dockerInspectMock, + dockerRunDetached: vi.fn(), })); - vi.mock("../../adapters/openshell/runtime", () => ({ captureOpenshell: captureOpenshellMock, getOpenshellBinary: vi.fn(() => "openshell"), runOpenshell: runOpenshellMock, })); - vi.mock("../../credentials/store", () => ({ + deleteCredential: vi.fn(), + getCredential: vi.fn(() => null), prompt: vi.fn(), + saveCredential: vi.fn(), })); - vi.mock("../../domain/sandbox/destroy", () => ({ getSandboxDeleteOutcome: vi.fn(() => ({ alreadyGone: false, gatewayUnreachable: false })), })); @@ -165,7 +167,6 @@ vi.mock("../../inference/nim", () => ({ stopNimContainer: vi.fn(), stopNimContainerByName: vi.fn(), })); - vi.mock("../../policy", async (importOriginal) => ({ ...(await importOriginal()), applyPreset: applyPresetMock, @@ -176,7 +177,6 @@ vi.mock("../../policy", async (importOriginal) => ({ removePreset: removePresetMock, resolveAgentBaselinePolicy: resolveTestAgentBaselinePolicy, })); - vi.mock("../../runner", () => ({ ROOT: "/repo", run: vi.fn(() => ({ status: 0 })), From 88ff07b7ad08dbb4194a94d1fe2c338ae3ebb612 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 20:15:07 -0700 Subject: [PATCH 060/117] fix(snapshot): keep clone transaction dormant Signed-off-by: Aaron Erickson --- ...hot-managed-clone-handoff-dormancy.test.ts | 19 +++++-- .../snapshot-managed-clone-providers.test.ts | 55 ++++++++++--------- .../actions/sandbox/snapshot/dependencies.ts | 11 ---- 3 files changed, 45 insertions(+), 40 deletions(-) diff --git a/src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts b/src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts index 5e07f125a1f..081e45ab1c8 100644 --- a/src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts +++ b/src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts @@ -6,16 +6,27 @@ import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; describe("managed snapshot clone handoff activation boundary", () => { - it("exposes the PR3.9 dependency seam while production restore stays fail-closed", () => { + it("keeps PR3.9 operations off the production dependency graph", () => { const dependencies = readFileSync( new URL("./snapshot/dependencies.ts", import.meta.url), "utf8", ); + const handoff = readFileSync( + new URL("../../onboard/workload/clone.ts", import.meta.url), + "utf8", + ); + const providerTransaction = readFileSync( + new URL("./snapshot/managed-clone-providers.ts", import.meta.url), + "utf8", + ); const productionAction = readFileSync(new URL("./snapshot.ts", import.meta.url), "utf8"); - expect(dependencies).toContain("prepareManagedWorkloadCloneHandoff"); - expect(dependencies).toContain("prepareManagedCloneProviderTransaction"); - expect(dependencies).toContain("revalidateManagedCloneMutationAuthority"); + expect(handoff).toContain("prepareManagedWorkloadCloneHandoff"); + expect(providerTransaction).toContain("prepareManagedCloneProviderTransaction"); + expect(providerTransaction).toContain("revalidateManagedCloneMutationAuthority"); + expect(dependencies).not.toContain("prepareManagedWorkloadCloneHandoff"); + expect(dependencies).not.toContain("prepareManagedCloneProviderTransaction"); + expect(dependencies).not.toContain("revalidateManagedCloneMutationAuthority"); expect(productionAction).toContain("rejectManagedSnapshotCloneUntilRebind"); expect(productionAction).not.toContain("prepareManagedWorkloadCloneHandoff"); expect(productionAction).not.toContain("ManagedWorkloadCloneHandoff"); diff --git a/src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts b/src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts index b187c2ba1c0..e06cb4523c5 100644 --- a/src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts +++ b/src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts @@ -160,32 +160,37 @@ function providerRunner(initial: readonly LiveBinding[] = []) { let failDelete = false; const run = vi.fn((args: string[]) => { commands.push(args.join(" ")); - if (args[0] === "provider" && args[1] === "get") { - const name = args[2] ?? ""; - const binding = live.get(name); - return binding - ? { status: 0, stdout: providerMetadata(binding), stderr: "" } - : { status: 1, stdout: "", stderr: `provider '${name}' not found` }; + switch (args.slice(0, 2).join(" ")) { + case "provider get": { + const name = args[2] ?? ""; + const binding = live.get(name); + return binding + ? { status: 0, stdout: providerMetadata(binding), stderr: "" } + : { status: 1, stdout: "", stderr: `provider '${name}' not found` }; + } + case "provider create": { + const binding = { + providerName: args[3] ?? "", + providerType: args[5] ?? "", + providerEnvKey: args[7] ?? "", + }; + const outcome = createBehavior?.(binding) ?? { status: 0, materialize: binding }; + (outcome.materialize === undefined ? [] : [outcome.materialize]).forEach((materialized) => + live.set(binding.providerName, { ...materialized }), + ); + return { status: outcome.status, stdout: "", stderr: "" }; + } + case "provider delete": { + (failDelete ? [] : [args[2] ?? ""]).forEach((name) => live.delete(name)); + return failDelete + ? { status: 1, stdout: "", stderr: "gateway unavailable" } + : { status: 0, stdout: "", stderr: "" }; + } + default: + return args.slice(0, 3).join(" ") === "sandbox provider detach" + ? { status: 0, stdout: "", stderr: "" } + : { status: 1, stdout: "", stderr: "unsupported test command" }; } - if (args[0] === "provider" && args[1] === "create") { - const binding = { - providerName: args[3] ?? "", - providerType: args[5] ?? "", - providerEnvKey: args[7] ?? "", - }; - const outcome = createBehavior?.(binding) ?? { status: 0, materialize: binding }; - if (outcome.materialize) live.set(binding.providerName, { ...outcome.materialize }); - return { status: outcome.status, stdout: "", stderr: "" }; - } - if (args[0] === "provider" && args[1] === "delete") { - if (failDelete) return { status: 1, stdout: "", stderr: "gateway unavailable" }; - live.delete(args[2] ?? ""); - return { status: 0, stdout: "", stderr: "" }; - } - if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "detach") { - return { status: 0, stdout: "", stderr: "" }; - } - return { status: 1, stdout: "", stderr: "unsupported test command" }; }); return { commands, diff --git a/src/lib/actions/sandbox/snapshot/dependencies.ts b/src/lib/actions/sandbox/snapshot/dependencies.ts index d4e13ec60f3..957d60dd625 100644 --- a/src/lib/actions/sandbox/snapshot/dependencies.ts +++ b/src/lib/actions/sandbox/snapshot/dependencies.ts @@ -11,10 +11,6 @@ export type { PreparedManagedWorkloadCloneHandoff, PrepareManagedWorkloadCloneHandoffInput, } from "../../../onboard/workload/clone"; -export { - ManagedWorkloadCloneError, - prepareManagedWorkloadCloneHandoff, -} from "../../../onboard/workload/clone"; export { backupSandboxStateWithManagedAuthority } from "./backup-authority"; export type { ManagedCloneProviderBinding, @@ -26,13 +22,6 @@ export type { PreparedManagedCloneProvider, PreparedManagedCloneProviderTransaction, } from "./managed-clone-providers"; -export { - cleanupManagedCloneProviderTransaction, - ManagedCloneProviderTransactionError, - prepareManagedCloneProviderTransaction, - provisionManagedCloneProviderTransaction, - revalidateManagedCloneMutationAuthority, -} from "./managed-clone-providers"; export { ManagedSnapshotProfileRestoreError, prepareManagedSnapshotProfileRestore, From 101e408ba78577289bb955abb474afe5cf528d6d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 20:18:13 -0700 Subject: [PATCH 061/117] test(snapshot): satisfy Hermes broker growth guard Signed-off-by: Aaron Erickson --- ...apshot-hermes-managed-clone-broker.test.ts | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/src/lib/actions/sandbox/snapshot-hermes-managed-clone-broker.test.ts b/src/lib/actions/sandbox/snapshot-hermes-managed-clone-broker.test.ts index 1076d616665..7b2cd338903 100644 --- a/src/lib/actions/sandbox/snapshot-hermes-managed-clone-broker.test.ts +++ b/src/lib/actions/sandbox/snapshot-hermes-managed-clone-broker.test.ts @@ -104,33 +104,34 @@ function providerRunner() { const live = new Map(); const createCredentials = new Map(); const run = vi.fn((args: string[], options: { env?: NodeJS.ProcessEnv } = {}) => { - if (args[0] === "provider" && args[1] === "get") { - const name = args[2] ?? ""; - const binding = live.get(name); - return binding - ? { - status: 0, - stdout: providerMetadata(name, binding.type, binding.credential), - stderr: "", - } - : { status: 1, stdout: "", stderr: `provider '${name}' not found` }; + switch (args.slice(0, 2).join(" ")) { + case "provider get": { + const name = args[2] ?? ""; + const binding = live.get(name); + return binding + ? { + status: 0, + stdout: providerMetadata(name, binding.type, binding.credential), + stderr: "", + } + : { status: 1, stdout: "", stderr: `provider '${name}' not found` }; + } + case "provider create": { + const name = args[3] ?? ""; + const type = args[5] ?? ""; + const credential = args[7] ?? ""; + live.set(name, { type, credential }); + createCredentials.set(name, options.env?.[credential] ?? ""); + return { status: 0, stdout: "", stderr: "" }; + } + case "provider delete": + live.delete(args[2] ?? ""); + return { status: 0, stdout: "", stderr: "" }; + default: + return args.slice(0, 3).join(" ") === "sandbox provider detach" + ? { status: 0, stdout: "", stderr: "" } + : { status: 1, stdout: "", stderr: "unsupported test command" }; } - if (args[0] === "provider" && args[1] === "create") { - const name = args[3] ?? ""; - const type = args[5] ?? ""; - const credential = args[7] ?? ""; - live.set(name, { type, credential }); - createCredentials.set(name, options.env?.[credential] ?? ""); - return { status: 0, stdout: "", stderr: "" }; - } - if (args[0] === "provider" && args[1] === "delete") { - live.delete(args[2] ?? ""); - return { status: 0, stdout: "", stderr: "" }; - } - if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "detach") { - return { status: 0, stdout: "", stderr: "" }; - } - return { status: 1, stdout: "", stderr: "unsupported test command" }; }); return { createCredentials, live, run }; } From f251a72691bcadff47edfff13ca4187f5aec2f07 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 20:21:07 -0700 Subject: [PATCH 062/117] test(onboard): satisfy bootstrap protocol growth guard Signed-off-by: Aaron Erickson --- .../onboard/managed-bootstrap/adapter.test.ts | 84 ++++++++++--------- 1 file changed, 45 insertions(+), 39 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/adapter.test.ts b/src/lib/onboard/managed-bootstrap/adapter.test.ts index 40ec9d744b0..cc15f292524 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.test.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.test.ts @@ -281,18 +281,19 @@ function adapterFor(agent: ManagedStartupAgent): Fixture { }), finalizeBootstrap: vi.fn(async (input) => { order.push(input.outcome); - if (input.outcome === "rollback") return rolledBackReceipt(input.handle, input.snapshot); - return { - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox: input.handle.sandbox, - bootstrapIdentity: IDENTITY, - outcome: "committed" as const, - restoredRuntimeId: null, - restoredSpecHash: null, - heldWorkloadRemoved: false, - alreadyRolledBack: false, - finalizedAt: "2026-07-29T12:03:00.000Z", - }; + return input.outcome === "rollback" + ? rolledBackReceipt(input.handle, input.snapshot) + : { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: input.handle.sandbox, + bootstrapIdentity: IDENTITY, + outcome: "committed" as const, + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: "2026-07-29T12:03:00.000Z", + }; }), }; return { adapter, order, raw }; @@ -445,35 +446,40 @@ describe("managed bootstrap adapter contract", () => { ] as const)("runs exact incomplete-create cleanup when create %s", async (failureMode) => { const fixture = adapterFor("langchain-deepagents-code"); const original = fixture.adapter.createHeldWorkload; - if (failureMode === "throws") { - vi.mocked(original).mockImplementationOnce(async (input) => { - await input.launch({ - heldWorkloadArgv: renderManagedBootstrapHeldCommand( - input.request, - input.bootstrapIdentity as string, - input.plan.intendedWorkloadArgv, - ), - bootstrapIdentity: input.bootstrapIdentity as string, + switch (failureMode) { + case "throws": + vi.mocked(original).mockImplementationOnce(async (input) => { + await input.launch({ + heldWorkloadArgv: renderManagedBootstrapHeldCommand( + input.request, + input.bootstrapIdentity as string, + input.plan.intendedWorkloadArgv, + ), + bootstrapIdentity: input.bootstrapIdentity as string, + }); + throw new Error("create failed after materialization"); }); - throw new Error("create failed after materialization"); - }); - } else if (failureMode === "returns without launch") { - vi.mocked(original).mockResolvedValueOnce(handleFor(requestFor("langchain-deepagents-code"))); - } else { - vi.mocked(original).mockImplementationOnce(async (input) => { - const receipt = await input.launch({ - heldWorkloadArgv: renderManagedBootstrapHeldCommand( - input.request, - input.bootstrapIdentity as string, - input.plan.intendedWorkloadArgv, - ), - bootstrapIdentity: input.bootstrapIdentity as string, + break; + case "returns without launch": + vi.mocked(original).mockResolvedValueOnce( + handleFor(requestFor("langchain-deepagents-code")), + ); + break; + default: + vi.mocked(original).mockImplementationOnce(async (input) => { + const receipt = await input.launch({ + heldWorkloadArgv: renderManagedBootstrapHeldCommand( + input.request, + input.bootstrapIdentity as string, + input.plan.intendedWorkloadArgv, + ), + bootstrapIdentity: input.bootstrapIdentity as string, + }); + return { + ...handleFor(requestFor("langchain-deepagents-code"), receipt), + sandbox: { ...receipt.sandbox, sandboxId: "wrong-owner" }, + }; }); - return { - ...handleFor(requestFor("langchain-deepagents-code"), receipt), - sandbox: { ...receipt.sandbox, sandboxId: "wrong-owner" }, - }; - }); } const failure = await captureFailure( From b7561d160bcea0ff07c3f917e2ebb5ae292834e6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 20:22:24 -0700 Subject: [PATCH 063/117] feat(onboard): persist managed bootstrap transactions Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/README.md | 16 +- src/lib/onboard/managed-bootstrap/adapter.ts | 8 +- .../managed-bootstrap/docker-journal.test.ts | 118 +++++ .../managed-bootstrap/docker-journal.ts | 435 +++++++++++++++++- .../onboard/managed-bootstrap/docker.test.ts | 109 +++-- src/lib/onboard/managed-bootstrap/docker.ts | 327 +++++++++---- 6 files changed, 887 insertions(+), 126 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 61a2efbf8f8..227b063bbd6 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -39,11 +39,17 @@ all three names, both launch-spec hashes, image identity, profile fingerprint, and sandbox ID and then enter the destructive cutover. Rollback publishes `rollback-authorized` before exact replacement deletion; commit publishes `shared-state-committed` before exact backup deletion. Cleanup is bound to full -runtime IDs. Mutable OpenShell names are read only to detect ownership reuse, -and unsafe name-only deletion returns a typed retention error. The dormant -adapter assumes the protocol's single coordinator; multi-process -lease/arbitration remains an explicit production-activation gate. Activation -must also inject the selected gateway's canonical state root. +runtime IDs. Its private state root now retains enumerable, versioned unfinished +records containing the provider and sandbox identities, plan and profile +fingerprints, exact original and replacement IDs, rollback target, and phase. +Exact commit and cleanup receipts are durable terminal records, so adapter +recreation does not depend on process-local transaction sets or tombstone maps. +Enumeration reconstructs only unfinished records; the following recovery slice +owns phase reconciliation and cross-surface resume or rollback. Mutable +OpenShell names are read only to detect ownership reuse, and unsafe name-only +deletion returns a typed retention error. Multi-process lease/arbitration remains +an explicit production-activation gate. Activation must also inject the selected +gateway's canonical state root. The current image definitions still do not package `nemoclaw-managed-startup-hold`, `managed-startup-image-runtime.cjs`, or the diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index 22b6fc27650..ee51a04e9df 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -929,13 +929,15 @@ function normalizePreparedReplacement( }); } +export function createManagedBootstrapPlanFingerprint(plan: ManagedBootstrapExpectedPlan): string { + return createHash("sha256").update(canonicalJson(plan), "utf8").digest("hex"); +} + export function createManagedBootstrapPreparedAuthority( transaction: ManagedBootstrapPreparedTransaction, ): ManagedBootstrapPreparedAuthority { const { handle, snapshot, prepared } = transaction; - const planFingerprint = createHash("sha256") - .update(canonicalJson(handle.plan), "utf8") - .digest("hex"); + const planFingerprint = createManagedBootstrapPlanFingerprint(handle.plan); const bound = Object.freeze({ schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, phase: "prepared" as const, diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index 91a398efa51..1f0c4d05a95 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -9,10 +9,14 @@ import { afterEach, describe, expect, it } from "vitest"; import { createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapFinalizationRecord, type DockerManagedBootstrapJournal, + parseDockerManagedBootstrapFinalizationRecord, parseDockerManagedBootstrapJournal, + serializeDockerManagedBootstrapFinalizationRecord, serializeDockerManagedBootstrapJournal, } from "./docker-journal"; @@ -22,11 +26,13 @@ const journal = Object.freeze({ schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, phase: "staged", bootstrapIdentity: IDENTITY, + providerId: "docker", sandbox: { sandboxName: "alpha", sandboxId: "sandbox-alpha", driverId: "docker", }, + planFingerprint: "9".repeat(64), profileFingerprint: "2".repeat(64), imageReference: `registry.example/image@sha256:${"3".repeat(64)}`, runtimeImageContentId: `sha256:${"4".repeat(64)}`, @@ -37,7 +43,59 @@ const journal = Object.freeze({ backupName: "openshell-alpha-backup", originalSpecHash: "7".repeat(64), replacementSpecHash: "8".repeat(64), + rollbackTargetRuntimeId: "5".repeat(64), + rollbackTargetSpecHash: "7".repeat(64), + preparationReceipt: { + schemaVersion: 1, + sandbox: { + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + driverId: "docker", + }, + bootstrapIdentity: IDENTITY, + authorityFingerprint: "a".repeat(64), + recordId: "prepared-alpha", + recordedAt: "2026-07-31T19:59:59.000Z", + }, + commitReceipt: null, } satisfies DockerManagedBootstrapJournal); +const finalization = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase: "committed", + bootstrapIdentity: IDENTITY, + providerId: "docker", + sandbox: journal.sandbox, + planFingerprint: journal.planFingerprint, + profileFingerprint: journal.profileFingerprint, + imageReference: journal.imageReference, + commitReceipt: { + schemaVersion: 1, + sandbox: journal.sandbox, + runtimeId: journal.replacementRuntimeId, + image: { + repository: "registry.example/image", + manifestDigest: `sha256:${"3".repeat(64)}` as const, + }, + runtimeImageContentId: journal.runtimeImageContentId, + originalSpecHash: journal.originalSpecHash, + replacementSpecHash: journal.replacementSpecHash, + profileFingerprint: journal.profileFingerprint, + bootstrapIdentity: IDENTITY, + transactionPending: false, + completedAt: "2026-07-31T20:00:00.000Z", + }, + cleanupReceipt: { + schemaVersion: 1, + sandbox: journal.sandbox, + bootstrapIdentity: IDENTITY, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: "2026-07-31T20:00:01.000Z", + }, +} satisfies DockerManagedBootstrapFinalizationRecord); afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); @@ -60,6 +118,9 @@ describe("Docker managed bootstrap journal", () => { ); expect(store.transition(IDENTITY, "staged", "cutover").phase).toBe("cutover"); + expect(store.recordCompletion(IDENTITY, finalization.commitReceipt).commitReceipt).toEqual( + finalization.commitReceipt, + ); expect(store.transition(IDENTITY, "cutover", "shared-state-committed").phase).toBe( "shared-state-committed", ); @@ -96,4 +157,61 @@ describe("Docker managed bootstrap journal", () => { serializeDockerManagedBootstrapJournal(Object.freeze({ ...journal, phase: "staged" })), ).toBe(`${JSON.stringify(journal)}\n`); }); + + it("enumerates unfinished records and persists exact terminal receipts across restart", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const first = createFileDockerManagedBootstrapJournalStore(root); + first.create(journal); + expect(first.listUnfinished()).toEqual([journal]); + + first.recordFinalization(finalization); + expect(first.listUnfinished()).toEqual([]); + const restarted = createFileDockerManagedBootstrapJournalStore(root); + expect(restarted.loadFinalization(IDENTITY)).toEqual(finalization); + expect( + parseDockerManagedBootstrapFinalizationRecord( + serializeDockerManagedBootstrapFinalizationRecord(finalization), + ), + ).toEqual(finalization); + expect(() => + restarted.recordFinalization({ + ...finalization, + cleanupReceipt: { ...finalization.cleanupReceipt, finalizedAt: "2026-07-31T20:00:02.000Z" }, + }), + ).toThrow("finalization record changed"); + }); + + it("persists the exact completion receipt for restart reconstruction", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const first = createFileDockerManagedBootstrapJournalStore(root); + first.create(journal); + first.transition(IDENTITY, "staged", "cutover"); + const completed = first.recordCompletion(IDENTITY, finalization.commitReceipt); + expect(completed.commitReceipt).toEqual(finalization.commitReceipt); + + const restarted = createFileDockerManagedBootstrapJournalStore(root); + expect(restarted.listUnfinished()).toEqual([completed]); + expect(restarted.recordCompletion(IDENTITY, finalization.commitReceipt)).toEqual(completed); + expect(() => + restarted.recordCompletion(IDENTITY, { + ...finalization.commitReceipt, + completedAt: "2026-07-31T20:00:02.000Z", + }), + ).toThrow("completion receipt changed"); + }); + + it("fails closed when enumeration encounters an unsupported state entry", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + fs.writeFileSync( + path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, "unexpected.json"), + "{}\n", + { mode: 0o600 }, + ); + expect(() => store.listUnfinished()).toThrow("unsupported entry"); + }); }); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index e3a7e72b3a2..45592a9ed30 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -4,12 +4,19 @@ import fs from "node:fs"; import path from "node:path"; -import type { ManagedBootstrapSandboxIdentity } from "./adapter"; +import type { + ManagedBootstrapCompletionReceipt, + ManagedBootstrapDurablePreparationReceipt, + ManagedBootstrapFinalizationReceipt, + ManagedBootstrapSandboxIdentity, +} from "./adapter"; -export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 1 as const; +export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 2 as const; export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY = "managed-bootstrap"; +export const DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION = 1 as const; const SHA256_RE = /^[a-f0-9]{64}$/u; +const MANIFEST_DIGEST_RE = /^sha256:[a-f0-9]{64}$/u; const MAX_JOURNAL_BYTES = 32 * 1024; const JOURNAL_DIRECTORY_MODE = 0o700; const JOURNAL_FILE_MODE = 0o600; @@ -28,7 +35,9 @@ export interface DockerManagedBootstrapJournal { readonly schemaVersion: typeof DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION; readonly phase: DockerManagedBootstrapJournalPhase; readonly bootstrapIdentity: string; + readonly providerId: string; readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly planFingerprint: string; readonly profileFingerprint: string; readonly imageReference: string; readonly runtimeImageContentId: string; @@ -39,17 +48,41 @@ export interface DockerManagedBootstrapJournal { readonly backupName: string; readonly originalSpecHash: string; readonly replacementSpecHash: string; + readonly rollbackTargetRuntimeId: string; + readonly rollbackTargetSpecHash: string; + readonly preparationReceipt: ManagedBootstrapDurablePreparationReceipt | null; + readonly commitReceipt: ManagedBootstrapCompletionReceipt | null; +} + +export interface DockerManagedBootstrapFinalizationRecord { + readonly schemaVersion: typeof DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION; + readonly phase: "committed" | "rolled-back"; + readonly bootstrapIdentity: string; + readonly providerId: string; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly planFingerprint: string; + readonly profileFingerprint: string; + readonly imageReference: string; + readonly commitReceipt: ManagedBootstrapCompletionReceipt | null; + readonly cleanupReceipt: ManagedBootstrapFinalizationReceipt; } export interface DockerManagedBootstrapJournalStore { create(journal: DockerManagedBootstrapJournal): void; load(bootstrapIdentity: string): DockerManagedBootstrapJournal | null; + listUnfinished(): readonly DockerManagedBootstrapJournal[]; transition( bootstrapIdentity: string, expected: DockerManagedBootstrapJournalPhase, next: DockerManagedBootstrapJournalPhase, ): DockerManagedBootstrapJournal; + recordCompletion( + bootstrapIdentity: string, + receipt: ManagedBootstrapCompletionReceipt, + ): DockerManagedBootstrapJournal; remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]): void; + recordFinalization(record: DockerManagedBootstrapFinalizationRecord): void; + loadFinalization(bootstrapIdentity: string): DockerManagedBootstrapFinalizationRecord | null; } /** @@ -128,15 +161,21 @@ export function normalizeDockerManagedBootstrapJournal( const expectedKeys = [ "backupName", "bootstrapIdentity", + "commitReceipt", "imageReference", "originalName", "originalRuntimeId", "originalSpecHash", "phase", + "planFingerprint", + "preparationReceipt", "profileFingerprint", + "providerId", "replacementRuntimeId", "replacementSpecHash", "replacementStagingName", + "rollbackTargetRuntimeId", + "rollbackTargetSpecHash", "runtimeImageContentId", "sandbox", "schemaVersion", @@ -151,7 +190,9 @@ export function normalizeDockerManagedBootstrapJournal( schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, phase: exactPhase(journal.phase), bootstrapIdentity: exactSha256(journal.bootstrapIdentity, "bootstrap identity"), + providerId: exactString(journal.providerId, "provider ID"), sandbox: exactSandbox(journal.sandbox), + planFingerprint: exactSha256(journal.planFingerprint, "plan fingerprint"), profileFingerprint: exactSha256(journal.profileFingerprint, "profile fingerprint"), imageReference: exactString(journal.imageReference, "image reference"), runtimeImageContentId: exactString(journal.runtimeImageContentId, "runtime image content ID"), @@ -166,6 +207,20 @@ export function normalizeDockerManagedBootstrapJournal( backupName: exactString(journal.backupName, "backup name", 253), originalSpecHash: exactSha256(journal.originalSpecHash, "original spec hash"), replacementSpecHash: exactSha256(journal.replacementSpecHash, "replacement spec hash"), + rollbackTargetRuntimeId: exactSha256( + journal.rollbackTargetRuntimeId, + "rollback target runtime ID", + ), + rollbackTargetSpecHash: exactSha256( + journal.rollbackTargetSpecHash, + "rollback target spec hash", + ), + preparationReceipt: + journal.preparationReceipt === null + ? null + : exactPreparationReceipt(journal.preparationReceipt), + commitReceipt: + journal.commitReceipt === null ? null : exactCompletionReceipt(journal.commitReceipt), } satisfies DockerManagedBootstrapJournal); if (normalized.originalRuntimeId === normalized.replacementRuntimeId) { fail("original and replacement runtime IDs must differ"); @@ -176,6 +231,33 @@ export function normalizeDockerManagedBootstrapJournal( ) { fail("original, staging, and backup names must be distinct"); } + if ( + normalized.providerId !== normalized.sandbox.driverId || + normalized.rollbackTargetRuntimeId !== normalized.originalRuntimeId || + normalized.rollbackTargetSpecHash !== normalized.originalSpecHash + ) { + fail("provider or rollback authority does not match the transaction identity"); + } + if ( + (normalized.preparationReceipt !== null && + (normalized.preparationReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + normalized.preparationReceipt.sandbox.sandboxName !== normalized.sandbox.sandboxName || + normalized.preparationReceipt.sandbox.sandboxId !== normalized.sandbox.sandboxId || + normalized.preparationReceipt.sandbox.driverId !== normalized.sandbox.driverId)) || + (normalized.commitReceipt !== null && + (normalized.commitReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + normalized.commitReceipt.sandbox.sandboxName !== normalized.sandbox.sandboxName || + normalized.commitReceipt.sandbox.sandboxId !== normalized.sandbox.sandboxId || + normalized.commitReceipt.sandbox.driverId !== normalized.sandbox.driverId || + normalized.commitReceipt.runtimeId !== normalized.replacementRuntimeId || + normalized.commitReceipt.profileFingerprint !== normalized.profileFingerprint || + normalized.commitReceipt.originalSpecHash !== normalized.originalSpecHash || + normalized.commitReceipt.replacementSpecHash !== normalized.replacementSpecHash || + `${normalized.commitReceipt.image.repository}@${normalized.commitReceipt.image.manifestDigest}` !== + normalized.imageReference)) + ) { + fail("durable preparation or commit receipt does not match the transaction identity"); + } return normalized; } @@ -211,6 +293,249 @@ export function parseDockerManagedBootstrapJournal(text: string): DockerManagedB return journal; } +function exactTimestamp(value: unknown, label: string): string { + const timestamp = exactString(value, label, 128); + const parsed = new Date(timestamp); + if (!Number.isFinite(parsed.getTime()) || parsed.toISOString() !== timestamp) { + fail(`${label} must be one canonical timestamp`); + } + return timestamp; +} + +function exactBoolean(value: unknown, label: string): boolean { + if (typeof value !== "boolean") fail(`${label} must be boolean`); + return value; +} + +function exactNullableSha256(value: unknown, label: string): string | null { + return value === null ? null : exactSha256(value, label); +} + +function exactImage(value: unknown): ManagedBootstrapCompletionReceipt["image"] { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("completion image identity must be an object"); + } + const image = value as Record; + if (Object.keys(image).sort().join(",") !== "manifestDigest,repository") { + fail("completion image identity schema is invalid"); + } + const manifestDigest = exactString(image.manifestDigest, "completion manifest digest", 128); + if (!MANIFEST_DIGEST_RE.test(manifestDigest)) { + fail("completion manifest digest must be canonical sha256"); + } + return Object.freeze({ + repository: exactString(image.repository, "completion image repository"), + manifestDigest: manifestDigest as `sha256:${string}`, + }); +} + +function exactPreparationReceipt(value: unknown): ManagedBootstrapDurablePreparationReceipt { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("durable preparation receipt must be an object"); + } + const receipt = value as Record; + const expectedKeys = [ + "authorityFingerprint", + "bootstrapIdentity", + "recordId", + "recordedAt", + "sandbox", + "schemaVersion", + ]; + if ( + Object.keys(receipt).sort().join(",") !== expectedKeys.sort().join(",") || + receipt.schemaVersion !== 1 + ) { + fail("durable preparation receipt schema is invalid"); + } + return Object.freeze({ + schemaVersion: 1, + sandbox: exactSandbox(receipt.sandbox), + bootstrapIdentity: exactSha256( + receipt.bootstrapIdentity, + "durable preparation bootstrap identity", + ), + authorityFingerprint: exactSha256( + receipt.authorityFingerprint, + "durable preparation authority fingerprint", + ), + recordId: exactString(receipt.recordId, "durable preparation record ID", 1024), + recordedAt: exactTimestamp(receipt.recordedAt, "durable preparation timestamp"), + }); +} + +function exactCompletionReceipt(value: unknown): ManagedBootstrapCompletionReceipt { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("commit receipt must be an object"); + } + const receipt = value as Record; + const expectedKeys = [ + "bootstrapIdentity", + "completedAt", + "image", + "originalSpecHash", + "profileFingerprint", + "replacementSpecHash", + "runtimeId", + "runtimeImageContentId", + "sandbox", + "schemaVersion", + "transactionPending", + ]; + if ( + Object.keys(receipt).sort().join(",") !== expectedKeys.sort().join(",") || + receipt.schemaVersion !== 1 + ) { + fail("commit receipt schema is invalid"); + } + return Object.freeze({ + schemaVersion: 1, + sandbox: exactSandbox(receipt.sandbox), + runtimeId: exactSha256(receipt.runtimeId, "commit runtime ID"), + image: exactImage(receipt.image), + runtimeImageContentId: exactString( + receipt.runtimeImageContentId, + "commit runtime image content ID", + ), + originalSpecHash: exactSha256(receipt.originalSpecHash, "commit original spec hash"), + replacementSpecHash: exactSha256(receipt.replacementSpecHash, "commit replacement spec hash"), + profileFingerprint: exactSha256(receipt.profileFingerprint, "commit profile fingerprint"), + bootstrapIdentity: exactSha256(receipt.bootstrapIdentity, "commit bootstrap identity"), + transactionPending: exactBoolean(receipt.transactionPending, "commit transaction pending"), + completedAt: exactTimestamp(receipt.completedAt, "commit completion timestamp"), + }); +} + +function exactCleanupReceipt(value: unknown): ManagedBootstrapFinalizationReceipt { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("cleanup receipt must be an object"); + } + const receipt = value as Record; + const expectedKeys = [ + "alreadyRolledBack", + "bootstrapIdentity", + "finalizedAt", + "heldWorkloadRemoved", + "outcome", + "restoredRuntimeId", + "restoredSpecHash", + "sandbox", + "schemaVersion", + ]; + if ( + Object.keys(receipt).sort().join(",") !== expectedKeys.sort().join(",") || + receipt.schemaVersion !== 1 || + !["committed", "rolled-back"].includes(String(receipt.outcome)) + ) { + fail("cleanup receipt schema is invalid"); + } + return Object.freeze({ + schemaVersion: 1, + sandbox: exactSandbox(receipt.sandbox), + bootstrapIdentity: exactSha256(receipt.bootstrapIdentity, "cleanup bootstrap identity"), + outcome: receipt.outcome as "committed" | "rolled-back", + restoredRuntimeId: exactNullableSha256(receipt.restoredRuntimeId, "restored runtime ID"), + restoredSpecHash: exactNullableSha256(receipt.restoredSpecHash, "restored spec hash"), + heldWorkloadRemoved: exactBoolean(receipt.heldWorkloadRemoved, "held workload removed"), + alreadyRolledBack: exactBoolean(receipt.alreadyRolledBack, "already rolled back"), + finalizedAt: exactTimestamp(receipt.finalizedAt, "cleanup finalization timestamp"), + }); +} + +export function normalizeDockerManagedBootstrapFinalizationRecord( + value: unknown, +): DockerManagedBootstrapFinalizationRecord { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("finalization record must be an object"); + } + const record = value as Record; + const expectedKeys = [ + "bootstrapIdentity", + "cleanupReceipt", + "commitReceipt", + "imageReference", + "phase", + "planFingerprint", + "profileFingerprint", + "providerId", + "sandbox", + "schemaVersion", + ]; + if ( + Object.keys(record).sort().join(",") !== expectedKeys.sort().join(",") || + record.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION || + !["committed", "rolled-back"].includes(String(record.phase)) + ) { + fail("finalization record schema is invalid"); + } + const phase = record.phase as "committed" | "rolled-back"; + const sandbox = exactSandbox(record.sandbox); + const commitReceipt = + record.commitReceipt === null ? null : exactCompletionReceipt(record.commitReceipt); + const cleanupReceipt = exactCleanupReceipt(record.cleanupReceipt); + const normalized = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase, + bootstrapIdentity: exactSha256(record.bootstrapIdentity, "finalization bootstrap identity"), + providerId: exactString(record.providerId, "finalization provider ID"), + sandbox, + planFingerprint: exactSha256(record.planFingerprint, "finalization plan fingerprint"), + profileFingerprint: exactSha256(record.profileFingerprint, "finalization profile fingerprint"), + imageReference: exactString(record.imageReference, "finalization image reference"), + commitReceipt, + cleanupReceipt, + } satisfies DockerManagedBootstrapFinalizationRecord); + if ( + normalized.providerId !== sandbox.driverId || + normalized.bootstrapIdentity !== cleanupReceipt.bootstrapIdentity || + normalized.phase !== cleanupReceipt.outcome || + JSON.stringify(normalized.sandbox) !== JSON.stringify(cleanupReceipt.sandbox) || + (phase === "committed") !== (commitReceipt !== null) || + (commitReceipt !== null && + (commitReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + commitReceipt.profileFingerprint !== normalized.profileFingerprint || + JSON.stringify(commitReceipt.sandbox) !== JSON.stringify(normalized.sandbox) || + `${commitReceipt.image.repository}@${commitReceipt.image.manifestDigest}` !== + normalized.imageReference)) + ) { + fail("finalization receipts do not match their durable transaction identity"); + } + return normalized; +} + +export function serializeDockerManagedBootstrapFinalizationRecord( + record: DockerManagedBootstrapFinalizationRecord, +): string { + const serialized = `${JSON.stringify(normalizeDockerManagedBootstrapFinalizationRecord(record))}\n`; + if (Buffer.byteLength(serialized, "utf8") > MAX_JOURNAL_BYTES) { + fail("serialized finalization record exceeds its bounded transport"); + } + return serialized; +} + +export function parseDockerManagedBootstrapFinalizationRecord( + text: string, +): DockerManagedBootstrapFinalizationRecord { + if ( + text.length === 0 || + text.includes("\0") || + Buffer.byteLength(text, "utf8") > MAX_JOURNAL_BYTES + ) { + fail("serialized finalization record is empty or too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("serialized finalization record is not valid JSON"); + } + const record = normalizeDockerManagedBootstrapFinalizationRecord(parsed); + if (serializeDockerManagedBootstrapFinalizationRecord(record) !== text) { + fail("serialized finalization record is not canonical"); + } + return record; +} + function assertDirectory(directory: string): void { fs.mkdirSync(directory, { recursive: true, mode: JOURNAL_DIRECTORY_MODE }); const stat = fs.lstatSync(directory); @@ -228,6 +553,10 @@ function decisionPath(target: string): string { return `${target}.decision`; } +function finalizationPath(target: string): string { + return `${target}.finalized`; +} + function readPrivateFile(target: string, label: string): string | null { let stat: fs.Stats; try { @@ -300,6 +629,15 @@ function atomicWrite( } } +function sameSerializedJournal( + left: DockerManagedBootstrapJournal, + right: DockerManagedBootstrapJournal, +): boolean { + return ( + serializeDockerManagedBootstrapJournal(left) === serializeDockerManagedBootstrapJournal(right) + ); +} + export function createFileDockerManagedBootstrapJournalStore( stateRoot: string, ): DockerManagedBootstrapJournalStore { @@ -325,9 +663,26 @@ export function createFileDockerManagedBootstrapJournalStore( } return decided; }; + const loadFinalization = ( + bootstrapIdentity: string, + ): DockerManagedBootstrapFinalizationRecord | null => { + assertDirectory(directory); + const contents = readPrivateFile( + finalizationPath(journalPath(directory, bootstrapIdentity)), + "finalization", + ); + return contents === null ? null : parseDockerManagedBootstrapFinalizationRecord(contents); + }; return Object.freeze({ create(journal: DockerManagedBootstrapJournal) { const normalized = normalizeDockerManagedBootstrapJournal(journal); + if ( + normalized.phase !== "staged" || + normalized.preparationReceipt === null || + normalized.commitReceipt !== null + ) { + fail("a new journal requires staged durable preparation authority"); + } assertDirectory(directory); const target = journalPath(directory, normalized.bootstrapIdentity); if (readPrivateFile(decisionPath(target), "decision") !== null) { @@ -336,6 +691,34 @@ export function createFileDockerManagedBootstrapJournalStore( atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(normalized), true); }, load, + listUnfinished() { + assertDirectory(directory); + const identities: string[] = []; + for (const name of fs.readdirSync(directory)) { + const match = name.match(/^([a-f0-9]{64})\.json$/u); + if (match) { + identities.push(match[1]); + continue; + } + if ( + /^\.[a-f0-9]{64}\.json\.[0-9]+\.[a-f0-9]+\.tmp$/u.test(name) || + /^[a-f0-9]{64}\.json\.(?:decision|finalized)$/u.test(name) + ) { + continue; + } + fail(`journal directory contains an unsupported entry: ${name}`); + } + return Object.freeze( + identities + .sort() + .filter((identity) => loadFinalization(identity) === null) + .map((identity) => { + const journal = load(identity); + if (!journal) fail(`enumerated journal ${identity} disappeared`); + return journal; + }), + ); + }, transition( bootstrapIdentity: string, expected: DockerManagedBootstrapJournalPhase, @@ -370,6 +753,33 @@ export function createFileDockerManagedBootstrapJournalStore( atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(updated), false); return updated; }, + recordCompletion( + bootstrapIdentity: string, + receipt: ManagedBootstrapCompletionReceipt, + ): DockerManagedBootstrapJournal { + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const current = load(bootstrapIdentity); + if (!current || current.phase !== "cutover") { + fail(`completion recording requires phase cutover, found ${current?.phase ?? "absent"}`); + } + const updated = normalizeDockerManagedBootstrapJournal({ + ...current, + commitReceipt: receipt, + }); + if (current.commitReceipt !== null) { + if (!sameSerializedJournal(current, updated)) { + fail("completion receipt changed for this bootstrap identity"); + } + return current; + } + atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(updated), false); + const persisted = load(bootstrapIdentity); + if (!persisted || !sameSerializedJournal(persisted, updated)) { + fail("completion receipt was not durably re-readable"); + } + return persisted; + }, remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]) { assertDirectory(directory); const target = journalPath(directory, bootstrapIdentity); @@ -385,5 +795,26 @@ export function createFileDockerManagedBootstrapJournalStore( fs.unlinkSync(target); fsyncDirectory(directory); }, + recordFinalization(record: DockerManagedBootstrapFinalizationRecord) { + const normalized = normalizeDockerManagedBootstrapFinalizationRecord(record); + assertDirectory(directory); + const target = finalizationPath(journalPath(directory, normalized.bootstrapIdentity)); + const serialized = serializeDockerManagedBootstrapFinalizationRecord(normalized); + const existing = readPrivateFile(target, "finalization"); + if (existing !== null) { + if (existing !== serialized) + fail("finalization record changed for this bootstrap identity"); + return; + } + try { + atomicWrite(directory, target, serialized, true); + } catch (error) { + if (readPrivateFile(target, "finalization") !== serialized) throw error; + } + if (readPrivateFile(target, "finalization") !== serialized) { + fail("finalization record was not durably re-readable"); + } + }, + loadFinalization, }); } diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 4716da993ed..8fc2dfc37c3 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -12,22 +12,25 @@ import { createManagedStartupRootApplyRequest } from "../managed-startup/root-ap import { createManagedBootstrapPreparedAuthority, MANAGED_BOOTSTRAP_SCHEMA_VERSION, - type ManagedBootstrapCompletionReceipt, type ManagedBootstrapDurablePreparationReceipt, type ManagedBootstrapHeldWorkloadHandle, type ManagedBootstrapObservedSnapshot, ManagedBootstrapOwnerCleanupRequiredError, type ManagedBootstrapPreparedReplacementHandle, - type ManagedBootstrapReplacementHandle, } from "./adapter"; import { createDockerManagedBootstrapAdapter, type DockerManagedBootstrapDeps } from "./docker"; import type { + DockerManagedBootstrapFinalizationRecord, DockerManagedBootstrapJournal, DockerManagedBootstrapJournalStore, } from "./docker-journal"; import { DockerManagedBootstrapJournalAcknowledgementLostError } from "./docker-journal"; import { normalizeDockerManagedBootstrapLaunchSpec } from "./docker-spec"; -import { parseManagedBootstrapEnvelope } from "./envelope"; +import { + MANAGED_BOOTSTRAP_COMPLETION_FILE, + parseManagedBootstrapEnvelope, + serializeManagedBootstrapImageCompletion, +} from "./envelope"; const IDENTITY = "1".repeat(64); const OLD_ID = "2".repeat(64); @@ -156,6 +159,7 @@ function fixture(options: FixtureOptions = {}) { let original = originalInspect(agentInputs(options.agent)); let replacement: DockerContainerInspect | null = null; let journal: DockerManagedBootstrapJournal | null = null; + let finalization: DockerManagedBootstrapFinalizationRecord | null = null; let sharedState: "committed" | "none" | "pending" = options.sharedState ?? "none"; const events: string[] = []; const lostTransitions = new Set(["cutover", "shared-state-committed"]); @@ -175,6 +179,7 @@ function fixture(options: FixtureOptions = {}) { } }, load: () => copyJournal(), + listUnfinished: () => (journal && !finalization ? [structuredClone(journal)] : []), transition(_identity, expected, next) { if (!journal || journal.phase !== expected) throw new Error("stale journal transition"); journal = { ...journal, phase: next }; @@ -189,6 +194,20 @@ function fixture(options: FixtureOptions = {}) { } return structuredClone(journal); }, + recordCompletion(_identity, receipt) { + if (!journal || journal.phase !== "cutover") { + throw new Error("completion requires cutover journal"); + } + if ( + journal.commitReceipt !== null && + JSON.stringify(journal.commitReceipt) !== JSON.stringify(receipt) + ) { + throw new Error("completion changed"); + } + journal = { ...journal, commitReceipt: structuredClone(receipt) }; + events.push("journal:completion"); + return structuredClone(journal); + }, remove(_identity, expected) { if (!journal || !expected.includes(journal.phase)) throw new Error("stale journal remove"); journal = null; @@ -200,6 +219,14 @@ function fixture(options: FixtureOptions = {}) { ); } }, + recordFinalization(value) { + if (finalization && JSON.stringify(finalization) !== JSON.stringify(value)) { + throw new Error("finalization changed"); + } + finalization = structuredClone(value); + events.push(`finalization:${value.phase}`); + }, + loadFinalization: () => (finalization ? structuredClone(finalization) : null), }; const inspect = (reference: string): DockerContainerInspect => { const candidates = [original, replacement].filter( @@ -268,6 +295,20 @@ function fixture(options: FixtureOptions = {}) { ).toBe(IDENTITY); return ok(); } + if (source === `${NEW_ID}:${MANAGED_BOOTSTRAP_COMPLETION_FILE}`) { + fs.writeFileSync( + destination, + serializeManagedBootstrapImageCompletion({ + bootstrapIdentity: IDENTITY, + agent: options.agent ?? "hermes", + profileFingerprint: agentInputs(options.agent).request.profileFingerprint, + transactionPending: sharedState === "pending", + }), + { mode: 0o444 }, + ); + fs.chmodSync(destination, 0o444); + return ok(); + } const receipt = source.split(":")[1]; const expected = receipt?.includes("shared-state-commit") ? "committed" : "pending"; if (sharedState === expected) { @@ -344,6 +385,9 @@ function fixture(options: FixtureOptions = {}) { get journal() { return journal; }, + get finalization() { + return finalization; + }, get original() { return original; }, @@ -356,24 +400,6 @@ function fixture(options: FixtureOptions = {}) { }; } -function completion( - replacement: ManagedBootstrapReplacementHandle, -): ManagedBootstrapCompletionReceipt { - return { - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox, - runtimeId: replacement.replacementRuntimeId, - image: replacement.image, - runtimeImageContentId: replacement.runtimeImageContentId, - originalSpecHash: replacement.originalSpecHash, - replacementSpecHash: replacement.replacementSpecHash, - profileFingerprint: replacement.profileFingerprint, - bootstrapIdentity: replacement.bootstrapIdentity, - transactionPending: true, - completedAt: "2026-07-31T12:15:00.000Z", - }; -} - function durablePreparation( handle: ManagedBootstrapHeldWorkloadHandle, snapshot: ManagedBootstrapObservedSnapshot, @@ -420,23 +446,46 @@ describe("Docker managed bootstrap adapter", () => { replacementRuntimeId: NEW_ID, }); + const commitReceipt = await adapter.awaitBootstrap({ + handle, + snapshot, + replacement, + timeoutSecs: 1, + }); + expect(fake.events.indexOf("journal:completion")).toBeGreaterThan( + fake.events.indexOf(`start:${NEW_ID}`), + ); + const finalized = await adapter.finalizeBootstrap({ + outcome: "commit", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: commitReceipt, + }); + expect(finalized).toMatchObject({ outcome: "committed" }); + expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( + fake.events.indexOf(`rm:${OLD_ID}`), + ); + expect(fake.journal).toBeNull(); + expect(fake.finalization).toMatchObject({ phase: "committed", commitReceipt }); + expect(fake.sharedState).toBe("none"); + expect(fake.replacement?.Id).toBe(NEW_ID); + + const eventCount = fake.events.length; await expect( - adapter.finalizeBootstrap({ + createDockerManagedBootstrapAdapter(fake.deps).finalizeBootstrap({ outcome: "commit", handle, snapshot, prepared, durablePreparation: durable, replacement, - completion: completion(replacement), + completion: commitReceipt, }), - ).resolves.toMatchObject({ outcome: "committed" }); - expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( - fake.events.indexOf(`rm:${OLD_ID}`), - ); - expect(fake.journal).toBeNull(); - expect(fake.sharedState).toBe("none"); - expect(fake.replacement?.Id).toBe(NEW_ID); + ).resolves.toEqual(finalized); + expect(fake.events).toHaveLength(eventCount); }); it("recovers a failed cutover after adapter restart from exact journal authority", async () => { diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index ab99a258731..d0024838821 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -43,6 +43,7 @@ import { assertManagedBootstrapSafeProcessEnvironmentKey, attachManagedBootstrapRollbackError, createManagedBootstrapIdentity, + createManagedBootstrapPlanFingerprint, createManagedBootstrapPreparedAuthority, MANAGED_BOOTSTRAP_SCHEMA_VERSION, type ManagedBootstrapAdapter, @@ -65,11 +66,14 @@ import { } from "./adapter"; import { createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapFinalizationRecord, type DockerManagedBootstrapJournal, DockerManagedBootstrapJournalAcknowledgementLostError, type DockerManagedBootstrapJournalStore, parseDockerManagedBootstrapJournal, + serializeDockerManagedBootstrapFinalizationRecord, serializeDockerManagedBootstrapJournal, } from "./docker-journal"; import { @@ -145,12 +149,6 @@ type ResolvedDeps = Required< type DockerBootstrapTransaction = DockerManagedBootstrapJournal; -interface DockerBootstrapRollbackTombstone { - readonly profileFingerprint: string; - readonly imageReference: string; - readonly receipt: ManagedBootstrapFinalizationReceipt; -} - export interface DockerManagedBootstrapAdapter extends ManagedBootstrapAdapter {} function resolveDeps(deps: DockerManagedBootstrapDeps): ResolvedDeps { @@ -1417,6 +1415,33 @@ function sameDockerBootstrapJournal( ); } +function sameDockerBootstrapPreparedAuthority( + left: DockerBootstrapTransaction, + right: DockerBootstrapTransaction, +): boolean { + return sameDockerBootstrapJournal( + Object.freeze({ + ...left, + phase: "staged" as const, + preparationReceipt: null, + commitReceipt: null, + }), + Object.freeze({ + ...right, + phase: "staged" as const, + preparationReceipt: null, + commitReceipt: null, + }), + ); +} + +function sameDurablePreparationReceipt( + left: ManagedBootstrapDurablePreparationReceipt, + right: ManagedBootstrapDurablePreparationReceipt, +): boolean { + return JSON.stringify(left) === JSON.stringify(right); +} + function createDockerBootstrapJournalDurably( journal: DockerBootstrapTransaction, deps: ResolvedDeps, @@ -1458,6 +1483,27 @@ function transitionDockerBootstrapJournalDurably( return persisted; } +function recordDockerBootstrapCompletionDurably( + journal: DockerBootstrapTransaction, + receipt: ManagedBootstrapCompletionReceipt, + deps: ResolvedDeps, +): DockerBootstrapTransaction { + const expected = Object.freeze({ ...journal, commitReceipt: receipt }); + try { + deps.journalStore.recordCompletion(journal.bootstrapIdentity, receipt); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + if (!recovered || !sameDockerBootstrapJournal(recovered, expected)) throw error; + return recovered; + } + const persisted = deps.journalStore.load(journal.bootstrapIdentity); + if (!persisted || !sameDockerBootstrapJournal(persisted, expected)) { + throw new Error("Managed bootstrap Docker completion receipt was not durably re-readable."); + } + return persisted; +} + function removeDockerBootstrapJournalDurably( journal: DockerBootstrapTransaction, deps: ResolvedDeps, @@ -1481,6 +1527,7 @@ function assertDockerBootstrapTransactionAuthority( snapshot: ManagedBootstrapObservedSnapshot, prepared?: ManagedBootstrapPreparedReplacementHandle | null, replacement?: ManagedBootstrapReplacementHandle | null, + durablePreparation?: ManagedBootstrapDurablePreparationReceipt | null, ): void { const originalName = dockerContainerName( parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, @@ -1489,9 +1536,11 @@ function assertDockerBootstrapTransactionAuthority( if ( transaction.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION || transaction.bootstrapIdentity !== handle.bootstrapIdentity || + transaction.providerId !== expectedSandbox.driverId || transaction.sandbox.sandboxName !== expectedSandbox.sandboxName || transaction.sandbox.sandboxId !== expectedSandbox.sandboxId || transaction.sandbox.driverId !== expectedSandbox.driverId || + transaction.planFingerprint !== createManagedBootstrapPlanFingerprint(handle.plan) || transaction.profileFingerprint !== handle.plan.profile.fingerprint || transaction.imageReference !== expectedImageReference(snapshot.image.repository, snapshot.image.manifestDigest) || @@ -1502,6 +1551,18 @@ function assertDockerBootstrapTransactionAuthority( replacementStagingName(originalName, handle.bootstrapIdentity) || transaction.backupName !== backupName(originalName, handle.bootstrapIdentity) || transaction.originalSpecHash !== snapshot.specHash || + transaction.rollbackTargetRuntimeId !== snapshot.runtimeId || + transaction.rollbackTargetSpecHash !== snapshot.specHash || + (transaction.preparationReceipt !== null && + prepared !== undefined && + prepared !== null && + transaction.preparationReceipt.authorityFingerprint !== + createManagedBootstrapPreparedAuthority({ handle, snapshot, prepared }) + .authorityFingerprint) || + (durablePreparation !== undefined && + durablePreparation !== null && + (transaction.preparationReceipt === null || + !sameDurablePreparationReceipt(transaction.preparationReceipt, durablePreparation))) || (prepared !== undefined && prepared !== null && (transaction.originalRuntimeId !== prepared.originalRuntimeId || @@ -1692,8 +1753,64 @@ export function createDockerManagedBootstrapAdapter( dependencies: DockerManagedBootstrapDeps = {}, ): DockerManagedBootstrapAdapter { const deps = resolveDeps(dependencies); - const committedTransactions = new Set(); - const rollbackTombstones = new Map(); + const finalizationRecord = ( + handle: ManagedBootstrapHeldWorkloadHandle, + ): DockerManagedBootstrapFinalizationRecord | null => { + const record = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + if (!record) return null; + if ( + record.providerId !== handle.sandbox.driverId || + record.sandbox.sandboxName !== handle.sandbox.sandboxName || + record.sandbox.sandboxId !== handle.sandbox.sandboxId || + record.sandbox.driverId !== handle.sandbox.driverId || + record.planFingerprint !== createManagedBootstrapPlanFingerprint(handle.plan) || + record.profileFingerprint !== handle.plan.profile.fingerprint || + record.imageReference !== + expectedImageReference(handle.plan.image.repository, handle.plan.image.manifestDigest) + ) { + throw new Error("Managed bootstrap finalization record does not match its durable identity."); + } + return record; + }; + const persistFinalization = ( + handle: ManagedBootstrapHeldWorkloadHandle, + phase: "committed" | "rolled-back", + commitReceipt: ManagedBootstrapCompletionReceipt | null, + cleanupReceipt: ManagedBootstrapFinalizationReceipt, + ): ManagedBootstrapFinalizationReceipt => { + const record = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase, + bootstrapIdentity: handle.bootstrapIdentity, + providerId: handle.sandbox.driverId, + sandbox: handle.sandbox, + planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + handle.plan.image.repository, + handle.plan.image.manifestDigest, + ), + commitReceipt, + cleanupReceipt, + } satisfies DockerManagedBootstrapFinalizationRecord); + const serialized = serializeDockerManagedBootstrapFinalizationRecord(record); + try { + deps.journalStore.recordFinalization(record); + } catch (error) { + const recovered = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + if ( + !recovered || + serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized + ) { + throw error; + } + } + const persisted = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + if (!persisted || serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized) { + throw new Error("Managed bootstrap finalization receipt was not durably re-readable."); + } + return persisted.cleanupReceipt; + }; const completedRollback = ( handle: ManagedBootstrapHeldWorkloadHandle, alreadyRolledBack: boolean, @@ -1709,34 +1826,39 @@ export function createDockerManagedBootstrapAdapter( alreadyRolledBack, finalizedAt: deps.now().toISOString(), } satisfies ManagedBootstrapFinalizationReceipt); - rollbackTombstones.set(handle.bootstrapIdentity, { - profileFingerprint: handle.plan.profile.fingerprint, - imageReference: expectedImageReference( - handle.plan.image.repository, - handle.plan.image.manifestDigest, - ), - receipt, - }); - return receipt; + return persistFinalization(handle, "rolled-back", null, receipt); + }; + const completedCommit = ( + handle: ManagedBootstrapHeldWorkloadHandle, + commitReceipt: ManagedBootstrapCompletionReceipt, + ): ManagedBootstrapFinalizationReceipt => { + const cleanupReceipt = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + } satisfies ManagedBootstrapFinalizationReceipt); + return persistFinalization(handle, "committed", commitReceipt, cleanupReceipt); }; const priorRollback = ( handle: ManagedBootstrapHeldWorkloadHandle, ): ManagedBootstrapFinalizationReceipt | null => { - const tombstone = rollbackTombstones.get(handle.bootstrapIdentity); - if (!tombstone) return null; - const receipt = tombstone.receipt; - if ( - receipt.sandbox.sandboxName !== handle.sandbox.sandboxName || - receipt.sandbox.sandboxId !== handle.sandbox.sandboxId || - receipt.sandbox.driverId !== handle.sandbox.driverId || - tombstone.profileFingerprint !== handle.plan.profile.fingerprint || - tombstone.imageReference !== - expectedImageReference(handle.plan.image.repository, handle.plan.image.manifestDigest) - ) { - throw new Error("Managed bootstrap rollback tombstone does not match its durable identity."); + const finalized = finalizationRecord(handle); + if (!finalized) return null; + if (finalized.phase === "committed") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: handle.bootstrapIdentity, + cleanupRuntimeId: finalized.commitReceipt?.runtimeId ?? "unknown", + detail: "rollback is no longer legal after the durable finalization receipt", + }); } return Object.freeze({ - ...receipt, + ...finalized.cleanupReceipt, alreadyRolledBack: true, }); }; @@ -1758,10 +1880,7 @@ export function createDockerManagedBootstrapAdapter( const finalized = priorRollback(handle); if (finalized) return finalized; const journal = deps.journalStore.load(handle.bootstrapIdentity); - if ( - committedTransactions.has(handle.bootstrapIdentity) || - journal?.phase === "shared-state-committed" - ) { + if (journal?.phase === "shared-state-committed") { throw new ManagedBootstrapDurableCommitCleanupPendingError({ bootstrapIdentity: handle.bootstrapIdentity, cleanupRuntimeId: journal?.originalRuntimeId ?? snapshot?.runtimeId ?? "unknown", @@ -1869,15 +1988,21 @@ export function createDockerManagedBootstrapAdapter( detail: "durable Docker cutover lacks its coordinator-recorded prepared authority", }); } - const stagedJournal = Object.freeze({ ...journal, phase: "staged" as const }); - if (!sameDockerBootstrapJournal(stagedJournal, preparedAuthority)) { + if (!sameDockerBootstrapPreparedAuthority(journal, preparedAuthority)) { throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: handle.bootstrapIdentity, runtimeId: journal.replacementRuntimeId, detail: "durable Docker cutover changed its prepared rollback authority", }); } - assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); + assertDockerBootstrapTransactionAuthority( + journal, + handle, + snapshot, + prepared, + replacement, + durablePreparation, + ); const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); if (!original) { throw new ManagedBootstrapCommitStateIndeterminateError({ @@ -2097,14 +2222,14 @@ export function createDockerManagedBootstrapAdapter( return completedRollback(handle, false); }; const commitBootstrapNow = ( + handle: ManagedBootstrapHeldWorkloadHandle, receipt: ManagedBootstrapCompletionReceipt, transaction: DockerBootstrapTransaction, input: { readonly sharedStateStatus: "committed" | "none"; readonly sharedStateTransaction: ReturnType; }, - ): void => { - if (committedTransactions.has(receipt.bootstrapIdentity)) return; + ): ManagedBootstrapFinalizationReceipt => { if ( transaction.phase !== "shared-state-committed" || transaction.replacementRuntimeId !== receipt.runtimeId || @@ -2196,7 +2321,7 @@ export function createDockerManagedBootstrapAdapter( } } removeDockerBootstrapJournalDurably(transaction, deps); - committedTransactions.add(receipt.bootstrapIdentity); + return completedCommit(handle, receipt); }; const finalizeBootstrap = async ( input: Parameters[0], @@ -2208,6 +2333,21 @@ export function createDockerManagedBootstrapAdapter( if (!completion || !snapshot || !prepared || !durablePreparation || !replacement) { throw new Error("Managed bootstrap commit requires one complete cutover receipt."); } + const finalized = finalizationRecord(handle); + if (finalized) { + if ( + finalized.phase !== "committed" || + !finalized.commitReceipt || + JSON.stringify(finalized.commitReceipt) !== JSON.stringify(completion) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: "durable finalization cannot change outcome or commit receipt", + }); + } + return finalized.cleanupReceipt; + } const preparedAuthority = transactionFromPreparedAuthority(handle, snapshot, prepared); assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); const sharedTransaction = managedSharedStateTransaction( @@ -2225,19 +2365,6 @@ export function createDockerManagedBootstrapAdapter( let journal = deps.journalStore.load(handle.bootstrapIdentity); if (!journal) { - if (committedTransactions.has(completion.bootstrapIdentity)) { - return Object.freeze({ - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox: handle.sandbox, - bootstrapIdentity: handle.bootstrapIdentity, - outcome: "committed", - restoredRuntimeId: null, - restoredSpecHash: null, - heldWorkloadRemoved: false, - alreadyRolledBack: false, - finalizedAt: deps.now().toISOString(), - }); - } const originalPresence = probeExactDockerContainerAbsence(snapshot.runtimeId, deps); if (originalPresence === "unknown") { throw new ManagedBootstrapCommitStateIndeterminateError({ @@ -2271,33 +2398,34 @@ export function createDockerManagedBootstrapAdapter( detail: "the retired-journal replacement does not match the exact completion receipt", }); } - committedTransactions.add(completion.bootstrapIdentity); - return Object.freeze({ - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox: handle.sandbox, + return completedCommit(handle, completion); + } + + if (!sameDockerBootstrapPreparedAuthority(journal, preparedAuthority)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: handle.bootstrapIdentity, - outcome: "committed", - restoredRuntimeId: null, - restoredSpecHash: null, - heldWorkloadRemoved: false, - alreadyRolledBack: false, - finalizedAt: deps.now().toISOString(), + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker commit changed its prepared rollback authority", }); } - + assertDockerBootstrapTransactionAuthority( + journal, + handle, + snapshot, + prepared, + replacement, + durablePreparation, + ); if ( - !sameDockerBootstrapJournal( - Object.freeze({ ...journal, phase: "staged" as const }), - preparedAuthority, - ) + journal.commitReceipt === null || + JSON.stringify(journal.commitReceipt) !== JSON.stringify(completion) ) { throw new ManagedBootstrapCommitStateIndeterminateError({ - bootstrapIdentity: handle.bootstrapIdentity, + bootstrapIdentity: journal.bootstrapIdentity, runtimeId: journal.replacementRuntimeId, - detail: "durable Docker commit changed its prepared rollback authority", + detail: "commit requires the exact durable completion receipt", }); } - assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); if (journal.phase === "staged" || journal.phase === "rollback-authorized") { throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: journal.bootstrapIdentity, @@ -2376,21 +2504,10 @@ export function createDockerManagedBootstrapAdapter( }); } - commitBootstrapNow(completion, journal, { + return commitBootstrapNow(handle, completion, journal, { sharedStateStatus: sharedStatus === "committed" ? "committed" : "none", sharedStateTransaction: sharedTransaction, }); - return Object.freeze({ - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox: handle.sandbox, - bootstrapIdentity: handle.bootstrapIdentity, - outcome: "committed", - restoredRuntimeId: null, - restoredSpecHash: null, - heldWorkloadRemoved: false, - alreadyRolledBack: false, - finalizedAt: deps.now().toISOString(), - }); }; return { async createHeldWorkload(input) { @@ -2639,7 +2756,9 @@ export function createDockerManagedBootstrapAdapter( schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, phase: "staged", bootstrapIdentity: handle.bootstrapIdentity, + providerId: handle.sandbox.driverId, sandbox: Object.freeze({ ...handle.sandbox }), + planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), profileFingerprint: handle.plan.profile.fingerprint, imageReference: expectedImageReference( snapshot.image.repository, @@ -2653,6 +2772,10 @@ export function createDockerManagedBootstrapAdapter( backupName: backupContainerName, originalSpecHash: snapshot.specHash, replacementSpecHash: expectedActivatedSpecHash, + rollbackTargetRuntimeId: snapshot.runtimeId, + rollbackTargetSpecHash: snapshot.specHash, + preparationReceipt: null, + commitReceipt: null, }); requestFile = writeProtectedEnvelope(handle.bootstrapIdentity, request); @@ -2732,9 +2855,20 @@ export function createDockerManagedBootstrapAdapter( async activateBootstrapReplacement({ handle, snapshot, prepared, durablePreparation }) { const authority = transactionFromPreparedAuthority(handle, snapshot, prepared); assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); + const durableAuthority = Object.freeze({ + ...authority, + preparationReceipt: durablePreparation, + }); const existingJournal = deps.journalStore.load(handle.bootstrapIdentity); if (existingJournal) { - assertDockerBootstrapTransactionAuthority(existingJournal, handle, snapshot, prepared); + assertDockerBootstrapTransactionAuthority( + existingJournal, + handle, + snapshot, + prepared, + null, + durablePreparation, + ); throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: existingJournal.bootstrapIdentity, runtimeId: existingJournal.replacementRuntimeId, @@ -2764,7 +2898,7 @@ export function createDockerManagedBootstrapAdapter( ); } - let journal = createDockerBootstrapJournalDurably(authority, deps); + let journal = createDockerBootstrapJournalDurably(durableAuthority, deps); const originalAtFence = inspectExact(snapshot.runtimeId, deps); const replacementAtFence = inspectExact(prepared.preparedRuntimeId, deps); assertTransactionOriginal(journal, originalAtFence); @@ -2937,7 +3071,19 @@ export function createDockerManagedBootstrapAdapter( "Managed bootstrap Docker image completion identities do not match the transaction.", ); } - return Object.freeze({ + if (afterWaitJournal.commitReceipt !== null) { + if ( + afterWaitJournal.commitReceipt.transactionPending !== imageCompletion.transactionPending + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: afterWaitJournal.bootstrapIdentity, + runtimeId: afterWaitJournal.replacementRuntimeId, + detail: "durable completion disagrees with the image-owned transaction receipt", + }); + } + return afterWaitJournal.commitReceipt; + } + const completion = Object.freeze({ schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, sandbox: handle.sandbox, runtimeId: replacement.replacementRuntimeId, @@ -2950,6 +3096,15 @@ export function createDockerManagedBootstrapAdapter( transactionPending: imageCompletion.transactionPending, completedAt: deps.now().toISOString(), }); + const completedJournal = recordDockerBootstrapCompletionDurably( + afterWaitJournal, + completion, + deps, + ); + if (completedJournal.commitReceipt === null) { + throw new Error("Managed bootstrap Docker completion receipt disappeared after recording."); + } + return completedJournal.commitReceipt; }, finalizeBootstrap, From 01a554daac1c03ee5e53cc7a7b8010af56e59085 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 20:29:51 -0700 Subject: [PATCH 064/117] test(onboard): linearize Docker bootstrap fixtures Signed-off-by: Aaron Erickson --- .../onboard/managed-bootstrap/docker.test.ts | 255 ++++++++++-------- 1 file changed, 149 insertions(+), 106 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 4716da993ed..f0ce289ec0a 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -152,6 +152,10 @@ type FixtureOptions = { sharedState?: "committed" | "none" | "pending"; }; +function failFixture(message: string): never { + throw new Error(message); +} + function fixture(options: FixtureOptions = {}) { let original = originalInspect(agentInputs(options.agent)); let replacement: DockerContainerInspect | null = null; @@ -167,37 +171,46 @@ function fixture(options: FixtureOptions = {}) { create(value) { journal = structuredClone(value); events.push("journal:staged"); - if (loseCreateAck) { - loseCreateAck = false; - throw new DockerManagedBootstrapJournalAcknowledgementLostError( - "lost journal create acknowledgement", - ); + switch (loseCreateAck) { + case true: + loseCreateAck = false; + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal create acknowledgement", + ); } }, load: () => copyJournal(), transition(_identity, expected, next) { - if (!journal || journal.phase !== expected) throw new Error("stale journal transition"); - journal = { ...journal, phase: next }; + const current = + journal !== null && journal.phase === expected + ? journal + : failFixture("stale journal transition"); + journal = { ...current, phase: next }; events.push(`journal:${next}`); - if (next === "cutover" && options.failAfterCutoverFence) { - throw new Error("injected crash after durable cutover fence"); + switch (true) { + case next === "cutover" && options.failAfterCutoverFence === true: + throw new Error("injected crash after durable cutover fence"); + case options.lostAcks === true && lostTransitions.delete(next): + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal transition acknowledgement", + ); + default: + return structuredClone(journal); } - if (options.lostAcks && lostTransitions.delete(next)) { - throw new DockerManagedBootstrapJournalAcknowledgementLostError( - "lost journal transition acknowledgement", - ); - } - return structuredClone(journal); }, remove(_identity, expected) { - if (!journal || !expected.includes(journal.phase)) throw new Error("stale journal remove"); + const current = journal; + void (current !== null && expected.includes(current.phase) + ? current + : failFixture("stale journal remove")); journal = null; events.push("journal:removed"); - if (loseRemoveAck) { - loseRemoveAck = false; - throw new DockerManagedBootstrapJournalAcknowledgementLostError( - "lost journal remove acknowledgement", - ); + switch (loseRemoveAck) { + case true: + loseRemoveAck = false; + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal remove acknowledgement", + ); } }, }; @@ -209,93 +222,104 @@ function fixture(options: FixtureOptions = {}) { (value) => value.Id === reference || String(value.Name ?? "").replace(/^\/+/u, "") === reference, ); - if (!found) throw new Error(`No such container: ${reference}`); - return structuredClone(found); + return found ? structuredClone(found) : failFixture(`No such container: ${reference}`); }; const dockerCapture: NonNullable = vi.fn((args) => { - if (args[0] === "image") { - return JSON.stringify([{ Id: CONFIG_ID, RepoDigests: [IMAGE] }]); + switch (args[0]) { + case "image": + return JSON.stringify([{ Id: CONFIG_ID, RepoDigests: [IMAGE] }]); + default: + return JSON.stringify([inspect(String(args[3] ?? ""))]); } - return JSON.stringify([inspect(String(args[3] ?? ""))]); }); const dockerRun: NonNullable = vi.fn( (args: readonly string[]) => { - if (args[0] === "create") { - events.push("create:replacement"); - const name = String(args[args.indexOf("--name") + 1] ?? ""); - const entrypoint = String(args[args.indexOf("--entrypoint") + 1] ?? ""); - const imageIndex = args.indexOf(IMAGE); - const env: string[] = []; - args.forEach((value, index) => { - if (value === "--env") env.push(String(args[index + 1] ?? "")); - }); - replacement = { - ...structuredClone(original), - Id: NEW_ID, - Name: `/${name}`, - Config: { - ...structuredClone(original.Config), - Image: IMAGE, - Env: env, - Entrypoint: [entrypoint], - Cmd: args.slice(imageIndex + 1), - }, - State: { Running: false, Paused: false, Restarting: false, Dead: false }, - }; - return options.lostAcks - ? { status: 1, stdout: "", stderr: "lost create acknowledgement" } - : ok(NEW_ID); - } - if (args[0] === "ps") return ok(original ? OLD_ID : ""); - if (args[0] === "inspect") { - const id = String(args[3] ?? ""); - try { - inspect(id); - return ok(`[{"Id":"${id}"}]`); - } catch { - return { status: 1, stderr: `Error response from daemon: No such container: ${id}` }; + switch (args[0]) { + case "create": { + events.push("create:replacement"); + const name = String(args[args.indexOf("--name") + 1] ?? ""); + const entrypoint = String(args[args.indexOf("--entrypoint") + 1] ?? ""); + const imageIndex = args.indexOf(IMAGE); + const env = args.flatMap((value, index) => + value === "--env" ? [String(args[index + 1] ?? "")] : [], + ); + replacement = { + ...structuredClone(original), + Id: NEW_ID, + Name: `/${name}`, + Config: { + ...structuredClone(original.Config), + Image: IMAGE, + Env: env, + Entrypoint: [entrypoint], + Cmd: args.slice(imageIndex + 1), + }, + State: { Running: false, Paused: false, Restarting: false, Dead: false }, + }; + return options.lostAcks + ? { status: 1, stdout: "", stderr: "lost create acknowledgement" } + : ok(NEW_ID); } - } - if (args[0] === "cp") { - const sourceIndex = args[1] === "-a" ? 2 : 1; - const source = String(args[sourceIndex] ?? ""); - const destination = String(args[sourceIndex + 1] ?? ""); - if (!source.includes(":")) { - events.push("stage:envelope"); - expect(fs.statSync(source).mode & 0o777).toBe(0o400); - expect( - parseManagedBootstrapEnvelope(fs.readFileSync(source, "utf8")).bootstrapIdentity, - ).toBe(IDENTITY); - return ok(); + case "ps": + return ok(original ? OLD_ID : ""); + case "inspect": { + const id = String(args[3] ?? ""); + try { + inspect(id); + return ok(`[{"Id":"${id}"}]`); + } catch { + return { status: 1, stderr: `Error response from daemon: No such container: ${id}` }; + } } - const receipt = source.split(":")[1]; - const expected = receipt?.includes("shared-state-commit") ? "committed" : "pending"; - if (sharedState === expected) { - fs.mkdirSync(destination, { recursive: true }); - return ok(); + case "cp": { + const sourceIndex = args[1] === "-a" ? 2 : 1; + const source = String(args[sourceIndex] ?? ""); + const destination = String(args[sourceIndex + 1] ?? ""); + const copyIntoContainer = () => { + events.push("stage:envelope"); + expect(fs.statSync(source).mode & 0o777).toBe(0o400); + expect( + parseManagedBootstrapEnvelope(fs.readFileSync(source, "utf8")).bootstrapIdentity, + ).toBe(IDENTITY); + return ok(); + }; + const copyFromContainer = () => { + const receipt = source.split(":")[1]; + const expected = receipt?.includes("shared-state-commit") ? "committed" : "pending"; + return sharedState === expected + ? (() => { + fs.mkdirSync(destination, { recursive: true }); + return ok(); + })() + : { + status: 1, + stderr: `Error response from daemon: Could not find the file ${receipt} in container ${NEW_ID}`, + }; + }; + return source.includes(":") ? copyFromContainer() : copyIntoContainer(); } - return { - status: 1, - stderr: `Error response from daemon: Could not find the file ${receipt} in container ${NEW_ID}`, - }; - } - if (args[0] === "run" && args.includes("--shared-state-transaction-status")) { - return ok(`${sharedState}\n`); - } - if (args[0] === "run" && args.includes("--rollback-shared-state-transaction")) { - sharedState = "none"; - events.push("shared:rollback"); - return ok(); - } - if (args[0] === "exec" && args.includes("--commit-shared-state-transaction")) { - sharedState = "committed"; - events.push("shared:commit"); - return ok(); - } - if (args[0] === "exec" && args.includes("--clear-shared-state-commit-receipt")) { - sharedState = "none"; - events.push("shared:clear"); - return ok(); + case "run": + switch (true) { + case args.includes("--shared-state-transaction-status"): + return ok(`${sharedState}\n`); + case args.includes("--rollback-shared-state-transaction"): + sharedState = "none"; + events.push("shared:rollback"); + return ok(); + } + break; + case "exec": + switch (true) { + case args.includes("--commit-shared-state-transaction"): + sharedState = "committed"; + events.push("shared:commit"); + return ok(); + case args.includes("--clear-shared-state-commit-receipt"): + sharedState = "none"; + events.push("shared:clear"); + return ok(); + } + break; } throw new Error(`unexpected Docker command: ${args.join(" ")}`); }, @@ -307,21 +331,34 @@ function fixture(options: FixtureOptions = {}) { dockerStop: vi.fn((id) => { events.push(`stop:${id}`); const target = id === OLD_ID ? original : replacement; - if (target?.State) target.State = { ...target.State, Running: false }; + [target] + .filter((value): value is DockerContainerInspect => value?.State !== undefined) + .forEach((value) => { + value.State = { ...value.State, Running: false }; + }); return options.lostAcks ? { status: 1, stderr: "lost stop acknowledgement" } : ok(); }), dockerRename: vi.fn((id, name) => { events.push(`rename:${id}:${name}`); const target = id === OLD_ID ? original : replacement; - if (target) target.Name = `/${name}`; + [target] + .filter((value): value is DockerContainerInspect => value !== null) + .forEach((value) => { + value.Name = `/${name}`; + }); return options.lostAcks ? { status: 1, stderr: "lost rename acknowledgement" } : ok(); }), dockerStart: vi.fn((id) => { events.push(`start:${id}`); const target = id === OLD_ID ? original : replacement; - if (target?.State && !(id === NEW_ID && options.failStart)) { - target.State = { ...target.State, Running: true }; - } + [target] + .filter( + (value): value is DockerContainerInspect => + value?.State !== undefined && !(id === NEW_ID && options.failStart), + ) + .forEach((value) => { + value.State = { ...value.State, Running: true }; + }); return id === NEW_ID && options.failStart ? { status: 1, stderr: "injected start failure" } : options.lostAcks @@ -330,8 +367,14 @@ function fixture(options: FixtureOptions = {}) { }), dockerRm: vi.fn((id) => { events.push(`rm:${id}`); - if (id === OLD_ID) original = null as unknown as DockerContainerInspect; - if (id === NEW_ID) replacement = null; + switch (id) { + case OLD_ID: + original = null as unknown as DockerContainerInspect; + break; + case NEW_ID: + replacement = null; + break; + } return options.lostAcks ? { status: 1, stderr: "lost rm acknowledgement" } : ok(); }), runCaptureOpenshell: vi.fn(() => `Name: alpha\nID: ${options.ownerId ?? "sandbox-alpha"}\n`), From a5de8609afc403a9ee9a61fabebe94be7608f7e5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 20:39:36 -0700 Subject: [PATCH 065/117] feat(onboard): bind managed create to runtime providers Signed-off-by: Aaron Erickson --- .../adapters/openshell/sandbox-identity.ts | 16 + src/lib/onboard.ts | 43 ++- .../sandbox-gpu-create-flow.ts | 2 + .../docker-gpu-local-inference.test.ts | 50 ++- src/lib/onboard/docker-gpu-local-inference.ts | 54 ++- ...ocker-gpu-sandbox-create-lifecycle.test.ts | 22 +- src/lib/onboard/docker-gpu-sandbox-create.ts | 339 +++++++++++++----- ...ker-startup-command-sandbox-create.test.ts | 103 +++++- src/lib/onboard/managed-bootstrap/README.md | 28 +- .../managed-bootstrap/docker-runtime.ts | 294 +++++++++++++++ src/lib/onboard/managed-bootstrap/index.ts | 5 + .../managed-bootstrap/runtime-create.ts | 146 ++++++++ src/lib/onboard/runtime-provider/contract.ts | 13 +- src/lib/onboard/runtime-provider/registry.ts | 3 +- .../runtime-provider-contract.test.ts | 76 ++++ src/lib/onboard/sandbox-create-launch.test.ts | 46 +++ src/lib/onboard/sandbox-create-launch.ts | 29 +- .../onboard/sandbox-gpu-create-flow.test.ts | 154 ++++++++ src/lib/onboard/sandbox-gpu-create-flow.ts | 120 +++++-- .../onboard/sandbox-gpu-create-run-attempt.ts | 302 ++++++++++++---- 20 files changed, 1572 insertions(+), 273 deletions(-) create mode 100644 src/lib/onboard/managed-bootstrap/docker-runtime.ts create mode 100644 src/lib/onboard/managed-bootstrap/runtime-create.ts diff --git a/src/lib/adapters/openshell/sandbox-identity.ts b/src/lib/adapters/openshell/sandbox-identity.ts index 1820a8f8f7d..dbcc47696c4 100644 --- a/src/lib/adapters/openshell/sandbox-identity.ts +++ b/src/lib/adapters/openshell/sandbox-identity.ts @@ -14,3 +14,19 @@ export function parseOpenShellSandboxId(output: string): string | null { ? (matches[0] as string) : null; } + +export function resolveOpenShellSandboxId( + sandboxName: string, + runCaptureOpenshell: (args: string[], options?: Record) => string, +): string { + const output = runCaptureOpenshell(["sandbox", "get", sandboxName], { + ignoreError: false, + }); + const sandboxId = parseOpenShellSandboxId(output); + if (!sandboxId) { + throw new Error( + `OpenShell sandbox '${sandboxName}' did not return one exact durable sandbox ID.`, + ); + } + return sandboxId; +} diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 7b0e01c02b1..1c5b2b7597c 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2712,7 +2712,7 @@ async function createSandboxWithBaseImageResolution( recreateRuntime.advance("creating"); const { createResult, - dockerGpuCreatePatch, + runtimePatch, route: selectedGpuRoute, firstCreateOutput, registryImageRef, @@ -2768,20 +2768,33 @@ async function createSandboxWithBaseImageResolution( } if (effectiveSandboxGpuConfig.sandboxGpuEnabled) { - dockerGpuLocalInference.verifyGpuSandboxLocalInferenceAfterReady( - effectiveSandboxGpuConfig, - provider, - { - sandboxName, - dockerDriverGateway, - selectedRoute: selectedGpuRoute, - verifyDirectSandboxGpu, - verifyGpuOrExit: dockerGpuCreatePatch.verifyGpuOrExit, - selectedMode: dockerGpuCreatePatch.selectedMode, - runCaptureOpenshell, - log: console.log, - }, - ); + try { + dockerGpuLocalInference.verifyGpuSandboxLocalInferenceAfterReady( + effectiveSandboxGpuConfig, + provider, + { + sandboxName, + dockerDriverGateway, + selectedRoute: selectedGpuRoute, + verifyDirectSandboxGpu, + verifyGpuOrExit: runtimePatch.verifyGpuOrExit, + selectedMode: runtimePatch.selectedMode, + runCaptureOpenshell, + log: console.log, + }, + ); + await runtimePatch.commitAfterReady(); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + try { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); + } catch (rollbackError) { + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + throw failure; + } } let actualDashboardPort = 0; diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index 7061252f12c..eebacbacd3d 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -64,8 +64,10 @@ export function createGpuPatchFixture() { maybeApplyDuringCreate: vi.fn(), createFailureMessage: vi.fn(() => null), exitOnPatchError: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), ensureApplied: vi.fn(), waitForSupervisorReconnectIfNeeded: vi.fn(), + commitAfterReady: vi.fn(), selectedMode: vi.fn(() => null), printReadinessFailureIfEnabled: vi.fn(), verifyGpuOrExit: vi.fn(() => VERIFIED_GPU_PROOF), diff --git a/src/lib/onboard/docker-gpu-local-inference.test.ts b/src/lib/onboard/docker-gpu-local-inference.test.ts index 60dfe28c639..3d72de11bee 100644 --- a/src/lib/onboard/docker-gpu-local-inference.test.ts +++ b/src/lib/onboard/docker-gpu-local-inference.test.ts @@ -311,10 +311,10 @@ describe("verifyGpuSandboxAfterReady", () => { }; } - it("runs the GPU proof and the runtime inference gate when the patch is active", () => { + it("runs the GPU proof and the runtime inference gate when the patch is active", async () => { const log = vi.fn(); const verifyDirectSandboxGpu = vi.fn(); - verifyGpuSandboxAfterReady( + await verifyGpuSandboxAfterReady( GPU_CONFIG, "vllm-local", baseOptions({ @@ -327,12 +327,12 @@ describe("verifyGpuSandboxAfterReady", () => { expect(log).toHaveBeenCalledWith(expect.stringContaining("reached local inference")); }); - it("captures the CUDA-usability proof onto the config for status persistence (#4231)", () => { + it("captures the CUDA-usability proof onto the config for status persistence (#4231)", async () => { const proof = { status: "verified" as const, cudaVerified: true, at: "t" }; const config: { sandboxGpuEnabled: boolean; sandboxGpuProof?: typeof proof | null } = { sandboxGpuEnabled: true, }; - verifyGpuSandboxAfterReady( + await verifyGpuSandboxAfterReady( config, "vllm-local", baseOptions({ @@ -343,45 +343,37 @@ describe("verifyGpuSandboxAfterReady", () => { expect(config.sandboxGpuProof).toEqual(proof); }); - it("does not duplicate proof diagnostics when Docker GPU patch verifier handles them", () => { + it("does not duplicate proof diagnostics when Docker GPU patch verifier handles them", async () => { const proofError = new Error("process.exit"); const verifyGpuOrExit = vi.fn(() => { throw proofError; }); const logError = vi.fn(); - expect(() => + await expect( verifyGpuSandboxAfterReady( GPU_CONFIG, "ollama-local", baseOptions({ verifyGpuOrExit, logError }), ), - ).toThrow(proofError); + ).rejects.toBe(proofError); expect(logError).not.toHaveBeenCalled(); }); - it("routes failure diagnostics through the provided error sink and exits", () => { + it("routes failure diagnostics through the provided error sink and throws for rollback", async () => { const logError = vi.fn(); - const exitSpy = vi.spyOn(process, "exit").mockImplementation((() => { - throw new Error("process.exit"); - }) as never); - try { - expect(() => - verifyGpuSandboxAfterReady( - GPU_CONFIG, - "ollama-local", - baseOptions({ - logError, - deps: { execInSandbox: execEmitting("HTTP_000"), sleep: vi.fn() }, - }), - ), - ).toThrow("process.exit"); - expect(exitSpy).toHaveBeenCalledWith(1); - expect(logError).toHaveBeenCalledWith( - expect.stringContaining("Local inference reachability check failed"), - ); - } finally { - exitSpy.mockRestore(); - } + await expect( + verifyGpuSandboxAfterReady( + GPU_CONFIG, + "ollama-local", + baseOptions({ + logError, + deps: { execInSandbox: execEmitting("HTTP_000"), sleep: vi.fn() }, + }), + ), + ).rejects.toThrow("GPU sandbox local inference reachability failed"); + expect(logError).toHaveBeenCalledWith( + expect.stringContaining("Local inference reachability check failed"), + ); }); }); diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index da58f8c44d8..447ac8069ca 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -11,6 +11,7 @@ import { import type { DockerGpuPatchMode } from "./docker-gpu-patch-types"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; +import type { ManagedBootstrapRuntimePatch } from "./managed-bootstrap/runtime-create"; import { executeSandboxCommandForVerification } from "./sandbox-verification-exec"; const { @@ -385,9 +386,9 @@ export type GpuSandboxAfterReadyOptions = { verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult; verifyGpuOrExit?: ( verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult, - ) => SandboxGpuProofResult; + ) => Promise; reportGpuProofFailure?: boolean; - selectedMode: () => DockerGpuPatchMode | null; + selectedMode: ManagedBootstrapRuntimePatch["selectedMode"]; runCaptureOpenshell: (args: string[], opts?: Record) => string; env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform; @@ -396,33 +397,47 @@ export type GpuSandboxAfterReadyOptions = { deps?: DockerGpuSandboxInferenceVerifyDeps; }; +function asDockerGpuPatchMode( + selected: ReturnType, +): DockerGpuPatchMode | null { + if (!selected || !["gpus", "nvidia-runtime", "cdi", "startup-command"].includes(selected.kind)) { + return null; + } + return { + kind: selected.kind as DockerGpuPatchMode["kind"], + label: selected.label, + device: selected.device, + args: [...selected.args], + }; +} + /** * Post-readiness GPU sandbox verification orchestrator (kept out of the * ~12k-line onboard.ts entrypoint per the codebase-growth guardrail). Runs the * direct GPU proof, then — only when the Docker GPU patch is active for a local * inference provider — gates on local inference reachability from the sandbox - * runtime (#4509). Exits the process with actionable output if either proof - * fails. + * runtime (#4509). Throws with actionable output if either proof fails so the + * caller can complete rollback before selecting a terminal exit status. */ -export function verifyGpuSandboxAfterReady( +export async function verifyGpuSandboxAfterReady( config: DockerGpuLocalInferenceConfig, provider: string | null | undefined, options: GpuSandboxAfterReadyOptions, -): void { - verifyGpuSandboxAccessAfterReady(config, options); +): Promise { + await verifyGpuSandboxAccessAfterReady(config, options); verifyGpuSandboxLocalInferenceAfterReady(config, provider, options); } -export function verifyGpuSandboxAccessAfterReady( +export async function verifyGpuSandboxAccessAfterReady( config: DockerGpuLocalInferenceConfig, options: GpuSandboxAfterReadyOptions, -): SandboxGpuProofResult { +): Promise { try { // Capture the CUDA-usability proof result and write it back onto the shared // config so onboarding can persist it to the registry and `status` can // report proven usability rather than mere configuration (#4231). const proof = options.verifyGpuOrExit - ? options.verifyGpuOrExit(options.verifyDirectSandboxGpu) + ? await options.verifyGpuOrExit(options.verifyDirectSandboxGpu) : options.verifyDirectSandboxGpu(options.sandboxName); config.sandboxGpuProof = proof; return proof; @@ -431,11 +446,16 @@ export function verifyGpuSandboxAccessAfterReady( // prints the richer Error-phase / patched-container diagnostics before // rethrowing. Avoid a second generic proof-failure block in that path. if (!options.verifyGpuOrExit && options.reportGpuProofFailure !== false) { - printDockerGpuProofFailure(options.sandboxName, error, options.selectedMode(), { - runCaptureOpenshell: options.runCaptureOpenshell, - additionalSummaryLines: adaptDockerGpuRouteForPatch(options.selectedRoute) - .additionalSummaryLines, - }); + printDockerGpuProofFailure( + options.sandboxName, + error, + asDockerGpuPatchMode(options.selectedMode()), + { + runCaptureOpenshell: options.runCaptureOpenshell, + additionalSummaryLines: adaptDockerGpuRouteForPatch(options.selectedRoute) + .additionalSummaryLines, + }, + ); } throw error; } @@ -469,6 +489,8 @@ export function verifyGpuSandboxLocalInferenceAfterReady( verification, options.logError ?? ((message) => console.error(message)), ); - process.exit(1); + throw new Error( + `GPU sandbox local inference reachability failed for ${verification.endpoint}.`, + ); } } diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index fb94eba175e..262cd25d109 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -42,7 +42,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { vi.restoreAllMocks(); }); - it("defers backup removal until waitForSupervisorReconnectIfNeeded sees supervisorReady=true", () => { + it("retains the backup after reconnect and removes it only after post-Ready commit", async () => { const deps = makeDeps(); const result = deferredCreateResult(); const recreatePatch = vi.fn(() => result); @@ -84,13 +84,16 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.waitForSupervisorReconnectIfNeeded(); expect(waitForSupervisor).toHaveBeenCalledTimes(1); + expect(finalizeBackup).not.toHaveBeenCalled(); + + await patch.commitAfterReady(); expect(finalizeBackup).toHaveBeenCalledTimes(1); expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: true }, deps); expect(capturePreRollbackDiagnostics).not.toHaveBeenCalled(); expect(onPatchFailureExit).not.toHaveBeenCalled(); }); - it("refuses compatibility success when the backup container cannot be removed", () => { + it("refuses compatibility success when the backup container cannot be removed", async () => { const deps = makeDeps(); const result = deferredCreateResult(); const onPatchFailureExit = vi.fn(); @@ -113,11 +116,14 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); patch.waitForSupervisorReconnectIfNeeded(); + expect(onPatchFailureExit).not.toHaveBeenCalled(); + + await patch.commitAfterReady(); expect(onPatchFailureExit).toHaveBeenCalledOnce(); expect(onPatchFailureExit.mock.calls[0]?.[1]).toEqual( expect.objectContaining({ - message: expect.stringContaining("backup container"), + message: expect.stringContaining("rollback backup"), }), ); expect(onPatchFailureExit.mock.calls[0]?.[2]).toEqual( @@ -245,7 +251,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(onPatchFailureExit).not.toHaveBeenCalled(); }); - it("records patchError when recreate throws and exitOnPatchError reports it via printDockerGpuPatchFailureAndExit", () => { + it("records patchError when recreate throws and exitOnPatchError reports it via printDockerGpuPatchFailureAndExit", async () => { const deps = makeDeps(); const recreatePatch = vi.fn(() => { throw new Error("docker rename failed"); @@ -271,7 +277,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); expect(patch.createFailureMessage()).toMatch(/Docker GPU patch failed/); - patch.exitOnPatchError(); + await patch.exitOnPatchError(); expect(onPatchFailureExit).toHaveBeenCalledTimes(1); // Supervisor wait must be skipped because needsSupervisorWait stayed false. patch.waitForSupervisorReconnectIfNeeded(); @@ -279,7 +285,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(finalizeBackup).not.toHaveBeenCalled(); }); - it("hard-stops a structured failed GPU proof on the compatibility route", () => { + it("hard-stops a structured failed GPU proof on the compatibility route", async () => { const deps = makeDeps(); const patch = createDockerGpuSandboxCreatePatch({ route: "compatibility", @@ -291,7 +297,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { }, }); - expect(() => + await expect( patch.verifyGpuOrExit(() => ({ status: "failed", cudaVerified: false, @@ -299,6 +305,6 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { detail: "No devices were found", at: "2026-07-07T00:00:00.000Z", })), - ).toThrow("Sandbox GPU proof returned failed status: nvidia-smi when available"); + ).rejects.toThrow("Sandbox GPU proof returned failed status: nvidia-smi when available"); }); }); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index b683cbb1c77..ed5f8dcf74e 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -42,7 +42,7 @@ export { type DockerGpuSandboxCreateDeps = Pick< DockerGpuPatchDeps, - "runOpenshell" | "runCaptureOpenshell" | "sleep" | "dockerCapture" + "runOpenshell" | "runCaptureOpenshell" | "sleep" | "dockerCapture" | "dockerRun" | "dockerStop" >; type WaitSupervisorFn = typeof waitForOpenShellSupervisorReconnect; @@ -61,6 +61,11 @@ type PatchFailureExitFn = ( type DockerGpuSandboxCreatePatchOptions = { route: SelectedDockerGpuRoute; persistStartupCommand?: boolean; + /** + * A managed bootstrap owns the one permitted recreation after Ready. Keep + * route diagnostics/proof active without running the legacy recreator. + */ + externalRecreation?: boolean; sandboxName: string; gpuDevice?: string | null; openshellSandboxCommand?: readonly string[] | null; @@ -91,12 +96,26 @@ type DockerGpuSandboxCreatePatchOptions = { }; }; +export interface DockerManagedBootstrapDeferredCutover { + readonly selectedMode: DockerGpuPatchMode; + readonly failureContext: DockerGpuPatchFailureContext; + rollback(): Promise; + commit(): Promise; +} + export type DockerGpuSandboxCreatePatch = { maybeApplyDuringCreate: () => void; createFailureMessage: () => string | null; - exitOnPatchError: () => void; - ensureApplied: () => void; + exitOnPatchError: () => Promise; + attachManagedBootstrapCutover: (cutover: DockerManagedBootstrapDeferredCutover) => void; + rollbackManagedStartupAfterCreateFailure: () => Promise; + ensureApplied: () => Promise; waitForSupervisorReconnectIfNeeded: () => void; + /** + * Irreversibly commit managed shared state and remove any recreation backup. + * Call only after the authoritative Ready gate and required GPU proof pass. + */ + commitAfterReady: () => Promise; selectedMode: () => DockerGpuPatchMode | null; /** * Print the Docker GPU readiness-failure block (including the Error-phase @@ -106,14 +125,14 @@ export type DockerGpuSandboxCreatePatch = { printReadinessFailureIfEnabled: () => void; /** * Run the GPU proof while distinguishing "sandbox in terminal phase" from - * "proof failed inside a live sandbox". Calls `process.exit(1)` for the - * former and rethrows after printing diagnostics for the latter so the - * onboarding flow surfaces the right failure cause (#4316). Returns the - * CUDA-usability proof result on success so callers can persist it (#4231). + * "proof failed inside a live sandbox". Awaits rollback and throws after + * printing diagnostics so the onboarding flow can select the terminal exit + * status without racing the rollback (#4316). Returns the CUDA-usability + * proof result on success so callers can persist it (#4231). */ verifyGpuOrExit: ( verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult, - ) => SandboxGpuProofResult; + ) => Promise; }; export function createDockerGpuSandboxCreatePatch( @@ -121,8 +140,12 @@ export function createDockerGpuSandboxCreatePatch( ): DockerGpuSandboxCreatePatch { const routeAdapter = adaptDockerGpuRouteForPatch(options.route); let result: DockerGpuPatchResult | null = null; + let managedBootstrapCutover: DockerManagedBootstrapDeferredCutover | null = null; let patchError: unknown = null; let needsSupervisorWait = false; + let cutoverFinalized = false; + let cutoverFinalization: Promise | null = null; + let cutoverFinalizationOutcome: "commit" | "rollback" | null = null; const findContainerIds = options.overrides?.findContainerIds ?? findOpenShellDockerSandboxContainerIds; @@ -145,7 +168,10 @@ export function createDockerGpuSandboxCreatePatch( backend: options.backend, dockerDesktopWsl: options.dockerDesktopWsl ?? isDockerDesktopWslRuntime(), }; - const patchEnabled = routeAdapter.enabled || options.persistStartupCommand === true; + const recreationEnabled = + options.externalRecreation !== true && + (routeAdapter.enabled || options.persistStartupCommand === true); + const patchEnabled = recreationEnabled; const patchTarget = routeAdapter.enabled ? "NVIDIA GPU access" : "restart-safe startup"; const recreateSelectedPatch = createDockerSandboxRecreator({ gpuEnabled: routeAdapter.enabled, @@ -156,21 +182,85 @@ export function createDockerGpuSandboxCreatePatch( recreateStartup: recreateStartupPatch, }); + const applyPatch = (deps: DockerGpuPatchDeps): void => { + if (!recreationEnabled) return; + result = recreateSelectedPatch(false, deps); + needsSupervisorWait = true; + console.log(` ✓ Docker container mode selected: ${result.mode.label}`); + }; + + const rollbackAfterFailure = async (): Promise => { + if (cutoverFinalized || (!managedBootstrapCutover && !result)) return null; + if (cutoverFinalization) { + try { + if (cutoverFinalizationOutcome !== "rollback") { + throw new Error("Managed startup rollback raced an in-progress commit finalization."); + } + await cutoverFinalization; + return null; + } catch (error) { + return error instanceof Error ? error : new Error(String(error)); + } + } + const finalization = (async () => { + await managedBootstrapCutover?.rollback(); + if (result) finalizeBackup({ result, supervisorReady: false }, options.deps); + cutoverFinalized = true; + needsSupervisorWait = false; + })(); + cutoverFinalization = finalization; + cutoverFinalizationOutcome = "rollback"; + try { + await finalization; + return null; + } catch (error) { + return error instanceof Error ? error : new Error(String(error)); + } finally { + if (!cutoverFinalized) { + cutoverFinalization = null; + cutoverFinalizationOutcome = null; + } + } + }; + + const reportPatchErrorAndExit = async (): Promise => { + if (!patchError) return; + const rollbackError = await rollbackAfterFailure(); + if (rollbackError) { + patchError = new Error( + `${patchError instanceof Error ? patchError.message : String(patchError)}; managed startup rollback failed: ${rollbackError.message}`, + ); + } + onPatchFailureExit(options.sandboxName, patchError, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }); + }; + const selectedMode = (): DockerGpuPatchMode | null => + managedBootstrapCutover?.selectedMode ?? result?.mode ?? null; + const failureContext = (): DockerGpuPatchFailureContext => + managedBootstrapCutover?.failureContext ?? buildFailureContext(options.sandboxName, result); + return { maybeApplyDuringCreate() { if (!patchEnabled || result || patchError) return; const containerIds = findContainerIds(options.sandboxName); if (containerIds.length === 0) return; + if (containerIds.length !== 1) { + patchError = new Error( + `Docker recreation observed ${String(containerIds.length)} matching containers; refusing an ambiguous replacement.`, + ); + return; + } console.log( - ` OpenShell Docker container detected; recreating it with ${patchTarget} before readiness wait...`, + ` OpenShell Docker container detected; applying ${patchTarget} before readiness wait...`, ); try { - result = recreateSelectedPatch(false, { + applyPatch({ runCaptureOpenshell: options.deps.runCaptureOpenshell, sleep: options.deps.sleep, }); - needsSupervisorWait = true; - console.log(` ✓ Docker container mode selected: ${result.mode.label}`); } catch (error) { patchError = error; } @@ -183,33 +273,40 @@ export function createDockerGpuSandboxCreatePatch( : "Docker startup-command patch failed while OpenShell sandbox create was still waiting."; }, - exitOnPatchError() { - if (!patchError) return; - onPatchFailureExit(options.sandboxName, patchError, { + async exitOnPatchError() { + await reportPatchErrorAndExit(); + }, + + attachManagedBootstrapCutover(cutover) { + if (managedBootstrapCutover || result || cutoverFinalized) { + throw new Error("Managed bootstrap cutover may be attached exactly once."); + } + managedBootstrapCutover = cutover; + }, + + async rollbackManagedStartupAfterCreateFailure() { + const rollbackError = await rollbackAfterFailure(); + if (!rollbackError) return; + onPatchFailureExit(options.sandboxName, rollbackError, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, additionalSummaryLines: routeAdapter.additionalSummaryLines, }); }, - ensureApplied() { + async ensureApplied() { if (!patchEnabled || result) return; - console.log(` Recreating OpenShell Docker sandbox container with ${patchTarget}...`); + console.log(` Applying ${patchTarget} to the OpenShell Docker sandbox...`); try { - result = recreateSelectedPatch(false, options.deps); - needsSupervisorWait = true; - console.log(` ✓ Docker container mode selected: ${result.mode.label}`); + applyPatch(options.deps); } catch (error) { - onPatchFailureExit(options.sandboxName, error, { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - }); + patchError = error; + await reportPatchErrorAndExit(); } }, waitForSupervisorReconnectIfNeeded() { - if (!needsSupervisorWait) return; + if (!needsSupervisorWait || cutoverFinalized) return; const supervisorReconnectTimeoutSecs = getDockerGpuSupervisorReconnectTimeoutSecs( options.timeoutSecs, ); @@ -221,14 +318,17 @@ export function createDockerGpuSandboxCreatePatch( supervisorReconnectTimeoutSecs, { runOpenshell: options.deps.runOpenshell, - // Pass `runCaptureOpenshell` so the supervisor-reconnect wait can - // short-circuit on a terminal sandbox phase instead of burning - // the full reconnect timeout window when the patched container - // crashed on startup (#4316). runCaptureOpenshell: options.deps.runCaptureOpenshell, sleep: options.deps.sleep, }, ); + if (supervisorReady) { + // Reconnect is necessary but not sufficient for cutover. Keep both the + // managed shared-state receipt and recreation backup until the caller + // accepts authoritative Ready and any required GPU proof. + needsSupervisorWait = false; + return; + } if (!supervisorReady && result) { try { captureFailedClone(options.sandboxName, result, options.deps); @@ -239,40 +339,13 @@ export function createDockerGpuSandboxCreatePatch( } } const finalizeOutcome = result - ? finalizeBackup({ result, supervisorReady }, options.deps) + ? finalizeBackup({ result, supervisorReady: false }, options.deps) : null; - if (supervisorReady) { - if (finalizeOutcome && !finalizeOutcome.backupRemoved) { - onPatchFailureExit( - options.sandboxName, - new Error( - "OpenShell supervisor reconnected, but the recreated backup container could not be removed.", - ), - { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - context: { - sandboxName: options.sandboxName, - oldContainerId: result?.oldContainerId, - newContainerId: result?.newContainerId, - backupContainerName: result?.backupContainerName, - selectedMode: result?.mode ?? null, - rolledBack: false, - }, - }, - ); - } - return; - } - const failureMessage = (() => { - if (!finalizeOutcome) { - return "OpenShell supervisor did not reconnect to the recreated container."; - } - return finalizeOutcome.rolledBack - ? "OpenShell supervisor did not reconnect to the recreated container; pre-patch sandbox restored." - : "OpenShell supervisor did not reconnect to the recreated container and rollback failed; pre-patch sandbox was NOT restored."; - })(); + cutoverFinalized = true; + needsSupervisorWait = false; + const failureMessage = finalizeOutcome?.rolledBack + ? "OpenShell supervisor did not reconnect to the recreated container; pre-patch sandbox restored." + : "OpenShell supervisor did not reconnect to the recreated container and rollback failed; pre-patch sandbox was NOT restored."; onPatchFailureExit(options.sandboxName, new Error(failureMessage), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, @@ -288,21 +361,108 @@ export function createDockerGpuSandboxCreatePatch( }); }, + async commitAfterReady() { + if (cutoverFinalized || (!managedBootstrapCutover && !result)) return; + if (needsSupervisorWait) { + const error = new Error( + "Managed startup cannot commit before the recreated OpenShell supervisor reconnects.", + ); + const rollbackError = await rollbackAfterFailure(); + onPatchFailureExit( + options.sandboxName, + rollbackError + ? new Error(`${error.message} Rollback failed: ${rollbackError.message}`) + : error, + { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }, + ); + return; + } + if (cutoverFinalization) { + if (cutoverFinalizationOutcome !== "commit") { + throw new Error("Managed startup commit raced an in-progress rollback finalization."); + } + await cutoverFinalization; + return; + } + const finalization = (async () => { + if (managedBootstrapCutover) { + try { + await managedBootstrapCutover.commit(); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + let rollbackError: Error | null = null; + try { + await managedBootstrapCutover.rollback(); + cutoverFinalized = true; + needsSupervisorWait = false; + } catch (rollbackFailure) { + rollbackError = + rollbackFailure instanceof Error + ? rollbackFailure + : new Error(String(rollbackFailure)); + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + onPatchFailureExit(options.sandboxName, failure, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: { + ...failureContext(), + rolledBack: rollbackError === null, + }, + }); + return; + } + } + const finalizeOutcome = result + ? finalizeBackup({ result, supervisorReady: true }, options.deps) + : null; + cutoverFinalized = true; + if (!finalizeOutcome || finalizeOutcome.backupRemoved) return; + onPatchFailureExit( + options.sandboxName, + new Error("Managed startup passed Ready, but its rollback backup could not be removed."), + { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: failureContext(), + }, + ); + })(); + cutoverFinalization = finalization; + cutoverFinalizationOutcome = "commit"; + try { + await finalization; + } finally { + if (!cutoverFinalized) { + cutoverFinalization = null; + cutoverFinalizationOutcome = null; + } + } + }, + selectedMode() { - return result?.mode ?? null; + return selectedMode(); }, printReadinessFailureIfEnabled() { if (!routeAdapter.enabled) return; - printDockerGpuReadinessFailure(options.sandboxName, result?.mode ?? null, { + printDockerGpuReadinessFailure(options.sandboxName, selectedMode(), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, - context: buildFailureContext(options.sandboxName, result), + context: failureContext(), additionalSummaryLines: routeAdapter.additionalSummaryLines, }); }, - verifyGpuOrExit(verifyDirectSandboxGpu) { + async verifyGpuOrExit(verifyDirectSandboxGpu) { // Before issuing GPU proof commands through `openshell sandbox exec`, // confirm the sandbox is still in a live phase. A sandbox that // transitioned to Error after the readiness wait succeeded (e.g. the @@ -312,7 +472,7 @@ export function createDockerGpuSandboxCreatePatch( // container/Error-phase classification instead of running the proof // (#4316). const sandboxName = options.sandboxName; - const failureContext = buildFailureContext(sandboxName, result); + const currentFailureContext = failureContext(); if (routeAdapter.enabled && options.deps.runCaptureOpenshell) { const list = options.deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true, @@ -321,20 +481,22 @@ export function createDockerGpuSandboxCreatePatch( if (phase) { console.error(""); console.error(` Skipping GPU proof: sandbox '${sandboxName}' is in ${phase} phase.`); - printDockerGpuProofFailure( - sandboxName, - new Error( - `Sandbox '${sandboxName}' entered ${phase} phase after readiness; GPU proof skipped.`, - ), - result?.mode ?? null, - { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - context: failureContext, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - }, + const failure = new Error( + `Sandbox '${sandboxName}' entered ${phase} phase after readiness; GPU proof skipped.`, ); - process.exit(1); + printDockerGpuProofFailure(sandboxName, failure, selectedMode(), { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + context: currentFailureContext, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }); + const rollbackError = await rollbackAfterFailure(); + if (rollbackError) { + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + throw failure; } } try { @@ -346,13 +508,20 @@ export function createDockerGpuSandboxCreatePatch( } return proof; } catch (error) { - printDockerGpuProofFailure(sandboxName, error, result?.mode ?? null, { + const failure = error instanceof Error ? error : new Error(String(error)); + printDockerGpuProofFailure(sandboxName, failure, selectedMode(), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, - context: routeAdapter.enabled ? failureContext : null, + context: routeAdapter.enabled ? currentFailureContext : null, additionalSummaryLines: routeAdapter.additionalSummaryLines, }); - throw error; + const rollbackError = await rollbackAfterFailure(); + if (rollbackError) { + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + throw failure; } }, }; diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts index 17cd02f6432..ec0ad3cb306 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts @@ -68,7 +68,7 @@ describe("Docker startup-command sandbox creation", () => { vi.restoreAllMocks(); }); - it("uses the startup-command recreation path with DCode's exact resource limits", () => { + it("uses the startup-command recreation path with DCode's exact resource limits", async () => { const dockerCaptureOutput: Record = { ps: "old-container-id\n", inspect: JSON.stringify([inspectFixture()]), @@ -102,7 +102,7 @@ describe("Docker startup-command sandbox creation", () => { }, }); - patch.ensureApplied(); + await patch.ensureApplied(); expect(recreatePatch).not.toHaveBeenCalled(); expect(dockerRunDetached.mock.calls[0]?.[0]).toEqual( @@ -156,7 +156,102 @@ describe("Docker startup-command sandbox creation", () => { expect(context.rolledBack).toBe(true); }); - it("reports startup-command creation failures through the composed patch boundary", () => { + it("defers a driver-owned managed cutover until the authoritative caller commits", async () => { + const deps = makeDeps(); + let releaseCommit = () => {}; + const commit = vi.fn( + () => + new Promise((resolve) => { + releaseCommit = resolve; + }), + ); + const rollback = vi.fn(async () => {}); + const patch = createDockerGpuSandboxCreatePatch({ + route: "native", + externalRecreation: true, + sandboxName: "alpha", + timeoutSecs: 60, + deps, + }); + patch.attachManagedBootstrapCutover({ + selectedMode: { + kind: "startup-command", + label: "managed bootstrap", + device: "", + args: [], + }, + failureContext: { sandboxName: "alpha" }, + commit, + rollback, + }); + patch.maybeApplyDuringCreate(); + await patch.ensureApplied(); + patch.waitForSupervisorReconnectIfNeeded(); + expect(commit).not.toHaveBeenCalled(); + const firstCommit = patch.commitAfterReady(); + const duplicateCommit = patch.commitAfterReady(); + expect(commit).toHaveBeenCalledOnce(); + expect(rollback).not.toHaveBeenCalled(); + releaseCommit(); + await Promise.all([firstCommit, duplicateCommit]); + }); + + it("rolls back a driver-owned cutover before reporting commit failure", async () => { + const deps = makeDeps(); + const events: string[] = []; + const commit = vi.fn(async () => { + events.push("commit"); + throw new Error("receipt validation failed"); + }); + const rollback = vi.fn(async () => { + events.push("rollback"); + }); + const onPatchFailureExit = vi.fn(() => { + events.push("exit"); + }); + const patch = createDockerGpuSandboxCreatePatch({ + route: "native", + externalRecreation: true, + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { onPatchFailureExit }, + }); + patch.attachManagedBootstrapCutover({ + selectedMode: { + kind: "startup-command", + label: "managed bootstrap", + device: "", + args: [], + }, + failureContext: { + sandboxName: "alpha", + oldContainerId: "held-container", + newContainerId: "replacement-container", + }, + commit, + rollback, + }); + + await patch.commitAfterReady(); + + expect(events).toEqual(["commit", "rollback", "exit"]); + expect(onPatchFailureExit).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ message: "receipt validation failed" }), + expect.objectContaining({ + context: expect.objectContaining({ + oldContainerId: "held-container", + newContainerId: "replacement-container", + rolledBack: true, + }), + }), + ); + await patch.rollbackManagedStartupAfterCreateFailure(); + expect(rollback).toHaveBeenCalledOnce(); + }); + + it("reports startup-command creation failures through the composed patch boundary", async () => { const deps = makeDeps(); const onPatchFailureExit = vi.fn(); const patch = createDockerGpuSandboxCreatePatch({ @@ -177,7 +272,7 @@ describe("Docker startup-command sandbox creation", () => { patch.maybeApplyDuringCreate(); expect(patch.createFailureMessage()).toMatch(/startup-command patch failed/); - patch.exitOnPatchError(); + await patch.exitOnPatchError(); expect(onPatchFailureExit).toHaveBeenCalledWith( "alpha", expect.objectContaining({ message: "startup recreate failed" }), diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 61a2efbf8f8..2e94e359a00 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -1,8 +1,9 @@ # Managed bootstrap protocol -This directory defines a dormant, driver-neutral transaction contract and its -first driver adapter. It does not register a runtime provider or change sandbox -creation, onboarding, snapshot, clone, or restore behavior. +This directory defines a dormant, driver-neutral transaction contract, its +first driver adapter, and an injectable sandbox-create lifecycle. Production +runtime bundles still report bootstrap as unsupported, so this integration does +not change onboarding, snapshot, clone, or restore behavior. The protocol binds one random bootstrap identity to: @@ -27,8 +28,9 @@ root-owned request, verifies an identity-bound completion, clears its private bootstrap variables and file descriptors, and then uses `exec "$@"` to preserve the captured supervisor argument boundaries. -The trampoline is intentionally not an entrypoint, and no production TypeScript -module imports this protocol or the Docker adapter. +The trampoline is intentionally not an entrypoint. The launch renderer can +produce its identity-bound hold command only when a caller supplies a managed +startup request. The Docker adapter creates and validates a stopped replacement under an identity-derived staging name while the original remains running. It stages the @@ -45,10 +47,16 @@ adapter assumes the protocol's single coordinator; multi-process lease/arbitration remains an explicit production-activation gate. Activation must also inject the selected gateway's canonical state root. +The runtime-provider bundle is the only bootstrap registration boundary. The +candidate Docker surface owns create routing, replacement construction, +native-to-compatibility fallback evidence, and deferred commit or rollback. +Central onboarding accepts that provider-neutral surface without a Docker or +Podman selection branch. Tests register an MXC-style surface through the same +bundle and render held launches for OpenClaw, Hermes, and DCode. + The current image definitions still do not package `nemoclaw-managed-startup-hold`, `managed-startup-image-runtime.cjs`, or the -shared-state bootstrap modes consumed by this adapter. A later activation slice -must add those prerequisites and wire the coordinator into Docker create as one -boundary. The same contract is exercised for OpenClaw, Hermes, and DCode without -a provider-specific central switch. Until that complete boundary lands, every -registered runtime provider keeps bootstrap unsupported. +shared-state bootstrap modes consumed by this adapter. Later persistence and +qualification slices must add those prerequisites and provide the canonical +durable authority store. Until that complete boundary passes protected E2E, +every production runtime provider keeps bootstrap unsupported. diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.ts new file mode 100644 index 00000000000..21932a2207f --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.ts @@ -0,0 +1,294 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { RuntimeProviderBootstrapSurface } from "../runtime-provider/contract"; +import { detectTegraDeviceGroupGids } from "../docker-gpu-jetson-groups"; +import { buildDockerGpuMode, selectDockerGpuPatchMode } from "../docker-gpu-patch-mode"; +import type { DockerGpuPatchMode } from "../docker-gpu-patch-types"; +import { renderCompatibilityFallbackCreateArgs } from "../docker-gpu-route"; +import { + createDockerGpuSandboxCreatePatch, + isDockerDesktopWslRuntime, +} from "../docker-gpu-sandbox-create"; +import { + isImmutableDockerImageId, + queryOpenShellDockerSandboxContainers, + queryOpenShellDockerSandboxRuntimeSnapshot, +} from "../openshell-docker-sandbox-containers"; +import * as sandboxGpuCreateAttempt from "../sandbox-gpu-create-attempt"; +import { + activateManagedBootstrapSequence, + finalizeManagedBootstrapSequence, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + prepareManagedBootstrapSequence, +} from "./adapter"; +import { createDockerManagedBootstrapAdapter } from "./docker"; +import type { + ManagedBootstrapRuntimeCompatibilityLaunchInput, + ManagedBootstrapRuntimeCreateLaunchResult, + ManagedBootstrapRuntimeCreateLifecycle, + ManagedBootstrapRuntimeCreateLifecycleInput, + ManagedBootstrapRuntimeOnboardRoutingInput, +} from "./runtime-create"; + +type SupportedBootstrapSurface = Extract< + RuntimeProviderBootstrapSurface, + { readonly supported: true } +>; + +function dockerReplacementOptions( + mode: DockerGpuPatchMode, + input: ManagedBootstrapRuntimeCreateLifecycleInput, +) { + const backend = input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic"; + return { + values: { + gpuModeArgs: [...mode.args], + gpuModeDevice: mode.device, + gpuModeKind: mode.kind, + gpuModeLabel: mode.label, + requiredUlimits: input.requiredLimits.map( + (limit) => `${limit.name}=${limit.soft}:${limit.hard}`, + ), + extraGroupGids: + backend === "jetson" && input.route === "compatibility" ? detectTegraDeviceGroupGids() : [], + }, + }; +} + +function selectedDockerMode( + input: ManagedBootstrapRuntimeCreateLifecycleInput, + dockerDesktopWsl: boolean | undefined, +): DockerGpuPatchMode { + const backend = input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic"; + if (input.route !== "compatibility" || !input.sandboxGpuConfig.sandboxGpuEnabled) { + return buildDockerGpuMode("startup-command"); + } + const selection = selectDockerGpuPatchMode( + { + image: `${input.image.repository}@${input.image.manifestDigest}`, + device: input.sandboxGpuConfig.sandboxGpuDevice, + backend, + dockerDesktopWsl, + }, + input.dependencies, + ); + if (selection.mode) return selection.mode; + throw new Error( + backend === "jetson" + ? "Docker did not accept the Jetson NVIDIA runtime GPU mode for managed bootstrap." + : "Docker did not accept a compatibility GPU mode for managed bootstrap.", + ); +} + +function createDockerLifecycle( + providerId: string, + input: ManagedBootstrapRuntimeCreateLifecycleInput, +): ManagedBootstrapRuntimeCreateLifecycle { + if (input.providerId !== providerId) { + throw new Error( + `Managed bootstrap provider '${providerId}' cannot run authority for '${input.providerId}'.`, + ); + } + const dockerDesktopWsl = + input.route === "compatibility" ? isDockerDesktopWslRuntime() : undefined; + const mode = selectedDockerMode(input, dockerDesktopWsl); + const backend = input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic"; + const persistStartupCommand = + input.persistStartupCommand && (input.route !== "native" || input.requiredLimits.length > 0); + const patch = createDockerGpuSandboxCreatePatch({ + route: input.route, + persistStartupCommand, + externalRecreation: true, + sandboxName: input.sandboxName, + gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, + openshellSandboxCommand: input.heldWorkloadArgv, + requiredUlimits: input.requiredLimits, + timeoutSecs: input.timeoutSecs, + backend, + dockerDesktopWsl, + deps: input.dependencies, + ...(input.onPatchFailure + ? { + overrides: { + onPatchFailureExit: (_sandboxName: string, error: unknown) => + input.onPatchFailure?.(error), + }, + } + : {}), + }); + const adapter = input.adapterOverride ?? createDockerManagedBootstrapAdapter(input.dependencies); + const createPlan = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandboxName: input.sandboxName, + driverId: providerId, + image: input.image, + profile: { + agent: input.request.agent, + fingerprint: input.request.profileFingerprint, + }, + agentIdentity: input.agentIdentity, + intendedWorkloadArgv: input.intendedWorkloadArgv, + expectedSupervisorArgv: input.expectedSupervisorArgv, + metadata: {}, + } as const; + const replacementOptions = dockerReplacementOptions(mode, input); + + return { + launchArgv: input.launchArgv, + patch, + async prepareNetwork() { + if (input.route !== "compatibility") return; + const { enforceDockerGpuPatchPreserveNetwork } = await import( + "../docker-gpu-local-inference" + ); + await enforceDockerGpuPatchPreserveNetwork( + input.network.inferenceProvider, + input.sandboxGpuConfig, + { + dockerDriverGateway: input.network.dockerDriverGateway, + selectedRoute: input.route, + gatewayPort: input.network.gatewayPort, + log: console.log, + }, + ); + }, + async runCreate( + launch: (input: { + readonly heldWorkloadArgv: readonly string[]; + readonly bootstrapIdentity: string; + }) => Promise>, + ): Promise { + const launchState: { value?: ManagedBootstrapRuntimeCreateLaunchResult } = {}; + const prepared = await prepareManagedBootstrapSequence(adapter, { + create: { + bootstrapIdentity: input.bootstrapIdentity, + plan: createPlan, + request: input.request, + launch: async (launchInput) => { + const launched = await launch(launchInput); + launchState.value = launched; + return launched.receipt; + }, + }, + request: input.request, + replacementOptions, + }); + const activated = await activateManagedBootstrapSequence(adapter, { + transaction: prepared, + authorityStore: input.authorityStore, + timeoutSecs: input.timeoutSecs, + }); + const launched = launchState.value; + if (!launched) { + await finalizeManagedBootstrapSequence(adapter, { + outcome: "rollback", + transaction: activated, + }); + throw new Error("Managed bootstrap did not return its OpenShell create receipt."); + } + let finalized = false; + patch.attachManagedBootstrapCutover({ + selectedMode: mode, + failureContext: { + sandboxName: input.sandboxName, + oldContainerId: activated.snapshot.runtimeId, + newContainerId: activated.replacement.replacementRuntimeId, + backupContainerName: null, + selectedMode: mode, + }, + async rollback() { + if (finalized) return; + await finalizeManagedBootstrapSequence(adapter, { + outcome: "rollback", + transaction: activated, + }); + finalized = true; + }, + async commit() { + if (finalized) return; + await finalizeManagedBootstrapSequence(adapter, { + outcome: "commit", + transaction: activated, + }); + finalized = true; + }, + }); + return launched.value; + }, + }; +} + +function createDockerOnboardRouting(input: ManagedBootstrapRuntimeOnboardRoutingInput) { + const baseline = input.nativeFallbackEnabled + ? queryOpenShellDockerSandboxContainers(input.sandboxName) + : null; + const inspectNativeRuntime = () => { + const snapshot = queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName); + return snapshot.ok + ? { + imageId: snapshot.imageId, + bookkeepingImageRef: snapshot.bookkeepingImageRef, + stateError: snapshot.stateError, + nativeGpuAttachmentState: snapshot.nativeGpuAttachmentState, + } + : null; + }; + return { + nativeFallbackHasCleanBaseline: baseline?.ok === true && baseline.ids.length === 0, + inspectNativeRuntime, + isNativeCreateRoutingFailure: (output: string, sawProgress: boolean): boolean => + sandboxGpuCreateAttempt.isNativeGpuCreateRoutingFailure(output, { sawProgress }), + isTrustedNativeRuntimeError: (error: string): boolean => + sandboxGpuCreateAttempt.isTrustedNativeGpuRuntimeError(error), + isNativeReadinessRoutingFailure: (failure: { + readonly failurePhase: string | null; + readonly runtimeError: string; + }): boolean => sandboxGpuCreateAttempt.isNativeGpuReadinessRoutingFailure(failure), + prepareCompatibilityLaunch: ( + compatibility: ManagedBootstrapRuntimeCompatibilityLaunchInput, + ) => { + const runtime = compatibility.runtimeSnapshot; + const imageId = + runtime?.imageId ?? + (compatibility.prebuildImageId && isImmutableDockerImageId(compatibility.prebuildImageId) + ? compatibility.prebuildImageId.toLowerCase() + : null); + let registryImageRef = compatibility.currentRegistryImageRef; + if ( + !registryImageRef && + runtime?.bookkeepingImageRef && + !isImmutableDockerImageId(runtime.bookkeepingImageRef) + ) { + registryImageRef = runtime.bookkeepingImageRef; + } + const createArgs = renderCompatibilityFallbackCreateArgs(compatibility.createArgs, { + imageRef: imageId, + allowUnbuiltSource: compatibility.allowUnbuiltSource, + compatibilityPolicyPath: compatibility.compatibilityPolicyPath, + }); + return { + createArgv: input.openshellArgv([ + "sandbox", + "create", + ...createArgs, + "--", + ...compatibility.startupCommand, + ]), + registryImageRef, + }; + }, + }; +} + +/** Candidate Docker surface. Production activation remains a later qualification slice. */ +export function createDockerManagedBootstrapSurface( + providerId = "docker", +): SupportedBootstrapSurface { + return { + providerId, + supported: true, + createLifecycle: (input) => createDockerLifecycle(providerId, input), + createOnboardRouting: createDockerOnboardRouting, + }; +} diff --git a/src/lib/onboard/managed-bootstrap/index.ts b/src/lib/onboard/managed-bootstrap/index.ts index 17099572608..5f03ff33572 100644 --- a/src/lib/onboard/managed-bootstrap/index.ts +++ b/src/lib/onboard/managed-bootstrap/index.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +export { resolveOpenShellSandboxId } from "../../adapters/openshell/sandbox-identity"; export { activateManagedBootstrapSequence, finalizeManagedBootstrapSequence, @@ -20,3 +21,7 @@ export { serializeManagedBootstrapEnvelope, serializeManagedBootstrapImageCompletion, } from "./envelope"; +export type { + ManagedBootstrapRuntimeCreateLifecycle, + ManagedBootstrapRuntimePatch, +} from "./runtime-create"; diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.ts b/src/lib/onboard/managed-bootstrap/runtime-create.ts new file mode 100644 index 00000000000..8cb6cad25d7 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/runtime-create.ts @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxGpuProofResult } from "../../state/registry"; +import type { ManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import type { SandboxGpuConfig } from "../sandbox-gpu-mode"; +import type { + ManagedBootstrapAdapter, + ManagedBootstrapAgentIdentity, + ManagedBootstrapAuthorityStore, + ManagedBootstrapCreateReceipt, + ManagedBootstrapImageIdentity, +} from "./adapter"; + +export interface ManagedBootstrapRuntimeCommandResult { + readonly status?: number | null; + readonly stdout?: string | Buffer | null; + readonly stderr?: string | Buffer | null; + readonly error?: Error | null; +} + +export interface ManagedBootstrapRuntimeDependencies { + readonly runCaptureOpenshell?: (args: string[], options?: Record) => string; + readonly runOpenshell?: ( + args: string[], + options?: Record, + ) => ManagedBootstrapRuntimeCommandResult; + readonly sleep?: (seconds: number) => void; +} + +export type ManagedBootstrapRuntimeRoute = "none" | "native" | "compatibility"; + +export interface ManagedBootstrapRuntimeLimit { + readonly name: string; + readonly soft: number; + readonly hard: number; +} + +/** Provider-neutral lifecycle surface consumed by sandbox-create coordinators. */ +export interface ManagedBootstrapRuntimePatch { + maybeApplyDuringCreate(): void | Promise; + createFailureMessage(): string | null; + exitOnPatchError(): void | Promise; + rollbackManagedStartupAfterCreateFailure(): void | Promise; + ensureApplied(): void | Promise; + waitForSupervisorReconnectIfNeeded(): void | Promise; + commitAfterReady(): void | Promise; + selectedMode(): { + readonly kind: string; + readonly label: string; + readonly device: string; + readonly args: readonly string[]; + } | null; + printReadinessFailureIfEnabled(): void; + verifyGpuOrExit( + verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult, + ): Promise; +} + +export interface ManagedBootstrapRuntimeCreateLifecycleInput { + readonly providerId: string; + readonly bootstrapIdentity: string; + readonly request: ManagedStartupRootApplyRequest; + readonly image: ManagedBootstrapImageIdentity; + readonly agentIdentity: ManagedBootstrapAgentIdentity; + readonly intendedWorkloadArgv: readonly string[]; + readonly expectedSupervisorArgv: readonly string[]; + readonly launchArgv: readonly string[]; + readonly heldWorkloadArgv: readonly string[]; + readonly authorityStore: ManagedBootstrapAuthorityStore; + /** Protected tests can wrap the real provider adapter at this boundary. */ + readonly adapterOverride?: ManagedBootstrapAdapter; + readonly route: ManagedBootstrapRuntimeRoute; + readonly persistStartupCommand: boolean; + readonly sandboxName: string; + readonly sandboxGpuConfig: SandboxGpuConfig; + readonly requiredLimits: readonly ManagedBootstrapRuntimeLimit[]; + readonly timeoutSecs: number; + readonly onPatchFailure?: (error: unknown) => never; + readonly network: { + readonly inferenceProvider: string; + readonly dockerDriverGateway: boolean; + readonly gatewayPort: number; + }; + readonly dependencies: ManagedBootstrapRuntimeDependencies; +} + +export interface ManagedBootstrapRuntimeCreateLaunchResult { + readonly value: T; + readonly receipt: ManagedBootstrapCreateReceipt; +} + +export interface ManagedBootstrapRuntimeCreateLifecycle { + readonly launchArgv: readonly string[]; + readonly patch: ManagedBootstrapRuntimePatch; + prepareNetwork(): Promise; + runCreate( + launch: (input: { + readonly heldWorkloadArgv: readonly string[]; + readonly bootstrapIdentity: string; + }) => Promise>, + ): Promise; +} + +export interface ManagedBootstrapRuntimeSnapshot { + readonly imageId: string | null; + readonly bookkeepingImageRef: string | null; + readonly stateError: string; + readonly nativeGpuAttachmentState: "present" | "absent" | "unknown"; +} + +export interface ManagedBootstrapRuntimeCompatibilityLaunchInput { + readonly createArgs: readonly string[]; + readonly currentRegistryImageRef: string | null; + readonly prebuildImageId: string | null; + readonly allowUnbuiltSource: boolean; + readonly compatibilityPolicyPath: string; + readonly startupCommand: readonly string[]; + readonly runtimeSnapshot: ManagedBootstrapRuntimeSnapshot | null; +} + +export interface ManagedBootstrapRuntimeCompatibilityLaunch { + readonly createArgv: readonly string[]; + readonly registryImageRef: string | null; +} + +/** Provider-owned native-to-compatibility evidence and launch preparation. */ +export interface ManagedBootstrapRuntimeOnboardRouting { + readonly nativeFallbackHasCleanBaseline: boolean; + inspectNativeRuntime(): ManagedBootstrapRuntimeSnapshot | null; + isNativeCreateRoutingFailure(output: string, sawProgress: boolean): boolean; + isTrustedNativeRuntimeError(error: string): boolean; + isNativeReadinessRoutingFailure(input: { + readonly failurePhase: string | null; + readonly runtimeError: string; + }): boolean; + prepareCompatibilityLaunch( + input: ManagedBootstrapRuntimeCompatibilityLaunchInput, + ): ManagedBootstrapRuntimeCompatibilityLaunch; +} + +export interface ManagedBootstrapRuntimeOnboardRoutingInput { + readonly sandboxName: string; + readonly openshellArgv: (args: string[]) => string[]; + readonly nativeFallbackEnabled: boolean; +} diff --git a/src/lib/onboard/runtime-provider/contract.ts b/src/lib/onboard/runtime-provider/contract.ts index 8888d95f83a..8938b6244ff 100644 --- a/src/lib/onboard/runtime-provider/contract.ts +++ b/src/lib/onboard/runtime-provider/contract.ts @@ -3,6 +3,12 @@ import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; import type { ManagedImageSelectionPolicy } from "../workload/source"; +import type { + ManagedBootstrapRuntimeCreateLifecycle, + ManagedBootstrapRuntimeCreateLifecycleInput, + ManagedBootstrapRuntimeOnboardRouting, + ManagedBootstrapRuntimeOnboardRoutingInput, +} from "../managed-bootstrap/runtime-create"; export const RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION = 1 as const; export const RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION = 1 as const; @@ -261,7 +267,12 @@ export type RuntimeProviderMutationAuthoritySurface = export type RuntimeProviderBootstrapSurface = | RuntimeProviderSupportedSurface<{ - prepare(sandbox: SandboxEntry): unknown; + createLifecycle( + input: ManagedBootstrapRuntimeCreateLifecycleInput, + ): ManagedBootstrapRuntimeCreateLifecycle; + createOnboardRouting( + input: ManagedBootstrapRuntimeOnboardRoutingInput, + ): ManagedBootstrapRuntimeOnboardRouting; }> | RuntimeProviderUnsupportedSurface; diff --git a/src/lib/onboard/runtime-provider/registry.ts b/src/lib/onboard/runtime-provider/registry.ts index 332792d2dc2..1d0a31ae6aa 100644 --- a/src/lib/onboard/runtime-provider/registry.ts +++ b/src/lib/onboard/runtime-provider/registry.ts @@ -337,7 +337,8 @@ function validateMutationAuthoritySurface( function validateBootstrapSurface(surface: Record): void { if (surface.supported === true) { - requireFunction(surface, "prepare", "bootstrap"); + requireFunction(surface, "createLifecycle", "bootstrap"); + requireFunction(surface, "createOnboardRouting", "bootstrap"); } } diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index bf2a47fe4b4..3ac4da9e4fc 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -21,6 +21,7 @@ import { loadAgent } from "../../agent/defs"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; import { cloneSandboxWorkloadReceipt } from "../../state/registry/workload"; import { MANAGED_IMAGE_REPOSITORIES } from "../managed-image/contract"; +import { createDockerManagedBootstrapSurface } from "../managed-bootstrap/docker-runtime"; import { encodeManagedStartupProfile, type ManagedStartupProfile, @@ -145,6 +146,28 @@ describe("RuntimeProviderBundle registry contract", () => { } }); + it("validates the dormant Docker bootstrap candidate through the same bundle registry", () => { + const docker = createDockerRuntimeProviderBundle(); + const providers = createRuntimeProviderBundleRegistry([ + [ + "docker", + { + ...docker, + bootstrap: createDockerManagedBootstrapSurface(), + }, + ], + ]); + + expect(providers.docker?.bootstrap).toMatchObject({ + providerId: "docker", + supported: true, + }); + expect(CURRENT_RUNTIME_PROVIDER_BUNDLES.docker?.bootstrap).toMatchObject({ + providerId: "docker", + supported: false, + }); + }); + it("deeply clones and freezes every registered nested value", () => { const source = mxcBundle(); const registry = createRuntimeProviderBundleRegistry([["mxc", source]]); @@ -170,6 +193,59 @@ describe("RuntimeProviderBundle registry contract", () => { }).toThrow(TypeError); }); + it("registers an MXC-style managed-bootstrap provider through the bundle surface", () => { + const bundle = mxcBundle(); + const createLifecycle = vi.fn(() => ({ + launchArgv: ["mxc", "create"], + patch: { + maybeApplyDuringCreate: vi.fn(), + createFailureMessage: vi.fn(() => null), + exitOnPatchError: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + ensureApplied: vi.fn(), + waitForSupervisorReconnectIfNeeded: vi.fn(), + commitAfterReady: vi.fn(), + selectedMode: vi.fn(() => null), + printReadinessFailureIfEnabled: vi.fn(), + verifyGpuOrExit: vi.fn(async (verify) => verify("alpha")), + }, + prepareNetwork: vi.fn(async () => undefined), + runCreate: vi.fn(), + })); + const createOnboardRouting = vi.fn(() => ({ + nativeFallbackHasCleanBaseline: false, + inspectNativeRuntime: vi.fn(() => null), + isNativeCreateRoutingFailure: vi.fn(() => false), + isTrustedNativeRuntimeError: vi.fn(() => false), + isNativeReadinessRoutingFailure: vi.fn(() => false), + prepareCompatibilityLaunch: vi.fn(() => ({ createArgv: [], registryImageRef: null })), + })); + const providers = createRuntimeProviderBundleRegistry([ + [ + "mxc", + replaceSurface(bundle, "bootstrap", { + providerId: "mxc", + supported: true, + createLifecycle, + createOnboardRouting, + }), + ], + ]); + const registered = providers.mxc!; + expectSupportedSurface(registered.bootstrap); + + const routing = registered.bootstrap.createOnboardRouting({ + sandboxName: "alpha", + openshellArgv: (args) => args, + nativeFallbackEnabled: false, + }); + + expect(registered.identity.id).toBe("mxc"); + expect(routing.nativeFallbackHasCleanBaseline).toBe(false); + expect(createOnboardRouting).toHaveBeenCalledOnce(); + expect(createLifecycle).not.toHaveBeenCalled(); + }); + it("rejects an omitted managed platform without changing legacy receipt acceptance", () => { const { platform: _omittedPlatform, ...managedWithoutPlatform } = MANAGED_RECEIPT; const persistedManaged = cloneSandboxWorkloadReceipt(managedWithoutPlatform); diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index d39c4611479..d6370c84709 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -8,9 +8,12 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; import { loadAgent } from "../agent/defs"; import { SANDBOX_BUILD_CONTEXT_PREFIX } from "../sandbox/build-context"; import { createOpenshellCliHelpers } from "./openshell-cli"; +import { encodeManagedStartupProfile } from "./managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; import { buildSandboxRuntimeEnvArgs, prepareSandboxCreateLaunch, @@ -64,6 +67,49 @@ describe("buildSandboxRuntimeEnvArgs", () => { }); describe("prepareSandboxCreateLaunch", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("renders one identity-bound held launch for %s without exposing the startup profile", (agentName) => { + const request = createManagedStartupRootApplyRequest({ + agent: agentName, + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agentName)), + }); + const result = prepareSandboxCreateLaunch({ + agent: loadAgent(agentName), + chatUiUrl: "", + createArgs: ["--name", `${agentName}-sandbox`], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: disabledHermesDashboardState, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + openshellArgv: (args) => ["openshell", ...args], + buildEnv: () => ({}), + managedStartupRootApplyRequest: request, + }); + + expect(result.intendedSandboxStartupCommand).toEqual([ + "env", + ...result.envArgs, + "nemoclaw-start", + ]); + expect(result.managedBootstrapIdentity).toMatch(/^[a-f0-9]{64}$/u); + expect(result.sandboxStartupCommand).toEqual([ + ...result.intendedSandboxStartupCommand.slice(0, -1), + "/usr/local/bin/nemoclaw-managed-startup-hold", + "--agent", + agentName, + "--profile-fingerprint", + request.profileFingerprint, + "--bootstrap-identity", + result.managedBootstrapIdentity, + ]); + expect(result.createArgv.join("\n")).not.toContain(request.encodedProfile); + }); + it("builds the sandbox create command and runtime env envelope", () => { const openshellShellCommand = vi.fn((args: string[]) => `openshell ${args.join(" ")}`); const result = prepareSandboxCreateLaunch({ diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index a0232e221bd..59ff3e0a328 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -9,6 +9,11 @@ import { appendExtraPlaceholderKeysEnvArg } from "./extra-placeholder-keys"; import type { HermesDashboardOnboardState } from "./hermes-dashboard"; import { appendHermesDashboardEnvArgs } from "./hermes-dashboard"; import { appendHostProxyEnvArgs } from "./host-proxy-env"; +import { + createManagedBootstrapIdentity, + renderManagedBootstrapHeldCommand, +} from "./managed-bootstrap/adapter"; +import type { ManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; import { appendOpenClawRuntimeEnvArgs } from "./openclaw-runtime-env"; import { prebuildSandboxImageIfEligible, @@ -57,6 +62,8 @@ export interface SandboxCreateLaunchInput { openshellShellCommand: OpenshellShellCommand; openshellArgv?: OpenshellArgv; buildEnv?(): Record; + /** Dormant until a complete runtime bundle and durable authority store are selected. */ + managedStartupRootApplyRequest?: ManagedStartupRootApplyRequest | null; } export interface SandboxCreateLaunch { @@ -66,6 +73,9 @@ export interface SandboxCreateLaunch { envArgs: string[]; sandboxEnv: Record; sandboxStartupCommand: string[]; + intendedSandboxStartupCommand: string[]; + managedBootstrapIdentity: string | null; + managedStartupRootApplyRequest: ManagedStartupRootApplyRequest | null; } export interface SandboxCreateLaunchWithPrebuildInput extends SandboxCreateLaunchInput { @@ -196,7 +206,21 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San // from openshell because bash returns the status of the last pipeline // command (awk, always 0) unless pipefail is set. Removing the pipe // lets the real exit code flow through to run(). - const sandboxStartupCommand = ["env", ...envArgs, "nemoclaw-start"]; + const intendedSandboxStartupCommand = ["env", ...envArgs, "nemoclaw-start"]; + const managedStartupRootApplyRequest = input.managedStartupRootApplyRequest ?? null; + const managedBootstrapIdentity = managedStartupRootApplyRequest + ? createManagedBootstrapIdentity() + : null; + const sandboxStartupCommand = + managedStartupRootApplyRequest && managedBootstrapIdentity + ? [ + ...renderManagedBootstrapHeldCommand( + managedStartupRootApplyRequest, + managedBootstrapIdentity, + intendedSandboxStartupCommand, + ), + ] + : intendedSandboxStartupCommand; const openshellArgs = ["sandbox", "create", ...input.createArgs, "--", ...sandboxStartupCommand]; const createCommand = renderSandboxCreateCommand( input.createArgs, @@ -214,6 +238,9 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San envArgs, sandboxEnv, sandboxStartupCommand, + intendedSandboxStartupCommand, + managedBootstrapIdentity, + managedStartupRootApplyRequest, }; } diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index c7671ff4e60..720306078f4 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -3,6 +3,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { createInMemoryRuntimeProviderBundle } from "../../../test/helpers/runtime-provider-bundle"; + const mocks = vi.hoisted(() => ({ streamSandboxCreate: vi.fn(), waitForCreatedSandboxReadyWithTrace: vi.fn(), @@ -59,11 +62,23 @@ import { setupGpuFlowMocks, VERIFIED_GPU_PROOF as VERIFIED_PROOF, } from "./__test-helpers__/sandbox-gpu-create-flow"; +import type { + ManagedBootstrapRuntimeCreateLifecycleInput, + ManagedBootstrapRuntimePatch, +} from "./managed-bootstrap/runtime-create"; +import { encodeManagedStartupProfile } from "./managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; +import { prepareSandboxCreateLaunch } from "./sandbox-create-launch"; import { runSandboxGpuCreateFlow, type SandboxGpuCreateFlowDeps, type SandboxGpuCreateFlowInput, } from "./sandbox-gpu-create-flow"; +import type { + RuntimeProviderBootstrapSurface, + RuntimeProviderBundle, +} from "./runtime-provider/contract"; +import { createRuntimeProviderBundleRegistry } from "./runtime-provider/registry"; const FAILED_PROOF: SandboxGpuProofResult = { status: "failed", @@ -150,6 +165,145 @@ function createSourceInput(): SandboxGpuCreateFlowInput { beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); +describe("runSandboxGpuCreateFlow provider-owned managed create", () => { + it("runs an MXC-style bundle without a Docker branch in central orchestration", async () => { + const input = createInput(); + input.sandboxGpuConfig = { + mode: "0", + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }; + input.gpuRoutePlan = "none"; + input.initialGpuRoute = "none"; + const request = createManagedStartupRootApplyRequest({ + agent: "openclaw", + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")), + }); + const launch = prepareSandboxCreateLaunch({ + agent: null, + sandboxName: "alpha", + chatUiUrl: "", + createArgs: ["--name", "alpha"], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: { config: null, enabled: false }, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + openshellArgv: (args) => ["openshell", ...args], + buildEnv: () => ({}), + managedStartupRootApplyRequest: request, + }); + input.createArgv = launch.createArgv; + input.sandboxEnv = launch.sandboxEnv; + input.sandboxStartupCommand = launch.sandboxStartupCommand; + const patch = createPatch() as unknown as ManagedBootstrapRuntimePatch; + const createLifecycle = vi.fn( + (lifecycleInput: ManagedBootstrapRuntimeCreateLifecycleInput) => ({ + launchArgv: ["mxc-launch", ...lifecycleInput.launchArgv.slice(1)], + patch, + prepareNetwork: vi.fn(async () => undefined), + runCreate: async ( + start: (held: { + readonly heldWorkloadArgv: readonly string[]; + readonly bootstrapIdentity: string; + }) => Promise<{ readonly value: T }>, + ): Promise => + ( + await start({ + heldWorkloadArgv: lifecycleInput.heldWorkloadArgv, + bootstrapIdentity: lifecycleInput.bootstrapIdentity, + }) + ).value, + }), + ); + const source = createInMemoryRuntimeProviderBundle({ + providerId: "mxc", + workloadProfile: { + support: null, + hostArchitectures: [], + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: true, + }, + }); + const registered = createRuntimeProviderBundleRegistry([ + [ + "mxc", + { + ...source, + bootstrap: { + providerId: "mxc", + supported: true, + createLifecycle, + createOnboardRouting: vi.fn(() => ({ + nativeFallbackHasCleanBaseline: false, + inspectNativeRuntime: vi.fn(() => null), + isNativeCreateRoutingFailure: vi.fn(() => false), + isTrustedNativeRuntimeError: vi.fn(() => false), + isNativeReadinessRoutingFailure: vi.fn(() => false), + prepareCompatibilityLaunch: vi.fn(() => ({ + createArgv: [], + registryImageRef: null, + })), + })), + }, + }, + ], + ]); + const runtimeProvider = registered.mxc as RuntimeProviderBundle & { + readonly bootstrap: Extract; + }; + input.managedBootstrap = { + bootstrapIdentity: launch.managedBootstrapIdentity!, + runtimeProvider, + authorityStore: { + async recordPreparedAuthority(authority) { + return { + schemaVersion: 1, + sandbox: authority.sandbox, + bootstrapIdentity: authority.bootstrapIdentity, + authorityFingerprint: authority.authorityFingerprint, + recordId: "mxc-record-alpha", + recordedAt: "2026-07-31T00:00:00.000Z", + }; + }, + }, + request, + image: { + repository: "registry.example/nemoclaw-openclaw", + manifestDigest: `sha256:${"d".repeat(64)}`, + }, + agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + intendedWorkloadArgv: launch.intendedSandboxStartupCommand, + expectedSupervisorArgv: ["/mxc/supervisor"], + }; + const deps = createDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementation((args) => + args[1] === "get" ? "ID: mxc-alpha\n" : "alpha Ready", + ); + + const result = await runSandboxGpuCreateFlow(input, deps); + + expect(result).toMatchObject({ route: "none", runtimePatch: patch }); + expect(createLifecycle).toHaveBeenCalledWith( + expect.objectContaining({ providerId: "mxc", route: "none" }), + ); + expect(mocks.streamSandboxCreate).toHaveBeenCalledWith( + "mxc-launch", + input.createArgv.slice(1), + input.sandboxEnv, + expect.anything(), + ); + expect(mocks.createDockerGpuSandboxCreatePatch).not.toHaveBeenCalled(); + expect(mocks.queryOpenShellDockerSandboxContainers).not.toHaveBeenCalled(); + expect(mocks.queryOpenShellDockerSandboxRuntimeSnapshot).not.toHaveBeenCalled(); + expect(mocks.enforceDockerGpuPatchPreserveNetwork).not.toHaveBeenCalled(); + }); +}); + describe("runSandboxGpuCreateFlow proof authorization", () => { it("does not retry compatibility when the native proof throws an exec/policy error (#6110)", async () => { const deps = createDeps(); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 98574891e0b..bb98a48b7cd 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -10,12 +10,23 @@ import type { DockerGpuPatchDeps, DockerUlimit } from "./docker-gpu-patch-types" import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { renderCompatibilityFallbackCreateArgs } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; -import type { DockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +import type { + ManagedBootstrapAdapter, + ManagedBootstrapAgentIdentity, + ManagedBootstrapAuthorityStore, + ManagedBootstrapImageIdentity, +} from "./managed-bootstrap/adapter"; +import type { ManagedBootstrapRuntimePatch } from "./managed-bootstrap/runtime-create"; +import type { ManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; import { isImmutableDockerImageId } from "./openshell-docker-sandbox-containers"; import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; import { createSandboxGpuCreateAttemptRunner } from "./sandbox-gpu-create-run-attempt"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; import type { SandboxPrebuildResult } from "./sandbox-prebuild"; +import type { + RuntimeProviderBootstrapSurface, + RuntimeProviderBundle, +} from "./runtime-provider/contract"; import { addTraceEvent } from "./tracing"; export { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; @@ -41,6 +52,18 @@ export interface SandboxGpuCreateFlowInput { restoreBackupPath: string | null; terminalAgent: boolean; persistStartupCommand?: boolean; + managedBootstrap?: { + readonly bootstrapIdentity: string; + readonly runtimeProvider: RuntimeProviderBundle & { + readonly bootstrap: Extract; + }; + readonly authorityStore: ManagedBootstrapAuthorityStore; + readonly request: ManagedStartupRootApplyRequest; + readonly image: ManagedBootstrapImageIdentity; + readonly agentIdentity: ManagedBootstrapAgentIdentity; + readonly intendedWorkloadArgv: readonly string[]; + readonly expectedSupervisorArgv: readonly string[]; + } | null; requiredUlimits?: readonly DockerUlimit[] | null; } @@ -50,11 +73,17 @@ export interface SandboxGpuCreateFlowDeps { sleep: Sleep; openshellArgv(args: string[]): string[]; verifyDirectSandboxGpu(sandboxName: string): SandboxGpuProofResult; + /** + * Protected failure-injection tests wrap the real driver adapter at one + * named transaction boundary. Production callers omit this factory and use + * the adapter selected by the managed-bootstrap runtime provider. + */ + createManagedBootstrapAdapter?: () => ManagedBootstrapAdapter; } export interface SandboxGpuCreateFlowResult { createResult: StreamSandboxCreateResult; - dockerGpuCreatePatch: DockerGpuSandboxCreatePatch; + runtimePatch: ManagedBootstrapRuntimePatch; route: SelectedDockerGpuRoute; firstCreateOutput: string; /** Mutable tag/reference retained only for registry and image-GC bookkeeping. */ @@ -105,46 +134,65 @@ export async function runSandboxGpuCreateFlow( throw new Error("Compatibility retry policy was not materialized."); } const nativeRuntimeSnapshot = attemptRunner.state.nativeRuntimeSnapshot; - const prebuildImageId = input.prebuild.imageId; - const imageId = - nativeRuntimeSnapshot?.imageId ?? - (prebuildImageId && isImmutableDockerImageId(prebuildImageId) - ? prebuildImageId.toLowerCase() - : null); - if ( - !registryImageRef && - nativeRuntimeSnapshot?.bookkeepingImageRef && - !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) - ) { - registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; + if (attemptRunner.managedRouting) { + const prepared = attemptRunner.managedRouting.prepareCompatibilityLaunch({ + createArgs: input.prebuild.createArgs, + currentRegistryImageRef: registryImageRef, + prebuildImageId: input.prebuild.imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + startupCommand: input.sandboxStartupCommand, + runtimeSnapshot: nativeRuntimeSnapshot, + }); + attemptRunner.state.compatibilityArgv = [...prepared.createArgv]; + registryImageRef = prepared.registryImageRef; + } else { + const prebuildImageId = input.prebuild.imageId; + const imageId = + nativeRuntimeSnapshot?.imageId ?? + (prebuildImageId && isImmutableDockerImageId(prebuildImageId) + ? prebuildImageId.toLowerCase() + : null); + if ( + !registryImageRef && + nativeRuntimeSnapshot?.bookkeepingImageRef && + !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) + ) { + registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; + } + const compatibilityArgs = renderCompatibilityFallbackCreateArgs( + input.prebuild.createArgs, + { + imageRef: imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + }, + ); + attemptRunner.state.compatibilityArgv = deps.openshellArgv([ + "sandbox", + "create", + ...compatibilityArgs, + "--", + ...input.sandboxStartupCommand, + ]); } - const compatibilityArgs = renderCompatibilityFallbackCreateArgs(input.prebuild.createArgs, { - imageRef: imageId, - allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, - compatibilityPolicyPath: input.compatibilityPolicyPath, - }); - attemptRunner.state.compatibilityArgv = deps.openshellArgv([ - "sandbox", - "create", - ...compatibilityArgs, - "--", - ...input.sandboxStartupCommand, - ]); if (attemptRunner.state.compatibilityArgv.length === 0) { throw new Error("Compatibility sandbox create executable is missing."); } }, activateCompatibilityAttempt: async () => { - await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( - input.provider, - input.sandboxGpuConfig, - { - dockerDriverGateway: input.dockerDriverGateway, - selectedRoute: "compatibility", - gatewayPort: input.gatewayPort, - log: console.log, - }, - ); + if (!input.managedBootstrap) { + await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( + input.provider, + input.sandboxGpuConfig, + { + dockerDriverGateway: input.dockerDriverGateway, + selectedRoute: "compatibility", + gatewayPort: input.gatewayPort, + log: console.log, + }, + ); + } input.sandboxGpuConfig.sandboxGpuProof = null; }, traceEvent: addTraceEvent, diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 726758cc0ac..8183d24a0c1 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { printSandboxCreateRecoveryHints } from "../build-context"; +import { resolveOpenShellSandboxId } from "../adapters/openshell/sandbox-identity"; import { getSandboxDeleteOutcome } from "../domain/sandbox/destroy"; import { streamSandboxCreate } from "../sandbox/create-stream"; import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-ready-gate"; @@ -13,8 +14,8 @@ import { reportSandboxCreateFailure } from "./created-sandbox-failure"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +import type { ManagedBootstrapRuntimeSnapshot } from "./managed-bootstrap/runtime-create"; import { - type OpenShellDockerSandboxRuntimeSnapshotQuery, queryOpenShellDockerSandboxContainers, queryOpenShellDockerSandboxRuntimeSnapshot, } from "./openshell-docker-sandbox-containers"; @@ -28,7 +29,7 @@ import * as sandboxGpuPreflight from "./sandbox-gpu-preflight"; import * as sandboxReadinessTracing from "./sandbox-readiness-tracing"; import { addTraceEvent } from "./tracing"; -type NativeRuntimeSnapshot = Extract; +type NativeRuntimeSnapshot = ManagedBootstrapRuntimeSnapshot; export type SandboxGpuCreateAttemptState = { firstCreateOutput: string; @@ -41,6 +42,12 @@ export type SandboxGpuCreateAttemptState = { // Ready row. Require one confirmation poll before advancing to the GPU proof. const COMPATIBILITY_STABLE_READY_POLLS = 2; +class ManagedBootstrapCreateStreamFailure extends Error { + constructor(readonly result: Awaited>) { + super("Managed bootstrap held workload did not complete its create stream."); + } +} + export function createSandboxGpuCreateAttemptRunner( input: SandboxGpuCreateFlowInput, deps: SandboxGpuCreateFlowDeps, @@ -51,13 +58,26 @@ export function createSandboxGpuCreateAttemptRunner( allowUnbuiltCompatibilitySource: false, nativeRuntimeSnapshot: null, }; + const managedRouting = input.managedBootstrap?.runtimeProvider.bootstrap.createOnboardRouting({ + sandboxName: input.sandboxName, + openshellArgv: deps.openshellArgv, + nativeFallbackEnabled: + input.initialGpuRoute === "native" && input.gpuRoutePlan === "native-with-fallback", + }); const nativeFallbackBaseline = - input.initialGpuRoute === "native" && input.gpuRoutePlan === "native-with-fallback" + !managedRouting && + input.initialGpuRoute === "native" && + input.gpuRoutePlan === "native-with-fallback" ? queryOpenShellDockerSandboxContainers(input.sandboxName) : null; const nativeFallbackHasCleanBaseline = - nativeFallbackBaseline?.ok === true && nativeFallbackBaseline.ids.length === 0; - const inspectNativeRuntime = () => queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName); + managedRouting?.nativeFallbackHasCleanBaseline ?? + (nativeFallbackBaseline?.ok === true && nativeFallbackBaseline.ids.length === 0); + const inspectNativeRuntime = (): NativeRuntimeSnapshot | null => { + if (managedRouting) return managedRouting.inspectNativeRuntime(); + const snapshot = queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName); + return snapshot.ok ? snapshot : null; + }; const runAttempt = async (route: SelectedDockerGpuRoute) => { const compatibility = route === "compatibility"; @@ -70,73 +90,199 @@ export function createSandboxGpuCreateAttemptRunner( ); } const hasRequiredUlimits = (input.requiredUlimits?.length ?? 0) > 0; - const dockerGpuCreatePatch = createDockerGpuSandboxCreatePatch({ - route, - // The startup clone preserves native CDI devices, so DCode can apply its - // exact required limits without replacing the native GPU envelope. - // Other native routes are not swapped solely to persist a command. - persistStartupCommand: - input.persistStartupCommand === true && (route !== "native" || hasRequiredUlimits), - sandboxName: input.sandboxName, - gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, - openshellSandboxCommand: input.sandboxStartupCommand, - requiredUlimits: input.requiredUlimits, - timeoutSecs: input.sandboxReadyTimeoutSecs, - backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", - deps, - }); + const managedBootstrap = input.managedBootstrap ?? null; const attemptArgv = state.compatibilityArgv ?? input.createArgv; - const [createExecutable, ...createExecutableArgs] = attemptArgv; + const managedLifecycle = managedBootstrap + ? managedBootstrap.runtimeProvider.bootstrap.createLifecycle({ + providerId: managedBootstrap.runtimeProvider.identity.id, + bootstrapIdentity: managedBootstrap.bootstrapIdentity, + request: managedBootstrap.request, + image: managedBootstrap.image, + agentIdentity: managedBootstrap.agentIdentity, + intendedWorkloadArgv: managedBootstrap.intendedWorkloadArgv, + expectedSupervisorArgv: managedBootstrap.expectedSupervisorArgv, + launchArgv: attemptArgv, + heldWorkloadArgv: input.sandboxStartupCommand, + authorityStore: managedBootstrap.authorityStore, + ...(deps.createManagedBootstrapAdapter + ? { adapterOverride: deps.createManagedBootstrapAdapter() } + : {}), + route, + persistStartupCommand: input.persistStartupCommand === true, + sandboxName: input.sandboxName, + sandboxGpuConfig: input.sandboxGpuConfig, + requiredLimits: input.requiredUlimits ?? [], + timeoutSecs: input.sandboxReadyTimeoutSecs, + network: { + inferenceProvider: input.provider, + dockerDriverGateway: input.dockerDriverGateway, + gatewayPort: input.gatewayPort, + }, + dependencies: { + runCaptureOpenshell: deps.runCaptureOpenshell, + runOpenshell: deps.runOpenshell, + sleep: deps.sleep, + }, + }) + : null; + const runtimePatch = + managedLifecycle?.patch ?? + createDockerGpuSandboxCreatePatch({ + route, + // The startup clone preserves native CDI devices, so DCode can apply its + // exact required limits without replacing the native GPU envelope. + // Other native routes are not swapped solely to persist a command. + persistStartupCommand: + input.persistStartupCommand === true && (route !== "native" || hasRequiredUlimits), + externalRecreation: false, + sandboxName: input.sandboxName, + gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, + openshellSandboxCommand: input.sandboxStartupCommand, + requiredUlimits: input.requiredUlimits, + timeoutSecs: input.sandboxReadyTimeoutSecs, + backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", + deps, + }); + await managedLifecycle?.prepareNetwork(); + const [createExecutable, ...createExecutableArgs] = managedLifecycle?.launchArgv ?? attemptArgv; if (!createExecutable) throw new Error("Sandbox create executable is missing."); - const createResult = await streamSandboxCreate( - createExecutable, - createExecutableArgs, - input.sandboxEnv, - { + const streamCreate = () => + streamSandboxCreate(createExecutable, createExecutableArgs, input.sandboxEnv, { readyCheck: () => { const list = deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); return isSandboxReady(list, input.sandboxName); }, - onPoll: () => dockerGpuCreatePatch.maybeApplyDuringCreate(), + onPoll: () => runtimePatch.maybeApplyDuringCreate(), readyCheckOutputPatterns: getReadyCheckOutputPatternsForAgent( input.terminalAgent, input.sandboxEnv, ), - failureCheck: dockerGpuCreatePatch.createFailureMessage, + failureCheck: runtimePatch.createFailureMessage, traceEvent: addTraceEvent, initialPhase: compatibility && (input.prebuild.imageRef || state.compatibilityArgv) ? "create" : undefined, - }, - ); + }); + let createResult: Awaited>; + let managedIncompleteCreateRecovered = false; + if (managedBootstrap && managedLifecycle) { + try { + createResult = await managedLifecycle.runCreate( + async ({ heldWorkloadArgv, bootstrapIdentity }) => { + if ( + bootstrapIdentity !== managedBootstrap.bootstrapIdentity || + heldWorkloadArgv.length !== input.sandboxStartupCommand.length || + heldWorkloadArgv.some((value, index) => value !== input.sandboxStartupCommand[index]) + ) { + throw new Error( + "Managed bootstrap launch does not match the rendered identity-bound hold.", + ); + } + const result = await streamCreate(); + const createFailure = + result.status === 0 ? null : classifySandboxCreateFailure(result.output); + if (result.status !== 0 && createFailure?.kind !== "sandbox_create_incomplete") { + throw new ManagedBootstrapCreateStreamFailure(result); + } + if (createFailure?.kind === "sandbox_create_incomplete") { + const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ + sandboxName: input.sandboxName, + timeoutSecs: input.sandboxReadyTimeoutSecs, + runCaptureOpenshell: deps.runCaptureOpenshell, + isSandboxReady, + getSandboxFailurePhase, + stableReadyPolls: 1, + sleep: deps.sleep, + }); + if (!readiness.ready) { + throw new Error( + `Managed bootstrap incomplete create did not reach authoritative Ready state (${readiness.reason}).`, + ); + } + } else { + const list = deps.runCaptureOpenshell(["sandbox", "list"], { + ignoreError: true, + }); + if (!isSandboxReady(list, input.sandboxName)) { + throw new Error( + "Managed bootstrap create completed without an authoritative Ready sandbox.", + ); + } + } + let sandboxId: string; + try { + sandboxId = resolveOpenShellSandboxId(input.sandboxName, deps.runCaptureOpenshell); + } catch (error) { + throw new Error( + createFailure?.kind === "sandbox_create_incomplete" + ? "Managed bootstrap incomplete create did not return one exact durable sandbox identity after Ready." + : "Managed bootstrap create did not return one exact durable sandbox identity after Ready.", + { cause: error }, + ); + } + managedIncompleteCreateRecovered = createFailure?.kind === "sandbox_create_incomplete"; + return { + value: result, + receipt: { + sandbox: { + sandboxName: input.sandboxName, + sandboxId, + driverId: managedBootstrap.runtimeProvider.identity.id, + }, + ready: true, + readyAt: new Date().toISOString(), + }, + }; + }, + ); + } catch (error) { + if (!(error instanceof ManagedBootstrapCreateStreamFailure)) throw error; + createResult = error.result; + } + } else { + createResult = await streamCreate(); + } if (!state.firstCreateOutput) state.firstCreateOutput = createResult.output; - dockerGpuCreatePatch.exitOnPatchError(); + await runtimePatch.exitOnPatchError(); if (createResult.status !== 0) { const failure = classifySandboxCreateFailure(createResult.output); if (failure.kind === "sandbox_create_incomplete") { console.warn(""); - console.warn( - ` Create stream exited with code ${createResult.status} after sandbox was created.`, - ); - console.warn(" Checking whether the sandbox reaches Ready state..."); + if (managedIncompleteCreateRecovered) { + console.warn( + ` Create stream exited with code ${createResult.status}; the exact durable sandbox reached Ready and completed managed bootstrap.`, + ); + } else { + console.warn( + ` Create stream exited with code ${createResult.status} after sandbox was created.`, + ); + console.warn(" Checking whether the sandbox reaches Ready state..."); + } } else if ( route === "native" && input.gpuRoutePlan === "native-with-fallback" && nativeFallbackHasCleanBaseline && (() => { if ( - sandboxGpuCreateAttempt.isNativeGpuCreateRoutingFailure(createResult.output, { - sawProgress: createResult.sawProgress, - }) + managedRouting + ? managedRouting.isNativeCreateRoutingFailure( + createResult.output, + createResult.sawProgress, + ) + : sandboxGpuCreateAttempt.isNativeGpuCreateRoutingFailure(createResult.output, { + sawProgress: createResult.sawProgress, + }) ) { state.allowUnbuiltCompatibilitySource = input.prebuild.imageRef === null; return true; } const snapshot = inspectNativeRuntime(); if ( - snapshot.ok && - sandboxGpuCreateAttempt.isTrustedNativeGpuRuntimeError(snapshot.stateError) + snapshot && + (managedRouting + ? managedRouting.isTrustedNativeRuntimeError(snapshot.stateError) + : sandboxGpuCreateAttempt.isTrustedNativeGpuRuntimeError(snapshot.stateError)) ) { state.nativeRuntimeSnapshot = snapshot; return true; @@ -144,6 +290,7 @@ export function createSandboxGpuCreateAttemptRunner( return false; })() ) { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); return { ok: false, route, @@ -152,6 +299,7 @@ export function createSandboxGpuCreateAttemptRunner( fallbackEligible: true, } as const; } else { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); reportSandboxCreateFailure( { sandboxName: input.sandboxName, @@ -171,8 +319,8 @@ export function createSandboxGpuCreateAttemptRunner( ); } } - dockerGpuCreatePatch.ensureApplied(); - dockerGpuCreatePatch.waitForSupervisorReconnectIfNeeded(); + await runtimePatch.ensureApplied(); + await runtimePatch.waitForSupervisorReconnectIfNeeded(); console.log(" Waiting for sandbox to become ready..."); const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ sandboxName: input.sandboxName, @@ -197,13 +345,19 @@ export function createSandboxGpuCreateAttemptRunner( const runtimeSnapshot = canClassifyNativeReadiness ? inspectNativeRuntime() : null; if ( canClassifyNativeReadiness && - runtimeSnapshot?.ok && - sandboxGpuCreateAttempt.isNativeGpuReadinessRoutingFailure({ - failurePhase: readiness.failurePhase, - runtimeError: runtimeSnapshot.stateError, - }) + runtimeSnapshot && + (managedRouting + ? managedRouting.isNativeReadinessRoutingFailure({ + failurePhase: readiness.failurePhase, + runtimeError: runtimeSnapshot.stateError, + }) + : sandboxGpuCreateAttempt.isNativeGpuReadinessRoutingFailure({ + failurePhase: readiness.failurePhase, + runtimeError: runtimeSnapshot.stateError, + })) ) { state.nativeRuntimeSnapshot = runtimeSnapshot; + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); return { ok: false, route, @@ -214,10 +368,11 @@ export function createSandboxGpuCreateAttemptRunner( fallbackEligible: true, } as const; } + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); printSandboxCreateFailureDiagnostics(input.sandboxName, { backupPath: input.restoreBackupPath, }); - if (compatibility) dockerGpuCreatePatch.printReadinessFailureIfEnabled(); + if (compatibility) runtimePatch.printReadinessFailureIfEnabled(); else { const deletion = deps.runOpenshell(["sandbox", "delete", input.sandboxName], { ignoreError: true, @@ -240,27 +395,32 @@ export function createSandboxGpuCreateAttemptRunner( route === "native" && input.gpuRoutePlan === "native-with-fallback" && nativeFallbackHasCleanBaseline; - const proof: SandboxGpuProofResult = dockerGpuLocalInference.verifyGpuSandboxAccessAfterReady( - input.sandboxGpuConfig, - { - sandboxName: input.sandboxName, - dockerDriverGateway: input.dockerDriverGateway, - selectedRoute: route, - verifyDirectSandboxGpu: deps.verifyDirectSandboxGpu, - verifyGpuOrExit: deferNativeProofFailure - ? undefined - : dockerGpuCreatePatch.verifyGpuOrExit, - reportGpuProofFailure: !deferNativeProofFailure, - selectedMode: dockerGpuCreatePatch.selectedMode, - runCaptureOpenshell: deps.runCaptureOpenshell, - log: console.log, - }, - ); + let proof: SandboxGpuProofResult; + try { + proof = await dockerGpuLocalInference.verifyGpuSandboxAccessAfterReady( + input.sandboxGpuConfig, + { + sandboxName: input.sandboxName, + dockerDriverGateway: input.dockerDriverGateway, + selectedRoute: route, + verifyDirectSandboxGpu: deps.verifyDirectSandboxGpu, + verifyGpuOrExit: deferNativeProofFailure ? undefined : runtimePatch.verifyGpuOrExit, + reportGpuProofFailure: !deferNativeProofFailure, + selectedMode: runtimePatch.selectedMode, + runCaptureOpenshell: deps.runCaptureOpenshell, + log: console.log, + }, + ); + } catch (error) { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); + throw error; + } if (deferNativeProofFailure && proof.status === "failed") { if (sandboxGpuPreflight.isExplicitNvidiaSmiDriverProofFailure(proof)) { const snapshot = inspectNativeRuntime(); - if (snapshot.ok && snapshot.nativeGpuAttachmentState === "absent") { + if (snapshot?.nativeGpuAttachmentState === "absent") { state.nativeRuntimeSnapshot = snapshot; + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); return { ok: false, route, @@ -272,6 +432,7 @@ export function createSandboxGpuCreateAttemptRunner( } as const; } } + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); console.error(""); console.error(" Native sandbox GPU proof failed."); console.error( @@ -283,15 +444,22 @@ export function createSandboxGpuCreateAttemptRunner( process.exit(1); } if (proof.status === "failed") { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); throw new Error("Sandbox GPU proof returned failed status."); } } + // GPU-enabled cutover stays reversible until the caller also proves the + // configured host-local inference path. Non-GPU workloads have completed + // their final authoritative Ready gate here. + if (!input.sandboxGpuConfig.sandboxGpuEnabled) { + await runtimePatch.commitAfterReady(); + } return { ok: true, route, - value: { createResult, dockerGpuCreatePatch }, + value: { createResult, runtimePatch }, } as const; }; - return { state, runAttempt }; + return { state, managedRouting, runAttempt }; } From 844bfb65cef5db7654e55b96998068c925250a95 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 20:39:45 -0700 Subject: [PATCH 066/117] fix(snapshot): pin files during authority hashing Signed-off-by: Aaron Erickson --- .../onboard/runtime-provider/snapshot.test.ts | 1 - src/lib/state/sandbox.ts | 22 +++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/runtime-provider/snapshot.test.ts b/src/lib/onboard/runtime-provider/snapshot.test.ts index 68de9202464..c604b7e0307 100644 --- a/src/lib/onboard/runtime-provider/snapshot.test.ts +++ b/src/lib/onboard/runtime-provider/snapshot.test.ts @@ -15,7 +15,6 @@ import { createRuntimeProviderSnapshotSurface, observeDockerRuntimeSnapshot, observeOpenShellRuntimeSnapshot, - RuntimeProviderSnapshotError, type RuntimeProviderSnapshotObservation, } from "./snapshot"; diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index d05dc3e7fc5..04c2ab01377 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -13,6 +13,7 @@ import { createHash } from "node:crypto"; import { chmodSync, closeSync, + constants, existsSync, fstatSync, lstatSync, @@ -1540,6 +1541,13 @@ function snapshotManifestAuthority(manifest: RebuildManifest): RebuildManifest { } function hashSnapshotTree(backupPath: string): string { + if (typeof constants.O_NOFOLLOW !== "number") { + throw new Error("snapshot hashing requires O_NOFOLLOW support"); + } + const openFlags = + constants.O_RDONLY | + constants.O_NOFOLLOW | + (typeof constants.O_NONBLOCK === "number" ? constants.O_NONBLOCK : 0); const hash = createHash("sha256"); const visit = (directory: string, relativeDirectory: string): void => { const entries = readdirSync(directory, { withFileTypes: true }).sort((left, right) => @@ -1565,7 +1573,7 @@ function hashSnapshotTree(backupPath: string): string { throw new Error(`snapshot contains unsupported entry '${relativePath}'`); } hash.update(JSON.stringify(["file", relativePath, stat.size]), "utf8"); - const descriptor = openSync(fullPath, "r"); + const descriptor = openSync(fullPath, openFlags); try { const opened = fstatSync(descriptor); if (!opened.isFile() || opened.dev !== stat.dev || opened.ino !== stat.ino) { @@ -1578,7 +1586,17 @@ function hashSnapshotTree(backupPath: string): string { hash.update(buffer.subarray(0, bytesRead)); } const after = fstatSync(descriptor); - if (after.size !== opened.size || after.mtimeMs !== opened.mtimeMs) { + const pathAfter = lstatSync(fullPath); + if ( + after.size !== opened.size || + after.mtimeMs !== opened.mtimeMs || + pathAfter.isSymbolicLink() || + !pathAfter.isFile() || + pathAfter.dev !== opened.dev || + pathAfter.ino !== opened.ino || + pathAfter.size !== opened.size || + pathAfter.mtimeMs !== opened.mtimeMs + ) { throw new Error(`snapshot entry '${relativePath}' changed while it was read`); } } finally { From 3c47ca5c3f3fc12d5328826c8ae8b6e7478e1edf Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 20:51:04 -0700 Subject: [PATCH 067/117] feat(onboard): persist shared-state commit receipts Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/README.md | 5 + .../managed-bootstrap/docker-shared-state.ts | 3 +- ...d-startup-shared-state-transaction.test.ts | 233 +++++++++ .../onboard/managed-startup/image-runtime.ts | 53 +- .../shared-state-transaction.ts | 460 +++++++++++++++++- 5 files changed, 738 insertions(+), 16 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 227b063bbd6..758faff9261 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -44,6 +44,11 @@ records containing the provider and sandbox identities, plan and profile fingerprints, exact original and replacement IDs, rollback target, and phase. Exact commit and cleanup receipts are durable terminal records, so adapter recreation does not depend on process-local transaction sets or tombstone maps. +The image-owned shared-state transaction uses the same identity-bound model: a +commit atomically moves its pending manifest and backups into a durable receipt +namespace, compacts that state to an exact commit receipt, and rejects rollback +after a restart. The provider may retire that receipt only after it proves the +external rollback backup is gone, leaving the next bootstrap attempt unblocked. Enumeration reconstructs only unfinished records; the following recovery slice owns phase reconciliation and cross-surface resume or rollback. Mutable OpenShell names are read only to detect ownership reuse, and unsafe name-only diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts index 5953fc2b356..e86214f31b2 100644 --- a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts @@ -17,6 +17,7 @@ import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "../docker-gpu-pat import { MANAGED_STARTUP_RUNTIME_EXECUTABLE } from "../managed-startup/image-runtime"; import { MANAGED_STARTUP_AGENTS, type ManagedStartupAgent } from "../managed-startup/profile"; import { + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, } from "../managed-startup/shared-state-transaction"; @@ -24,8 +25,6 @@ import { isImmutableDockerImageId } from "../openshell-docker-sandbox-containers import { cleanupTempDir, secureTempFile } from "../temp-files"; const RECEIPT_TEMP_PREFIX = "nemoclaw-managed-startup-receipt"; -const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY = - "/var/lib/nemoclaw/managed-startup-shared-state-commit-v1"; const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; const DURABLE_IDENTITY_RE = /^[a-f0-9]{64}$/u; const NEUTRALIZED_PROCESS_INJECTION_ENV = [ diff --git a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts index 0d28be2cbe2..3358a070eed 100644 --- a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts +++ b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts @@ -10,9 +10,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; import type { SandboxMessagingPlan } from "../messaging/manifest"; import type { ManagedStartupAgent, ManagedStartupProfile } from "./managed-startup/profile"; +import { fingerprintManagedStartupProfile } from "./managed-startup/profile"; import { beginManagedStartupSharedStateTransaction, + clearManagedStartupSharedStateCommitReceipt, commitManagedStartupSharedStateTransaction, + getManagedStartupSharedStateTransactionStatus, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, type ManagedStartupSharedTransactionOptions, rollbackManagedStartupSharedStateTransaction, } from "./managed-startup/shared-state-transaction"; @@ -67,6 +71,13 @@ describe("managed startup shared-state transaction", () => { ); } + function commitReceiptDirectory(): string { + return path.join( + path.dirname(transactionDirectory), + path.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY), + ); + } + it.each([ "openclaw", "hermes", @@ -246,6 +257,180 @@ describe("managed startup shared-state transaction", () => { expect(commitManagedStartupSharedStateTransaction("openclaw", options)).toBe(false); }); + it("fsyncs every transaction namespace before exposing a pending receipt", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "before\n"); + const open = vi.spyOn(fs, "openSync"); + const fsync = vi.spyOn(fs, "fsyncSync"); + + expect( + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options), + ).toBe(true); + + const transactionParent = path.dirname(transactionDirectory); + const backupDirectory = path.join(transactionDirectory, "backups"); + expect(open).toHaveBeenCalledWith(transactionParent, fs.constants.O_RDONLY); + expect(open).toHaveBeenCalledWith(transactionDirectory, fs.constants.O_RDONLY); + expect(open).toHaveBeenCalledWith(backupDirectory, fs.constants.O_RDONLY); + // File contents plus parent, backup, and manifest directory entries all + // reach stable storage before the transaction is returned as pending. + expect(fsync.mock.calls.length).toBeGreaterThanOrEqual(6); + }); + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("persists one exact compact %s bootstrap commit across fresh calls, forbids rollback, and retires it for the next attempt", (agent) => { + const profile = managedStartupE2eProfile(agent); + const bootstrapIdentity = "b".repeat(64); + const nextBootstrapIdentity = "d".repeat(64); + const boundOptions = { ...options, bootstrapIdentity }; + const root = agentRoot(agent); + fs.mkdirSync(root); + const config = path.join( + root, + agent === "openclaw" ? "openclaw.json" : agent === "hermes" ? "config.yaml" : "config.toml", + ); + fs.writeFileSync(config, "before\n"); + + expect(beginManagedStartupSharedStateTransaction(profile, boundOptions)).toBe(true); + fs.writeFileSync(config, "committed\n"); + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("pending"); + expect(commitManagedStartupSharedStateTransaction(agent, boundOptions)).toBe(true); + + const receiptDirectory = commitReceiptDirectory(); + const receiptFile = path.join(receiptDirectory, "receipt.json"); + expect(fs.existsSync(transactionDirectory)).toBe(false); + expect(fs.readdirSync(receiptDirectory)).toEqual(["receipt.json"]); + expect(mode(receiptDirectory)).toBe(0o700); + expect(mode(receiptFile)).toBe(0o400); + expect(JSON.parse(fs.readFileSync(receiptFile, "utf8"))).toEqual({ + schemaVersion: 1, + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }); + + // These calls reconstruct state solely from the image-owned receipt. + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("committed"); + expect(commitManagedStartupSharedStateTransaction(agent, boundOptions)).toBe(true); + expect(() => rollbackManagedStartupSharedStateTransaction(agent, boundOptions)).toThrow( + /durably committed and cannot be rolled back/u, + ); + expect(() => + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: "e".repeat(64), + bootstrapIdentity, + }, + options, + ), + ).toThrow(/different bootstrap attempt/u); + expect(fs.readFileSync(config, "utf8")).toBe("committed\n"); + + expect(clearManagedStartupSharedStateCommitReceipt(agent, boundOptions)).toBe(true); + expect(fs.existsSync(receiptDirectory)).toBe(false); + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("none"); + + const nextOptions = { ...options, bootstrapIdentity: nextBootstrapIdentity }; + expect(beginManagedStartupSharedStateTransaction(profile, nextOptions)).toBe(true); + expect(rollbackManagedStartupSharedStateTransaction(agent, nextOptions)).toBe(true); + }); + + it.each([ + "during-compact-receipt-write", + "before-backup-removal", + "during-backup-removal", + "after-backup-removal", + "after-manifest-removal", + ] as const)("recovers an atomically established commit interrupted %s", (interruption) => { + const profile = managedStartupE2eProfile("openclaw"); + const bootstrapIdentity = "b".repeat(64); + const boundOptions = { ...options, bootstrapIdentity }; + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "before\n"); + beginManagedStartupSharedStateTransaction(profile, boundOptions); + fs.writeFileSync(path.join(root, "openclaw.json"), "committed\n"); + + const originalRmSync = fs.rmSync.bind(fs); + const rm = vi.spyOn(fs, "rmSync").mockImplementation((( + target: fs.PathLike, + removeOptions?: fs.RmDirOptions, + ) => { + if (String(target).endsWith(`${path.sep}backups`)) { + throw new Error("injected post-rename cleanup interruption"); + } + return originalRmSync(target, removeOptions); + }) as typeof fs.rmSync); + expect(() => commitManagedStartupSharedStateTransaction("openclaw", boundOptions)).toThrow( + /injected post-rename cleanup interruption/u, + ); + rm.mockRestore(); + + expect(fs.existsSync(transactionDirectory)).toBe(false); + const committedDirectory = commitReceiptDirectory(); + const backups = path.join(committedDirectory, "backups"); + const manifest = path.join(committedDirectory, "manifest.json"); + if (interruption === "during-compact-receipt-write") { + fs.renameSync( + path.join(committedDirectory, "receipt.json"), + path.join(committedDirectory, ".receipt.json.1234567890abcdef12345678"), + ); + } else if (interruption === "during-backup-removal") { + const [firstBackup] = fs.readdirSync(backups); + expect(firstBackup).toBeTruthy(); + fs.unlinkSync(path.join(backups, firstBackup!)); + } else if (interruption === "after-backup-removal") { + originalRmSync(backups, { force: false, recursive: true }); + } else if (interruption === "after-manifest-removal") { + fs.unlinkSync(manifest); + } + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent: "openclaw", + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("committed"); + expect(commitManagedStartupSharedStateTransaction("openclaw", boundOptions)).toBe(true); + expect(fs.readdirSync(committedDirectory)).toEqual(["receipt.json"]); + expect(clearManagedStartupSharedStateCommitReceipt("openclaw", boundOptions)).toBe(true); + }); + it("resumes the same pending profile idempotently and rejects profile drift", () => { const root = agentRoot("openclaw"); fs.mkdirSync(root); @@ -262,6 +447,54 @@ describe("managed startup shared-state transaction", () => { expect(rollbackManagedStartupSharedStateTransaction("openclaw", options)).toBe(true); }); + it("validates a directly mounted copied receipt under an unchanged 0755 image parent", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "{}\n"); + const profile = managedStartupE2eProfile("openclaw"); + const bootstrapIdentity = "b".repeat(64); + const boundOptions = { ...options, bootstrapIdentity }; + beginManagedStartupSharedStateTransaction(profile, boundOptions); + + const imageParent = path.join(temporaryRoot, "image-var-lib-nemoclaw"); + const copiedReceipt = path.join(imageParent, "managed-startup-shared-state-transaction-v1"); + fs.mkdirSync(imageParent, { mode: 0o755 }); + fs.chmodSync(imageParent, 0o755); + fs.cpSync(transactionDirectory, copiedReceipt, { + recursive: true, + preserveTimestamps: true, + }); + + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent: "openclaw", + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + { + ...boundOptions, + transactionDirectory: copiedReceipt, + }, + ), + ).toBe("pending"); + + fs.chmodSync(imageParent, 0o700); + expect(() => + getManagedStartupSharedStateTransactionStatus( + { + agent: "openclaw", + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + { + ...boundOptions, + transactionDirectory: copiedReceipt, + }, + ), + ).toThrow(/must be .* mode 755/u); + }); + it("rejects planted target and ancestor symlinks before creating a receipt", () => { const outside = path.join(temporaryRoot, "outside"); fs.mkdirSync(outside); diff --git a/src/lib/onboard/managed-startup/image-runtime.ts b/src/lib/onboard/managed-startup/image-runtime.ts index f8e5c7f4edd..898783c7f30 100644 --- a/src/lib/onboard/managed-startup/image-runtime.ts +++ b/src/lib/onboard/managed-startup/image-runtime.ts @@ -35,7 +35,9 @@ import { } from "./root-apply"; import { beginManagedStartupSharedStateTransaction, + clearManagedStartupSharedStateCommitReceipt, commitManagedStartupSharedStateTransaction, + getManagedStartupSharedStateTransactionStatus, MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, rollbackManagedStartupSharedStateTransaction, } from "./shared-state-transaction"; @@ -1540,7 +1542,7 @@ function readCliAgent(argv: readonly string[], expectedLength = 2): string { const index = argv.indexOf("--agent"); if (index < 0 || index + 1 >= argv.length || argv.length !== expectedLength) { fail( - "usage: managed-startup-image-runtime [--apply-root-stdin|--wait-for-completion|--verify-completion|--begin-shared-state-transaction|--commit-shared-state-transaction] --agent ", + "usage: managed-startup-image-runtime [--apply-root-stdin|--wait-for-completion|--verify-completion|--begin-shared-state-transaction|--commit-shared-state-transaction|--clear-shared-state-commit-receipt|--shared-state-transaction-status] --agent ", ); } return argv[index + 1] as string; @@ -1554,6 +1556,14 @@ function readCliFingerprint(argv: readonly string[]): string { return argv[index + 1] as string; } +function readCliBootstrapIdentity(argv: readonly string[]): string { + const index = argv.indexOf("--bootstrap-identity"); + if (index < 0 || index + 1 >= argv.length || !SHA256_RE.test(String(argv[index + 1] ?? ""))) { + fail("managed bootstrap identity argument is missing or invalid"); + } + return argv[index + 1] as string; +} + export async function main(argv: readonly string[] = process.argv.slice(2)): Promise { if (argv.length === 1 && argv[0] === "--internal-write-openclaw-hash") { internalWriteOpenClawHash(); @@ -1600,29 +1610,58 @@ export async function main(argv: readonly string[] = process.argv.slice(2)): Pro return; } if ( - argv.length === 4 && + (argv.length === 4 || argv.length === 6) && argv[0] === "--rollback-shared-state-transaction" && - argv[3] === "--read-only-receipt" + argv[argv.length - 1] === "--read-only-receipt" ) { requireRoot(); - const agent = exactAgent(readCliAgent(argv, 4)); + const agent = exactAgent(readCliAgent(argv, argv.length)); const rolledBack = rollbackManagedStartupSharedStateTransaction(agent, { transactionDirectory: MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, readOnlyReceipt: true, + bootstrapIdentity: argv.length === 6 ? readCliBootstrapIdentity(argv) : null, }); if (!rolledBack) fail("read-only shared-state rollback receipt is missing"); console.log(`[managed-startup] verified and restored ${agent} shared state`); return; } - if (argv.length === 3 && argv[0] === "--commit-shared-state-transaction") { + if ((argv.length === 3 || argv.length === 5) && argv[0] === "--commit-shared-state-transaction") { requireRoot(); - const agent = exactAgent(readCliAgent(argv, 3)); - if (!commitManagedStartupSharedStateTransaction(agent)) { + const agent = exactAgent(readCliAgent(argv, argv.length)); + if ( + !commitManagedStartupSharedStateTransaction(agent, { + bootstrapIdentity: argv.length === 5 ? readCliBootstrapIdentity(argv) : null, + }) + ) { fail("managed startup transaction is missing at commit"); } console.log(`[managed-startup] committed ${agent} shared state`); return; } + if (argv.length === 5 && argv[0] === "--clear-shared-state-commit-receipt") { + requireRoot(); + const agent = exactAgent(readCliAgent(argv, 5)); + const bootstrapIdentity = readCliBootstrapIdentity(argv); + if (!clearManagedStartupSharedStateCommitReceipt(agent, { bootstrapIdentity })) { + fail("managed startup durable commit receipt is missing at cleanup"); + } + console.log(`[managed-startup] cleared ${agent} durable shared-state commit receipt`); + return; + } + if (argv.length === 7 && argv[0] === "--shared-state-transaction-status") { + requireRoot(); + const agent = exactAgent(readCliAgent(argv, 7)); + const profileFingerprint = readCliFingerprint(argv); + const bootstrapIdentity = readCliBootstrapIdentity(argv); + process.stdout.write( + `${getManagedStartupSharedStateTransactionStatus({ + agent, + profileFingerprint, + bootstrapIdentity, + })}\n`, + ); + return; + } const result = await applyManagedStartupImageProfile(readCliAgent(argv)); console.log( result.adapterApplied diff --git a/src/lib/onboard/managed-startup/shared-state-transaction.ts b/src/lib/onboard/managed-startup/shared-state-transaction.ts index d921ec36d89..68ac40bd7e6 100644 --- a/src/lib/onboard/managed-startup/shared-state-transaction.ts +++ b/src/lib/onboard/managed-startup/shared-state-transaction.ts @@ -20,14 +20,19 @@ const MAX_TRANSACTION_FILES = 128; const MAX_TRANSACTION_FILE_BYTES = 8 * 1024 * 1024; const MAX_TRANSACTION_TOTAL_BYTES = 32 * 1024 * 1024; const MAX_MANIFEST_BYTES = 256 * 1024; +const MAX_COMMIT_RECEIPT_BYTES = 4096; const TRANSACTION_PARENT_DIRECTORY_MODE = 0o755; const TRANSACTION_DIRECTORY_MODE = 0o700; const TRANSACTION_FILE_MODE = 0o400; +const ATOMIC_TEMPORARY_FILE_MODE = 0o600; export const MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY = "/var/lib/nemoclaw/managed-startup-shared-state-transaction-v1"; export const MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY = "/run/nemoclaw/managed-startup-shared-rollback-receipt-v1"; +export const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY = + "/var/lib/nemoclaw/managed-startup-shared-state-commit-v1"; +const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE = "receipt.json"; interface FilePresentReceipt { readonly path: string; @@ -66,13 +71,23 @@ interface TransactionManifest { readonly schemaVersion: typeof TRANSACTION_SCHEMA_VERSION; readonly agent: ManagedStartupAgent; readonly profileFingerprint: string; + readonly bootstrapIdentity: string | null; readonly files: readonly FileReceipt[]; readonly directories: readonly DirectoryReceipt[]; } +interface CommitReceipt { + readonly schemaVersion: typeof TRANSACTION_SCHEMA_VERSION; + readonly agent: ManagedStartupAgent; + readonly profileFingerprint: string; + readonly bootstrapIdentity: string; +} + export interface ManagedStartupSharedTransactionOptions { readonly sandboxRoot?: string; readonly transactionDirectory?: string; + /** Test/helper seam. Production derives the fixed image-owned commit receipt path. */ + readonly commitReceiptDirectory?: string; /** Test seam. Production always retains the root:root defaults. */ readonly trustedUid?: number; /** Test seam. Production always retains the root:root defaults. */ @@ -82,6 +97,8 @@ export interface ManagedStartupSharedTransactionOptions { * so ownership may reflect the Docker CLI user instead of container root. */ readonly readOnlyReceipt?: boolean; + /** One-attempt identity for managed bootstrap; null for legacy root application. */ + readonly bootstrapIdentity?: string | null; } interface ResolvedOptions { @@ -90,9 +107,12 @@ interface ResolvedOptions { readonly transactionDirectory: string; readonly backupDirectory: string; readonly manifestFile: string; + readonly commitReceiptDirectory: string; + readonly commitReceiptFile: string; readonly trustedUid: number; readonly trustedGid: number; readonly readOnlyReceipt: boolean; + readonly bootstrapIdentity: string | null; } interface StableFile { @@ -109,11 +129,28 @@ function resolveOptions(options: ManagedStartupSharedTransactionOptions = {}): R const transactionDirectory = path.resolve( options.transactionDirectory ?? MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, ); + const commitReceiptDirectory = path.resolve( + options.commitReceiptDirectory ?? + (options.transactionDirectory + ? path.join( + path.dirname(transactionDirectory), + path.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY), + ) + : MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY), + ); if ( transactionDirectory === sandboxRoot || - transactionDirectory.startsWith(`${sandboxRoot}${path.sep}`) + transactionDirectory.startsWith(`${sandboxRoot}${path.sep}`) || + commitReceiptDirectory === sandboxRoot || + commitReceiptDirectory.startsWith(`${sandboxRoot}${path.sep}`) || + path.dirname(commitReceiptDirectory) !== path.dirname(transactionDirectory) || + commitReceiptDirectory === transactionDirectory ) { - fail("transaction receipts must not be stored in sandbox-shared state"); + fail("transaction and commit receipts require distinct paths outside sandbox-shared state"); + } + const bootstrapIdentity = options.bootstrapIdentity ?? null; + if (bootstrapIdentity !== null && !/^[a-f0-9]{64}$/u.test(bootstrapIdentity)) { + fail("bootstrap identity must encode 32 lowercase-hex bytes"); } return { sandboxRoot, @@ -121,9 +158,15 @@ function resolveOptions(options: ManagedStartupSharedTransactionOptions = {}): R transactionDirectory, backupDirectory: path.join(transactionDirectory, "backups"), manifestFile: path.join(transactionDirectory, "manifest.json"), + commitReceiptDirectory, + commitReceiptFile: path.join( + commitReceiptDirectory, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE, + ), trustedUid: options.trustedUid ?? 0, trustedGid: options.trustedGid ?? 0, readOnlyReceipt: options.readOnlyReceipt ?? false, + bootstrapIdentity, }; } @@ -482,16 +525,63 @@ function atomicWriteTrustedFile( } } +function fsyncDirectory(directory: string): void { + const descriptor = fs.openSync(directory, fs.constants.O_RDONLY); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + function canonicalManifest(manifest: TransactionManifest): string { return `${JSON.stringify(manifest, null, 2)}\n`; } +function canonicalCommitReceipt(receipt: CommitReceipt): string { + return `${JSON.stringify(receipt, null, 2)}\n`; +} + function requireExactKeys(record: Record, keys: readonly string[]): void { if (Object.keys(record).sort().join(",") !== [...keys].sort().join(",")) { fail("transaction manifest contains unexpected fields"); } } +function parseCommitReceipt(text: string): CommitReceipt { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("commit receipt is not valid JSON"); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + fail("commit receipt must be an object"); + } + const record = parsed as Record; + requireExactKeys(record, ["agent", "bootstrapIdentity", "profileFingerprint", "schemaVersion"]); + if ( + record.schemaVersion !== TRANSACTION_SCHEMA_VERSION || + !["openclaw", "hermes", "langchain-deepagents-code"].includes(String(record.agent)) || + typeof record.profileFingerprint !== "string" || + !/^[a-f0-9]{64}$/u.test(record.profileFingerprint) || + typeof record.bootstrapIdentity !== "string" || + !/^[a-f0-9]{64}$/u.test(record.bootstrapIdentity) + ) { + fail("commit receipt has an invalid envelope"); + } + const receipt: CommitReceipt = { + schemaVersion: TRANSACTION_SCHEMA_VERSION, + agent: record.agent as ManagedStartupAgent, + profileFingerprint: record.profileFingerprint, + bootstrapIdentity: record.bootstrapIdentity, + }; + if (canonicalCommitReceipt(receipt) !== text) { + fail("commit receipt is not canonical"); + } + return receipt; +} + function safeMetadata(value: unknown): value is number { return Number.isSafeInteger(value) && (value as number) >= 0; } @@ -509,6 +599,7 @@ function parseManifest(text: string): TransactionManifest { const record = parsed as Record; requireExactKeys(record, [ "agent", + "bootstrapIdentity", "directories", "files", "profileFingerprint", @@ -519,6 +610,11 @@ function parseManifest(text: string): TransactionManifest { !["openclaw", "hermes", "langchain-deepagents-code"].includes(String(record.agent)) || typeof record.profileFingerprint !== "string" || !/^[a-f0-9]{64}$/u.test(record.profileFingerprint) || + !( + record.bootstrapIdentity === null || + (typeof record.bootstrapIdentity === "string" && + /^[a-f0-9]{64}$/u.test(record.bootstrapIdentity)) + ) || !Array.isArray(record.files) || !Array.isArray(record.directories) || record.files.length > MAX_TRANSACTION_FILES || @@ -613,6 +709,7 @@ function parseManifest(text: string): TransactionManifest { schemaVersion: TRANSACTION_SCHEMA_VERSION, agent: record.agent as ManagedStartupAgent, profileFingerprint: record.profileFingerprint, + bootstrapIdentity: record.bootstrapIdentity as string | null, files, directories, }; @@ -639,9 +736,9 @@ function requireTrustedTransactionPath( } } -function requireReadOnlyReceiptMount(options: ResolvedOptions): void { +function requireReadOnlyReceiptMount(target: string, options: ResolvedOptions): void { if (!options.readOnlyReceipt) return; - const probe = path.join(options.transactionDirectory, ".nemoclaw-write-probe"); + const probe = path.join(target, ".nemoclaw-write-probe"); let descriptor: number | undefined; try { descriptor = fs.openSync( @@ -664,7 +761,7 @@ function loadManifest(options: ResolvedOptions): TransactionManifest | null { requireTransactionBoundaries(options); if (!pathExistsNoFollow(options.transactionDirectory)) return null; requireTrustedTransactionPath(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE, options); - requireReadOnlyReceiptMount(options); + requireReadOnlyReceiptMount(options.transactionDirectory, options); requireTrustedTransactionPath(options.backupDirectory, TRANSACTION_DIRECTORY_MODE, options); requireTrustedTransactionPath(options.manifestFile, TRANSACTION_FILE_MODE, options); const stable = readStableFile(options.manifestFile, MAX_MANIFEST_BYTES); @@ -679,6 +776,59 @@ function loadManifest(options: ResolvedOptions): TransactionManifest | null { return parseManifest(stable.bytes.toString("utf8")); } +function transactionOptionsAt( + options: ResolvedOptions, + transactionDirectory: string, +): ResolvedOptions { + return { + ...options, + transactionDirectory, + backupDirectory: path.join(transactionDirectory, "backups"), + manifestFile: path.join(transactionDirectory, "manifest.json"), + }; +} + +function loadCommitReceipt( + options: ResolvedOptions, +): { readonly receipt: CommitReceipt; readonly compact: boolean } | null { + requireTransactionBoundaries(options); + if (!pathExistsNoFollow(options.commitReceiptDirectory)) return null; + requireTrustedTransactionPath( + options.commitReceiptDirectory, + TRANSACTION_DIRECTORY_MODE, + options, + ); + if (pathExistsNoFollow(options.commitReceiptFile)) { + requireReadOnlyReceiptMount(options.commitReceiptDirectory, options); + requireTrustedTransactionPath(options.commitReceiptFile, TRANSACTION_FILE_MODE, options); + const stable = readStableFile(options.commitReceiptFile, MAX_COMMIT_RECEIPT_BYTES); + if ( + (!options.readOnlyReceipt && + (Number(stable.stat.uid) !== options.trustedUid || + Number(stable.stat.gid) !== options.trustedGid)) || + Number(stable.stat.mode & 0o7777n) !== TRANSACTION_FILE_MODE + ) { + fail("commit receipt ownership changed while it was read"); + } + return { receipt: parseCommitReceipt(stable.bytes.toString("utf8")), compact: true }; + } + const stagedOptions = transactionOptionsAt(options, options.commitReceiptDirectory); + const staged = loadManifest(stagedOptions); + if (!staged || staged.bootstrapIdentity === null) { + fail("durable commit staging receipt is incomplete"); + } + verifyAllBackups(staged.files, stagedOptions); + return { + receipt: { + schemaVersion: TRANSACTION_SCHEMA_VERSION, + agent: staged.agent, + profileFingerprint: staged.profileFingerprint, + bootstrapIdentity: staged.bootstrapIdentity, + }, + compact: false, + }; +} + function verifyBackup(receipt: FilePresentReceipt, options: ResolvedOptions): Buffer { const backupPath = path.join(options.backupDirectory, receipt.backup); requireTrustedTransactionPath(backupPath, TRANSACTION_FILE_MODE, options); @@ -742,11 +892,162 @@ function directoryMatchesReceipt(target: string, receipt: DirectoryPresentReceip function removeTransactionDirectory(options: ResolvedOptions): void { requireTrustedTransactionPath(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE, options); fs.rmSync(options.transactionDirectory, { force: false, recursive: true }); + fsyncDirectory(options.transactionParentDirectory); if (pathExistsNoFollow(options.transactionDirectory)) { fail("transaction directory remained after cleanup"); } } +function assertCommitReceiptMatches( + receipt: CommitReceipt, + expected: { + readonly agent: ManagedStartupAgent; + readonly profileFingerprint?: string; + readonly bootstrapIdentity: string; + }, +): void { + if ( + receipt.agent !== expected.agent || + (expected.profileFingerprint !== undefined && + receipt.profileFingerprint !== expected.profileFingerprint) || + receipt.bootstrapIdentity !== expected.bootstrapIdentity + ) { + fail("durable commit receipt belongs to a different bootstrap attempt"); + } +} + +function loadCommitStagingManifest(options: ResolvedOptions): TransactionManifest | null { + if (!pathExistsNoFollow(options.manifestFile)) return null; + requireTrustedTransactionPath(options.manifestFile, TRANSACTION_FILE_MODE, options); + const stable = readStableFile(options.manifestFile, MAX_MANIFEST_BYTES); + if ( + Number(stable.stat.uid) !== options.trustedUid || + Number(stable.stat.gid) !== options.trustedGid || + Number(stable.stat.mode & 0o7777n) !== TRANSACTION_FILE_MODE + ) { + fail("durable commit staging manifest ownership changed while it was read"); + } + return parseManifest(stable.bytes.toString("utf8")); +} + +function retireInterruptedCommitReceiptWrites( + receipt: CommitReceipt, + options: ResolvedOptions, +): void { + const temporaryPattern = new RegExp( + `^\\.${MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE.replace(".", "\\.")}\\.[a-f0-9]{24}$`, + "u", + ); + for (const entry of fs.readdirSync(options.commitReceiptDirectory)) { + if (!temporaryPattern.test(entry)) continue; + const target = path.join(options.commitReceiptDirectory, entry); + const stat = fs.lstatSync(target); + if ( + stat.isSymbolicLink() || + !stat.isFile() || + stat.nlink !== 1 || + stat.uid !== options.trustedUid || + stat.gid !== options.trustedGid || + ![ATOMIC_TEMPORARY_FILE_MODE, TRANSACTION_FILE_MODE].includes(modeOf(stat)) + ) { + fail("interrupted durable commit receipt write has unsafe metadata"); + } + const stable = readStableFile(target, MAX_COMMIT_RECEIPT_BYTES); + const mode = Number(stable.stat.mode & 0o7777n); + if ( + Number(stable.stat.uid) !== options.trustedUid || + Number(stable.stat.gid) !== options.trustedGid || + ![ATOMIC_TEMPORARY_FILE_MODE, TRANSACTION_FILE_MODE].includes(mode) + ) { + fail("interrupted durable commit receipt write changed during verification"); + } + if (stable.bytes.length > 0) { + let interruptedReceipt: CommitReceipt | null = null; + try { + interruptedReceipt = parseCommitReceipt(stable.bytes.toString("utf8")); + } catch { + // The atomic writer may have crashed after any partial write. The + // trusted 0700 directory and exact random temp-name shape bind this + // artifact to that interrupted write; the established receipt is now + // authoritative. + } + if (interruptedReceipt) assertCommitReceiptMatches(interruptedReceipt, receipt); + } + fs.unlinkSync(target); + fsyncDirectory(options.commitReceiptDirectory); + } +} + +function compactDurableCommitReceipt( + state: { readonly receipt: CommitReceipt; readonly compact: boolean }, + options: ResolvedOptions, +): void { + if (!state.compact) { + atomicWriteTrustedFile( + options.commitReceiptFile, + canonicalCommitReceipt(state.receipt), + TRANSACTION_FILE_MODE, + options.trustedUid, + options.trustedGid, + ); + fsyncDirectory(options.commitReceiptDirectory); + } + retireInterruptedCommitReceiptWrites(state.receipt, options); + const stagedOptions = transactionOptionsAt(options, options.commitReceiptDirectory); + const manifestExists = pathExistsNoFollow(stagedOptions.manifestFile); + const backupsExist = pathExistsNoFollow(stagedOptions.backupDirectory); + const unexpectedBeforeCleanup = fs + .readdirSync(options.commitReceiptDirectory) + .filter( + (entry) => + ![ + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE, + path.basename(stagedOptions.backupDirectory), + path.basename(stagedOptions.manifestFile), + ].includes(entry), + ); + if (unexpectedBeforeCleanup.length !== 0) { + fail("durable commit receipt directory contains unexpected artifacts"); + } + if (manifestExists) { + // The fsynced compact receipt is authoritative after commit. Validate the + // remaining manifest identity without requiring a complete backup tree: + // recursive backup deletion may have been interrupted at any point. + const staged = loadCommitStagingManifest(stagedOptions); + if (!staged || staged.bootstrapIdentity === null) { + fail("durable commit staging receipt disappeared during cleanup"); + } + assertCommitReceiptMatches(state.receipt, { + agent: staged.agent, + profileFingerprint: staged.profileFingerprint, + bootstrapIdentity: staged.bootstrapIdentity, + }); + } + if (backupsExist) { + requireTrustedTransactionPath( + stagedOptions.backupDirectory, + TRANSACTION_DIRECTORY_MODE, + options, + ); + fs.rmSync(stagedOptions.backupDirectory, { force: false, recursive: true }); + fsyncDirectory(options.commitReceiptDirectory); + } + if (manifestExists) { + requireTrustedTransactionPath(stagedOptions.manifestFile, TRANSACTION_FILE_MODE, options); + fs.unlinkSync(stagedOptions.manifestFile); + fsyncDirectory(options.commitReceiptDirectory); + } + const unexpected = fs + .readdirSync(options.commitReceiptDirectory) + .filter((entry) => entry !== MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE); + if (unexpected.length !== 0) { + fail("durable commit receipt directory contains unexpected artifacts"); + } + const verified = loadCommitReceipt(options); + if (!verified?.compact) fail("durable commit receipt did not compact successfully"); + assertCommitReceiptMatches(verified.receipt, state.receipt); +} + export function beginManagedStartupSharedStateTransaction( profile: ManagedStartupProfile, inputOptions: ManagedStartupSharedTransactionOptions = {}, @@ -758,9 +1059,25 @@ export function beginManagedStartupSharedStateTransaction( } requireTransactionBoundaries(options); const profileFingerprint = fingerprintManagedStartupProfile(profile); + const committed = loadCommitReceipt(options); + if (committed) { + if (options.bootstrapIdentity === null) { + fail("a durable managed bootstrap commit receipt already exists"); + } + assertCommitReceiptMatches(committed.receipt, { + agent: profile.agent, + profileFingerprint, + bootstrapIdentity: options.bootstrapIdentity, + }); + fail("this managed bootstrap attempt is already durably committed"); + } const pending = loadManifest(options); if (pending) { - if (pending.agent !== profile.agent || pending.profileFingerprint !== profileFingerprint) { + if ( + pending.agent !== profile.agent || + pending.profileFingerprint !== profileFingerprint || + pending.bootstrapIdentity !== options.bootstrapIdentity + ) { fail("a pending managed startup transaction belongs to a different profile"); } verifyAllBackups(pending.files, options); @@ -780,6 +1097,7 @@ export function beginManagedStartupSharedStateTransaction( schemaVersion: TRANSACTION_SCHEMA_VERSION, agent: profile.agent, profileFingerprint, + bootstrapIdentity: options.bootstrapIdentity, files: snapshots.map(({ receipt }) => receipt), directories, }; @@ -801,9 +1119,11 @@ export function beginManagedStartupSharedStateTransaction( }; fs.chownSync(options.transactionDirectory, options.trustedUid, options.trustedGid); fs.chmodSync(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE); + fsyncDirectory(options.transactionParentDirectory); fs.mkdirSync(options.backupDirectory, { mode: TRANSACTION_DIRECTORY_MODE }); fs.chownSync(options.backupDirectory, options.trustedUid, options.trustedGid); fs.chmodSync(options.backupDirectory, TRANSACTION_DIRECTORY_MODE); + fsyncDirectory(options.transactionDirectory); for (const snapshot of snapshots) { if (snapshot.receipt.state !== "file" || snapshot.bytes === null) continue; atomicWriteTrustedFile( @@ -814,6 +1134,7 @@ export function beginManagedStartupSharedStateTransaction( options.trustedGid, ); } + fsyncDirectory(options.backupDirectory); atomicWriteTrustedFile( options.manifestFile, canonicalManifest(manifest), @@ -821,6 +1142,7 @@ export function beginManagedStartupSharedStateTransaction( options.trustedUid, options.trustedGid, ); + fsyncDirectory(options.transactionDirectory); loadManifest(options); } catch (error) { try { @@ -990,11 +1312,25 @@ export function rollbackManagedStartupSharedStateTransaction( ): boolean { const options = resolveOptions(inputOptions); requireTransactionIdentity(options); + const committed = loadCommitReceipt(options); + if (committed) { + if (options.bootstrapIdentity === null) { + fail("shared state is already durably committed"); + } + assertCommitReceiptMatches(committed.receipt, { + agent: expectedAgent, + bootstrapIdentity: options.bootstrapIdentity, + }); + fail("shared state is already durably committed and cannot be rolled back"); + } const manifest = loadManifest(options); if (!manifest) return false; if (manifest.agent !== expectedAgent) { fail(`pending transaction targets ${manifest.agent}, expected ${expectedAgent}`); } + if (manifest.bootstrapIdentity !== options.bootstrapIdentity) { + fail("pending transaction belongs to a different bootstrap attempt"); + } const backups = verifyAllBackups(manifest.files, options); ensureOriginalDirectories(manifest.directories, options); restoreFiles(manifest.files, backups, options); @@ -1015,11 +1351,121 @@ export function commitManagedStartupSharedStateTransaction( if (options.readOnlyReceipt) { fail("cannot commit a read-only rollback receipt"); } + const committed = loadCommitReceipt(options); + if (committed) { + if (options.bootstrapIdentity === null) { + fail("durable commit receipt is missing its expected bootstrap identity"); + } + assertCommitReceiptMatches(committed.receipt, { + agent: expectedAgent, + bootstrapIdentity: options.bootstrapIdentity, + }); + compactDurableCommitReceipt(committed, options); + return true; + } const manifest = loadManifest(options); if (!manifest) return false; if (manifest.agent !== expectedAgent) { fail(`pending transaction targets ${manifest.agent}, expected ${expectedAgent}`); } - removeTransactionDirectory(options); + if (manifest.bootstrapIdentity !== options.bootstrapIdentity) { + fail("pending transaction belongs to a different bootstrap attempt"); + } + if (manifest.bootstrapIdentity === null) { + removeTransactionDirectory(options); + return true; + } + verifyAllBackups(manifest.files, options); + if (pathExistsNoFollow(options.commitReceiptDirectory)) { + fail("durable commit receipt path appeared before transaction commit"); + } + try { + fs.renameSync(options.transactionDirectory, options.commitReceiptDirectory); + fsyncDirectory(options.transactionParentDirectory); + } catch (error) { + fail(`could not atomically establish durable commit state: ${(error as Error).message}`); + } + const renamed = loadCommitReceipt(options); + if (!renamed) fail("durable commit state disappeared after atomic rename"); + assertCommitReceiptMatches(renamed.receipt, { + agent: expectedAgent, + profileFingerprint: manifest.profileFingerprint, + bootstrapIdentity: manifest.bootstrapIdentity, + }); + compactDurableCommitReceipt(renamed, options); return true; } + +/** + * Retire one exact durable bootstrap commit only after the runtime owner has + * proven its external rollback backup is gone. This prevents a completed + * attempt's image-owned receipt from blocking a later legitimate bootstrap in + * the same persisted workload. + */ +export function clearManagedStartupSharedStateCommitReceipt( + expectedAgent: ManagedStartupAgent, + inputOptions: ManagedStartupSharedTransactionOptions = {}, +): boolean { + const options = resolveOptions(inputOptions); + requireTransactionIdentity(options); + if (options.readOnlyReceipt) { + fail("cannot clear a durable commit from a read-only receipt"); + } + if (options.bootstrapIdentity === null) { + fail("durable commit cleanup requires its bootstrap identity"); + } + const committed = loadCommitReceipt(options); + if (!committed) return false; + assertCommitReceiptMatches(committed.receipt, { + agent: expectedAgent, + bootstrapIdentity: options.bootstrapIdentity, + }); + compactDurableCommitReceipt(committed, options); + requireTrustedTransactionPath( + options.commitReceiptDirectory, + TRANSACTION_DIRECTORY_MODE, + options, + ); + requireTrustedTransactionPath(options.commitReceiptFile, TRANSACTION_FILE_MODE, options); + const entries = fs.readdirSync(options.commitReceiptDirectory); + if (entries.length !== 1 || entries[0] !== MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE) { + fail("durable commit receipt directory contains unexpected artifacts"); + } + fs.rmSync(options.commitReceiptDirectory, { force: false, recursive: true }); + fsyncDirectory(options.transactionParentDirectory); + if (pathExistsNoFollow(options.commitReceiptDirectory)) { + fail("durable commit receipt remained after cleanup"); + } + return true; +} + +export function getManagedStartupSharedStateTransactionStatus( + expected: { + readonly agent: ManagedStartupAgent; + readonly profileFingerprint: string; + readonly bootstrapIdentity: string; + }, + inputOptions: ManagedStartupSharedTransactionOptions = {}, +): "committed" | "none" | "pending" { + const options = resolveOptions({ + ...inputOptions, + bootstrapIdentity: expected.bootstrapIdentity, + }); + requireTransactionIdentity(options); + const manifest = loadManifest(options); + if (manifest) { + if ( + manifest.agent !== expected.agent || + manifest.profileFingerprint !== expected.profileFingerprint || + manifest.bootstrapIdentity !== expected.bootstrapIdentity + ) { + fail("pending transaction does not match the expected bootstrap identity"); + } + verifyAllBackups(manifest.files, options); + return "pending"; + } + const committed = loadCommitReceipt(options); + if (!committed) return "none"; + assertCommitReceiptMatches(committed.receipt, expected); + return "committed"; +} From ed6088240231390e66e279a7afec46716db40a99 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 20:55:42 -0700 Subject: [PATCH 068/117] fix(snapshot): address restore review findings Signed-off-by: Aaron Erickson --- src/lib/actions/maintenance.test.ts | 41 ++++++++++++++++++- src/lib/actions/maintenance.ts | 19 ++++++++- src/lib/actions/sandbox/snapshot.ts | 7 ++++ src/lib/onboard/lifecycle-contracts.md | 14 +++++-- .../onboard/runtime-provider/snapshot.test.ts | 13 ++++++ src/lib/onboard/runtime-provider/snapshot.ts | 3 +- src/lib/state/sandbox.ts | 13 +++--- 7 files changed, 96 insertions(+), 14 deletions(-) diff --git a/src/lib/actions/maintenance.test.ts b/src/lib/actions/maintenance.test.ts index b646d700f4a..515ef28c6d1 100644 --- a/src/lib/actions/maintenance.test.ts +++ b/src/lib/actions/maintenance.test.ts @@ -19,6 +19,7 @@ const mocks = vi.hoisted(() => ({ isSandboxContainerDefinitivelyAbsent: vi.fn(), openBackupShieldsWindow: vi.fn(), relockBackupShieldsWindow: vi.fn(), + withSandboxMutationLock: vi.fn(), })); vi.mock("../state/registry", () => ({ @@ -32,7 +33,7 @@ vi.mock("../state/sandbox", () => ({ BackupResult: {}, })); vi.mock("../state/mcp-lifecycle-lock", () => ({ - withSandboxMutationLock: vi.fn((_name, callback) => callback()), + withSandboxMutationLock: mocks.withSandboxMutationLock, })); vi.mock("./sandbox/snapshot/backup-authority", () => ({ backupSandboxStateWithManagedAuthority: (name: string) => mocks.backupSandboxState(name), @@ -108,6 +109,7 @@ describe("backupAll", () => { wasLocked: false, })); mocks.relockBackupShieldsWindow.mockReturnValue(true); + mocks.withSandboxMutationLock.mockImplementation((_name, callback) => callback()); }); afterEach(() => { @@ -225,6 +227,43 @@ describe("backupAll", () => { logSpy.mockRestore(); }); + it("counts a mutation-lock acquisition failure and continues with later sandboxes", async () => { + mocks.listSandboxes.mockReturnValue({ + sandboxes: [{ name: "alpha" }, { name: "beta" }], + defaultSandbox: "alpha", + }); + mocks.parseReadySandboxNames.mockReturnValue(new Set(["alpha", "beta"])); + mocks.withSandboxMutationLock + .mockRejectedValueOnce(new Error("Timed out waiting for the sandbox mutation lock")) + .mockImplementationOnce((_name, callback) => callback()); + mocks.backupSandboxState.mockReturnValue({ + success: true, + backedUpDirs: ["workspace"], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + manifest: { backupPath: "/backups/beta/timestamp" }, + }); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`exit:${code}`); + }) as never); + + await expect(backupAll()).rejects.toThrow("exit:1"); + + expect(mocks.withSandboxMutationLock.mock.calls.map(([name]) => name)).toEqual([ + "alpha", + "beta", + ]); + expect(mocks.backupSandboxState).toHaveBeenCalledOnce(); + expect(mocks.backupSandboxState).toHaveBeenCalledWith("beta"); + expect(logSpy.mock.calls.flat().join("\n")).toContain("1 backed up, 1 failed, 0 skipped"); + expect(errorSpy.mock.calls.flat().join("\n")).toContain( + "alpha: backup failed (mutation lock: Timed out waiting for the sandbox mutation lock)", + ); + }); + it("does not back up when gateway preflight exits", async () => { mocks.listSandboxes.mockReturnValue({ sandboxes: [{ name: "sb-good" }], diff --git a/src/lib/actions/maintenance.ts b/src/lib/actions/maintenance.ts index 227505a5cb7..1025b327fd7 100644 --- a/src/lib/actions/maintenance.ts +++ b/src/lib/actions/maintenance.ts @@ -31,6 +31,7 @@ import { openBackupShieldsWindow, relockBackupShieldsWindow, } from "./sandbox/backup-shields-window"; +import * as snapshotBackup from "./sandbox/snapshot/backup-authority"; import { backupStartedSandboxState, isSandboxContainerDefinitivelyAbsent, @@ -38,7 +39,6 @@ import { type StartedForBackup, startStoppedSandboxContainerForBackup, } from "./sandbox/stopped-sandbox-backup"; -import * as snapshotBackup from "./sandbox/snapshot/backup-authority"; const useColor = !process.env.NO_COLOR && !!process.stdout.isTTY; const trueColor = @@ -294,7 +294,22 @@ export async function backupAll(): Promise { } }; for (const sb of sandboxes) { - await withSandboxMutationLock(sb.name, () => backupRegisteredSandbox(sb)); + let enteredMutationLock = false; + try { + await withSandboxMutationLock(sb.name, () => { + enteredMutationLock = true; + return backupRegisteredSandbox(sb); + }); + } catch (error) { + // Callback failures retain the existing fail-fast behavior. A lock that + // could not be acquired is instead one failed sandbox attempt so the + // remaining backups, orphan confirmation, summary, and strict gate all + // still run. + if (enteredMutationLock) throw error; + const detail = error instanceof Error ? error.message : String(error); + console.error(` ${RD}✗${R} ${sb.name}: backup failed (mutation lock: ${detail})`); + failed++; + } } // The classification above is only as fresh as the pre-loop listing, and // the backup loop can run for minutes. Confirm with a second pinned listing diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 7cb0175de8e..2a1f1562a80 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -1317,6 +1317,13 @@ async function runSnapshotRestoreUnlocked( } else { console.log(` Restoring snapshot into '${sandboxName}'...`); } + if (Boolean(snapshotRestoreAuthority) !== Boolean(validateManagedRestoreBeforeMutation)) { + console.error( + ` Cannot restore managed snapshot '${sandboxName}': content authority and the runtime mutation fence must both be present.`, + ); + console.error(` Destination '${targetSandbox}' was not changed.`); + snapshotExit(1); + } const result = snapshotRestoreAuthority && validateManagedRestoreBeforeMutation ? sandboxState.restoreSandboxState(targetSandbox, backupPath, { diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index c55e537932c..870cd41ab4b 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -161,10 +161,18 @@ Provider inputs are detached and deeply frozen at the extension boundary, and ce Docker lifecycle inspection and GPU inspection remain inside the Docker provider adapter. The provider-neutral receipt can represent another provider, including an MXC-style implementation, without adding provider switches to snapshot or rebuild orchestration. -Legacy and custom-image snapshots retain their state-only backup and restore path. +Legacy and custom-image snapshots retain their state-only backup and restore path until the +[incremental runtime epic](https://github.com/NVIDIA/NemoClaw/issues/7744) completes managed +create finalization, clone/rebind, recovery, and activation for every supported agent. The path can +retire only after those slices prove managed authority before mutation and exact recovery across +OpenClaw, Hermes, and Deep Agents Code while legacy/custom-image restores keep parity coverage. This contract does not activate another runtime provider or managed-image onboarding path. -Ordinary onboard recreation and create finalization remain deferred because the replacement target is not registered when that restore currently runs; the raw state layer rejects a managed manifest unless both exact content authority and a runtime-validation fence are present. -Cross-provider clone and rebind, durable interrupted-restore recovery, ordinary recreate integration, and user-visible runtime activation remain separate review units. +Ordinary onboard recreation and create finalization remain deferred under +[#7744](https://github.com/NVIDIA/NemoClaw/issues/7744) because the replacement target is not +registered when that restore currently runs; the raw state layer rejects a managed manifest unless +both exact content authority and a runtime-validation fence are present. Cross-provider clone and +rebind, durable interrupted-restore recovery, ordinary recreate integration, and user-visible +runtime activation are separately reviewable units tracked by that epic. If provider proof fails after filesystem restoration, NemoClaw reports that state changed and requires the operator to retry the exact snapshot after the runtime stabilizes. ## Agent-specific differences diff --git a/src/lib/onboard/runtime-provider/snapshot.test.ts b/src/lib/onboard/runtime-provider/snapshot.test.ts index c604b7e0307..d473a4da700 100644 --- a/src/lib/onboard/runtime-provider/snapshot.test.ts +++ b/src/lib/onboard/runtime-provider/snapshot.test.ts @@ -578,6 +578,19 @@ function dockerLifecycleCapture( } describe("Docker provider snapshot evidence", () => { + it("normalizes Docker's explicit paused status", () => { + const observed = observeDockerRuntimeSnapshot( + sandbox({ openshellDriver: "docker" }), + "docker", + { + captureHostCommand: dockerLifecycleCapture(undefined, { status: "paused", paused: true }), + queryRuntimeSnapshot: () => dockerSnapshot(), + }, + ); + + expect(observed.lifecycleState).toBe("paused"); + }); + it("captures exact live container, lifecycle, and device selectors", () => { const queryRuntimeSnapshot = vi.fn(() => dockerSnapshot({ diff --git a/src/lib/onboard/runtime-provider/snapshot.ts b/src/lib/onboard/runtime-provider/snapshot.ts index 8eac713502a..3f088c1ef0c 100644 --- a/src/lib/onboard/runtime-provider/snapshot.ts +++ b/src/lib/onboard/runtime-provider/snapshot.ts @@ -12,8 +12,8 @@ import { queryOpenShellDockerSandboxRuntimeSnapshot, } from "../openshell-docker-sandbox-containers"; import { - RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION, RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION, + RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION, type RuntimeProviderCommandCapture, type RuntimeProviderManagedProfileRestoreAuthority, type RuntimeProviderRuntimeReceipt, @@ -327,6 +327,7 @@ function parseDockerLifecycle( const status = fields[1].trim().toLowerCase(); let state: RuntimeProviderSnapshotLifecycleState; if (status === "running") state = fields[2] ? "paused" : "running"; + else if (status === "paused" && fields[2] === true) state = "paused"; else if (["created", "exited", "dead"].includes(status) && fields[2] === false) state = "stopped"; else { throw new RuntimeProviderSnapshotError( diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 04c2ab01377..bcbb710d867 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -1551,7 +1551,7 @@ function hashSnapshotTree(backupPath: string): string { const hash = createHash("sha256"); const visit = (directory: string, relativeDirectory: string): void => { const entries = readdirSync(directory, { withFileTypes: true }).sort((left, right) => - left.name.localeCompare(right.name), + left.name === right.name ? 0 : left.name < right.name ? -1 : 1, ); for (const entry of entries) { const fullPath = path.join(directory, entry.name); @@ -1559,26 +1559,25 @@ function hashSnapshotTree(backupPath: string): string { relativeDirectory.split(path.sep).join(path.posix.sep), entry.name, ); - const stat = lstatSync(fullPath); - if (stat.isDirectory()) { + if (entry.isDirectory()) { hash.update(JSON.stringify(["directory", relativePath]), "utf8"); visit(fullPath, relativePath); continue; } - if (stat.isSymbolicLink()) { + if (entry.isSymbolicLink()) { hash.update(JSON.stringify(["symlink", relativePath, readlinkSync(fullPath)]), "utf8"); continue; } - if (!stat.isFile()) { + if (!entry.isFile()) { throw new Error(`snapshot contains unsupported entry '${relativePath}'`); } - hash.update(JSON.stringify(["file", relativePath, stat.size]), "utf8"); const descriptor = openSync(fullPath, openFlags); try { const opened = fstatSync(descriptor); - if (!opened.isFile() || opened.dev !== stat.dev || opened.ino !== stat.ino) { + if (!opened.isFile()) { throw new Error(`snapshot entry '${relativePath}' changed while it was opened`); } + hash.update(JSON.stringify(["file", relativePath, opened.size]), "utf8"); const buffer = Buffer.allocUnsafe(64 * 1024); for (;;) { const bytesRead = readSync(descriptor, buffer, 0, buffer.byteLength, null); From ed34be373dbe430ee8e49f01ceac06f2a9e1e14e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 20:58:22 -0700 Subject: [PATCH 069/117] refactor(onboard): extract managed GPU finalization Signed-off-by: Aaron Erickson --- src/lib/onboard.ts | 40 +++++---------- .../docker-gpu-local-inference.test.ts | 50 +++++++++++++++++++ src/lib/onboard/docker-gpu-local-inference.ts | 32 +++++++++++- 3 files changed, 94 insertions(+), 28 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 1c5b2b7597c..e8a7ab520de 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2768,33 +2768,19 @@ async function createSandboxWithBaseImageResolution( } if (effectiveSandboxGpuConfig.sandboxGpuEnabled) { - try { - dockerGpuLocalInference.verifyGpuSandboxLocalInferenceAfterReady( - effectiveSandboxGpuConfig, - provider, - { - sandboxName, - dockerDriverGateway, - selectedRoute: selectedGpuRoute, - verifyDirectSandboxGpu, - verifyGpuOrExit: runtimePatch.verifyGpuOrExit, - selectedMode: runtimePatch.selectedMode, - runCaptureOpenshell, - log: console.log, - }, - ); - await runtimePatch.commitAfterReady(); - } catch (error) { - const failure = error instanceof Error ? error : new Error(String(error)); - try { - await runtimePatch.rollbackManagedStartupAfterCreateFailure(); - } catch (rollbackError) { - ( - failure as Error & { managedBootstrapRollbackError?: unknown } - ).managedBootstrapRollbackError = rollbackError; - } - throw failure; - } + await dockerGpuLocalInference.verifyGpuSandboxLocalInferenceAndCommitAfterReady( + effectiveSandboxGpuConfig, + provider, + { + sandboxName, + dockerDriverGateway, + selectedRoute: selectedGpuRoute, + verifyDirectSandboxGpu, + runCaptureOpenshell, + log: console.log, + }, + runtimePatch, + ); } let actualDashboardPort = 0; diff --git a/src/lib/onboard/docker-gpu-local-inference.test.ts b/src/lib/onboard/docker-gpu-local-inference.test.ts index 3d72de11bee..702a4a7e757 100644 --- a/src/lib/onboard/docker-gpu-local-inference.test.ts +++ b/src/lib/onboard/docker-gpu-local-inference.test.ts @@ -11,6 +11,7 @@ import { shouldUseDockerGpuPatchHostNetwork, verifyDockerGpuSandboxLocalInference, verifyGpuSandboxAfterReady, + verifyGpuSandboxLocalInferenceAndCommitAfterReady, } from "./docker-gpu-local-inference"; const HOST_NETWORK_ENV = { @@ -377,6 +378,55 @@ describe("verifyGpuSandboxAfterReady", () => { }); }); +describe("verifyGpuSandboxLocalInferenceAndCommitAfterReady", () => { + function options() { + return { + ...gpuPatchOptions(), + verifyDirectSandboxGpu: vi.fn(), + runCaptureOpenshell: vi.fn(() => ""), + log: vi.fn(), + }; + } + + it("commits only after the runtime inference route is proven", async () => { + const runtimePatch = { + commitAfterReady: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + }; + await verifyGpuSandboxLocalInferenceAndCommitAfterReady( + GPU_CONFIG, + "ollama-local", + { + ...options(), + deps: { execInSandbox: execEmitting("HTTP_200"), sleep: vi.fn() }, + }, + runtimePatch, + ); + expect(runtimePatch.commitAfterReady).toHaveBeenCalledOnce(); + expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + }); + + it("rolls back before propagating an inference verification failure", async () => { + const runtimePatch = { + commitAfterReady: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + }; + await expect( + verifyGpuSandboxLocalInferenceAndCommitAfterReady( + GPU_CONFIG, + "ollama-local", + { + ...options(), + deps: { execInSandbox: execEmitting("HTTP_000"), sleep: vi.fn() }, + }, + runtimePatch, + ), + ).rejects.toThrow("GPU sandbox local inference reachability failed"); + expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce(); + expect(runtimePatch.commitAfterReady).not.toHaveBeenCalled(); + }); +}); + describe("printDockerGpuSandboxInferenceVerificationFailure", () => { it("surfaces endpoint, provider label, detail, and recovery hints", () => { const lines: string[] = []; diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index 447ac8069ca..b4140641999 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -464,7 +464,7 @@ export async function verifyGpuSandboxAccessAfterReady( export function verifyGpuSandboxLocalInferenceAfterReady( config: DockerGpuLocalInferenceConfig, provider: string | null | undefined, - options: GpuSandboxAfterReadyOptions, + options: Omit, ): void { if (options.selectedRoute !== "compatibility") return; const verification = verifyDockerGpuSandboxLocalInference(config, provider, { @@ -494,3 +494,33 @@ export function verifyGpuSandboxLocalInferenceAfterReady( ); } } + +/** + * Keep the managed create transaction reversible until the sandbox's real + * local-inference route is proven. Rollback failures are attached to the + * original verification failure so callers retain both pieces of evidence. + */ +export async function verifyGpuSandboxLocalInferenceAndCommitAfterReady( + config: DockerGpuLocalInferenceConfig, + provider: string | null | undefined, + options: Omit, + runtimePatch: Pick< + ManagedBootstrapRuntimePatch, + "commitAfterReady" | "rollbackManagedStartupAfterCreateFailure" + >, +): Promise { + try { + verifyGpuSandboxLocalInferenceAfterReady(config, provider, options); + await runtimePatch.commitAfterReady(); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + try { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); + } catch (rollbackError) { + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + throw failure; + } +} From 7c823d41ce56005016db9afe66164caa3ce134c9 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 21:07:04 -0700 Subject: [PATCH 070/117] fix(snapshot): address clone handoff review findings Signed-off-by: Aaron Erickson --- ...hot-managed-clone-handoff-dormancy.test.ts | 76 +++++++++++++++---- src/lib/messaging/plan-validation.test.ts | 14 ++++ src/lib/messaging/plan-validation.ts | 12 +++ src/lib/onboard/inference-route.ts | 2 +- src/lib/onboard/lifecycle-contracts.md | 5 +- 5 files changed, 91 insertions(+), 18 deletions(-) diff --git a/src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts b/src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts index 53647e6bfb2..e0615132e29 100644 --- a/src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts +++ b/src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts @@ -1,24 +1,70 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { readFileSync } from "node:fs"; +import { createHash } from "node:crypto"; -import { describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -describe("managed snapshot clone handoff activation boundary", () => { - it("keeps the PR3.9 contract unwired while production restore stays fail-closed", () => { - const dependencies = readFileSync( - new URL("./snapshot/dependencies.ts", import.meta.url), - "utf8", +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { MANAGED_IMAGE_REPOSITORIES } from "../../onboard/managed-image/contract"; +import { encodeManagedStartupProfile } from "../../onboard/managed-startup/profile"; +import * as fixture from "./snapshot-restore-test-fixture"; + +beforeEach(() => fixture.resetSnapshotRestoreMocks()); +afterEach(() => fixture.cleanupSnapshotRestoreMocks()); + +describe("managed snapshot clone activation boundary", () => { + it("rejects managed cross-sandbox restore before destination effects (#7744)", async () => { + const consoleError = vi.spyOn(console, "error").mockImplementation(() => undefined); + const encodedProfile = encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")); + fixture.getLatestBackupMock.mockReturnValue({ + snapshotVersion: 4, + timestamp: "2026-07-30T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + agentType: "openclaw", + workload: { + schemaVersion: 1, + kind: "managed-image", + reference: `${MANAGED_IMAGE_REPOSITORIES.openclaw}@sha256:${"a".repeat(64)}`, + platform: "linux/amd64", + release: "v0.0.100", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true, + }, + runtimeSnapshot: { + schemaVersion: 1, + providerId: "docker", + providerHandle: "snapshot-provider-handle", + lifecycleState: "running", + lifecycleGeneration: "snapshot-generation", + runtime: { + schemaVersion: 1, + providerId: "docker", + runtime: { kind: "docker-container", handle: "container-id" }, + acceleration: { kind: "none" }, + }, + }, + }); + fixture.getSandboxMock.mockImplementation((name) => + name === "alpha" ? { name: "alpha", agent: "openclaw", openshellDriver: "docker" } : null, ); - const productionAction = readFileSync(new URL("./snapshot.ts", import.meta.url), "utf8"); + const { runSandboxSnapshot } = await import("./snapshot"); - expect(dependencies).not.toContain("prepareManagedWorkloadCloneHandoff"); - expect(dependencies).not.toContain("ManagedWorkloadCloneHandoff"); - expect(productionAction).toContain("rejectManagedSnapshotCloneUntilRebind"); - expect(productionAction).not.toContain("prepareManagedWorkloadCloneHandoff"); - expect(productionAction).not.toContain("ManagedWorkloadCloneHandoff"); - expect(productionAction).not.toContain("prepareManagedCloneProviders"); - expect(productionAction).not.toContain("provisionManagedCloneProviders"); + await expect( + runSandboxSnapshot("alpha", { kind: "restore", to: "beta", force: true, yes: true }), + ).rejects.toMatchObject({ exitCode: 1 }); + + expect(consoleError.mock.calls.flat().join("\n")).toContain( + "requires managed-profile clone rebind", + ); + expect(fixture.lifecycleMock.events).not.toContain("delete"); + expect(fixture.streamSandboxCreateMock).not.toHaveBeenCalled(); + expect(fixture.restoreSandboxStateMock).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/messaging/plan-validation.test.ts b/src/lib/messaging/plan-validation.test.ts index 3879b920dea..f64bb7e932f 100644 --- a/src/lib/messaging/plan-validation.test.ts +++ b/src/lib/messaging/plan-validation.test.ts @@ -272,6 +272,20 @@ describe("parseSandboxMessagingPlan", () => { ).toBeNull(); }); + it.each([ + ["a disabled flag missing from disabledChannels", true, []], + ["disabledChannels membership with an enabled flag", false, ["telegram"]], + ] as const)("rejects %s", (_label, disabled, disabledChannels) => { + expect( + parseSandboxMessagingPlan( + makePlan({ + channels: [{ ...makePlan().channels[0], disabled }], + disabledChannels: [...disabledChannels], + }), + ), + ).toBeNull(); + }); + it.each([ ["disabledChannels", { disabledChannels: [" Telegram "] }], ["credentialBindings", { credentialBindings: [{ channelId: "Telegram" }] }], diff --git a/src/lib/messaging/plan-validation.ts b/src/lib/messaging/plan-validation.ts index bdf411e286c..f4a2a91e4ba 100644 --- a/src/lib/messaging/plan-validation.ts +++ b/src/lib/messaging/plan-validation.ts @@ -103,6 +103,18 @@ export function parseSandboxMessagingPlan( normalizedChannelIds.add(normalizedChannelId); } if (!value.disabledChannels.every(isCanonicalMessagingChannelId)) return null; + const disabledChannelIds = new Set(value.disabledChannels as string[]); + if ( + disabledChannelIds.size !== value.disabledChannels.length || + [...disabledChannelIds].some((channelId) => !normalizedChannelIds.has(channelId)) || + value.channels.some( + (channel) => + isObjectRecord(channel) && + (channel.disabled === true) !== disabledChannelIds.has(String(channel.channelId)), + ) + ) { + return null; + } if ( !hasCanonicalChannelReferences(value.credentialBindings) || !hasCanonicalChannelReferences(value.agentRender) || diff --git a/src/lib/onboard/inference-route.ts b/src/lib/onboard/inference-route.ts index 347ee7121a5..55b78b08d22 100644 --- a/src/lib/onboard/inference-route.ts +++ b/src/lib/onboard/inference-route.ts @@ -16,7 +16,7 @@ import { listSandboxes } from "../state/registry"; type RunCaptureOpenshell = (args: string[], options?: { ignoreError?: boolean }) => string | null; -/** Resolve the exact portable inference route shared by managed rebuild and clone paths. */ +/** Resolve the exact portable inference route used by managed clone preparation. */ export function resolveManagedStartupInferenceRoute( agentName: string, provider: string, diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 19c6ebd8de6..f3cb27bcf25 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -117,7 +117,7 @@ runtime mutation | **`--fresh` onboard** — `resolveOnboardEntryOptions`, `prepareFreshSession`, `createBaseImageResolutionContext` | Current flags/environment/prompts replace resumable intent. `--fresh` disables auto-resume and forces base-image resolution; it does not prove that the selected sandbox name is unused. | The first destructive effect is local: the prior onboard session is cleared before a new session is saved. A matching live sandbox can later reuse or recreate through the normal sandbox decision; `--fresh` does not itself delete it. | The new session and machine snapshot replace the old resume checkpoint. Credential and effect boundaries then match new onboard or live recreate. | The discarded resume checkpoint is not restored on later failure. Covered by `entry-options.test.ts`, `session-bootstrap.test.ts`, and base-image resolution tests. | | **Resume, re-onboard, or recreate** — `onboard()`, `prepareOnboardSession`, `decideSandboxResume`, live-sandbox handling in `createSandbox` | For `--resume`, the recorded session is authoritative and conflicting current name/provider/model/image/tool-disclosure hints are rejected. A new re-onboard run takes current flags, environment, and prompts as intent while registry/gateway state provides drift evidence. The machine resolves a complete secret-free create intent, including policy, messaging/provider, GPU, resource, disabled-channel, and agent inputs, before repair/removal or live recreation. | Ordinary live recreation conditionally backs up before provider cleanup, **delete**, image removal, and registry removal. A selected pre-upgrade backup suppresses a new one; an explicit override permits recreation without backup. Resume registry removal and `repair-and-recreate` now follow complete intent validation. Temporary policy/build artifacts remain materialization effects after the delete boundary. | Resume continues the recorded session/machine snapshot; non-resume re-onboard writes a new session first. OpenClaw records completed sandbox name, web search, messaging, and resource choices with explicit progress markers, including explicit `null` choices, while the complete create intent stays process-local and is not persisted or emitted. Raw credential values remain outside the session. A missing process value can be rebound only when the same OpenClaw session recorded successfully registering that provider and its live name, type, and credential-key binding still match; otherwise interactive resume requests it again and non-interactive resume exits with environment-variable guidance. Credentials are checked before mutation and again immediately before materialization. | Restore failures warn and can still publish the replacement; managed-DCode live-selection failure leaves a running, unregistered sandbox with manual-delete guidance. Checkpoint replay reuses an exact live sandbox after an interrupted create and backfills missing create/register receipts. Cancel rollback is not armed and there is no rebuild-style receipt rollback. Coverage: transition traces, create-intent characterization, checkpoint replay and resume guards, and sandbox-handler crash recovery. Gaps: early backup asymmetry and no rebuild-style cross-effect rollback. | | **Rebuild or installer-driven upgrade** — `rebuildSandbox` in `rebuild-pipeline.ts`; `upgradeSandboxes` | Registry state is authoritative. A matching session may fill guarded legacy gaps only when its selection agrees; an unrelated/global session is never used. Ambient provider/model selection is quarantined by `isolateAmbientRecreateEnv`, apart from narrowly scoped legacy recovery. Preflight assembles target config, messaging/policy/runtime inputs, recovery inputs, and a retained replacement context. Generic agents use `preflightRebuildImage`; DCode uses its specialized managed-context preflight instead and proves the live route only for normal live rebuild. Resource profile is not part of preflight. | Consent persistence, target-gateway selection/recovery, and target-preflight registry updates can precede disposable image build/probes. Backup is the first durable recovery checkpoint when available. Shields unlock, MCP detach/scrub, and NIM stop are destructive in-place effects before the **sandbox delete** boundary. Prepared context and mutation-edge conditions are rechecked before delete, proving buildability/input identity but not replacement health or atomic swap. | Durable checkpoints are the backup/recovery manifest when one exists and the rewritten recreate session; stale recovery can reach deletion without a manifest, making that session its first new durable checkpoint. Rollback receipts/snapshots are process-local. Credential metadata comes from the target or guarded fallback; raw credentials/providers are checked against current process/gateway state, while prepared installer recovery may reconstruct a missing gateway provider from a validated host credential. | In-process rollback best-effort restores registry/MCP retry metadata, but process death after non-MCP delete can still lose it. The inner onboard selects resource profile after deletion from non-quarantined ambient input. Covered by rebuild, image-preflight, DCode, and messaging tests. Gaps: post-delete resource intent plus health-before-delete and atomic swap. Closed issue #5801 records the original gap; #6835 fixed only the printed recovery path. | -| **Managed snapshot clone handoff (internal and dormant)** — `prepareManagedWorkloadCloneHandoff` | The current source registry row owns mutable operator intent; the selected snapshot owns immutable managed-workload and provider-runtime history. Preparation proves the selected runtime provider and its `clone` capability, exact current registry generation and live-identity fingerprint, snapshot/source workload equivalence, snapshot runtime generation, and the state layer's selected-manifest/payload digest. It then rebinds the secret-free startup profile, messaging intent, dashboard identity, and any provider-owned Hermes inference name for OpenClaw, Hermes, or DCode without a central Podman-specific switch. | None. The handoff is an inert planning artifact and performs no provider, sandbox, registry, filesystem, credential, or broker effect. Production snapshot restore continues to reject cross-sandbox managed-image restore through `rejectManagedSnapshotCloneUntilRebind`; no user-visible clone support is advertised by this slice. | The returned handoff is a deeply frozen, locally owned value carrying exact source registry compare-and-swap authority, immutable workload authority, provider runtime generation evidence, `SnapshotRestoreAuthority` content identity, rebound managed profile, and destination registry intent. It contains credential-presence metadata and provider names, never raw credential values or live handles. | There is intentionally no compensation because preparation has no effects. All-agent and Docker/MXC-style provider tests cover the dormant contract and canonical name boundaries. Provider materialization, destination creation/bootstrap, mutation-edge content/provider authority revalidation, rollback, recovery, protected E2E, and activation remain later slices. | +| **Managed snapshot clone handoff (internal and dormant)** — `prepareManagedWorkloadCloneHandoff` | The current source registry row owns mutable operator intent; the selected snapshot owns immutable managed-workload and provider-runtime history. Preparation proves the selected runtime provider and its `clone` capability, exact current registry generation and live-identity fingerprint, snapshot/source workload equivalence, snapshot runtime generation, and the state layer's selected-manifest/payload digest. It then rebinds the secret-free startup profile, messaging intent, dashboard identity, and any provider-owned Hermes inference name for OpenClaw, Hermes, or DCode without a central Podman-specific switch. | None. The handoff is an inert planning artifact and performs no provider, sandbox, registry, filesystem, credential, or broker effect. Production snapshot restore continues to reject cross-sandbox managed-image restore through `rejectManagedSnapshotCloneUntilRebind`; no user-visible clone support is advertised by this slice. | The returned handoff is a deeply frozen, locally owned value carrying exact source registry compare-and-swap authority, immutable workload authority, provider runtime generation evidence, `SnapshotRestoreAuthority` content identity, rebound managed profile, and destination registry intent. It contains credential-presence metadata and provider names, never raw credential values or live handles. | There is intentionally no compensation because preparation has no effects. `src/lib/onboard/managed-workload-clone-handoff.test.ts`, `src/lib/onboard/managed-startup-clone-rebinder.test.ts`, and `src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts` cover the all-agent, Docker/MXC-style provider, canonical-name, and fail-closed boundaries. Provider materialization, destination creation/bootstrap, mutation-edge content/provider authority revalidation, rollback, recovery, protected E2E, and activation remain tracked by [#7744](https://github.com/NVIDIA/NemoClaw/issues/7744). | | **Channel add/remove/start/stop** — `addSandboxChannel`, `removeSandboxChannel`, `sandboxChannelsSetEnabled` in `policy-channel.ts` | Add compiles and merges a manifest-derived channel delta with `MessagingWorkflowPlanner`. Start, stop, and remove transform the registry plan and rehydrate executable render/build/runtime/forward details from current manifests. | Token-backed add can mutate gateway credentials before policy and plan persistence; QR/in-sandbox-auth add skips that credential upsert. Start persists the enabled plan before policy; stop persists the disabled plan before the rebuild prompt. Remove clears QR-backed durable state when applicable, detaches gateway/bridge state, removes policy, then persists the plan. A queued rebuild has a separate delete boundary. | The compact registry messaging plan is authoritative; render/build/runtime/state/health entries and nested host-forward details are rehydrated rather than persisted. Session policy-preset sync is best-effort, and channel mutations do not rewrite `Session.messagingPlan`. Raw tokens stay in process/gateway bindings. | `rollbackChannelAdd`, re-disable after failed start, and fail-closed QR-state cleanup provide partial compensation. Covered by `policy-channel*.test.ts`, `workflow-planner.test.ts`, and channel integration tests. Gaps: channel add has a separate `--force` conflict policy; add/remove effects can precede plan persistence, and persistence failures are not fully rolled back. | | **Provider, model, or credential-binding change** — `runInferenceSet` | CLI intent plus registry/session metadata. Target resolution and OpenShell preparation occur before locking. The target is re-resolved in the mutating phase under the sandbox lifecycle and timer-bound shields locks; that phase validates provider/model syntax, selected agent, shields state, and local reachability before the first write. | First mutation is the gateway route, then a minimal registry write, API-family/config resolution, registry refresh, best-effort config/hash sync, matching-session update, and audit. An OpenClaw API-family change can then restart the managed gateway after the shields lock is released but while the outer sandbox lock remains held. No sandbox deletion. | Registry and matching session store logical provider/model/credential-environment metadata. Audit records the action, sandbox, and reason rather than credentials; raw values remain gateway-bound. | Forward-only; no rollback. `rebuild` is the repair path for degraded state. Covered by `inference-set*.test.ts`. Gap: several stores can diverge after a mid-sequence failure. | | **Credential rotation** — `configRotateToken` in `src/lib/sandbox/config.ts` | A session with `credentialEnv` selects the provider and binding. A non-null different `sandboxName` is rejected, but a legacy/null session name is accepted for the requested sandbox. The new value comes from a named environment variable, stdin, or a secret prompt; it is trimmed, then rejected when empty or still containing internal whitespace. | `saveCredential` first stages the value in the current process. OpenShell provider update is the first external mutation, with provider create as a fallback; audit follows. No sandbox deletion. | The logical binding is unchanged, so session and registry are not rewritten. The raw value exists only in process memory/environment and the gateway provider; audit records action/sandbox/reason without the value. | No rollback after a successful provider update; an audit failure can report failure after the credential is already active. Covered by the rotate-token cases in `test/config-set-nested-ssrf.test.ts`. Gap: a null-name legacy session is not strongly bound to the requested sandbox. | @@ -236,6 +236,7 @@ PR #5955 moved the rebuild messaging conflict check before destruction. | Session sanitation, sandbox prompt checkpoints, and no-secret persistence | `src/lib/state/onboard-session-sandbox-prompts.test.ts`, `src/lib/state/onboard-checkpoint.test.ts`, `machine/handlers/sandbox-create-intent-boundary.test.ts` | Tri-state decisions remain scoped to checkpointed sandbox choices. | | Versioned checkpoint schema, tri-state decisions, migration, and unknown-future fail-safe | `src/lib/state/onboard-checkpoint.test.ts`, `src/lib/state/onboard-checkpoint-migrate.test.ts` | Live decision reads still use legacy fields | | Resumable create replay, durable identity, and stale-binding fail-closed | `src/lib/onboard/checkpoint-replay.test.ts`, `src/lib/onboard/checkpoint-resume-guard.test.ts`, `machine/handlers/sandbox-checkpoint-crash-recovery.test.ts` | None at the sandbox-handler boundary. | -| Managed snapshot workload, content, and provider authority across explicit and rebuild flows | `src/lib/actions/sandbox/snapshot/backup-authority.test.ts`, `restore-authority.test.ts`, `managed-profile.test.ts`, `provider-lifecycle.test.ts`, and `snapshot-managed-provider-restore-order.test.ts` | Cross-provider clone and rebind, durable interrupted-restore recovery, and user-visible runtime activation remain separate review units. | +| Managed snapshot workload, content, and provider authority across explicit and rebuild flows | `src/lib/actions/sandbox/snapshot/backup-authority.test.ts`, `restore-authority.test.ts`, `managed-profile.test.ts`, `provider-lifecycle.test.ts`, and `snapshot-managed-provider-restore-order.test.ts` | Durable interrupted-restore recovery and user-visible runtime activation remain separate review units. | +| Dormant managed clone handoff and fail-closed production boundary | `src/lib/onboard/managed-workload-clone-handoff.test.ts`, `src/lib/onboard/managed-startup-clone-rebinder.test.ts`, and `src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts` | Provider materialization, destination bootstrap, rollback, recovery, protected E2E, and activation remain tracked by [#7744](https://github.com/NVIDIA/NemoClaw/issues/7744). | When lifecycle behavior changes one of these contracts, update the map and the narrow owning test in that same PR. Do not add source-text scans or production scaffolding solely to preserve current orchestration order. From 361b455f050cca3c01974fefd12806b5c13a5e86 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 21:07:50 -0700 Subject: [PATCH 071/117] test(runtime): advance managed bootstrap boundary Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/index.ts | 1 - test/runtime-provider-source-shape.test.ts | 11 ++++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/index.ts b/src/lib/onboard/managed-bootstrap/index.ts index 5f03ff33572..c55768afe02 100644 --- a/src/lib/onboard/managed-bootstrap/index.ts +++ b/src/lib/onboard/managed-bootstrap/index.ts @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -export { resolveOpenShellSandboxId } from "../../adapters/openshell/sandbox-identity"; export { activateManagedBootstrapSequence, finalizeManagedBootstrapSequence, diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index ece9d79adaf..ceebdd516c8 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -8,7 +8,7 @@ import { describe, expect, it } from "vitest"; const repoRoot = join(import.meta.dirname, ".."); describe("runtime provider central source boundary", () => { - // source-shape-contract: compatibility -- Migrated lifecycle and mutation consumers must stay provider-neutral while production selection excludes unqualified future providers and managed-bootstrap dependencies + // source-shape-contract: compatibility -- Migrated lifecycle and mutation consumers must stay provider-neutral while production selection excludes unqualified future providers and driver-specific bootstrap dependencies it("keeps migrated provider identities and implementations behind the one bundle composition", () => { const read = (relativePath: string) => readFileSync(join(repoRoot, relativePath), "utf8"); const driverNeutralActions = { @@ -53,7 +53,12 @@ describe("runtime provider central source boundary", () => { expect(driverNeutralActions["actions/sandbox/start.ts"]).toMatch( /resolved\.lifecycle\.verifyStarted\(/u, ); - expect(Object.values(providerContract).join("\n")).not.toMatch(/managed-bootstrap/u); + expect(providerContract.contract).toMatch( + /import type[\s\S]*from ["']\.\.\/managed-bootstrap\/runtime-create["']/u, + ); + expect( + [providerContract.current, providerContract.docker, providerContract.registry].join("\n"), + ).not.toMatch(/managed-bootstrap/u); expect(providerContract.current).not.toMatch(/\b(?:podman|mxc)\b/iu); }); @@ -97,7 +102,7 @@ describe("runtime provider central source boundary", () => { ); expect(bootstrapProtocol.join("\n")).not.toMatch(/\b(?:docker|podman|openshell|mxc)\b/iu); expect(activationSources.join("\n")).not.toMatch( - /(?:from\s+["'][^"']*managed-bootstrap|require\([^)]*managed-bootstrap)/u, + /(?:from\s+["'][^"']*managed-bootstrap\/(?:docker|docker-journal|docker-runtime)|require\([^)]*managed-bootstrap)/u, ); expect(dockerProvider).not.toMatch( /(?:from\s+["'][^"']*managed-bootstrap|require\([^)]*managed-bootstrap)/u, From c2e6f06f2276c0e9d449cf97dbec3c82cf330087 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 21:15:52 -0700 Subject: [PATCH 072/117] fix(snapshot): address provider transaction review Signed-off-by: Aaron Erickson --- .../snapshot-managed-clone-providers.test.ts | 46 +++++++++++++++++++ .../snapshot/managed-clone-providers.ts | 13 +++++- src/lib/onboard/lifecycle-contracts.md | 2 +- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts b/src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts index e06cb4523c5..cec4e673e20 100644 --- a/src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts +++ b/src/lib/actions/sandbox/snapshot-managed-clone-providers.test.ts @@ -332,6 +332,24 @@ describe("managed clone provider transaction", () => { ).toBe(false); }); + it("rejects a destination registered under another runtime provider", () => { + const profile = managedStartupE2eProfile("langchain-deepagents-code"); + const source = entry("source", profile); + const destination = entry("destination", profile, { openshellDriver: "mxc" }); + const runner = providerRunner(); + + expect(() => + prepareManagedCloneProviderTransaction({ + handoff: handoff(profile, source), + destination, + environment: {}, + runOpenshell: runner.run, + transactionId: "8".repeat(32), + }), + ).toThrow(/destination registry authority uses a different runtime provider/u); + expect(runner.run).not.toHaveBeenCalled(); + }); + it("fails closed on indeterminate provider inspection with bounded diagnostics", () => { const profile = managedStartupE2eProfile("openclaw"); const source = entry("source", profile); @@ -450,6 +468,34 @@ describe("managed clone provider transaction", () => { expect(runner.live.get(TOKEN_BINDING.providerName)?.providerType).toBe("other"); }); + it("bounds provider creation before exact-result reconciliation", () => { + const { prepared, runner, source } = prepareWithBinding({}); + + provisionManagedCloneProviderTransaction(prepared, { + ...authorityDeps(source), + environment: { RUNTIME_TOKEN: "test-only-runtime-token" }, + runOpenshell: runner.run, + }); + + expect(runner.run).toHaveBeenCalledWith( + [ + "provider", + "create", + "--name", + TOKEN_BINDING.providerName, + "--type", + TOKEN_BINDING.providerType, + "--credential", + TOKEN_BINDING.providerEnvKey, + ], + expect.objectContaining({ + maxBuffer: 64 * 1024, + suppressOutput: true, + timeout: 30_000, + }), + ); + }); + it("rolls back confirmed providers when a later credential disappears", () => { const first = { ...TOKEN_BINDING, providerName: "destination-first-token" }; const second = { diff --git a/src/lib/actions/sandbox/snapshot/managed-clone-providers.ts b/src/lib/actions/sandbox/snapshot/managed-clone-providers.ts index f87189523da..be1a595f64e 100644 --- a/src/lib/actions/sandbox/snapshot/managed-clone-providers.ts +++ b/src/lib/actions/sandbox/snapshot/managed-clone-providers.ts @@ -25,6 +25,7 @@ import type { SandboxEntry } from "../../../state/registry/types"; import * as sandboxState from "../../../state/sandbox"; const PROVIDER_PROBE_DIAGNOSTIC_LIMIT = 64 * 1024; +const PROVIDER_CREATE_TIMEOUT_MS = 30_000; const PROVIDER_PROBE_TIMEOUT_MS = 5_000; const PROVIDER_TYPE_PATTERN = /^[A-Za-z0-9._:-]{1,64}$/u; const PROVIDER_ENV_KEY_PATTERN = /^[A-Z_][A-Z0-9_]{0,127}$/u; @@ -375,6 +376,12 @@ export function prepareManagedCloneProviderTransaction(input: { if (input.destination && input.destination.name !== destinationSandboxName) { fail("destination registry authority names a different sandbox"); } + const destinationProviderId = input.destination + ? normalizeRuntimeProviderIdentity(input.destination.openshellDriver) + : null; + if (destinationProviderId !== null && destinationProviderId !== input.handoff.providerId) { + fail("destination registry authority uses a different runtime provider"); + } const desired = mergeBindings([ ...applicationBindings({ profile: input.handoff.rebound.profile, @@ -416,11 +423,11 @@ export function prepareManagedCloneProviderTransaction(input: { } let destinationRegistryAuthority: SandboxRebuildAuthority | undefined; - if (input.destination) { + if (input.destination && destinationProviderId !== null) { try { destinationRegistryAuthority = captureSandboxRebuildAuthority( input.destination, - normalizeRuntimeProviderIdentity(input.destination.openshellDriver), + destinationProviderId, ); } catch (error) { fail("destination has no exact managed registry authority", error); @@ -551,8 +558,10 @@ export function provisionManagedCloneProviderTransaction( { ignoreError: true, env: { [provider.binding.providerEnvKey]: credential }, + maxBuffer: PROVIDER_PROBE_DIAGNOSTIC_LIMIT, stdio: ["ignore", "pipe", "pipe"], suppressOutput: true, + timeout: PROVIDER_CREATE_TIMEOUT_MS, }, ); } catch (error) { diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 54197352d28..9a10af3fb8d 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -117,7 +117,7 @@ runtime mutation | **`--fresh` onboard** — `resolveOnboardEntryOptions`, `prepareFreshSession`, `createBaseImageResolutionContext` | Current flags/environment/prompts replace resumable intent. `--fresh` disables auto-resume and forces base-image resolution; it does not prove that the selected sandbox name is unused. | The first destructive effect is local: the prior onboard session is cleared before a new session is saved. A matching live sandbox can later reuse or recreate through the normal sandbox decision; `--fresh` does not itself delete it. | The new session and machine snapshot replace the old resume checkpoint. Credential and effect boundaries then match new onboard or live recreate. | The discarded resume checkpoint is not restored on later failure. Covered by `entry-options.test.ts`, `session-bootstrap.test.ts`, and base-image resolution tests. | | **Resume, re-onboard, or recreate** — `onboard()`, `prepareOnboardSession`, `decideSandboxResume`, live-sandbox handling in `createSandbox` | For `--resume`, the recorded session is authoritative and conflicting current name/provider/model/image/tool-disclosure hints are rejected. A new re-onboard run takes current flags, environment, and prompts as intent while registry/gateway state provides drift evidence. The machine resolves a complete secret-free create intent, including policy, messaging/provider, GPU, resource, disabled-channel, and agent inputs, before repair/removal or live recreation. | Ordinary live recreation conditionally backs up before provider cleanup, **delete**, image removal, and registry removal. A selected pre-upgrade backup suppresses a new one; an explicit override permits recreation without backup. Resume registry removal and `repair-and-recreate` now follow complete intent validation. Temporary policy/build artifacts remain materialization effects after the delete boundary. | Resume continues the recorded session/machine snapshot; non-resume re-onboard writes a new session first. OpenClaw records completed sandbox name, web search, messaging, and resource choices with explicit progress markers, including explicit `null` choices, while the complete create intent stays process-local and is not persisted or emitted. Raw credential values remain outside the session. A missing process value can be rebound only when the same OpenClaw session recorded successfully registering that provider and its live name, type, and credential-key binding still match; otherwise interactive resume requests it again and non-interactive resume exits with environment-variable guidance. Credentials are checked before mutation and again immediately before materialization. | Restore failures warn and can still publish the replacement; managed-DCode live-selection failure leaves a running, unregistered sandbox with manual-delete guidance. Checkpoint replay reuses an exact live sandbox after an interrupted create and backfills missing create/register receipts. Cancel rollback is not armed and there is no rebuild-style receipt rollback. Coverage: transition traces, create-intent characterization, checkpoint replay and resume guards, and sandbox-handler crash recovery. Gaps: early backup asymmetry and no rebuild-style cross-effect rollback. | | **Rebuild or installer-driven upgrade** — `rebuildSandbox` in `rebuild-pipeline.ts`; `upgradeSandboxes` | Registry state is authoritative. A matching session may fill guarded legacy gaps only when its selection agrees; an unrelated/global session is never used. Ambient provider/model selection is quarantined by `isolateAmbientRecreateEnv`, apart from narrowly scoped legacy recovery. Preflight assembles target config, messaging/policy/runtime inputs, recovery inputs, and a retained replacement context. Generic agents use `preflightRebuildImage`; DCode uses its specialized managed-context preflight instead and proves the live route only for normal live rebuild. Resource profile is not part of preflight. | Consent persistence, target-gateway selection/recovery, and target-preflight registry updates can precede disposable image build/probes. Backup is the first durable recovery checkpoint when available. Shields unlock, MCP detach/scrub, and NIM stop are destructive in-place effects before the **sandbox delete** boundary. Prepared context and mutation-edge conditions are rechecked before delete, proving buildability/input identity but not replacement health or atomic swap. | Durable checkpoints are the backup/recovery manifest when one exists and the rewritten recreate session; stale recovery can reach deletion without a manifest, making that session its first new durable checkpoint. Rollback receipts/snapshots are process-local. Credential metadata comes from the target or guarded fallback; raw credentials/providers are checked against current process/gateway state, while prepared installer recovery may reconstruct a missing gateway provider from a validated host credential. | In-process rollback best-effort restores registry/MCP retry metadata, but process death after non-MCP delete can still lose it. The inner onboard selects resource profile after deletion from non-quarantined ambient input. Covered by rebuild, image-preflight, DCode, and messaging tests. Gaps: post-delete resource intent plus health-before-delete and atomic swap. Closed issue #5801 records the original gap; #6835 fixed only the printed recovery path. | -| **Managed snapshot clone handoff and provider transaction (internal and dormant)** — `prepareManagedWorkloadCloneHandoff`; `prepareManagedCloneProviderTransaction` | The current source registry row owns mutable operator intent; the selected snapshot owns immutable managed-workload and provider-runtime history. Handoff preparation proves the selected runtime provider and its `clone` capability, exact current registry generation and live-identity fingerprint, snapshot/source workload equivalence, snapshot runtime generation, and the state layer's selected-manifest/payload digest. It rebinds the secret-free startup profile, messaging intent, dashboard identity, and provider-owned contributions for OpenClaw, Hermes, or DCode without a central Podman-specific switch. Provider preparation then resolves active application bindings plus provider-contributed bindings, treating a live exact provider as reusable only when the destination registry independently proves that same logical binding. | The handoff and provider plan are inert. The internal materializer can create only bindings proven absent at preflight; it never updates or deletes an existing destination-owned provider. Immediately before each create it revalidates the source and optional destination registry rows plus the exact `SnapshotRestoreAuthority`. Production snapshot restore does not invoke this transaction and continues to reject cross-sandbox managed-image restore through `rejectManagedSnapshotCloneUntilRebind`; no user-visible clone support is advertised. | Both plans are deeply frozen and secret-free. A successful create produces an exact process-local ownership receipt; a non-zero create reconciled to an exact provider remains ambiguous and unowned. The receipt ledger remembers completed cleanup so a repeated cleanup cannot delete a later same-name provider. Raw credentials exist only in the explicit apply environment and one OpenShell child environment. | Failure rolls back only providers confirmed created by the exact in-process receipt, preserves collisions and ambiguous creates, reports incomplete cleanup for retry, and never rewrites a reused provider. All-agent and Docker/MXC-style handoff tests plus provider race, force-replace, disappearing-credential, rollback, and idempotent-cleanup tests cover the dormant contract. Destination creation/bootstrap, filesystem mutation-edge invocation, Hermes broker activation, durable recovery, protected E2E, and user-visible activation remain later slices. | +| **Managed snapshot clone handoff and provider transaction (internal and dormant)** — `prepareManagedWorkloadCloneHandoff`; `prepareManagedCloneProviderTransaction` | The current source registry row owns mutable operator intent; the selected snapshot owns immutable managed-workload and provider-runtime history. Handoff preparation proves the selected runtime provider and its `clone` capability, exact current registry generation and live-identity fingerprint, snapshot/source workload equivalence, snapshot runtime generation, and the state layer's selected-manifest/payload digest. It rebinds the secret-free startup profile, messaging intent, dashboard identity, and provider-owned contributions for OpenClaw, Hermes, or DCode without a central Podman-specific switch. Provider preparation then resolves active application bindings plus provider-contributed bindings, treating a live exact provider as reusable only when the destination registry independently proves that same logical binding. | The handoff and provider plan are inert. The internal materializer can create only bindings proven absent at preflight; it never updates or deletes an existing destination-owned provider. Immediately before each create it revalidates the source and optional destination registry rows plus the exact `SnapshotRestoreAuthority`. Production snapshot restore does not invoke this transaction and continues to reject cross-sandbox managed-image restore through `rejectManagedSnapshotCloneUntilRebind`; no user-visible clone support is advertised. | Both plans are deeply frozen and secret-free. A successful create produces an exact process-local ownership receipt; a non-zero create reconciled to an exact provider remains ambiguous and unowned. The receipt ledger remembers completed cleanup so a repeated cleanup cannot delete a later same-name provider. Raw credentials exist only in the explicit apply environment and one OpenShell child environment. | Failure rolls back only providers confirmed created by the exact in-process receipt, preserves collisions and ambiguous creates, reports incomplete cleanup for retry, and never rewrites a reused provider. All-agent and Docker/MXC-style handoff tests plus provider race, force-replace, disappearing-credential, rollback, and idempotent-cleanup tests cover the dormant contract. This PR intentionally covers only the dormant contract. Epic [#7744](https://github.com/NVIDIA/NemoClaw/issues/7744) tracks destination creation/bootstrap, filesystem mutation-edge invocation, Hermes broker activation, durable recovery, protected E2E, and user-visible activation. | | **Channel add/remove/start/stop** — `addSandboxChannel`, `removeSandboxChannel`, `sandboxChannelsSetEnabled` in `policy-channel.ts` | Add compiles and merges a manifest-derived channel delta with `MessagingWorkflowPlanner`. Start, stop, and remove transform the registry plan and rehydrate executable render/build/runtime/forward details from current manifests. | Token-backed add can mutate gateway credentials before policy and plan persistence; QR/in-sandbox-auth add skips that credential upsert. Start persists the enabled plan before policy; stop persists the disabled plan before the rebuild prompt. Remove clears QR-backed durable state when applicable, detaches gateway/bridge state, removes policy, then persists the plan. A queued rebuild has a separate delete boundary. | The compact registry messaging plan is authoritative; render/build/runtime/state/health entries and nested host-forward details are rehydrated rather than persisted. Session policy-preset sync is best-effort, and channel mutations do not rewrite `Session.messagingPlan`. Raw tokens stay in process/gateway bindings. | `rollbackChannelAdd`, re-disable after failed start, and fail-closed QR-state cleanup provide partial compensation. Covered by `policy-channel*.test.ts`, `workflow-planner.test.ts`, and channel integration tests. Gaps: channel add has a separate `--force` conflict policy; add/remove effects can precede plan persistence, and persistence failures are not fully rolled back. | | **Provider, model, or credential-binding change** — `runInferenceSet` | CLI intent plus registry/session metadata. Target resolution and OpenShell preparation occur before locking. The target is re-resolved in the mutating phase under the sandbox lifecycle and timer-bound shields locks; that phase validates provider/model syntax, selected agent, shields state, and local reachability before the first write. | First mutation is the gateway route, then a minimal registry write, API-family/config resolution, registry refresh, best-effort config/hash sync, matching-session update, and audit. An OpenClaw API-family change can then restart the managed gateway after the shields lock is released but while the outer sandbox lock remains held. No sandbox deletion. | Registry and matching session store logical provider/model/credential-environment metadata. Audit records the action, sandbox, and reason rather than credentials; raw values remain gateway-bound. | Forward-only; no rollback. `rebuild` is the repair path for degraded state. Covered by `inference-set*.test.ts`. Gap: several stores can diverge after a mid-sequence failure. | | **Credential rotation** — `configRotateToken` in `src/lib/sandbox/config.ts` | A session with `credentialEnv` selects the provider and binding. A non-null different `sandboxName` is rejected, but a legacy/null session name is accepted for the requested sandbox. The new value comes from a named environment variable, stdin, or a secret prompt; it is trimmed, then rejected when empty or still containing internal whitespace. | `saveCredential` first stages the value in the current process. OpenShell provider update is the first external mutation, with provider create as a fallback; audit follows. No sandbox deletion. | The logical binding is unchanged, so session and registry are not rewritten. The raw value exists only in process memory/environment and the gateway provider; audit records action/sandbox/reason without the value. | No rollback after a successful provider update; an audit failure can report failure after the credential is already active. Covered by the rotate-token cases in `test/config-set-nested-ssrf.test.ts`. Gap: a null-name legacy session is not strongly bound to the requested sandbox. | From 9e64898342ea98b9254989c86a09b43598726ffd Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 21:16:03 -0700 Subject: [PATCH 073/117] docs(snapshot): clarify legacy restore retention Signed-off-by: Aaron Erickson --- src/lib/onboard/lifecycle-contracts.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 870cd41ab4b..36ce0c7c201 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -161,11 +161,11 @@ Provider inputs are detached and deeply frozen at the extension boundary, and ce Docker lifecycle inspection and GPU inspection remain inside the Docker provider adapter. The provider-neutral receipt can represent another provider, including an MXC-style implementation, without adding provider switches to snapshot or rebuild orchestration. -Legacy and custom-image snapshots retain their state-only backup and restore path until the -[incremental runtime epic](https://github.com/NVIDIA/NemoClaw/issues/7744) completes managed -create finalization, clone/rebind, recovery, and activation for every supported agent. The path can -retire only after those slices prove managed authority before mutation and exact recovery across -OpenClaw, Hermes, and Deep Agents Code while legacy/custom-image restores keep parity coverage. +Legacy and custom-image snapshots retain their state-only backup and restore path. The managed +authority path may become the default for managed images only after the +[incremental runtime epic](https://github.com/NVIDIA/NemoClaw/issues/7744) completes create +finalization, clone/rebind, recovery, and activation for every supported agent with authority proven +before mutation. Any later consolidation must preserve legacy and custom-image restore parity. This contract does not activate another runtime provider or managed-image onboarding path. Ordinary onboard recreation and create finalization remain deferred under [#7744](https://github.com/NVIDIA/NemoClaw/issues/7744) because the replacement target is not From 380925ac8212c5bba7dfb40d6f96a3422e8fc7d1 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 21:34:20 -0700 Subject: [PATCH 074/117] fix(hermes): address broker transaction review Signed-off-by: Aaron Erickson --- .../host/runtime-refresh-credentials.ts | 18 +++ agents/hermes/host/tool-gateway-broker.ts | 80 +++++++++--- package.json | 1 + src/lib/hermes-tool-gateway-broker.ts | 27 +++- test/hermes-tool-gateway-broker.test.ts | 116 +++++++++++++++++- ...s-tool-gateway-runtime-credentials.test.ts | 45 +++++++ .../openshell-policy-boundary.test.ts | 18 +++ 7 files changed, 281 insertions(+), 24 deletions(-) diff --git a/agents/hermes/host/runtime-refresh-credentials.ts b/agents/hermes/host/runtime-refresh-credentials.ts index f257a5a40ab..34e4f38d4c6 100644 --- a/agents/hermes/host/runtime-refresh-credentials.ts +++ b/agents/hermes/host/runtime-refresh-credentials.ts @@ -41,6 +41,24 @@ class RuntimeRefreshCredentialStore { return this.register(state, nextRefreshToken); } + replace(state, nextRefreshToken) { + const sandbox = String(state?.sandbox || "").trim(); + if (!sandbox) return null; + const hadPrevious = this.credentials.has(sandbox); + const previous = this.credentials.get(sandbox); + if (!this.register(state, nextRefreshToken)) return null; + const replacement = this.credentials.get(sandbox); + let pending = true; + return () => { + if (!pending) return false; + pending = false; + if (this.credentials.get(sandbox) !== replacement) return false; + if (hadPrevious) this.credentials.set(sandbox, previous); + else this.credentials.delete(sandbox); + return true; + }; + } + unregister(sandboxName) { const sandbox = String(sandboxName || "").trim(); return sandbox ? this.credentials.delete(sandbox) : false; diff --git a/agents/hermes/host/tool-gateway-broker.ts b/agents/hermes/host/tool-gateway-broker.ts index d2da24381ad..a814349fa59 100755 --- a/agents/hermes/host/tool-gateway-broker.ts +++ b/agents/hermes/host/tool-gateway-broker.ts @@ -75,6 +75,8 @@ const UPSTREAM_REQUEST_TIMEOUT_MS = readPositiveIntEnv( 1000, ); const STAGED_CLONE_BINDING_TTL_MS = 5 * 60 * 1000; +const CONTROL_REQUEST_TIMEOUT_MS = 1_000; +const BROKER_SHUTDOWN_TIMEOUT_MS = 1_000; const DEFAULT_INFERENCE_BASE_URL = "https://inference-api.nousresearch.com/v1"; const TRUSTED_INFERENCE_BASE_URLS = new Set([DEFAULT_INFERENCE_BASE_URL]); @@ -296,13 +298,16 @@ function atomicWriteJson(file, value) { fs.chmodSync(file, 0o600); } -function updateOpenshellRefreshProvider(state, refreshToken) { +function updateOpenshellRefreshProvider(state) { const providerName = String(state.provider_name || ""); if (!providerName) return; const providerCredential = - typeof state.broker_token === "string" && state.broker_token - ? state.broker_token - : refreshToken; + typeof state.broker_token === "string" ? state.broker_token.trim() : ""; + if (!providerCredential) { + throw Object.assign(new Error("broker_credential_unavailable"), { + code: "broker_credential_unavailable", + }); + } const result = spawnSync( OPENSHELL_BIN, ["provider", "update", providerName, "--credential", CREDENTIAL_ENV], @@ -406,7 +411,7 @@ async function refreshAccessToken(refreshToken, loaded, deadlineAtMs = null) { }); if (nextDigest !== digest) { - updateOpenshellRefreshProvider(loaded.state, nextRefreshToken); + updateOpenshellRefreshProvider(loaded.state); const nextState = { ...loaded.state, refresh_token_sha256: nextDigest, @@ -609,7 +614,8 @@ function activateStagedCloneBinding(sandbox, activationToken, deadlineAtMs) { inference_agent_key_rotated_at: new Date().toISOString(), updated_at: new Date().toISOString(), }; - if (!runtimeRefreshCredentials.register(nextState, stagedRefreshToken)) { + const restoreRuntimeCredential = runtimeRefreshCredentials.replace(nextState, stagedRefreshToken); + if (!restoreRuntimeCredential) { throw Object.assign(new Error("staged_runtime_registration_failed"), { code: "staged_runtime_registration_failed", }); @@ -624,7 +630,7 @@ function activateStagedCloneBinding(sandbox, activationToken, deadlineAtMs) { atomicWriteJson(loaded.file, nextState); loaded.state = nextState; } catch (error) { - runtimeRefreshCredentials.unregister(sandbox); + restoreRuntimeCredential(); throw error; } runtimeRefreshCredentials.unregister(staged.runtime_credential_state.sandbox); @@ -781,9 +787,15 @@ function readControlJson(req) { return new Promise((resolve, reject) => { const chunks = []; let size = 0; + const timeout = setTimeout(() => { + reject(new Error("control_request_timeout")); + req.destroy(); + }, CONTROL_REQUEST_TIMEOUT_MS); + timeout.unref?.(); req.on("data", (chunk) => { size += chunk.length; if (size > 16_384) { + clearTimeout(timeout); reject(new Error("control_request_too_large")); req.destroy(); return; @@ -791,13 +803,17 @@ function readControlJson(req) { chunks.push(chunk); }); req.on("end", () => { + clearTimeout(timeout); try { resolve(JSON.parse(Buffer.concat(chunks).toString("utf8"))); } catch { reject(new Error("control_request_invalid")); } }); - req.on("error", reject); + req.on("error", (error) => { + clearTimeout(timeout); + reject(error); + }); }); } @@ -873,14 +889,17 @@ async function handleControlRequest(req, res) { if (req.url === "/credentials/register") { const loaded = loadStateForSandbox(sandbox); const refreshToken = String(payload?.refresh_token || "").trim(); - if (!loaded || !runtimeRefreshCredentials.register(loaded.state, refreshToken)) { + const restoreRuntimeCredential = loaded + ? runtimeRefreshCredentials.replace(loaded.state, refreshToken) + : null; + if (!loaded || !restoreRuntimeCredential) { sendText(res, 409, "credential does not match destination broker state"); return; } try { await ensureInferenceAgentKey(loaded, refreshToken); } catch (error) { - runtimeRefreshCredentials.unregister(sandbox); + restoreRuntimeCredential(); throw error; } sendJson(res, 200, { ok: true }); @@ -1076,7 +1095,9 @@ if (CONTROL_SOCKET_PATH) { } catch (error) { if (error?.code !== "ENOENT") throw error; } - fs.mkdirSync(path.dirname(CONTROL_SOCKET_PATH), { recursive: true, mode: 0o700 }); + const controlSocketDirectory = path.dirname(CONTROL_SOCKET_PATH); + fs.mkdirSync(controlSocketDirectory, { recursive: true, mode: 0o700 }); + fs.chmodSync(controlSocketDirectory, 0o700); controlServer = http.createServer((req, res) => { handleControlRequest(req, res).catch((error) => { console.error(`Hermes tool gateway control error: ${error?.message || error}`); @@ -1084,11 +1105,18 @@ if (CONTROL_SOCKET_PATH) { else res.end(); }); }); - controlServer.listen(CONTROL_SOCKET_PATH, () => { - fs.chmodSync(CONTROL_SOCKET_PATH, 0o600); - preflightControlReady = true; - maybeRunPreflightProbe(); - }); + controlServer.requestTimeout = CONTROL_REQUEST_TIMEOUT_MS; + controlServer.headersTimeout = CONTROL_REQUEST_TIMEOUT_MS; + const previousUmask = process.umask(0o177); + try { + controlServer.listen(CONTROL_SOCKET_PATH, () => { + fs.chmodSync(CONTROL_SOCKET_PATH, 0o600); + preflightControlReady = true; + maybeRunPreflightProbe(); + }); + } finally { + process.umask(previousUmask); + } if (PREFLIGHT_PROBE) controlServer.on("error", () => finishPreflightProbe(2)); } @@ -1119,8 +1147,17 @@ if (!PREFLIGHT_PROBE) { refreshTimer.unref?.(); } +let brokerClosing = false; + function closeBroker() { + if (brokerClosing) return; + brokerClosing = true; + let exited = false; + let shutdownTimer; const exit = () => { + if (exited) return; + exited = true; + clearTimeout(shutdownTimer); if (CONTROL_SOCKET_PATH) { try { fs.unlinkSync(CONTROL_SOCKET_PATH); @@ -1130,8 +1167,15 @@ function closeBroker() { } process.exit(0); }; - if (controlServer) controlServer.close(() => server.close(exit)); - else server.close(exit); + shutdownTimer = setTimeout(() => { + controlServer?.closeAllConnections(); + server.closeAllConnections(); + exit(); + }, BROKER_SHUTDOWN_TIMEOUT_MS); + shutdownTimer.unref?.(); + const closePublicServer = () => server.close(exit); + if (controlServer) controlServer.close(closePublicServer); + else closePublicServer(); } process.on("SIGTERM", closeBroker); diff --git a/package.json b/package.json index 0e335c3d561..92b617dee5c 100644 --- a/package.json +++ b/package.json @@ -107,6 +107,7 @@ "files": [ ".version", ".source-revision", + "agents/hermes/host/", "bin/", "dist/", "src/lib/messaging/channels/**/policy/*.{yaml,yml}", diff --git a/src/lib/hermes-tool-gateway-broker.ts b/src/lib/hermes-tool-gateway-broker.ts index 836a88821d0..0cad0463661 100644 --- a/src/lib/hermes-tool-gateway-broker.ts +++ b/src/lib/hermes-tool-gateway-broker.ts @@ -228,6 +228,7 @@ function brokerControlRequest(route, payload) { function registerHermesToolGatewayRuntimeCredential(refreshToken, exactSandboxName = null) { const digest = hashRefreshToken(refreshToken); let matched = false; + if (exactSandboxName === null) ensurePrivateDir(HERMES_TOOL_GATEWAY_STATE_DIR); const stateNames = exactSandboxName === null ? fs.readdirSync(HERMES_TOOL_GATEWAY_STATE_DIR) @@ -366,7 +367,12 @@ function stageHermesToolGatewayCloneBinding(sandboxName, refreshToken, options = return Object.freeze({ activationToken, brokerToken, requestId }); } -function activateHermesToolGatewayCloneBinding(sandboxName, refreshToken, stagedBinding) { +function activateHermesToolGatewayCloneBinding( + sandboxName, + refreshToken, + stagedBinding, + deps = {}, +) { const sandbox = validateName(sandboxName, "sandbox name"); const normalized = String(refreshToken || "").trim(); const activationToken = String(stagedBinding?.activationToken || "").trim(); @@ -374,23 +380,33 @@ function activateHermesToolGatewayCloneBinding(sandboxName, refreshToken, staged if (!normalized || !isValidActivationToken(activationToken) || !brokerToken) { throw new Error("Hermes staged destination credential binding is incomplete"); } - const state = persistHermesToolGatewayProviderState( + const previousState = (deps.readState ?? readHermesToolGatewayProviderState)(sandbox); + const previousStateSnapshot = previousState ? structuredClone(previousState) : null; + const state = (deps.persistState ?? persistHermesToolGatewayProviderState)( sandbox, normalized, brokerToken, getHermesInferenceProviderName(sandbox), ); - const response = brokerControlJsonRequest("credentials/activate", { + const response = (deps.controlRequest ?? brokerControlJsonRequest)("credentials/activate", { sandbox, activation_token: activationToken, deadline_at_ms: newControlDeadline(), }); - const reconciled = response ?? brokerControlStatus({ activation_token: activationToken }); + const reconciled = + response ?? (deps.controlStatus ?? brokerControlStatus)({ activation_token: activationToken }); if (reconciled?.state === "activated") { return state; } if (reconciled?.state === "discarded" || reconciled?.state === "staged") { - removeHermesToolGatewayProviderState(sandbox); + if (previousStateSnapshot) { + (deps.writeState ?? atomicWriteJson)(state.file, previousStateSnapshot); + } else if (!(deps.removeState ?? removeHermesToolGatewayProviderState)(sandbox)) { + throw Object.assign( + new Error("Hermes managed-tool gateway broker activation cleanup failed"), + { code: "hermes_clone_activation_cleanup_failed" }, + ); + } } else { throw Object.assign( new Error("Hermes managed-tool gateway broker activation outcome is unknown"), @@ -837,6 +853,7 @@ module.exports = { getHermesToolGatewayBrokerToken, persistHermesToolGatewayProviderState, removeHermesToolGatewayProviderState, + registerHermesToolGatewayRuntimeCredential, registerHermesToolGatewayRefreshProvider, probeHermesToolGatewayBrokerStart, preflightHermesToolGatewayCloneBinding, diff --git a/test/hermes-tool-gateway-broker.test.ts b/test/hermes-tool-gateway-broker.test.ts index ea517d23190..30246d1d210 100644 --- a/test/hermes-tool-gateway-broker.test.ts +++ b/test/hermes-tool-gateway-broker.test.ts @@ -4,6 +4,7 @@ import { type ChildProcess, spawn } from "node:child_process"; import crypto from "node:crypto"; +import { once } from "node:events"; import fs from "node:fs"; import http from "node:http"; import { createRequire } from "node:module"; @@ -248,6 +249,86 @@ describe("Hermes managed-tool gateway broker", () => { expect(unlinkState).not.toHaveBeenCalled(); }); + it("creates the private state directory before a broad runtime credential scan", ({ + resources, + }) => { + const previousHome = process.env.HOME; + const home = resources.temporaryDirectory("nemoclaw-hermes-empty-state-"); + try { + process.env.HOME = home; + delete require.cache[require.resolve(BROKER_WRAPPER)]; + const broker = require(BROKER_WRAPPER); + + expect(fs.existsSync(broker.HERMES_TOOL_GATEWAY_STATE_DIR)).toBe(false); + expect(broker.registerHermesToolGatewayRuntimeCredential("test-only-refresh")).toBe(false); + expect(fs.statSync(broker.HERMES_TOOL_GATEWAY_STATE_DIR).mode & 0o777).toBe(0o700); + } finally { + previousHome === undefined + ? Reflect.deleteProperty(process.env, "HOME") + : Reflect.set(process.env, "HOME", previousHome); + delete require.cache[require.resolve(BROKER_WRAPPER)]; + } + }); + + it("restores a prior destination broker binding and reports cleanup failure", () => { + delete require.cache[require.resolve(BROKER_WRAPPER)]; + const broker = require(BROKER_WRAPPER); + const previousState = { + sandbox: "destination", + provider_name: "destination-hermes-tool-gateway", + inference_provider_name: "destination-hermes-inference", + broker_token: "prior-broker-token", + refresh_token_sha256: sha256("prior-refresh-token"), + }; + const stagedBinding = { + activationToken: `nc_activate_${"a".repeat(32)}`, + brokerToken: "next-broker-token", + }; + const writeState = vi.fn(); + const removeState = vi.fn(() => true); + const persistence = () => ({ + file: "/test-only/destination.json", + brokerToken: stagedBinding.brokerToken, + }); + + expect(() => + broker.activateHermesToolGatewayCloneBinding( + "destination", + "next-refresh-token", + stagedBinding, + { + readState: () => previousState, + persistState: persistence, + controlRequest: () => ({ state: "staged" }), + writeState, + removeState, + }, + ), + ).toThrow("could not activate destination credentials"); + expect(writeState).toHaveBeenCalledExactlyOnceWith( + "/test-only/destination.json", + previousState, + ); + expect(removeState).not.toHaveBeenCalled(); + + expect(() => + broker.activateHermesToolGatewayCloneBinding( + "new-destination", + "next-refresh-token", + stagedBinding, + { + readState: () => null, + persistState: () => ({ + file: "/test-only/new-destination.json", + brokerToken: stagedBinding.brokerToken, + }), + controlRequest: () => ({ state: "discarded" }), + removeState: () => false, + }, + ), + ).toThrow("activation cleanup failed"); + }); + it("removes broker state only for the exact registry identity", () => { delete require.cache[require.resolve(BROKER_WRAPPER)]; const broker = require(BROKER_WRAPPER); @@ -586,6 +667,7 @@ describe("Hermes managed-tool gateway broker", () => { // exceed that limit before the socket name is appended. const socketDir = resources.ownDirectory(fs.mkdtempSync("/tmp/nc-hermes-broker-")); const controlSocket = path.join(socketDir, "control.sock"); + fs.chmodSync(socketDir, 0o777); fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); fs.mkdirSync(binDir, { recursive: true }); const openshellLog = path.join(tmp, "openshell.log"); @@ -694,7 +776,8 @@ describe("Hermes managed-tool gateway broker", () => { return ( response.status === 200 && fs.existsSync(controlSocket) && - (fs.statSync(controlSocket).mode & 0o777) === 0o600 + (fs.statSync(controlSocket).mode & 0o777) === 0o600 && + (fs.statSync(socketDir).mode & 0o777) === 0o700 ); }, ); @@ -766,6 +849,7 @@ describe("Hermes managed-tool gateway broker", () => { "/credentials/stage", discardedPayload, ); + expect(discardedStage.status, `${discardedStage.body}\n${output}`).toBe(200); const discarded = JSON.parse(discardedStage.body) as { activation_token: string }; await expect( controlRequest(controlSocket, "/credentials/discard", { @@ -833,5 +917,35 @@ describe("Hermes managed-tool gateway broker", () => { ); expect(output).not.toContain("source-refresh-token"); expect(output).not.toContain("destination-refresh-token"); + + const openIncompleteControlRequest = async (): Promise => { + const socket = net.createConnection(controlSocket); + await once(socket, "connect"); + socket.write( + [ + "POST /credentials/register HTTP/1.1", + "Host: localhost", + "Content-Type: application/json", + "Content-Length: 200", + "Connection: keep-alive", + "", + '{"sandbox":"destination"', + ].join("\r\n"), + ); + return socket; + }; + + const timedOutRequest = await openIncompleteControlRequest(); + const requestTimeoutStartedAt = Date.now(); + await once(timedOutRequest, "close", { signal: AbortSignal.timeout(3_000) }); + expect(Date.now() - requestTimeoutStartedAt).toBeLessThan(3_000); + + const shutdownRequest = await openIncompleteControlRequest(); + const childExit = once(child, "exit", { signal: AbortSignal.timeout(3_000) }); + const shutdownStartedAt = Date.now(); + child.kill("SIGTERM"); + await childExit; + shutdownRequest.destroy(); + expect(Date.now() - shutdownStartedAt).toBeLessThan(3_000); }); }); diff --git a/test/hermes-tool-gateway-runtime-credentials.test.ts b/test/hermes-tool-gateway-runtime-credentials.test.ts index fe0ca7a44e2..49c17a8a24a 100644 --- a/test/hermes-tool-gateway-runtime-credentials.test.ts +++ b/test/hermes-tool-gateway-runtime-credentials.test.ts @@ -2,14 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 import { createHash } from "node:crypto"; +import { createRequire } from "node:module"; import { describe, expect, it } from "vitest"; +const require = createRequire(import.meta.url); const { RuntimeRefreshCredentialStore } = require("../agents/hermes/host/runtime-refresh-credentials.ts") as { RuntimeRefreshCredentialStore: new ( hashCredential: (value: string) => string, ) => { register(state: Record, refreshToken: string): boolean; + replace(state: Record, refreshToken: string): (() => boolean) | null; resolve(state: Record): string | null; unregister(sandboxName: string): boolean; }; @@ -53,4 +56,46 @@ describe("Hermes tool-gateway runtime credentials", () => { expect(store.register(destination, "wrong-refresh")).toBe(false); expect(store.resolve(destination)).toBeNull(); }); + + it("restores the prior credential after a replacement transaction fails", () => { + const store = new RuntimeRefreshCredentialStore(sha256); + const priorToken = "test-only-prior-refresh"; + const nextToken = "test-only-next-refresh"; + const priorState = { + sandbox: "destination", + refresh_token_sha256: sha256(priorToken), + }; + const nextState = { + sandbox: "destination", + refresh_token_sha256: sha256(nextToken), + }; + + expect(store.register(priorState, priorToken)).toBe(true); + const restorePrior = store.replace(nextState, nextToken); + expect(restorePrior).toBeTypeOf("function"); + expect(store.resolve(nextState)).toBe(nextToken); + expect(restorePrior?.()).toBe(true); + expect(restorePrior?.()).toBe(false); + expect(store.resolve(priorState)).toBe(priorToken); + expect(store.resolve(nextState)).toBeNull(); + + const restoreWithoutClobber = store.replace(nextState, nextToken); + const concurrentToken = "test-only-concurrent-refresh"; + const concurrentState = { + sandbox: "destination", + refresh_token_sha256: sha256(concurrentToken), + }; + expect(store.register(concurrentState, concurrentToken)).toBe(true); + expect(restoreWithoutClobber?.()).toBe(false); + expect(store.resolve(concurrentState)).toBe(concurrentToken); + + const restoreAbsent = store.replace( + { sandbox: "new-clone", refresh_token_sha256: sha256(nextToken) }, + nextToken, + ); + expect(restoreAbsent?.()).toBe(true); + expect( + store.resolve({ sandbox: "new-clone", refresh_token_sha256: sha256(nextToken) }), + ).toBeNull(); + }); }); diff --git a/test/package-contract/openshell-policy-boundary.test.ts b/test/package-contract/openshell-policy-boundary.test.ts index 8ee05c74e37..cde1952677c 100644 --- a/test/package-contract/openshell-policy-boundary.test.ts +++ b/test/package-contract/openshell-policy-boundary.test.ts @@ -179,6 +179,24 @@ describe("OpenShell policy boundary package contract", () => { ).toBe(false); }); + it("ships the Hermes host broker with its canonical sandbox-name boundary", () => { + expect(packageFiles(repoRoot)).toContain("agents/hermes/host/"); + for (const file of [ + "managed-tool-gateway-matrix.json", + "runtime-refresh-credentials.ts", + "tool-gateway-broker.ts", + "tool-gateway-control-contract.ts", + ]) { + expect(fs.existsSync(path.join(repoRoot, "agents", "hermes", "host", file))).toBe(true); + } + + const controlContract = + require("../../agents/hermes/host/tool-gateway-control-contract.ts") as { + isValidName: (value: unknown) => boolean; + }; + expect(controlContract.isValidName("packaged-hermes-sandbox")).toBe(true); + }); + it("ships an out-of-tree runtime sandbox-policy schema validator", { timeout: 90_000 }, () => { const productionDependencyTree = spawnSync( "npm", From c3f9fdc6e0b3e2c91a3794d45c4213c57009737b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 22:11:37 -0700 Subject: [PATCH 075/117] fix(onboard): harden bootstrap dormancy contract Signed-off-by: Aaron Erickson --- scripts/managed-bootstrap-trampoline.sh | 31 ++--- src/lib/onboard/managed-bootstrap/README.md | 3 + test/managed-bootstrap-trampoline.test.ts | 1 + test/runtime-provider-source-shape.test.ts | 121 ++++++++++++++------ 4 files changed, 109 insertions(+), 47 deletions(-) diff --git a/scripts/managed-bootstrap-trampoline.sh b/scripts/managed-bootstrap-trampoline.sh index 7c621bc1b80..37c4a9b020b 100755 --- a/scripts/managed-bootstrap-trampoline.sh +++ b/scripts/managed-bootstrap-trampoline.sh @@ -14,8 +14,9 @@ fail() { exit 1 } -[ "$(/usr/bin/id -u)" -eq 0 ] && [ "$(/usr/bin/id -g)" -eq 0 ] \ - || fail "must run as root" +if [ "$(/usr/bin/id -u)" -ne 0 ] || [ "$(/usr/bin/id -g)" -ne 0 ]; then + fail "must run as root" +fi [ "$#" -ge 16 ] \ || fail "managed bootstrap arguments are incomplete" [ "$1" = "--agent" ] || fail "agent argument is missing" @@ -53,19 +54,22 @@ esac case "$_nemoclaw_agent_uid:$_nemoclaw_agent_gid" in *[!0-9:]* | :* | *:) fail "agent uid/gid must be numeric" ;; esac -[ "$(/usr/bin/id -u sandbox)" = "$_nemoclaw_agent_uid" ] \ - && [ "$(/usr/bin/id -g sandbox)" = "$_nemoclaw_agent_gid" ] \ - || fail "agent identity does not match the image sandbox account" -[ "$_nemoclaw_agent_workdir" = "/sandbox" ] \ - && [ -d "$_nemoclaw_agent_workdir" ] \ - && [ ! -L "$_nemoclaw_agent_workdir" ] \ - || fail "agent workdir does not match the image sandbox workspace" +if [ "$(/usr/bin/id -u sandbox)" != "$_nemoclaw_agent_uid" ] \ + || [ "$(/usr/bin/id -g sandbox)" != "$_nemoclaw_agent_gid" ]; then + fail "agent identity does not match the image sandbox account" +fi +if [ "$_nemoclaw_agent_workdir" != "/sandbox" ] \ + || [ ! -d "$_nemoclaw_agent_workdir" ] \ + || [ -L "$_nemoclaw_agent_workdir" ]; then + fail "agent workdir does not match the image sandbox workspace" +fi [ "$_nemoclaw_request" = "/var/lib/nemoclaw-managed-bootstrap-request.json" ] \ || fail "request file path is not the fixed bootstrap path" _nemoclaw_runtime="/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs" -[ -f "$_nemoclaw_runtime" ] && [ ! -L "$_nemoclaw_runtime" ] \ - || fail "managed startup runtime is missing" +if [ ! -f "$_nemoclaw_runtime" ] || [ -L "$_nemoclaw_runtime" ]; then + fail "managed startup runtime is missing" +fi if [ -L "$_nemoclaw_request" ]; then fail "bootstrap request path is a symbolic link" fi @@ -86,8 +90,9 @@ if [ -e "$_nemoclaw_request" ]; then --profile-fingerprint "$_nemoclaw_fingerprint" \ --bootstrap-identity "$_nemoclaw_bootstrap_identity" /usr/bin/rm -f -- "$_nemoclaw_request" - [ ! -e "$_nemoclaw_request" ] && [ ! -L "$_nemoclaw_request" ] \ - || fail "bootstrap runtime did not consume its request" + if [ -e "$_nemoclaw_request" ] || [ -L "$_nemoclaw_request" ]; then + fail "bootstrap runtime did not consume its request" + fi fi /usr/bin/env -i \ HOME="/root" \ diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 79b17729eac..b63abbca9b1 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -1,3 +1,6 @@ + + + # Managed bootstrap protocol This directory defines a dormant, driver-neutral transaction contract. It does diff --git a/test/managed-bootstrap-trampoline.test.ts b/test/managed-bootstrap-trampoline.test.ts index 92e173da6c6..ea8c284f0b3 100644 --- a/test/managed-bootstrap-trampoline.test.ts +++ b/test/managed-bootstrap-trampoline.test.ts @@ -88,6 +88,7 @@ esac executable( supervisor, `#!/bin/sh +set -e test ! -e "$REQUEST" test "$#" -eq 3 test "$1" = "supervise" diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 9992230a0df..67c7cdb5336 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -1,16 +1,30 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { execFileSync } from "node:child_process"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; const repoRoot = join(import.meta.dirname, ".."); +const byCodeUnit = (left: string, right: string): number => + left < right ? -1 : left > right ? 1 : 0; + +function trackedPaths(...pathspecs: readonly string[]): string[] { + return execFileSync("git", ["ls-files", "-z", "--", ...pathspecs], { + cwd: repoRoot, + encoding: "utf8", + }) + .split("\0") + .filter(Boolean) + .sort(byCodeUnit); +} + +const read = (relativePath: string): string => readFileSync(join(repoRoot, relativePath), "utf8"); describe("runtime provider central source boundary", () => { // source-shape-contract: compatibility -- Migrated lifecycle and mutation consumers must stay provider-neutral while production selection excludes unqualified future providers and managed-bootstrap dependencies it("keeps migrated provider identities and implementations behind the one bundle composition", () => { - const read = (relativePath: string) => readFileSync(join(repoRoot, relativePath), "utf8"); const driverNeutralActions = { "actions/inference-set.ts": read("src/lib/actions/inference-set.ts"), "actions/sandbox/destroy-execution.ts": read("src/lib/actions/sandbox/destroy-execution.ts"), @@ -59,31 +73,75 @@ describe("runtime provider central source boundary", () => { // source-shape-contract: security -- The bootstrap protocol and image-owned trampoline must remain dormant until a later provider slice supplies runtime packaging and exact activation it("keeps managed bootstrap provider-neutral, image-owned, and dormant", () => { - const bootstrapProtocol = [ - readFileSync(join(repoRoot, "src/lib/onboard/managed-bootstrap/adapter.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/managed-bootstrap/envelope.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/managed-bootstrap/index.ts"), "utf8"), - ]; - const activationSources = [ - readFileSync(join(repoRoot, "src/lib/onboard.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/docker-gpu-sandbox-create.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/sandbox-create-launch.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/sandbox-create-step.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/sandbox-gpu-create-flow.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/sandbox-gpu-create-run-attempt.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/runtime-provider/contract.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/runtime-provider/current.ts"), "utf8"), - readFileSync(join(repoRoot, "src/lib/onboard/runtime-provider/registry.ts"), "utf8"), - ]; const dockerProvider = readFileSync( join(repoRoot, "src/lib/onboard/runtime-provider/docker.ts"), "utf8", ); - const managedDockerfiles = [ - readFileSync(join(repoRoot, "Dockerfile"), "utf8"), - readFileSync(join(repoRoot, "agents/hermes/Dockerfile"), "utf8"), - readFileSync(join(repoRoot, "agents/langchain-deepagents-code/Dockerfile"), "utf8"), - ]; + const productionPaths = trackedPaths( + "src/lib/onboard.ts", + "src/lib/onboard", + "scripts", + "agents", + ".github/workflows", + "Dockerfile", + "Dockerfile.base", + ); + const bootstrapProtocolPaths = productionPaths.filter( + (path) => + path.startsWith("src/lib/onboard/managed-bootstrap/") && + path.endsWith(".ts") && + !path.endsWith(".test.ts"), + ); + const activationPaths = productionPaths.filter( + (path) => + (path === "src/lib/onboard.ts" || path.startsWith("src/lib/onboard/")) && + path.endsWith(".ts") && + !path.endsWith(".test.ts") && + !path.startsWith("src/lib/onboard/managed-bootstrap/"), + ); + const providerPaths = activationPaths.filter((path) => + path.startsWith("src/lib/onboard/runtime-provider/"), + ); + const dockerfilePaths = productionPaths.filter((path) => + /(?:^|\/)Dockerfile(?:\.base)?$/u.test(path), + ); + const packagingPaths = productionPaths.filter( + (path) => + dockerfilePaths.includes(path) || + path.startsWith("scripts/") || + path.startsWith("agents/") || + path.startsWith(".github/workflows/"), + ); + const bootstrapProtocol = bootstrapProtocolPaths.map(read); + const activationSources = activationPaths.map(read); + const providerSources = providerPaths.map(read); + const packagingSources = packagingPaths.map((path) => [path, read(path)] as const); + const bootstrapLoad = + /(?:from\s*|import\s*|import\s*\(\s*|require\s*\(\s*)["'][^"']*managed-bootstrap/iu; + const packagedBootstrapAsset = + /(?:nemoclaw-managed-bootstrap|managed-bootstrap-trampoline|managed-startup-image-runtime\.cjs|nemoclaw-managed-startup-hold)/u; + + expect(bootstrapProtocolPaths).toEqual([ + "src/lib/onboard/managed-bootstrap/adapter.ts", + "src/lib/onboard/managed-bootstrap/envelope.ts", + "src/lib/onboard/managed-bootstrap/index.ts", + ]); + expect(providerPaths).toEqual([ + "src/lib/onboard/runtime-provider/access.ts", + "src/lib/onboard/runtime-provider/contract.ts", + "src/lib/onboard/runtime-provider/current.ts", + "src/lib/onboard/runtime-provider/docker.ts", + "src/lib/onboard/runtime-provider/registry.ts", + "src/lib/onboard/runtime-provider/snapshot.ts", + ]); + expect(dockerfilePaths).toEqual([ + "Dockerfile", + "Dockerfile.base", + "agents/hermes/Dockerfile", + "agents/hermes/Dockerfile.base", + "agents/langchain-deepagents-code/Dockerfile", + "agents/langchain-deepagents-code/Dockerfile.base", + ]); expect(bootstrapProtocol.join("\n")).not.toMatch( /from\s+["'][^"']*(?:docker|podman)[^"']*["']/iu, @@ -92,18 +150,13 @@ describe("runtime provider central source boundary", () => { /(?:driverId|providerId)\s*(?:===|!==)\s*["'](?:docker|podman)["']/iu, ); expect(bootstrapProtocol.join("\n")).not.toMatch(/\b(?:docker|podman|openshell|mxc)\b/iu); - expect(activationSources.join("\n")).not.toMatch( - /(?:from\s+["'][^"']*managed-bootstrap|require\([^)]*managed-bootstrap)/u, - ); - expect(dockerProvider).not.toMatch( - /(?:from\s+["'][^"']*managed-bootstrap|require\([^)]*managed-bootstrap)/u, - ); + expect(activationSources.join("\n")).not.toMatch(bootstrapLoad); + expect(providerSources.join("\n")).not.toMatch(/managed-bootstrap/iu); expect(dockerProvider.match(/bootstrap:\s*unsupported\(/gu)).toHaveLength(2); - - for (const dockerfile of managedDockerfiles) { - expect(dockerfile).not.toContain("nemoclaw-managed-bootstrap"); - expect(dockerfile).not.toContain("managed-startup-image-runtime.cjs"); - expect(dockerfile).not.toContain("nemoclaw-managed-startup-hold"); - } + expect( + packagingSources + .filter(([, source]) => packagedBootstrapAsset.test(source)) + .map(([path]) => path), + ).toEqual(["scripts/managed-bootstrap-trampoline.sh"]); }); }); From e58701096b4db46df691c0b0593d3c944e7f5e81 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 22:31:58 -0700 Subject: [PATCH 076/117] test(onboard): update managed cutover fixtures Signed-off-by: Aaron Erickson --- src/lib/onboard/docker-gpu-route-consumers.test.ts | 10 +++++----- test/onboard-prepared-build-context.test.ts | 9 ++++++--- test/onboard-sandbox-recreation.test.ts | 5 +++++ test/onboard-terminal-dashboard.test.ts | 9 ++++++--- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/lib/onboard/docker-gpu-route-consumers.test.ts b/src/lib/onboard/docker-gpu-route-consumers.test.ts index f2454e5d45d..b11215d340e 100644 --- a/src/lib/onboard/docker-gpu-route-consumers.test.ts +++ b/src/lib/onboard/docker-gpu-route-consumers.test.ts @@ -137,7 +137,7 @@ describe("selected route consumers", () => { expect(reverifyBridgeReachability).not.toHaveBeenCalled(); }); - it("skips compatibility-only inference gates after native wins", () => { + it("skips compatibility-only inference gates after native wins", async () => { const execInSandbox = vi.fn(); expect( verifyDockerGpuSandboxLocalInference(GPU_CONFIG, "ollama-local", { @@ -149,7 +149,7 @@ describe("selected route consumers", () => { ).toEqual({ status: "skipped", reason: "not-docker-gpu-patch" }); const verifyDirectSandboxGpu = vi.fn(); - verifyGpuSandboxAfterReady(GPU_CONFIG, "ollama-local", { + await verifyGpuSandboxAfterReady(GPU_CONFIG, "ollama-local", { sandboxName: "alpha", dockerDriverGateway: true, selectedRoute: "native", @@ -162,11 +162,11 @@ describe("selected route consumers", () => { expect(execInSandbox).not.toHaveBeenCalled(); }); - it("defers native proof diagnostics while automatic fallback owns recovery", () => { + it("defers native proof diagnostics while automatic fallback owns recovery", async () => { const proofError = new Error("native CUDA proof failed"); const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); try { - expect(() => + await expect( verifyGpuSandboxAfterReady(GPU_CONFIG, "ollama-local", { sandboxName: "alpha", dockerDriverGateway: true, @@ -178,7 +178,7 @@ describe("selected route consumers", () => { selectedMode: () => null, runCaptureOpenshell: vi.fn(() => ""), }), - ).toThrow(proofError); + ).rejects.toThrow(proofError); expect(consoleError).not.toHaveBeenCalled(); } finally { consoleError.mockRestore(); diff --git a/test/onboard-prepared-build-context.test.ts b/test/onboard-prepared-build-context.test.ts index 0d91d612ae4..5babec1765c 100644 --- a/test/onboard-prepared-build-context.test.ts +++ b/test/onboard-prepared-build-context.test.ts @@ -98,12 +98,15 @@ let stageCalls = 0; dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch = () => ({ maybeApplyDuringCreate: () => {}, createFailureMessage: () => null, - exitOnPatchError: () => {}, - ensureApplied: () => {}, + exitOnPatchError: async () => {}, + attachManagedBootstrapCutover: () => {}, + rollbackManagedStartupAfterCreateFailure: async () => {}, + ensureApplied: async () => {}, waitForSupervisorReconnectIfNeeded: () => {}, + commitAfterReady: async () => {}, selectedMode: () => null, printReadinessFailureIfEnabled: () => {}, - verifyGpuOrExit: (verify) => verify(sandboxName), + verifyGpuOrExit: async (verify) => verify(sandboxName), }); buildContextStage.stageCreateSandboxBuildContext = () => { diff --git a/test/onboard-sandbox-recreation.test.ts b/test/onboard-sandbox-recreation.test.ts index 11b35a58b6e..c0a4eb2f6f3 100644 --- a/test/onboard-sandbox-recreation.test.ts +++ b/test/onboard-sandbox-recreation.test.ts @@ -1203,6 +1203,7 @@ const fs = require("node:fs"); const commands = []; let sandboxListCalls = 0; +let dockerPsCalls = 0; const keepAlive = setInterval(() => {}, 1000); runner.run = (command, opts = {}) => { _deleted = _deleted || _n(command).includes("sandbox delete"); @@ -1210,6 +1211,10 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { + if (_n(command).startsWith("docker ps -a --no-trunc ")) { + dockerPsCalls += 1; + if (dockerPsCalls === 1) return "a".repeat(64); + } if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) { sandboxListCalls += 1; diff --git a/test/onboard-terminal-dashboard.test.ts b/test/onboard-terminal-dashboard.test.ts index 61baa83310a..0dd2686eadd 100644 --- a/test/onboard-terminal-dashboard.test.ts +++ b/test/onboard-terminal-dashboard.test.ts @@ -67,12 +67,15 @@ const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, "") dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch = () => ({ maybeApplyDuringCreate: () => {}, createFailureMessage: () => null, - exitOnPatchError: () => {}, - ensureApplied: () => {}, + exitOnPatchError: async () => {}, + attachManagedBootstrapCutover: () => {}, + rollbackManagedStartupAfterCreateFailure: async () => {}, + ensureApplied: async () => {}, waitForSupervisorReconnectIfNeeded: () => {}, + commitAfterReady: async () => {}, selectedMode: () => null, printReadinessFailureIfEnabled: () => {}, - verifyGpuOrExit: (verify) => verify(sandboxName), + verifyGpuOrExit: async (verify) => verify(sandboxName), }); agentOnboard.createAgentSandbox = () => { From 1fb671dab6db7456dfaafb92d5ed814add983bd4 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 22:40:41 -0700 Subject: [PATCH 077/117] fix(bootstrap): harden Docker transaction review contracts Signed-off-by: Aaron Erickson --- .../managed-bootstrap/docker-journal.test.ts | 84 ++- .../managed-bootstrap/docker-journal.ts | 99 +++- .../managed-bootstrap/docker-spec.test.ts | 18 + .../onboard/managed-bootstrap/docker-spec.ts | 8 +- .../managed-bootstrap/docker-test-fixture.ts | 479 ++++++++++++++++++ .../onboard/managed-bootstrap/docker.test.ts | 471 ++--------------- test/runtime-provider-source-shape.test.ts | 6 - 7 files changed, 700 insertions(+), 465 deletions(-) create mode 100644 src/lib/onboard/managed-bootstrap/docker-test-fixture.ts diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index 91a398efa51..7ed0d8bba6a 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -5,7 +5,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { createFileDockerManagedBootstrapJournalStore, @@ -39,6 +39,23 @@ const journal = Object.freeze({ replacementSpecHash: "8".repeat(64), } satisfies DockerManagedBootstrapJournal); +function readPinnedPrivateFile(target: string): { readonly mode: number; readonly text: string } { + const descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + const text = fs.readFileSync(descriptor, "utf8"); + const after = fs.fstatSync(descriptor, { bigint: true }); + expect(after.dev).toBe(before.dev); + expect(after.ino).toBe(before.ino); + expect(after.size).toBe(before.size); + expect(after.mtimeNs).toBe(before.mtimeNs); + expect(after.ctimeNs).toBe(before.ctimeNs); + return { mode: Number(before.mode & 0o777n), text }; + } finally { + fs.closeSync(descriptor); + } +} + afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); }); @@ -52,8 +69,9 @@ describe("Docker managed bootstrap journal", () => { const directory = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); const file = path.join(directory, `${IDENTITY}.json`); expect(fs.statSync(directory).mode & 0o777).toBe(0o700); - expect(fs.statSync(file).mode & 0o777).toBe(0o600); - expect(parseDockerManagedBootstrapJournal(fs.readFileSync(file, "utf8"))).toEqual(journal); + const persisted = readPinnedPrivateFile(file); + expect(persisted.mode).toBe(0o600); + expect(parseDockerManagedBootstrapJournal(persisted.text)).toEqual(journal); expect(() => store.create(journal)).toThrow("already exists"); expect(() => store.transition(IDENTITY, "staged", "shared-state-committed")).toThrow( "unsupported", @@ -77,7 +95,7 @@ describe("Docker managed bootstrap journal", () => { fs.writeFileSync(`${file}.decision`, "rollback-authorized\n", { mode: 0o600 }); expect(store.load(IDENTITY)?.phase).toBe("rollback-authorized"); - expect(parseDockerManagedBootstrapJournal(fs.readFileSync(file, "utf8")).phase).toBe( + expect(parseDockerManagedBootstrapJournal(readPinnedPrivateFile(file).text).phase).toBe( "rollback-authorized", ); fs.unlinkSync(`${file}.decision`); @@ -88,6 +106,64 @@ describe("Docker managed bootstrap journal", () => { store.remove(IDENTITY, ["rollback-authorized"]); }); + it("reconciles an exclusive decision collision by typed durable authority", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + store.transition(IDENTITY, "staged", "cutover"); + const target = path.join( + root, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, + `${IDENTITY}.json.decision`, + ); + const link = vi.spyOn(fs, "linkSync").mockImplementationOnce(() => { + fs.writeFileSync(target, "rollback-authorized\n", { flag: "wx", mode: 0o600 }); + throw Object.assign(new Error("exclusive decision collision"), { code: "EEXIST" }); + }); + try { + expect(store.transition(IDENTITY, "cutover", "rollback-authorized").phase).toBe( + "rollback-authorized", + ); + } finally { + link.mockRestore(); + } + }); + + it("preserves a primary journal write failure when temporary cleanup also fails", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + const rename = vi.spyOn(fs, "renameSync").mockImplementationOnce(() => { + throw new Error("primary journal rename failure"); + }); + const unlink = vi.spyOn(fs, "unlinkSync").mockImplementationOnce(() => { + throw new Error("temporary cleanup failure"); + }); + try { + expect(() => store.transition(IDENTITY, "staged", "cutover")).toThrow( + "primary journal rename failure", + ); + } finally { + rename.mockRestore(); + unlink.mockRestore(); + } + }); + + it.skipIf(process.platform === "win32")("refuses a symlink in place of journal authority", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + const file = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, `${IDENTITY}.json`); + const moved = `${file}.moved`; + fs.renameSync(file, moved); + fs.symlinkSync(moved, file); + + expect(() => store.load(IDENTITY)).toThrow("journal file ownership boundary is invalid"); + }); + it("rejects non-canonical authority", () => { expect(() => parseDockerManagedBootstrapJournal(`${JSON.stringify({ ...journal, phase: "unknown" })}\n`), diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index e3a7e72b3a2..c6043409d7d 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -64,6 +64,15 @@ export class DockerManagedBootstrapJournalAcknowledgementLostError extends Error } } +class DockerManagedBootstrapJournalExistsError extends Error { + constructor() { + super( + "Managed bootstrap Docker journal is invalid: journal already exists for this bootstrap identity", + ); + this.name = "DockerManagedBootstrapJournalExistsError"; + } +} + const ALLOWED_TRANSITIONS = new Set([ "staged->cutover", "cutover->rollback-authorized", @@ -228,25 +237,64 @@ function decisionPath(target: string): string { return `${target}.decision`; } +function sameStableMetadata(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.mode === right.mode && + left.nlink === right.nlink && + left.uid === right.uid && + left.gid === right.gid && + left.size === right.size && + left.mtimeNs === right.mtimeNs && + left.ctimeNs === right.ctimeNs + ); +} + function readPrivateFile(target: string, label: string): string | null { - let stat: fs.Stats; + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") { + fail(`cannot safely open ${label} because O_NOFOLLOW is unavailable`); + } + const nonblock = typeof fs.constants.O_NONBLOCK === "number" ? fs.constants.O_NONBLOCK : 0; + let descriptor: number; try { - stat = fs.lstatSync(target); + descriptor = fs.openSync(target, fs.constants.O_RDONLY | noFollow | nonblock); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + if ((error as NodeJS.ErrnoException).code === "ELOOP") { + fail(`${label} file ownership boundary is invalid`); + } throw error; } - if ( - !stat.isFile() || - stat.isSymbolicLink() || - stat.nlink !== 1 || - (stat.mode & 0o077) !== 0 || - stat.size <= 0 || - stat.size > MAX_JOURNAL_BYTES - ) { - fail(`${label} file ownership boundary is invalid`); + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + (before.mode & 0o077n) !== 0n || + before.size <= 0n || + before.size > BigInt(MAX_JOURNAL_BYTES) + ) { + fail(`${label} file ownership boundary is invalid`); + } + const contents = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < contents.length) { + const count = fs.readSync(descriptor, contents, offset, contents.length - offset, offset); + if (count === 0) break; + offset += count; + } + const overflow = Buffer.alloc(1); + const overflowCount = fs.readSync(descriptor, overflow, 0, 1, offset); + const after = fs.fstatSync(descriptor, { bigint: true }); + if (offset !== contents.length || overflowCount !== 0 || !sameStableMetadata(before, after)) { + fail(`${label} file changed during its stable read`); + } + return contents.toString("utf8"); + } finally { + fs.closeSync(descriptor); } - return fs.readFileSync(target, "utf8"); } function fsyncDirectory(directory: string): void { @@ -269,6 +317,7 @@ function atomicWrite( `.${path.basename(target)}.${process.pid}.${Date.now().toString(16)}.tmp`, ); let descriptor: number | null = null; + let primaryFailure: { readonly error: unknown } | null = null; try { descriptor = fs.openSync(temporary, "wx", JOURNAL_FILE_MODE); fs.writeFileSync(descriptor, contents, "utf8"); @@ -280,7 +329,7 @@ function atomicWrite( fs.linkSync(temporary, target); } catch (error) { if ((error as NodeJS.ErrnoException).code === "EEXIST") { - fail("journal already exists for this bootstrap identity"); + throw new DockerManagedBootstrapJournalExistsError(); } throw error; } @@ -290,14 +339,26 @@ function atomicWrite( } fs.chmodSync(target, JOURNAL_FILE_MODE); fsyncDirectory(directory); - } finally { - if (descriptor !== null) fs.closeSync(descriptor); + } catch (error) { + primaryFailure = { error }; + } + let cleanupFailure: { readonly error: unknown } | null = null; + if (descriptor !== null) { try { - fs.unlinkSync(temporary); + fs.closeSync(descriptor); } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + cleanupFailure = { error }; + } + } + try { + fs.unlinkSync(temporary); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT" && cleanupFailure === null) { + cleanupFailure = { error }; } } + if (primaryFailure !== null) throw primaryFailure.error; + if (cleanupFailure !== null) throw cleanupFailure.error; } export function createFileDockerManagedBootstrapJournalStore( @@ -358,9 +419,7 @@ export function createFileDockerManagedBootstrapJournalStore( atomicWrite(directory, decision, `${next}\n`, true); } catch (error) { if ( - !(error instanceof Error) || - error.message !== - "Managed bootstrap Docker journal is invalid: journal already exists for this bootstrap identity" || + !(error instanceof DockerManagedBootstrapJournalExistsError) || readPrivateFile(decision, "decision") !== `${next}\n` ) { throw error; diff --git a/src/lib/onboard/managed-bootstrap/docker-spec.test.ts b/src/lib/onboard/managed-bootstrap/docker-spec.test.ts index 57bcc8633ff..a5d711ca422 100644 --- a/src/lib/onboard/managed-bootstrap/docker-spec.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-spec.test.ts @@ -36,6 +36,24 @@ describe("managed bootstrap Docker launch spec", () => { ); }); + it("orders durable launch keys by code unit across host locale settings", () => { + const inspect = createDockerGpuInspectFixture(); + inspect.Config!.Labels = { + "com.nvidia.foo": "lower", + "com.nvidia.Foo": "upper", + "com.nvidia-foo": "punctuation", + }; + + const canonical = normalizeDockerManagedBootstrapLaunchSpec(inspect).canonicalJson; + + expect(canonical.indexOf('"com.nvidia-foo"')).toBeLessThan( + canonical.indexOf('"com.nvidia.Foo"'), + ); + expect(canonical.indexOf('"com.nvidia.Foo"')).toBeLessThan( + canonical.indexOf('"com.nvidia.foo"'), + ); + }); + it.each([ { name: "anonymous Config.Volumes whose data source cannot be proven", diff --git a/src/lib/onboard/managed-bootstrap/docker-spec.ts b/src/lib/onboard/managed-bootstrap/docker-spec.ts index bf96451f99a..c3cab4e3296 100644 --- a/src/lib/onboard/managed-bootstrap/docker-spec.ts +++ b/src/lib/onboard/managed-bootstrap/docker-spec.ts @@ -189,6 +189,10 @@ function assertUnsupportedDefaults(host: Record): void { } } +function byCodeUnit(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + function normalizedNetworkSettings( value: DockerContainerInspect["NetworkSettings"], ): DockerContainerInspect["NetworkSettings"] { @@ -196,7 +200,7 @@ function normalizedNetworkSettings( return { Networks: Object.fromEntries( Object.entries(networks) - .sort(([left], [right]) => left.localeCompare(right)) + .sort(([left], [right]) => byCodeUnit(left, right)) .map(([name, network]) => [ name, { @@ -212,7 +216,7 @@ function canonicalize(value: unknown): unknown { if (typeof value !== "object" || value === null) return value; return Object.fromEntries( Object.entries(value as Record) - .sort(([left], [right]) => left.localeCompare(right)) + .sort(([left], [right]) => byCodeUnit(left, right)) .map(([key, nested]) => [key, canonicalize(nested)]), ); } diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts new file mode 100644 index 00000000000..1c2a6871b6a --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -0,0 +1,479 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { expect, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import type { DockerContainerInspect } from "../docker-gpu-patch-types"; +import { encodeManagedStartupProfile, type ManagedStartupAgent } from "../managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import { + createManagedBootstrapPreparedAuthority, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapCompletionReceipt, + type ManagedBootstrapDurablePreparationReceipt, + type ManagedBootstrapHeldWorkloadHandle, + type ManagedBootstrapObservedSnapshot, + type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapReplacementHandle, +} from "./adapter"; +import type { DockerManagedBootstrapDeps } from "./docker"; +import { + type DockerManagedBootstrapJournal, + DockerManagedBootstrapJournalAcknowledgementLostError, + type DockerManagedBootstrapJournalPhase, + type DockerManagedBootstrapJournalStore, +} from "./docker-journal"; +import { normalizeDockerManagedBootstrapLaunchSpec } from "./docker-spec"; +import { parseManagedBootstrapEnvelope } from "./envelope"; + +export const IDENTITY = "1".repeat(64); +export const OLD_ID = "2".repeat(64); +export const NEW_ID = "3".repeat(64); +const CONFIG_ID = `sha256:${"4".repeat(64)}`; +const MANIFEST = `sha256:${"5".repeat(64)}` as const; +const REPOSITORY = "registry.example/nemoclaw/hermes"; +const IMAGE = `${REPOSITORY}@${MANIFEST}`; +const SUPERVISOR = ["/opt/openshell/bin/openshell-sandbox", "supervise"] as const; +export const SUPPORTED_AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; + +type FixtureCommandResult = { + readonly status: number; + readonly stdout?: string; + readonly stderr?: string; +}; + +export type DockerFixtureAcknowledgement = + | "container:create" + | "container:remove" + | "container:rename" + | "container:start" + | "container:stop" + | "journal:create" + | "journal:cutover" + | "journal:remove" + | "journal:rollback-authorized" + | "journal:staged" + | "journal:shared-state-committed"; + +export type DockerFixtureOptions = { + readonly agent?: ManagedStartupAgent; + readonly dockerStartResults?: Readonly>; + readonly journalTransitionFailures?: Partial< + Readonly> + >; + readonly lostAcknowledgements?: readonly DockerFixtureAcknowledgement[]; + readonly ownerId?: string; + readonly sharedState?: "committed" | "none" | "pending"; +}; + +function agentInputs(agent: ManagedStartupAgent = "hermes") { + const request = createManagedStartupRootApplyRequest({ + agent, + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agent, false, false)), + }); + const heldArgv = [ + "env", + "A=1", + "/usr/local/bin/nemoclaw-managed-startup-hold", + "--agent", + agent, + "--profile-fingerprint", + request.profileFingerprint, + "--bootstrap-identity", + IDENTITY, + ] as const; + return { + request, + heldArgv, + metadata: { "nemoclaw.ai/managed-profile": request.profileFingerprint }, + }; +} + +export const { heldArgv } = agentInputs(); +export const sandbox = { + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + driverId: "docker", +}; + +function shellArgv(argv: readonly string[]): string { + return argv.join(" "); +} + +function originalInspect(inputs = agentInputs()): DockerContainerInspect { + return { + Id: OLD_ID, + Image: CONFIG_ID, + Name: "/openshell-alpha", + Config: { + Image: IMAGE, + Env: ["A=1", `OPENSHELL_SANDBOX_COMMAND=${shellArgv(inputs.heldArgv)}`], + Labels: { + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-name": "alpha", + "openshell.ai/sandbox-id": "sandbox-alpha", + ...inputs.metadata, + }, + Entrypoint: [SUPERVISOR[0]], + Cmd: SUPERVISOR.slice(1), + User: "root", + WorkingDir: "/sandbox", + Hostname: "alpha", + }, + State: { Running: true, Paused: false, Restarting: false, Dead: false }, + HostConfig: { + Binds: ["/host/workspace:/sandbox:rw"], + NetworkMode: "openshell", + RestartPolicy: { Name: "unless-stopped" }, + CapDrop: ["NET_RAW"], + SecurityOpt: ["no-new-privileges"], + Ulimits: [{ Name: "nofile", Soft: 65_536, Hard: 65_536 }], + }, + NetworkSettings: { Networks: { openshell: { Aliases: ["openshell-alpha"] } } }, + }; +} + +export function authority(agent: ManagedStartupAgent = "hermes") { + const inputs = agentInputs(agent); + const inspect = originalInspect(inputs); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(inspect); + const plan = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandboxName: "alpha", + driverId: "docker", + image: { repository: REPOSITORY, manifestDigest: MANIFEST }, + profile: { agent, fingerprint: inputs.request.profileFingerprint }, + agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + intendedWorkloadArgv: ["env", "A=1", "nemoclaw-start"], + expectedSupervisorArgv: SUPERVISOR, + metadata: inputs.metadata, + }; + const handle: ManagedBootstrapHeldWorkloadHandle = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: inputs.heldArgv, + intendedWorkloadArgv: plan.intendedWorkloadArgv, + plan, + createReceipt: { sandbox, ready: true, readyAt: "2026-07-31T12:00:00.000Z" }, + }; + const snapshot: ManagedBootstrapObservedSnapshot = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + runtimeId: OLD_ID, + bootstrapIdentity: IDENTITY, + image: plan.image, + runtimeImageContentId: CONFIG_ID, + specHash: normalized.hash, + specCanonicalJson: normalized.canonicalJson, + agentIdentity: plan.agentIdentity, + supervisorArgv: SUPERVISOR, + heldWorkloadArgv: inputs.heldArgv, + metadata: inputs.metadata, + }; + return { handle, plan, request: inputs.request, snapshot }; +} + +function failFixture(message: string): never { + throw new Error(message); +} + +function readProtectedEnvelope(source: string): ReturnType { + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") throw new Error("test requires O_NOFOLLOW"); + const descriptor = fs.openSync(source, fs.constants.O_RDONLY | noFollow); + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + expect(Number(before.mode & 0o777n)).toBe(0o400); + const parsed = parseManagedBootstrapEnvelope(fs.readFileSync(descriptor, "utf8")); + const after = fs.fstatSync(descriptor, { bigint: true }); + expect(after.dev).toBe(before.dev); + expect(after.ino).toBe(before.ino); + expect(after.size).toBe(before.size); + expect(after.mtimeNs).toBe(before.mtimeNs); + expect(after.ctimeNs).toBe(before.ctimeNs); + return parsed; + } finally { + fs.closeSync(descriptor); + } +} + +export function fixture(options: DockerFixtureOptions = {}) { + let original = originalInspect(agentInputs(options.agent)); + let replacement: DockerContainerInspect | null = null; + let journal: DockerManagedBootstrapJournal | null = null; + let sharedState: "committed" | "none" | "pending" = options.sharedState ?? "none"; + const events: string[] = []; + const lostAcknowledgements = new Set(options.lostAcknowledgements ?? []); + const losesAcknowledgement = (operation: DockerFixtureAcknowledgement) => + lostAcknowledgements.has(operation); + const ok = (stdout = ""): FixtureCommandResult => ({ status: 0, stdout, stderr: "" }); + const copyJournal = () => (journal ? structuredClone(journal) : null); + const store: DockerManagedBootstrapJournalStore = { + create(value) { + journal = structuredClone(value); + events.push("journal:staged"); + if (losesAcknowledgement("journal:create")) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal create acknowledgement", + ); + } + }, + load: () => copyJournal(), + transition(_identity, expected, next) { + const current = + journal !== null && journal.phase === expected + ? journal + : failFixture("stale journal transition"); + journal = { ...current, phase: next }; + events.push(`journal:${next}`); + const injectedFailure = options.journalTransitionFailures?.[next]; + if (injectedFailure) throw injectedFailure; + if (losesAcknowledgement(`journal:${next}`)) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal transition acknowledgement", + ); + } + return structuredClone(journal); + }, + remove(_identity, expected) { + const current = journal; + void (current !== null && expected.includes(current.phase) + ? current + : failFixture("stale journal remove")); + journal = null; + events.push("journal:removed"); + if (losesAcknowledgement("journal:remove")) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal remove acknowledgement", + ); + } + }, + }; + const inspect = (reference: string): DockerContainerInspect => { + const candidates = [original, replacement].filter( + (value): value is DockerContainerInspect => value !== null, + ); + const found = candidates.find( + (value) => + value.Id === reference || String(value.Name ?? "").replace(/^\/+/u, "") === reference, + ); + return found ? structuredClone(found) : failFixture(`No such container: ${reference}`); + }; + const dockerCapture: NonNullable = vi.fn((args) => { + switch (args[0]) { + case "image": + return JSON.stringify([{ Id: CONFIG_ID, RepoDigests: [IMAGE] }]); + default: + return JSON.stringify([inspect(String(args[3] ?? ""))]); + } + }); + const dockerRun: NonNullable = vi.fn( + (args: readonly string[]) => { + switch (args[0]) { + case "create": { + events.push("create:replacement"); + const name = String(args[args.indexOf("--name") + 1] ?? ""); + const entrypoint = String(args[args.indexOf("--entrypoint") + 1] ?? ""); + const imageIndex = args.indexOf(IMAGE); + const env = args.flatMap((value, index) => + value === "--env" ? [String(args[index + 1] ?? "")] : [], + ); + replacement = { + ...structuredClone(original), + Id: NEW_ID, + Name: `/${name}`, + Config: { + ...structuredClone(original.Config), + Image: IMAGE, + Env: env, + Entrypoint: [entrypoint], + Cmd: args.slice(imageIndex + 1), + }, + State: { Running: false, Paused: false, Restarting: false, Dead: false }, + }; + return losesAcknowledgement("container:create") + ? { status: 1, stdout: "", stderr: "lost create acknowledgement" } + : ok(NEW_ID); + } + case "ps": + return ok(original ? OLD_ID : ""); + case "inspect": { + const id = String(args[3] ?? ""); + try { + inspect(id); + return ok(`[{"Id":"${id}"}]`); + } catch { + return { status: 1, stderr: `Error response from daemon: No such container: ${id}` }; + } + } + case "cp": { + const sourceIndex = args[1] === "-a" ? 2 : 1; + const source = String(args[sourceIndex] ?? ""); + const destination = String(args[sourceIndex + 1] ?? ""); + const copyIntoContainer = () => { + events.push("stage:envelope"); + expect(readProtectedEnvelope(source).bootstrapIdentity).toBe(IDENTITY); + return ok(); + }; + const copyFromContainer = () => { + const receipt = source.split(":")[1]; + const expected = receipt?.includes("shared-state-commit") ? "committed" : "pending"; + return sharedState === expected + ? (() => { + fs.mkdirSync(destination, { recursive: true }); + return ok(); + })() + : { + status: 1, + stderr: `Error response from daemon: Could not find the file ${receipt} in container ${NEW_ID}`, + }; + }; + return source.includes(":") ? copyFromContainer() : copyIntoContainer(); + } + case "run": + switch (true) { + case args.includes("--shared-state-transaction-status"): + return ok(`${sharedState}\n`); + case args.includes("--rollback-shared-state-transaction"): + sharedState = "none"; + events.push("shared:rollback"); + return ok(); + } + break; + case "exec": + switch (true) { + case args.includes("--commit-shared-state-transaction"): + sharedState = "committed"; + events.push("shared:commit"); + return ok(); + case args.includes("--clear-shared-state-commit-receipt"): + sharedState = "none"; + events.push("shared:clear"); + return ok(); + } + break; + } + throw new Error(`unexpected Docker command: ${args.join(" ")}`); + }, + ); + const deps: DockerManagedBootstrapDeps = { + journalStore: store, + dockerCapture, + dockerRun, + dockerStop: vi.fn((id) => { + events.push(`stop:${id}`); + const target = id === OLD_ID ? original : replacement; + [target] + .filter((value): value is DockerContainerInspect => value?.State !== undefined) + .forEach((value) => { + value.State = { ...value.State, Running: false }; + }); + return losesAcknowledgement("container:stop") + ? { status: 1, stderr: "lost stop acknowledgement" } + : ok(); + }), + dockerRename: vi.fn((id, name) => { + events.push(`rename:${id}:${name}`); + const target = id === OLD_ID ? original : replacement; + [target] + .filter((value): value is DockerContainerInspect => value !== null) + .forEach((value) => { + value.Name = `/${name}`; + }); + return losesAcknowledgement("container:rename") + ? { status: 1, stderr: "lost rename acknowledgement" } + : ok(); + }), + dockerStart: vi.fn((id) => { + events.push(`start:${id}`); + const result = options.dockerStartResults?.[id] ?? ok(); + const target = id === OLD_ID ? original : replacement; + [target] + .filter( + (value): value is DockerContainerInspect => + value?.State !== undefined && result.status === 0, + ) + .forEach((value) => { + value.State = { ...value.State, Running: true }; + }); + return losesAcknowledgement("container:start") + ? { status: 1, stderr: "lost start acknowledgement" } + : result; + }), + dockerRm: vi.fn((id) => { + events.push(`rm:${id}`); + switch (id) { + case OLD_ID: + original = null as unknown as DockerContainerInspect; + break; + case NEW_ID: + replacement = null; + break; + } + return losesAcknowledgement("container:remove") + ? { status: 1, stderr: "lost rm acknowledgement" } + : ok(); + }), + runCaptureOpenshell: vi.fn(() => `Name: alpha\nID: ${options.ownerId ?? "sandbox-alpha"}\n`), + runOpenshell: vi.fn(() => ok()), + now: () => new Date("2026-07-31T12:30:00.000Z"), + }; + return { + deps, + events, + get journal() { + return journal; + }, + get original() { + return original; + }, + get replacement() { + return replacement; + }, + get sharedState() { + return sharedState; + }, + }; +} + +export function completion( + replacement: ManagedBootstrapReplacementHandle, +): ManagedBootstrapCompletionReceipt { + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + runtimeId: replacement.replacementRuntimeId, + image: replacement.image, + runtimeImageContentId: replacement.runtimeImageContentId, + originalSpecHash: replacement.originalSpecHash, + replacementSpecHash: replacement.replacementSpecHash, + profileFingerprint: replacement.profileFingerprint, + bootstrapIdentity: replacement.bootstrapIdentity, + transactionPending: true, + completedAt: "2026-07-31T12:15:00.000Z", + }; +} + +export function durablePreparation( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, +): ManagedBootstrapDurablePreparationReceipt { + const preparedAuthority = createManagedBootstrapPreparedAuthority({ + handle, + snapshot, + prepared, + }); + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + authorityFingerprint: preparedAuthority.authorityFingerprint, + recordId: `test-authority-${handle.plan.profile.agent}`, + recordedAt: "2026-07-31T12:10:00.000Z", + }; +} diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index f0ce289ec0a..296f90df1b3 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -1,441 +1,38 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import fs from "node:fs"; - import { describe, expect, it, vi } from "vitest"; -import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; -import type { DockerContainerInspect } from "../docker-gpu-patch-types"; -import { encodeManagedStartupProfile, type ManagedStartupAgent } from "../managed-startup/profile"; -import { createManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import { ManagedBootstrapOwnerCleanupRequiredError } from "./adapter"; +import { createDockerManagedBootstrapAdapter } from "./docker"; import { - createManagedBootstrapPreparedAuthority, - MANAGED_BOOTSTRAP_SCHEMA_VERSION, - type ManagedBootstrapCompletionReceipt, - type ManagedBootstrapDurablePreparationReceipt, - type ManagedBootstrapHeldWorkloadHandle, - type ManagedBootstrapObservedSnapshot, - ManagedBootstrapOwnerCleanupRequiredError, - type ManagedBootstrapPreparedReplacementHandle, - type ManagedBootstrapReplacementHandle, -} from "./adapter"; -import { createDockerManagedBootstrapAdapter, type DockerManagedBootstrapDeps } from "./docker"; -import type { - DockerManagedBootstrapJournal, - DockerManagedBootstrapJournalStore, -} from "./docker-journal"; -import { DockerManagedBootstrapJournalAcknowledgementLostError } from "./docker-journal"; -import { normalizeDockerManagedBootstrapLaunchSpec } from "./docker-spec"; -import { parseManagedBootstrapEnvelope } from "./envelope"; - -const IDENTITY = "1".repeat(64); -const OLD_ID = "2".repeat(64); -const NEW_ID = "3".repeat(64); -const CONFIG_ID = `sha256:${"4".repeat(64)}`; -const MANIFEST = `sha256:${"5".repeat(64)}` as const; -const REPOSITORY = "registry.example/nemoclaw/hermes"; -const IMAGE = `${REPOSITORY}@${MANIFEST}`; -const SUPERVISOR = ["/opt/openshell/bin/openshell-sandbox", "supervise"] as const; -const SUPPORTED_AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; - -function agentInputs(agent: ManagedStartupAgent = "hermes") { - const request = createManagedStartupRootApplyRequest({ - agent, - encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agent, false, false)), - }); - const heldArgv = [ - "env", - "A=1", - "/usr/local/bin/nemoclaw-managed-startup-hold", - "--agent", - agent, - "--profile-fingerprint", - request.profileFingerprint, - "--bootstrap-identity", - IDENTITY, - ] as const; - return { - request, - heldArgv, - metadata: { "nemoclaw.ai/managed-profile": request.profileFingerprint }, - }; -} - -const { heldArgv } = agentInputs(); -const sandbox = { sandboxName: "alpha", sandboxId: "sandbox-alpha", driverId: "docker" }; - -function shellArgv(argv: readonly string[]): string { - return argv.join(" "); -} - -function originalInspect(inputs = agentInputs()): DockerContainerInspect { - return { - Id: OLD_ID, - Image: CONFIG_ID, - Name: "/openshell-alpha", - Config: { - Image: IMAGE, - Env: ["A=1", `OPENSHELL_SANDBOX_COMMAND=${shellArgv(inputs.heldArgv)}`], - Labels: { - "openshell.ai/managed-by": "openshell", - "openshell.ai/sandbox-name": "alpha", - "openshell.ai/sandbox-id": "sandbox-alpha", - ...inputs.metadata, - }, - Entrypoint: [SUPERVISOR[0]], - Cmd: SUPERVISOR.slice(1), - User: "root", - WorkingDir: "/sandbox", - Hostname: "alpha", - }, - State: { Running: true, Paused: false, Restarting: false, Dead: false }, - HostConfig: { - Binds: ["/host/workspace:/sandbox:rw"], - NetworkMode: "openshell", - RestartPolicy: { Name: "unless-stopped" }, - CapDrop: ["NET_RAW"], - SecurityOpt: ["no-new-privileges"], - Ulimits: [{ Name: "nofile", Soft: 65_536, Hard: 65_536 }], - }, - NetworkSettings: { Networks: { openshell: { Aliases: ["openshell-alpha"] } } }, - }; -} - -function authority(agent: ManagedStartupAgent = "hermes") { - const inputs = agentInputs(agent); - const inspect = originalInspect(inputs); - const normalized = normalizeDockerManagedBootstrapLaunchSpec(inspect); - const plan = { - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandboxName: "alpha", - driverId: "docker", - image: { repository: REPOSITORY, manifestDigest: MANIFEST }, - profile: { agent, fingerprint: inputs.request.profileFingerprint }, - agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, - intendedWorkloadArgv: ["env", "A=1", "nemoclaw-start"], - expectedSupervisorArgv: SUPERVISOR, - metadata: inputs.metadata, - }; - const handle: ManagedBootstrapHeldWorkloadHandle = { - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox, - bootstrapIdentity: IDENTITY, - heldWorkloadArgv: inputs.heldArgv, - intendedWorkloadArgv: plan.intendedWorkloadArgv, - plan, - createReceipt: { sandbox, ready: true, readyAt: "2026-07-31T12:00:00.000Z" }, - }; - const snapshot: ManagedBootstrapObservedSnapshot = { - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox, - runtimeId: OLD_ID, - bootstrapIdentity: IDENTITY, - image: plan.image, - runtimeImageContentId: CONFIG_ID, - specHash: normalized.hash, - specCanonicalJson: normalized.canonicalJson, - agentIdentity: plan.agentIdentity, - supervisorArgv: SUPERVISOR, - heldWorkloadArgv: inputs.heldArgv, - metadata: inputs.metadata, - }; - return { handle, plan, request: inputs.request, snapshot }; -} - -type FixtureOptions = { - agent?: ManagedStartupAgent; - failAfterCutoverFence?: boolean; - failStart?: boolean; - lostAcks?: boolean; - ownerId?: string; - sharedState?: "committed" | "none" | "pending"; -}; - -function failFixture(message: string): never { - throw new Error(message); -} - -function fixture(options: FixtureOptions = {}) { - let original = originalInspect(agentInputs(options.agent)); - let replacement: DockerContainerInspect | null = null; - let journal: DockerManagedBootstrapJournal | null = null; - let sharedState: "committed" | "none" | "pending" = options.sharedState ?? "none"; - const events: string[] = []; - const lostTransitions = new Set(["cutover", "shared-state-committed"]); - let loseCreateAck = options.lostAcks === true; - let loseRemoveAck = options.lostAcks === true; - const ok = (stdout = "") => ({ status: 0, stdout, stderr: "" }); - const copyJournal = () => (journal ? structuredClone(journal) : null); - const store: DockerManagedBootstrapJournalStore = { - create(value) { - journal = structuredClone(value); - events.push("journal:staged"); - switch (loseCreateAck) { - case true: - loseCreateAck = false; - throw new DockerManagedBootstrapJournalAcknowledgementLostError( - "lost journal create acknowledgement", - ); - } - }, - load: () => copyJournal(), - transition(_identity, expected, next) { - const current = - journal !== null && journal.phase === expected - ? journal - : failFixture("stale journal transition"); - journal = { ...current, phase: next }; - events.push(`journal:${next}`); - switch (true) { - case next === "cutover" && options.failAfterCutoverFence === true: - throw new Error("injected crash after durable cutover fence"); - case options.lostAcks === true && lostTransitions.delete(next): - throw new DockerManagedBootstrapJournalAcknowledgementLostError( - "lost journal transition acknowledgement", - ); - default: - return structuredClone(journal); - } - }, - remove(_identity, expected) { - const current = journal; - void (current !== null && expected.includes(current.phase) - ? current - : failFixture("stale journal remove")); - journal = null; - events.push("journal:removed"); - switch (loseRemoveAck) { - case true: - loseRemoveAck = false; - throw new DockerManagedBootstrapJournalAcknowledgementLostError( - "lost journal remove acknowledgement", - ); - } - }, - }; - const inspect = (reference: string): DockerContainerInspect => { - const candidates = [original, replacement].filter( - (value): value is DockerContainerInspect => value !== null, - ); - const found = candidates.find( - (value) => - value.Id === reference || String(value.Name ?? "").replace(/^\/+/u, "") === reference, - ); - return found ? structuredClone(found) : failFixture(`No such container: ${reference}`); - }; - const dockerCapture: NonNullable = vi.fn((args) => { - switch (args[0]) { - case "image": - return JSON.stringify([{ Id: CONFIG_ID, RepoDigests: [IMAGE] }]); - default: - return JSON.stringify([inspect(String(args[3] ?? ""))]); - } - }); - const dockerRun: NonNullable = vi.fn( - (args: readonly string[]) => { - switch (args[0]) { - case "create": { - events.push("create:replacement"); - const name = String(args[args.indexOf("--name") + 1] ?? ""); - const entrypoint = String(args[args.indexOf("--entrypoint") + 1] ?? ""); - const imageIndex = args.indexOf(IMAGE); - const env = args.flatMap((value, index) => - value === "--env" ? [String(args[index + 1] ?? "")] : [], - ); - replacement = { - ...structuredClone(original), - Id: NEW_ID, - Name: `/${name}`, - Config: { - ...structuredClone(original.Config), - Image: IMAGE, - Env: env, - Entrypoint: [entrypoint], - Cmd: args.slice(imageIndex + 1), - }, - State: { Running: false, Paused: false, Restarting: false, Dead: false }, - }; - return options.lostAcks - ? { status: 1, stdout: "", stderr: "lost create acknowledgement" } - : ok(NEW_ID); - } - case "ps": - return ok(original ? OLD_ID : ""); - case "inspect": { - const id = String(args[3] ?? ""); - try { - inspect(id); - return ok(`[{"Id":"${id}"}]`); - } catch { - return { status: 1, stderr: `Error response from daemon: No such container: ${id}` }; - } - } - case "cp": { - const sourceIndex = args[1] === "-a" ? 2 : 1; - const source = String(args[sourceIndex] ?? ""); - const destination = String(args[sourceIndex + 1] ?? ""); - const copyIntoContainer = () => { - events.push("stage:envelope"); - expect(fs.statSync(source).mode & 0o777).toBe(0o400); - expect( - parseManagedBootstrapEnvelope(fs.readFileSync(source, "utf8")).bootstrapIdentity, - ).toBe(IDENTITY); - return ok(); - }; - const copyFromContainer = () => { - const receipt = source.split(":")[1]; - const expected = receipt?.includes("shared-state-commit") ? "committed" : "pending"; - return sharedState === expected - ? (() => { - fs.mkdirSync(destination, { recursive: true }); - return ok(); - })() - : { - status: 1, - stderr: `Error response from daemon: Could not find the file ${receipt} in container ${NEW_ID}`, - }; - }; - return source.includes(":") ? copyFromContainer() : copyIntoContainer(); - } - case "run": - switch (true) { - case args.includes("--shared-state-transaction-status"): - return ok(`${sharedState}\n`); - case args.includes("--rollback-shared-state-transaction"): - sharedState = "none"; - events.push("shared:rollback"); - return ok(); - } - break; - case "exec": - switch (true) { - case args.includes("--commit-shared-state-transaction"): - sharedState = "committed"; - events.push("shared:commit"); - return ok(); - case args.includes("--clear-shared-state-commit-receipt"): - sharedState = "none"; - events.push("shared:clear"); - return ok(); - } - break; - } - throw new Error(`unexpected Docker command: ${args.join(" ")}`); - }, - ); - const deps: DockerManagedBootstrapDeps = { - journalStore: store, - dockerCapture, - dockerRun, - dockerStop: vi.fn((id) => { - events.push(`stop:${id}`); - const target = id === OLD_ID ? original : replacement; - [target] - .filter((value): value is DockerContainerInspect => value?.State !== undefined) - .forEach((value) => { - value.State = { ...value.State, Running: false }; - }); - return options.lostAcks ? { status: 1, stderr: "lost stop acknowledgement" } : ok(); - }), - dockerRename: vi.fn((id, name) => { - events.push(`rename:${id}:${name}`); - const target = id === OLD_ID ? original : replacement; - [target] - .filter((value): value is DockerContainerInspect => value !== null) - .forEach((value) => { - value.Name = `/${name}`; - }); - return options.lostAcks ? { status: 1, stderr: "lost rename acknowledgement" } : ok(); - }), - dockerStart: vi.fn((id) => { - events.push(`start:${id}`); - const target = id === OLD_ID ? original : replacement; - [target] - .filter( - (value): value is DockerContainerInspect => - value?.State !== undefined && !(id === NEW_ID && options.failStart), - ) - .forEach((value) => { - value.State = { ...value.State, Running: true }; - }); - return id === NEW_ID && options.failStart - ? { status: 1, stderr: "injected start failure" } - : options.lostAcks - ? { status: 1, stderr: "lost start acknowledgement" } - : ok(); - }), - dockerRm: vi.fn((id) => { - events.push(`rm:${id}`); - switch (id) { - case OLD_ID: - original = null as unknown as DockerContainerInspect; - break; - case NEW_ID: - replacement = null; - break; - } - return options.lostAcks ? { status: 1, stderr: "lost rm acknowledgement" } : ok(); - }), - runCaptureOpenshell: vi.fn(() => `Name: alpha\nID: ${options.ownerId ?? "sandbox-alpha"}\n`), - runOpenshell: vi.fn(() => ok()), - now: () => new Date("2026-07-31T12:30:00.000Z"), - }; - return { - deps, - events, - get journal() { - return journal; - }, - get original() { - return original; - }, - get replacement() { - return replacement; - }, - get sharedState() { - return sharedState; - }, - }; -} - -function completion( - replacement: ManagedBootstrapReplacementHandle, -): ManagedBootstrapCompletionReceipt { - return { - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox, - runtimeId: replacement.replacementRuntimeId, - image: replacement.image, - runtimeImageContentId: replacement.runtimeImageContentId, - originalSpecHash: replacement.originalSpecHash, - replacementSpecHash: replacement.replacementSpecHash, - profileFingerprint: replacement.profileFingerprint, - bootstrapIdentity: replacement.bootstrapIdentity, - transactionPending: true, - completedAt: "2026-07-31T12:15:00.000Z", - }; -} - -function durablePreparation( - handle: ManagedBootstrapHeldWorkloadHandle, - snapshot: ManagedBootstrapObservedSnapshot, - prepared: ManagedBootstrapPreparedReplacementHandle, -): ManagedBootstrapDurablePreparationReceipt { - const authority = createManagedBootstrapPreparedAuthority({ handle, snapshot, prepared }); - return { - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox: handle.sandbox, - bootstrapIdentity: handle.bootstrapIdentity, - authorityFingerprint: authority.authorityFingerprint, - recordId: `test-authority-${handle.plan.profile.agent}`, - recordedAt: "2026-07-31T12:10:00.000Z", - }; -} + authority, + completion, + durablePreparation, + fixture, + heldArgv, + IDENTITY, + NEW_ID, + OLD_ID, + SUPPORTED_AGENTS, +} from "./docker-test-fixture"; describe("Docker managed bootstrap adapter", () => { - it("journals both exact identities before cutover and reconciles lost acknowledgements", async () => { - const fake = fixture({ lostAcks: true, sharedState: "pending" }); + it("publishes durable commit authority before deleting the rollback backup after lost acknowledgements", async () => { + const fake = fixture({ + lostAcknowledgements: [ + "container:create", + "container:remove", + "container:rename", + "container:start", + "container:stop", + "journal:create", + "journal:cutover", + "journal:remove", + "journal:shared-state-committed", + ], + sharedState: "pending", + }); const adapter = createDockerManagedBootstrapAdapter(fake.deps); const { handle, request: rootRequest, snapshot } = authority(); const prepared = await adapter.prepareBootstrapReplacement({ @@ -482,8 +79,12 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.replacement?.Id).toBe(NEW_ID); }); - it("recovers a failed cutover after adapter restart from exact journal authority", async () => { - const fake = fixture({ failStart: true }); + it("publishes durable rollback authority before deleting the replacement after restart", async () => { + const fake = fixture({ + dockerStartResults: { + [NEW_ID]: { status: 1, stderr: "injected start failure" }, + }, + }); const first = createDockerManagedBootstrapAdapter(fake.deps); const { handle, request: rootRequest, snapshot } = authority(); const prepared = await first.prepareBootstrapReplacement({ @@ -525,7 +126,11 @@ describe("Docker managed bootstrap adapter", () => { }); it("recovers the pre-stop cutover crash state after adapter restart", async () => { - const fake = fixture({ failAfterCutoverFence: true }); + const fake = fixture({ + journalTransitionFailures: { + cutover: new Error("injected crash after durable cutover fence"), + }, + }); const { handle, request: rootRequest, snapshot } = authority(); const adapter = createDockerManagedBootstrapAdapter(fake.deps); const prepared = await adapter.prepareBootstrapReplacement({ diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index ece9d79adaf..b62e4d0dc57 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -79,10 +79,6 @@ describe("runtime provider central source boundary", () => { join(repoRoot, "src/lib/onboard/runtime-provider/docker.ts"), "utf8", ); - const dockerBootstrapAdapter = readFileSync( - join(repoRoot, "src/lib/onboard/managed-bootstrap/docker.ts"), - "utf8", - ); const managedDockerfiles = [ readFileSync(join(repoRoot, "Dockerfile"), "utf8"), readFileSync(join(repoRoot, "agents/hermes/Dockerfile"), "utf8"), @@ -103,8 +99,6 @@ describe("runtime provider central source boundary", () => { /(?:from\s+["'][^"']*managed-bootstrap|require\([^)]*managed-bootstrap)/u, ); expect(dockerProvider.match(/bootstrap:\s*unsupported\(/gu)).toHaveLength(2); - expect(dockerBootstrapAdapter).toContain('"rollback-authorized"'); - expect(dockerBootstrapAdapter).toContain('"shared-state-committed"'); expect(bootstrapProtocol[2]).not.toMatch(/from\s+["'][^"']*docker/u); for (const dockerfile of managedDockerfiles) { From 333b673d39db144a9742691bc950d431d21bb35b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 22:49:24 -0700 Subject: [PATCH 078/117] fix(onboard): reconcile managed bootstrap after restart Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/README.md | 18 +- .../onboard/managed-bootstrap/adapter.test.ts | 49 +++ src/lib/onboard/managed-bootstrap/adapter.ts | 102 ++++++ .../managed-bootstrap/docker-journal.test.ts | 2 + .../managed-bootstrap/docker-journal.ts | 19 +- .../managed-bootstrap/docker-runtime.ts | 6 +- .../onboard/managed-bootstrap/docker.test.ts | 154 +++++++++ src/lib/onboard/managed-bootstrap/docker.ts | 309 ++++++++++++++++++ src/lib/onboard/managed-bootstrap/index.ts | 2 + .../managed-bootstrap/runtime-create.ts | 2 + .../runtime-provider-contract.test.ts | 3 +- .../onboard/sandbox-gpu-create-flow.test.ts | 11 +- .../onboard/sandbox-gpu-create-run-attempt.ts | 3 +- 13 files changed, 663 insertions(+), 17 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 5c6bb38dbff..1ca2f12e5e6 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -51,12 +51,18 @@ commit atomically moves its pending manifest and backups into a durable receipt namespace, compacts that state to an exact commit receipt, and rejects rollback after a restart. The provider may retire that receipt only after it proves the external rollback backup is gone, leaving the next bootstrap attempt unblocked. -Enumeration reconstructs only unfinished records; the following recovery slice -owns phase reconciliation and cross-surface resume or rollback. Mutable -OpenShell names are read only to detect ownership reuse, and unsafe name-only -deletion returns a typed retention error. Multi-process lease/arbitration remains -an explicit production-activation gate. Activation must also inject the selected -gateway's canonical state root. +At lifecycle startup, the driver-neutral coordinator asks the selected provider +to reconcile every unfinished record before a new sandbox create begins. The +Docker provider then resumes the durable phase monotonically: pre-cutover work +rolls back, rollback-authorized work completes exact restore and cleanup, and +shared-state-committed work completes exact backup cleanup and commit. Recovery +persists an identity-bound finalization receipt before removing the active +journal, is idempotent across another interruption, and returns normalized, +provider-owned receipts in stable identity order. Mutable OpenShell names are +read only to detect ownership reuse, and unsafe name-only deletion returns a +typed retention error. Multi-process lease/arbitration remains an explicit +production-activation gate. Activation must also inject the selected gateway's +canonical state root. The runtime-provider bundle is the only bootstrap registration boundary. The candidate Docker surface owns create routing, replacement construction, diff --git a/src/lib/onboard/managed-bootstrap/adapter.test.ts b/src/lib/onboard/managed-bootstrap/adapter.test.ts index cc15f292524..6e3a5c1feb9 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.test.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.test.ts @@ -25,6 +25,7 @@ import { type ManagedBootstrapPreparedReplacementHandle, type ManagedBootstrapReplacementHandle, prepareManagedBootstrapSequence, + recoverManagedBootstrapTransactions, renderManagedBootstrapHeldCommand, } from "./adapter"; @@ -227,6 +228,7 @@ function adapterFor(agent: ManagedStartupAgent): Fixture { const order: string[] = []; const raw: Fixture["raw"] = { handle: null, snapshot: null, prepared: null }; const adapter: ManagedBootstrapAdapter = { + recoverUnfinishedTransactions: vi.fn(async () => []), createHeldWorkload: vi.fn(async (input) => { order.push("create"); const receipt = await input.launch({ @@ -657,6 +659,53 @@ describe("managed bootstrap adapter contract", () => { expect(fixture.adapter.finalizeBootstrap).not.toHaveBeenCalled(); }); + it("normalizes, freezes, and orders provider-owned restart recovery receipts", async () => { + const fixture = adapterFor("openclaw"); + const receipt = cleanupReceipt(); + const candidate = (bootstrapIdentity: string) => ({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: receipt.sandbox.driverId, + sourcePhase: "cutover", + sandbox: receipt.sandbox, + bootstrapIdentity, + outcome: "rolled-back" as const, + finalization: { ...receipt, bootstrapIdentity }, + }); + vi.mocked(fixture.adapter.recoverUnfinishedTransactions).mockResolvedValueOnce([ + candidate("b".repeat(64)), + candidate("a".repeat(64)), + ]); + + const recovered = await recoverManagedBootstrapTransactions(fixture.adapter); + + expect(recovered.map(({ bootstrapIdentity }) => bootstrapIdentity)).toEqual([ + "a".repeat(64), + "b".repeat(64), + ]); + expect(Object.isFrozen(recovered)).toBe(true); + expect(recovered.every((entry) => Object.isFrozen(entry.finalization))).toBe(true); + }); + + it("rejects recovery evidence whose provider does not own the durable sandbox", async () => { + const fixture = adapterFor("openclaw"); + const receipt = cleanupReceipt(); + vi.mocked(fixture.adapter.recoverUnfinishedTransactions).mockResolvedValueOnce([ + { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: "mxc", + sourcePhase: "cutover", + sandbox: receipt.sandbox, + bootstrapIdentity: IDENTITY, + outcome: "rolled-back", + finalization: receipt, + }, + ]); + + await expect(recoverManagedBootstrapTransactions(fixture.adapter)).rejects.toThrow( + "recovery provider does not own", + ); + }); + it.each([ "BASH_ENV=/sandbox/attacker", "ENV=/sandbox/attacker", diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index ee51a04e9df..bff6c6cb0ff 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -235,6 +235,21 @@ export interface ManagedBootstrapFinalizationReceipt { readonly finalizedAt: string; } +/** + * Driver-neutral evidence that one durable, process-orphaned transaction was + * reconciled without reconstructing authority from mutable runtime names. + */ +export interface ManagedBootstrapRecoveryReceipt { + readonly schemaVersion: typeof MANAGED_BOOTSTRAP_SCHEMA_VERSION; + readonly providerId: string; + /** Provider-owned phase name retained for diagnostics, never central routing. */ + readonly sourcePhase: string; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly bootstrapIdentity: string; + readonly outcome: "committed" | "rolled-back"; + readonly finalization: ManagedBootstrapFinalizationReceipt; +} + export class ManagedBootstrapDurableCommitCleanupPendingError extends Error { readonly bootstrapIdentity: string; readonly cleanupRuntimeId: string; @@ -306,6 +321,12 @@ export function attachManagedBootstrapRollbackError(failure: Error, rollbackErro } export interface ManagedBootstrapAdapter { + /** + * Enumerate durable unfinished records and reconcile each through the owning + * provider. Implementations must be restart-safe and idempotent. + */ + recoverUnfinishedTransactions(): Promise; + /** Return only after one durable sandbox/driver identity reports Ready. */ createHeldWorkload( input: ManagedBootstrapCreateInput, @@ -372,6 +393,87 @@ export interface ManagedBootstrapAdapter { }): Promise; } +function normalizeRecoveryReceipt( + candidate: ManagedBootstrapRecoveryReceipt, +): ManagedBootstrapRecoveryReceipt { + if ( + typeof candidate !== "object" || + candidate === null || + Array.isArray(candidate) || + candidate.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + !["committed", "rolled-back"].includes(String(candidate.outcome)) + ) { + protocolFail("recovery receipt has an invalid schema or outcome"); + } + assertOpaqueString(candidate.providerId, "recovery provider ID"); + assertOpaqueString(candidate.sourcePhase, "recovery source phase"); + assertSandboxIdentity(candidate.sandbox); + if (candidate.sandbox.driverId !== candidate.providerId) { + protocolFail("recovery provider does not own the recovered sandbox"); + } + if (!SHA256_RE.test(candidate.bootstrapIdentity)) { + protocolFail("recovery bootstrap identity must be lowercase SHA-256"); + } + const finalization = candidate.finalization; + if ( + typeof finalization !== "object" || + finalization === null || + Array.isArray(finalization) || + finalization.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + finalization.outcome !== candidate.outcome || + finalization.bootstrapIdentity !== candidate.bootstrapIdentity || + !isDeepStrictEqual(finalization.sandbox, candidate.sandbox) || + typeof finalization.heldWorkloadRemoved !== "boolean" || + typeof finalization.alreadyRolledBack !== "boolean" + ) { + protocolFail("recovery finalization does not match its durable identity"); + } + if ( + (finalization.restoredRuntimeId !== null && !SHA256_RE.test(finalization.restoredRuntimeId)) || + (finalization.restoredSpecHash !== null && !SHA256_RE.test(finalization.restoredSpecHash)) || + (finalization.restoredRuntimeId === null) !== (finalization.restoredSpecHash === null) || + (candidate.outcome === "committed" && + (finalization.restoredRuntimeId !== null || + finalization.heldWorkloadRemoved || + finalization.alreadyRolledBack)) + ) { + protocolFail("recovery finalization state is inconsistent"); + } + assertTimestamp(finalization.finalizedAt, "recovery finalization timestamp"); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: candidate.providerId, + sourcePhase: candidate.sourcePhase, + sandbox: Object.freeze({ ...candidate.sandbox }), + bootstrapIdentity: candidate.bootstrapIdentity, + outcome: candidate.outcome, + finalization: Object.freeze({ + ...finalization, + sandbox: Object.freeze({ ...candidate.sandbox }), + }), + }); +} + +/** Recover process-orphaned work without relying on coordinator WeakMap state. */ +export async function recoverManagedBootstrapTransactions( + adapter: ManagedBootstrapAdapter, +): Promise { + const candidates = await adapter.recoverUnfinishedTransactions(); + if (!Array.isArray(candidates)) { + protocolFail("provider recovery must return a receipt array"); + } + const receipts = candidates.map(normalizeRecoveryReceipt); + const identities = receipts.map(({ bootstrapIdentity }) => bootstrapIdentity); + if (new Set(identities).size !== identities.length) { + protocolFail("provider recovery returned duplicate bootstrap identities"); + } + return Object.freeze( + [...receipts].sort((left, right) => + left.bootstrapIdentity.localeCompare(right.bootstrapIdentity), + ), + ); +} + export interface ManagedBootstrapPreparationInput { readonly create: ManagedBootstrapCreateInput; readonly request: ManagedStartupRootApplyRequest; diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index 1f0c4d05a95..555c82c6d3e 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -27,6 +27,7 @@ const journal = Object.freeze({ phase: "staged", bootstrapIdentity: IDENTITY, providerId: "docker", + agent: "hermes", sandbox: { sandboxName: "alpha", sandboxId: "sandbox-alpha", @@ -64,6 +65,7 @@ const finalization = Object.freeze({ phase: "committed", bootstrapIdentity: IDENTITY, providerId: "docker", + agent: journal.agent, sandbox: journal.sandbox, planFingerprint: journal.planFingerprint, profileFingerprint: journal.profileFingerprint, diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index 45592a9ed30..10608935d4d 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -3,7 +3,7 @@ import fs from "node:fs"; import path from "node:path"; - +import type { ManagedStartupAgent } from "../managed-startup/profile"; import type { ManagedBootstrapCompletionReceipt, ManagedBootstrapDurablePreparationReceipt, @@ -11,9 +11,9 @@ import type { ManagedBootstrapSandboxIdentity, } from "./adapter"; -export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 2 as const; +export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 3 as const; export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY = "managed-bootstrap"; -export const DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION = 1 as const; +export const DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION = 2 as const; const SHA256_RE = /^[a-f0-9]{64}$/u; const MANIFEST_DIGEST_RE = /^sha256:[a-f0-9]{64}$/u; @@ -36,6 +36,7 @@ export interface DockerManagedBootstrapJournal { readonly phase: DockerManagedBootstrapJournalPhase; readonly bootstrapIdentity: string; readonly providerId: string; + readonly agent: ManagedStartupAgent; readonly sandbox: ManagedBootstrapSandboxIdentity; readonly planFingerprint: string; readonly profileFingerprint: string; @@ -59,6 +60,7 @@ export interface DockerManagedBootstrapFinalizationRecord { readonly phase: "committed" | "rolled-back"; readonly bootstrapIdentity: string; readonly providerId: string; + readonly agent: ManagedStartupAgent; readonly sandbox: ManagedBootstrapSandboxIdentity; readonly planFingerprint: string; readonly profileFingerprint: string; @@ -136,6 +138,13 @@ function exactPhase(value: unknown): DockerManagedBootstrapJournalPhase { return value as DockerManagedBootstrapJournalPhase; } +function exactAgent(value: unknown): ManagedStartupAgent { + if (!["openclaw", "hermes", "langchain-deepagents-code"].includes(String(value))) { + fail("agent is unsupported"); + } + return value as ManagedStartupAgent; +} + function exactSandbox(value: unknown): ManagedBootstrapSandboxIdentity { if (typeof value !== "object" || value === null || Array.isArray(value)) { fail("sandbox identity must be an object"); @@ -159,6 +168,7 @@ export function normalizeDockerManagedBootstrapJournal( } const journal = value as Record; const expectedKeys = [ + "agent", "backupName", "bootstrapIdentity", "commitReceipt", @@ -191,6 +201,7 @@ export function normalizeDockerManagedBootstrapJournal( phase: exactPhase(journal.phase), bootstrapIdentity: exactSha256(journal.bootstrapIdentity, "bootstrap identity"), providerId: exactString(journal.providerId, "provider ID"), + agent: exactAgent(journal.agent), sandbox: exactSandbox(journal.sandbox), planFingerprint: exactSha256(journal.planFingerprint, "plan fingerprint"), profileFingerprint: exactSha256(journal.profileFingerprint, "profile fingerprint"), @@ -450,6 +461,7 @@ export function normalizeDockerManagedBootstrapFinalizationRecord( } const record = value as Record; const expectedKeys = [ + "agent", "bootstrapIdentity", "cleanupReceipt", "commitReceipt", @@ -478,6 +490,7 @@ export function normalizeDockerManagedBootstrapFinalizationRecord( phase, bootstrapIdentity: exactSha256(record.bootstrapIdentity, "finalization bootstrap identity"), providerId: exactString(record.providerId, "finalization provider ID"), + agent: exactAgent(record.agent), sandbox, planFingerprint: exactSha256(record.planFingerprint, "finalization plan fingerprint"), profileFingerprint: exactSha256(record.profileFingerprint, "finalization profile fingerprint"), diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.ts index 21932a2207f..ada0c3457ac 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.ts @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { RuntimeProviderBootstrapSurface } from "../runtime-provider/contract"; import { detectTegraDeviceGroupGids } from "../docker-gpu-jetson-groups"; import { buildDockerGpuMode, selectDockerGpuPatchMode } from "../docker-gpu-patch-mode"; import type { DockerGpuPatchMode } from "../docker-gpu-patch-types"; @@ -15,12 +14,14 @@ import { queryOpenShellDockerSandboxContainers, queryOpenShellDockerSandboxRuntimeSnapshot, } from "../openshell-docker-sandbox-containers"; +import type { RuntimeProviderBootstrapSurface } from "../runtime-provider/contract"; import * as sandboxGpuCreateAttempt from "../sandbox-gpu-create-attempt"; import { activateManagedBootstrapSequence, finalizeManagedBootstrapSequence, MANAGED_BOOTSTRAP_SCHEMA_VERSION, prepareManagedBootstrapSequence, + recoverManagedBootstrapTransactions, } from "./adapter"; import { createDockerManagedBootstrapAdapter } from "./docker"; import type { @@ -137,6 +138,9 @@ function createDockerLifecycle( return { launchArgv: input.launchArgv, patch, + async recoverUnfinished() { + return recoverManagedBootstrapTransactions(adapter); + }, async prepareNetwork() { if (input.route !== "compatibility") return; const { enforceDockerGpuPatchPreserveNetwork } = await import( diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 582c5e3622e..38c06911c34 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -149,6 +149,10 @@ function authority(agent: ManagedStartupAgent = "hermes") { type FixtureOptions = { agent?: ManagedStartupAgent; failAfterCutoverFence?: boolean; + failAfterRollbackFence?: boolean; + failAfterSharedCommitFence?: boolean; + failAfterStagedFence?: boolean; + failRemoveOnce?: boolean; failStart?: boolean; lostAcks?: boolean; ownerId?: string; @@ -169,12 +173,16 @@ function fixture(options: FixtureOptions = {}) { const lostTransitions = new Set(["cutover", "shared-state-committed"]); let loseCreateAck = options.lostAcks === true; let loseRemoveAck = options.lostAcks === true; + let failRemoveOnce = options.failRemoveOnce === true; const ok = (stdout = "") => ({ status: 0, stdout, stderr: "" }); const copyJournal = () => (journal ? structuredClone(journal) : null); const store: DockerManagedBootstrapJournalStore = { create(value) { journal = structuredClone(value); events.push("journal:staged"); + if (options.failAfterStagedFence === true) { + throw new Error("injected crash after durable staged fence"); + } switch (loseCreateAck) { case true: loseCreateAck = false; @@ -195,6 +203,10 @@ function fixture(options: FixtureOptions = {}) { switch (true) { case next === "cutover" && options.failAfterCutoverFence === true: throw new Error("injected crash after durable cutover fence"); + case next === "rollback-authorized" && options.failAfterRollbackFence === true: + throw new Error("injected crash after durable rollback fence"); + case next === "shared-state-committed" && options.failAfterSharedCommitFence === true: + throw new Error("injected crash after durable shared-state commit fence"); case options.lostAcks === true && lostTransitions.delete(next): throw new DockerManagedBootstrapJournalAcknowledgementLostError( "lost journal transition acknowledgement", @@ -408,6 +420,10 @@ function fixture(options: FixtureOptions = {}) { }), dockerRm: vi.fn((id) => { events.push(`rm:${id}`); + if (failRemoveOnce) { + failRemoveOnce = false; + throw new Error("injected crash before exact Docker removal"); + } switch (id) { case OLD_ID: original = null as unknown as DockerContainerInspect; @@ -573,6 +589,144 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.original.State?.Running).toBe(false); }); + it.each([ + ["staged", { failAfterStagedFence: true }, "staged"], + ["cutover", { failAfterCutoverFence: true }, "cutover"], + ] as const)("reconciles a process restart from the durable %s phase", async (_label, options, phase) => { + const fake = fixture(options); + const first = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await first.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + await expect( + first.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }), + ).rejects.toThrow(`crash after durable ${phase} fence`); + expect(fake.journal?.phase).toBe(phase); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject([ + { + sourcePhase: phase, + outcome: "rolled-back", + finalization: { + restoredRuntimeId: OLD_ID, + heldWorkloadRemoved: false, + }, + }, + ]); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toEqual([]); + expect(fake.journal).toBeNull(); + expect(fake.finalization?.phase).toBe("rolled-back"); + expect(fake.replacement).toBeNull(); + expect(fake.original.Name).toBe("/openshell-alpha"); + expect(fake.original.State?.Running).toBe(true); + }); + + it("finishes a rollback-authorized transaction after shared-state rollback is interrupted", async () => { + const fake = fixture({ + agent: "openclaw", + failAfterRollbackFence: true, + sharedState: "pending", + }); + const first = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority("openclaw"); + const prepared = await first.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await first.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + await expect( + first.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: null, + }), + ).rejects.toThrow("crash after durable rollback fence"); + expect(fake.journal?.phase).toBe("rollback-authorized"); + expect(fake.sharedState).toBe("pending"); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject([ + { sourcePhase: "rollback-authorized", outcome: "rolled-back" }, + ]); + expect(fake.sharedState).toBe("none"); + expect(fake.replacement).toBeNull(); + expect(fake.original.State?.Running).toBe(true); + }); + + it("finishes exact commit cleanup after a process restart at the durable commit fence", async () => { + const fake = fixture({ + agent: "langchain-deepagents-code", + failRemoveOnce: true, + sharedState: "pending", + }); + const first = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority("langchain-deepagents-code"); + const prepared = await first.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await first.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + const completion = await first.awaitBootstrap({ + handle, + snapshot, + replacement, + timeoutSecs: 1, + }); + await expect( + first.finalizeBootstrap({ + outcome: "commit", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion, + }), + ).rejects.toThrow("crash before exact Docker removal"); + expect(fake.journal?.phase).toBe("shared-state-committed"); + expect(fake.sharedState).toBe("committed"); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject([ + { sourcePhase: "shared-state-committed", outcome: "committed" }, + ]); + expect(fake.journal).toBeNull(); + expect(fake.finalization?.phase).toBe("committed"); + expect(fake.sharedState).toBe("none"); + expect(fake.replacement?.State?.Running).toBe(true); + }); + it("recovers the pre-stop cutover crash state after adapter restart", async () => { const fake = fixture({ failAfterCutoverFence: true }); const { handle, request: rootRequest, snapshot } = authority(); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index d0024838821..977b016405a 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -59,6 +59,7 @@ import { type ManagedBootstrapObservedSnapshot, ManagedBootstrapOwnerCleanupRequiredError, type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapRecoveryReceipt, type ManagedBootstrapReplacementHandle, type ManagedBootstrapReplacementOptions, type ManagedBootstrapSandboxIdentity, @@ -1406,6 +1407,16 @@ function managedSharedStateTransaction( } as const; } +function recoveredManagedSharedStateTransaction(journal: DockerBootstrapTransaction) { + return { + agent: journal.agent, + bootstrapIdentity: journal.bootstrapIdentity, + containerId: journal.replacementRuntimeId, + image: journal.runtimeImageContentId, + profileFingerprint: journal.profileFingerprint, + } as const; +} + function sameDockerBootstrapJournal( left: DockerBootstrapTransaction, right: DockerBootstrapTransaction, @@ -1783,6 +1794,7 @@ export function createDockerManagedBootstrapAdapter( phase, bootstrapIdentity: handle.bootstrapIdentity, providerId: handle.sandbox.driverId, + agent: handle.plan.profile.agent, sandbox: handle.sandbox, planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), profileFingerprint: handle.plan.profile.fingerprint, @@ -1862,6 +1874,289 @@ export function createDockerManagedBootstrapAdapter( alreadyRolledBack: true, }); }; + const persistRecoveredFinalization = ( + journal: DockerBootstrapTransaction, + phase: "committed" | "rolled-back", + commitReceipt: ManagedBootstrapCompletionReceipt | null, + cleanupReceipt: ManagedBootstrapFinalizationReceipt, + ): ManagedBootstrapFinalizationReceipt => { + const record = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase, + bootstrapIdentity: journal.bootstrapIdentity, + providerId: journal.providerId, + agent: journal.agent, + sandbox: journal.sandbox, + planFingerprint: journal.planFingerprint, + profileFingerprint: journal.profileFingerprint, + imageReference: journal.imageReference, + commitReceipt, + cleanupReceipt, + } satisfies DockerManagedBootstrapFinalizationRecord); + const serialized = serializeDockerManagedBootstrapFinalizationRecord(record); + try { + deps.journalStore.recordFinalization(record); + } catch (error) { + const recovered = deps.journalStore.loadFinalization(journal.bootstrapIdentity); + if ( + !recovered || + serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized + ) { + throw error; + } + } + const persisted = deps.journalStore.loadFinalization(journal.bootstrapIdentity); + if (!persisted || serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized) { + throw new Error("Managed bootstrap recovered finalization was not durably re-readable."); + } + return persisted.cleanupReceipt; + }; + const recoveredReceipt = ( + journal: DockerBootstrapTransaction, + sourcePhase: DockerBootstrapTransaction["phase"], + finalization: ManagedBootstrapFinalizationReceipt, + ): ManagedBootstrapRecoveryReceipt => + Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: journal.providerId, + sourcePhase, + sandbox: journal.sandbox, + bootstrapIdentity: journal.bootstrapIdentity, + outcome: finalization.outcome, + finalization, + }); + const finishRecoveredRollback = ( + journal: DockerBootstrapTransaction, + sourcePhase: DockerBootstrapTransaction["phase"], + ): ManagedBootstrapRecoveryReceipt => { + const cleanupReceipt = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: journal.sandbox, + bootstrapIdentity: journal.bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: journal.originalRuntimeId, + restoredSpecHash: journal.originalSpecHash, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + } satisfies ManagedBootstrapFinalizationReceipt); + const finalization = persistRecoveredFinalization(journal, "rolled-back", null, cleanupReceipt); + removeDockerBootstrapJournalDurably(journal, deps); + return recoveredReceipt(journal, sourcePhase, finalization); + }; + const finishRecoveredCommit = ( + journal: DockerBootstrapTransaction, + sourcePhase: DockerBootstrapTransaction["phase"], + ): ManagedBootstrapRecoveryReceipt => { + if (journal.phase !== "shared-state-committed" || journal.commitReceipt === null) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable commit recovery requires its exact completion receipt and commit fence", + }); + } + const replacement = inspectTransactionRuntime(journal, journal.replacementRuntimeId, deps); + if (!replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "the exact committed replacement is absent during restart recovery", + }); + } + assertTransactionReplacement(journal, replacement); + if ( + dockerContainerName(replacement) !== journal.originalName || + !isStableRunning(replacement) || + normalizeDockerManagedBootstrapLaunchSpec(replacement).hash !== journal.replacementSpecHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "the committed replacement does not match its durable runtime authority", + }); + } + const sharedTransaction = recoveredManagedSharedStateTransaction(journal); + const sharedStatus = probeDockerManagedStartupSharedState( + { transaction: sharedTransaction, profileFingerprint: journal.profileFingerprint }, + deps, + ); + if (sharedStatus === "pending") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "shared state is pending after the durable commit fence", + }); + } + const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); + if (original) { + assertTransactionOriginal(journal, original); + if ( + dockerContainerName(original) !== journal.backupName || + !isExplicitlyStopped(original) || + normalizeDockerManagedBootstrapLaunchSpec({ + ...original, + Name: `/${journal.originalName}`, + }).hash !== journal.originalSpecHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "the exact rollback backup changed before recovered commit cleanup", + }); + } + if (sharedStatus === "none") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "the shared commit receipt was retired before exact backup absence was proven", + }); + } + const removed = deps.dockerRm(journal.originalRuntimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(journal.originalRuntimeId, deps) !== "absent" + ) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: journal.bootstrapIdentity, + cleanupRuntimeId: journal.originalRuntimeId, + detail: `${commandDetail(removed) || "Docker removal failed"}; exact backup absence was not proven`, + }); + } + } + if (sharedStatus === "committed") { + clearDockerManagedStartupSharedStateCommitReceipt(sharedTransaction, deps); + } + if (probeExactDockerContainerAbsence(journal.originalRuntimeId, deps) !== "absent") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: journal.bootstrapIdentity, + cleanupRuntimeId: journal.originalRuntimeId, + detail: "exact rollback-backup absence was not durable after restart recovery", + }); + } + const cleanupReceipt = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: journal.sandbox, + bootstrapIdentity: journal.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + } satisfies ManagedBootstrapFinalizationReceipt); + const finalization = persistRecoveredFinalization( + journal, + "committed", + journal.commitReceipt, + cleanupReceipt, + ); + removeDockerBootstrapJournalDurably(journal, deps); + return recoveredReceipt(journal, sourcePhase, finalization); + }; + const finishRecoveredRollbackPhase = ( + journal: DockerBootstrapTransaction, + sourcePhase: DockerBootstrapTransaction["phase"], + ): ManagedBootstrapRecoveryReceipt => { + const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); + if (!original) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "the exact rollback original is absent during restart recovery", + }); + } + assertTransactionOriginal(journal, original); + const replacement = inspectTransactionRuntime(journal, journal.replacementRuntimeId, deps); + if (replacement) assertTransactionReplacement(journal, replacement); + if (journal.phase === "staged") { + if ( + dockerContainerName(original) !== journal.originalName || + !isStableRunning(original) || + normalizeDockerManagedBootstrapLaunchSpec(original).hash !== journal.originalSpecHash || + (replacement !== null && + (dockerContainerName(replacement) !== journal.replacementStagingName || + !isExplicitlyStopped(replacement))) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "staged restart recovery does not match its pre-cutover fence", + }); + } + if (replacement) removeExactReplacement(journal, replacement, deps); + return finishRecoveredRollback(journal, sourcePhase); + } + if (journal.phase !== "cutover" && journal.phase !== "rollback-authorized") { + return finishRecoveredCommit(journal, sourcePhase); + } + let activeJournal = journal; + if (!replacement && dockerContainerName(original) !== journal.originalName) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "the replacement disappeared before exact rollback restoration was proven", + }); + } + if (replacement) { + const sharedTransaction = recoveredManagedSharedStateTransaction(journal); + const sharedStatus = probeDockerManagedStartupSharedState( + { transaction: sharedTransaction, profileFingerprint: journal.profileFingerprint }, + deps, + ); + if (sharedStatus === "committed") { + if (journal.phase === "rollback-authorized") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "shared state committed after durable rollback authorization", + }); + } + activeJournal = transitionDockerBootstrapJournalDurably( + journal, + "shared-state-committed", + deps, + ); + return finishRecoveredCommit(activeJournal, sourcePhase); + } + if (journal.phase === "cutover") { + activeJournal = transitionDockerBootstrapJournalDurably( + journal, + "rollback-authorized", + deps, + ); + } + if (sharedStatus === "pending") { + finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: false, + retainContainerAfterRollback: true, + }, + deps, + ); + } + } else if (journal.phase === "cutover") { + activeJournal = transitionDockerBootstrapJournalDurably(journal, "rollback-authorized", deps); + } + restoreOriginal(activeJournal, deps); + const restored = inspectExact(activeJournal.originalRuntimeId, deps); + if ( + !isStableRunning(restored) || + dockerContainerName(restored) !== activeJournal.originalName || + normalizeDockerManagedBootstrapLaunchSpec(restored).hash !== activeJournal.originalSpecHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: activeJournal.bootstrapIdentity, + runtimeId: activeJournal.originalRuntimeId, + detail: "restart recovery did not restore the exact original runtime and launch spec", + }); + } + return finishRecoveredRollback(activeJournal, sourcePhase); + }; const rollbackBootstrapNow = ({ handle, snapshot, @@ -2510,6 +2805,19 @@ export function createDockerManagedBootstrapAdapter( }); }; return { + async recoverUnfinishedTransactions() { + const receipts: ManagedBootstrapRecoveryReceipt[] = []; + for (const journal of deps.journalStore.listUnfinished()) { + const sourcePhase = journal.phase; + receipts.push( + journal.phase === "shared-state-committed" + ? finishRecoveredCommit(journal, sourcePhase) + : finishRecoveredRollbackPhase(journal, sourcePhase), + ); + } + return Object.freeze(receipts); + }, + async createHeldWorkload(input) { if ( input.plan.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || @@ -2757,6 +3065,7 @@ export function createDockerManagedBootstrapAdapter( phase: "staged", bootstrapIdentity: handle.bootstrapIdentity, providerId: handle.sandbox.driverId, + agent: handle.plan.profile.agent, sandbox: Object.freeze({ ...handle.sandbox }), planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), profileFingerprint: handle.plan.profile.fingerprint, diff --git a/src/lib/onboard/managed-bootstrap/index.ts b/src/lib/onboard/managed-bootstrap/index.ts index c55768afe02..abeddd0a06d 100644 --- a/src/lib/onboard/managed-bootstrap/index.ts +++ b/src/lib/onboard/managed-bootstrap/index.ts @@ -10,7 +10,9 @@ export { type ManagedBootstrapAuthorityStore, type ManagedBootstrapExpectedPlan, type ManagedBootstrapPreparedTransaction, + type ManagedBootstrapRecoveryReceipt, prepareManagedBootstrapSequence, + recoverManagedBootstrapTransactions, } from "./adapter"; export { MANAGED_BOOTSTRAP_COMPLETION_FILE, diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.ts b/src/lib/onboard/managed-bootstrap/runtime-create.ts index 8cb6cad25d7..1fe762d1a4e 100644 --- a/src/lib/onboard/managed-bootstrap/runtime-create.ts +++ b/src/lib/onboard/managed-bootstrap/runtime-create.ts @@ -10,6 +10,7 @@ import type { ManagedBootstrapAuthorityStore, ManagedBootstrapCreateReceipt, ManagedBootstrapImageIdentity, + ManagedBootstrapRecoveryReceipt, } from "./adapter"; export interface ManagedBootstrapRuntimeCommandResult { @@ -93,6 +94,7 @@ export interface ManagedBootstrapRuntimeCreateLaunchResult { export interface ManagedBootstrapRuntimeCreateLifecycle { readonly launchArgv: readonly string[]; readonly patch: ManagedBootstrapRuntimePatch; + recoverUnfinished(): Promise; prepareNetwork(): Promise; runCreate( launch: (input: { diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index 3ac4da9e4fc..6628831a980 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -20,8 +20,8 @@ import { stopSandbox } from "../../actions/sandbox/stop"; import { loadAgent } from "../../agent/defs"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; import { cloneSandboxWorkloadReceipt } from "../../state/registry/workload"; -import { MANAGED_IMAGE_REPOSITORIES } from "../managed-image/contract"; import { createDockerManagedBootstrapSurface } from "../managed-bootstrap/docker-runtime"; +import { MANAGED_IMAGE_REPOSITORIES } from "../managed-image/contract"; import { encodeManagedStartupProfile, type ManagedStartupProfile, @@ -209,6 +209,7 @@ describe("RuntimeProviderBundle registry contract", () => { printReadinessFailureIfEnabled: vi.fn(), verifyGpuOrExit: vi.fn(async (verify) => verify("alpha")), }, + recoverUnfinished: vi.fn(async () => []), prepareNetwork: vi.fn(async () => undefined), runCreate: vi.fn(), })); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index 720306078f4..87979a668c7 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -68,17 +68,17 @@ import type { } from "./managed-bootstrap/runtime-create"; import { encodeManagedStartupProfile } from "./managed-startup/profile"; import { createManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; +import type { + RuntimeProviderBootstrapSurface, + RuntimeProviderBundle, +} from "./runtime-provider/contract"; +import { createRuntimeProviderBundleRegistry } from "./runtime-provider/registry"; import { prepareSandboxCreateLaunch } from "./sandbox-create-launch"; import { runSandboxGpuCreateFlow, type SandboxGpuCreateFlowDeps, type SandboxGpuCreateFlowInput, } from "./sandbox-gpu-create-flow"; -import type { - RuntimeProviderBootstrapSurface, - RuntimeProviderBundle, -} from "./runtime-provider/contract"; -import { createRuntimeProviderBundleRegistry } from "./runtime-provider/registry"; const FAILED_PROOF: SandboxGpuProofResult = { status: "failed", @@ -205,6 +205,7 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { (lifecycleInput: ManagedBootstrapRuntimeCreateLifecycleInput) => ({ launchArgv: ["mxc-launch", ...lifecycleInput.launchArgv.slice(1)], patch, + recoverUnfinished: vi.fn(async () => []), prepareNetwork: vi.fn(async () => undefined), runCreate: async ( start: (held: { diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 8183d24a0c1..3210934ff57 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { printSandboxCreateRecoveryHints } from "../build-context"; import { resolveOpenShellSandboxId } from "../adapters/openshell/sandbox-identity"; +import { printSandboxCreateRecoveryHints } from "../build-context"; import { getSandboxDeleteOutcome } from "../domain/sandbox/destroy"; import { streamSandboxCreate } from "../sandbox/create-stream"; import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-ready-gate"; @@ -143,6 +143,7 @@ export function createSandboxGpuCreateAttemptRunner( backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", deps, }); + await managedLifecycle?.recoverUnfinished(); await managedLifecycle?.prepareNetwork(); const [createExecutable, ...createExecutableArgs] = managedLifecycle?.launchArgv ?? attemptArgv; if (!createExecutable) throw new Error("Sandbox create executable is missing."); From 0cf5aaccd4d3cd6a7f183530ea1160dee67ed9eb Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 22:51:31 -0700 Subject: [PATCH 079/117] test(onboard): inventory Docker bootstrap sources Signed-off-by: Aaron Erickson --- test/runtime-provider-source-shape.test.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 67c7cdb5336..c572f1a3b9c 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -86,12 +86,19 @@ describe("runtime provider central source boundary", () => { "Dockerfile", "Dockerfile.base", ); - const bootstrapProtocolPaths = productionPaths.filter( + const bootstrapSourcePaths = productionPaths.filter( (path) => path.startsWith("src/lib/onboard/managed-bootstrap/") && path.endsWith(".ts") && !path.endsWith(".test.ts"), ); + const bootstrapProtocolPaths = bootstrapSourcePaths.filter((path) => + [ + "src/lib/onboard/managed-bootstrap/adapter.ts", + "src/lib/onboard/managed-bootstrap/envelope.ts", + "src/lib/onboard/managed-bootstrap/index.ts", + ].includes(path), + ); const activationPaths = productionPaths.filter( (path) => (path === "src/lib/onboard.ts" || path.startsWith("src/lib/onboard/")) && @@ -121,8 +128,13 @@ describe("runtime provider central source boundary", () => { const packagedBootstrapAsset = /(?:nemoclaw-managed-bootstrap|managed-bootstrap-trampoline|managed-startup-image-runtime\.cjs|nemoclaw-managed-startup-hold)/u; - expect(bootstrapProtocolPaths).toEqual([ + expect(bootstrapSourcePaths).toEqual([ "src/lib/onboard/managed-bootstrap/adapter.ts", + "src/lib/onboard/managed-bootstrap/docker-journal.ts", + "src/lib/onboard/managed-bootstrap/docker-shared-state.ts", + "src/lib/onboard/managed-bootstrap/docker-spec.ts", + "src/lib/onboard/managed-bootstrap/docker-test-fixture.ts", + "src/lib/onboard/managed-bootstrap/docker.ts", "src/lib/onboard/managed-bootstrap/envelope.ts", "src/lib/onboard/managed-bootstrap/index.ts", ]); From b79f0340ea4ff6afe1afea9c238caade85b450a9 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 23:01:28 -0700 Subject: [PATCH 080/117] test(cli): exclude source fixtures from distribution Signed-off-by: Aaron Erickson --- tsconfig.src.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tsconfig.src.json b/tsconfig.src.json index da1287ae436..13c12fb1310 100644 --- a/tsconfig.src.json +++ b/tsconfig.src.json @@ -16,5 +16,10 @@ "types": ["node"] }, "include": ["src/**/*.ts"], - "exclude": ["node_modules", "nemoclaw", "src/**/*.test.ts"] + "exclude": [ + "node_modules", + "nemoclaw", + "src/**/*.test.ts", + "src/**/*-test-fixture.ts" + ] } From 298409989059bdfe2dd079acde54e991ea9aa8db Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 23:05:53 -0700 Subject: [PATCH 081/117] fix(onboard): make terminal recovery restart-safe Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/README.md | 28 +++---- .../managed-bootstrap/docker-journal.test.ts | 2 + .../managed-bootstrap/docker-journal.ts | 13 ++-- .../onboard/managed-bootstrap/docker.test.ts | 41 ++++++++--- src/lib/onboard/managed-bootstrap/docker.ts | 73 ++++++++++++++++--- 5 files changed, 116 insertions(+), 41 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 1ca2f12e5e6..b14748d1600 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -51,25 +51,27 @@ commit atomically moves its pending manifest and backups into a durable receipt namespace, compacts that state to an exact commit receipt, and rejects rollback after a restart. The provider may retire that receipt only after it proves the external rollback backup is gone, leaving the next bootstrap attempt unblocked. -At lifecycle startup, the driver-neutral coordinator asks the selected provider -to reconcile every unfinished record before a new sandbox create begins. The -Docker provider then resumes the durable phase monotonically: pre-cutover work -rolls back, rollback-authorized work completes exact restore and cleanup, and -shared-state-committed work completes exact backup cleanup and commit. Recovery -persists an identity-bound finalization receipt before removing the active -journal, is idempotent across another interruption, and returns normalized, -provider-owned receipts in stable identity order. Mutable OpenShell names are -read only to detect ownership reuse, and unsafe name-only deletion returns a -typed retention error. Multi-process lease/arbitration remains an explicit -production-activation gate. Activation must also inject the selected gateway's -canonical state root. +At managed create-lifecycle startup, the driver-neutral coordinator asks the +selected provider to reconcile every unfinished record before a new sandbox +create begins. The Docker provider then resumes the durable phase monotonically: +staged work rolls back without entering cutover; cutover work follows a proven +image-owned commit forward or durably authorizes rollback; rollback-authorized +work completes exact restore and cleanup; and shared-state-committed work +completes exact backup cleanup and commit. Recovery persists an identity-bound +finalization receipt before removing the active journal, is idempotent across +another interruption, and returns normalized, provider-owned receipts in stable +identity order. Mutable OpenShell names are read only to detect ownership reuse, +and unsafe name-only deletion returns a typed retention error. Multi-process +lease/arbitration remains an explicit production-activation gate. Activation +must also inject the selected gateway's canonical state root. The runtime-provider bundle is the only bootstrap registration boundary. The candidate Docker surface owns create routing, replacement construction, native-to-compatibility fallback evidence, and deferred commit or rollback. Central onboarding accepts that provider-neutral surface without a Docker or Podman selection branch. Tests register an MXC-style surface through the same -bundle and render held launches for OpenClaw, Hermes, and DCode. +bundle, render held launches for OpenClaw, Hermes, and DCode, and exercise +recovery phases across those three agents. The current image definitions still do not package `nemoclaw-managed-startup-hold`, `managed-startup-image-runtime.cjs`, or the diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index 555c82c6d3e..354a87a830e 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -168,6 +168,8 @@ describe("Docker managed bootstrap journal", () => { expect(first.listUnfinished()).toEqual([journal]); first.recordFinalization(finalization); + expect(first.listUnfinished()).toEqual([journal]); + first.remove(IDENTITY, ["staged"]); expect(first.listUnfinished()).toEqual([]); const restarted = createFileDockerManagedBootstrapJournalStore(root); expect(restarted.loadFinalization(IDENTITY)).toEqual(finalization); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index 10608935d4d..2eb77bc9035 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -722,14 +722,11 @@ export function createFileDockerManagedBootstrapJournalStore( fail(`journal directory contains an unsupported entry: ${name}`); } return Object.freeze( - identities - .sort() - .filter((identity) => loadFinalization(identity) === null) - .map((identity) => { - const journal = load(identity); - if (!journal) fail(`enumerated journal ${identity} disappeared`); - return journal; - }), + identities.sort().map((identity) => { + const journal = load(identity); + if (!journal) fail(`enumerated journal ${identity} disappeared`); + return journal; + }), ); }, transition( diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 38c06911c34..a24882b4c64 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -152,6 +152,7 @@ type FixtureOptions = { failAfterRollbackFence?: boolean; failAfterSharedCommitFence?: boolean; failAfterStagedFence?: boolean; + failJournalRemoveOnce?: boolean; failRemoveOnce?: boolean; failStart?: boolean; lostAcks?: boolean; @@ -173,6 +174,7 @@ function fixture(options: FixtureOptions = {}) { const lostTransitions = new Set(["cutover", "shared-state-committed"]); let loseCreateAck = options.lostAcks === true; let loseRemoveAck = options.lostAcks === true; + let failJournalRemoveOnce = options.failJournalRemoveOnce === true; let failRemoveOnce = options.failRemoveOnce === true; const ok = (stdout = "") => ({ status: 0, stdout, stderr: "" }); const copyJournal = () => (journal ? structuredClone(journal) : null); @@ -180,11 +182,10 @@ function fixture(options: FixtureOptions = {}) { create(value) { journal = structuredClone(value); events.push("journal:staged"); - if (options.failAfterStagedFence === true) { - throw new Error("injected crash after durable staged fence"); - } - switch (loseCreateAck) { - case true: + switch (true) { + case options.failAfterStagedFence === true: + throw new Error("injected crash after durable staged fence"); + case loseCreateAck: loseCreateAck = false; throw new DockerManagedBootstrapJournalAcknowledgementLostError( "lost journal create acknowledgement", @@ -192,7 +193,7 @@ function fixture(options: FixtureOptions = {}) { } }, load: () => copyJournal(), - listUnfinished: () => (journal && !finalization ? [structuredClone(journal)] : []), + listUnfinished: () => (journal ? [structuredClone(journal)] : []), transition(_identity, expected, next) { const current = journal !== null && journal.phase === expected @@ -234,6 +235,11 @@ function fixture(options: FixtureOptions = {}) { void (current !== null && expected.includes(current.phase) ? current : failFixture("stale journal remove")); + switch (true) { + case failJournalRemoveOnce: + failJournalRemoveOnce = false; + throw new Error("injected crash before terminal journal removal"); + } journal = null; events.push("journal:removed"); switch (loseRemoveAck) { @@ -420,9 +426,10 @@ function fixture(options: FixtureOptions = {}) { }), dockerRm: vi.fn((id) => { events.push(`rm:${id}`); - if (failRemoveOnce) { - failRemoveOnce = false; - throw new Error("injected crash before exact Docker removal"); + switch (true) { + case failRemoveOnce: + failRemoveOnce = false; + throw new Error("injected crash before exact Docker removal"); } switch (id) { case OLD_ID: @@ -527,6 +534,9 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( fake.events.indexOf(`rm:${OLD_ID}`), ); + expect(fake.events.indexOf("finalization:committed")).toBeLessThan( + fake.events.indexOf("journal:removed"), + ); expect(fake.journal).toBeNull(); expect(fake.finalization).toMatchObject({ phase: "committed", commitReceipt }); expect(fake.sharedState).toBe("none"); @@ -583,6 +593,9 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( fake.events.indexOf(`rm:${NEW_ID}`), ); + expect(fake.events.indexOf("finalization:rolled-back")).toBeLessThan( + fake.events.indexOf("journal:removed"), + ); expect(fake.journal).toBeNull(); expect(fake.replacement).toBeNull(); expect(fake.original.Name).toBe("/openshell-alpha"); @@ -679,6 +692,7 @@ describe("Docker managed bootstrap adapter", () => { it("finishes exact commit cleanup after a process restart at the durable commit fence", async () => { const fake = fixture({ agent: "langchain-deepagents-code", + failJournalRemoveOnce: true, failRemoveOnce: true, sharedState: "pending", }); @@ -718,7 +732,14 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.sharedState).toBe("committed"); const restarted = createDockerManagedBootstrapAdapter(fake.deps); - await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject([ + await expect(restarted.recoverUnfinishedTransactions()).rejects.toThrow( + "crash before terminal journal removal", + ); + expect(fake.journal?.phase).toBe("shared-state-committed"); + expect(fake.finalization?.phase).toBe("committed"); + + const resumed = createDockerManagedBootstrapAdapter(fake.deps); + await expect(resumed.recoverUnfinishedTransactions()).resolves.toMatchObject([ { sourcePhase: "shared-state-committed", outcome: "committed" }, ]); expect(fake.journal).toBeNull(); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index 977b016405a..3202b6907e1 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -1840,6 +1840,21 @@ export function createDockerManagedBootstrapAdapter( } satisfies ManagedBootstrapFinalizationReceipt); return persistFinalization(handle, "rolled-back", null, receipt); }; + const completeRollbackTransaction = ( + handle: ManagedBootstrapHeldWorkloadHandle, + journal: DockerBootstrapTransaction, + ): ManagedBootstrapFinalizationReceipt => { + let ownerCleanupFailure: { readonly error: unknown } | null = null; + try { + removeOwnedWorkload(handle.sandbox, deps, journal.originalRuntimeId); + } catch (error) { + ownerCleanupFailure = { error }; + } + const finalization = completedRollback(handle, false); + removeDockerBootstrapJournalDurably(journal, deps); + if (ownerCleanupFailure) throw ownerCleanupFailure.error; + return finalization; + }; const completedCommit = ( handle: ManagedBootstrapHeldWorkloadHandle, commitReceipt: ManagedBootstrapCompletionReceipt, @@ -1925,6 +1940,44 @@ export function createDockerManagedBootstrapAdapter( outcome: finalization.outcome, finalization, }); + const compactRecoveredFinalization = ( + journal: DockerBootstrapTransaction, + sourcePhase: DockerBootstrapTransaction["phase"], + ): ManagedBootstrapRecoveryReceipt | null => { + const finalization = deps.journalStore.loadFinalization(journal.bootstrapIdentity); + if (!finalization) return null; + const phaseMatches = + (finalization.phase === "committed" && + journal.phase === "shared-state-committed" && + finalization.commitReceipt !== null && + JSON.stringify(finalization.commitReceipt) === JSON.stringify(journal.commitReceipt)) || + (finalization.phase === "rolled-back" && + (journal.phase === "staged" || journal.phase === "rollback-authorized") && + finalization.commitReceipt === null); + if ( + !phaseMatches || + finalization.bootstrapIdentity !== journal.bootstrapIdentity || + finalization.providerId !== journal.providerId || + finalization.agent !== journal.agent || + finalization.sandbox.sandboxName !== journal.sandbox.sandboxName || + finalization.sandbox.sandboxId !== journal.sandbox.sandboxId || + finalization.sandbox.driverId !== journal.sandbox.driverId || + finalization.planFingerprint !== journal.planFingerprint || + finalization.profileFingerprint !== journal.profileFingerprint || + finalization.imageReference !== journal.imageReference + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: + journal.phase === "shared-state-committed" + ? journal.replacementRuntimeId + : journal.originalRuntimeId, + detail: "terminal finalization does not match its retained durable journal", + }); + } + removeDockerBootstrapJournalDurably(journal, deps); + return recoveredReceipt(journal, sourcePhase, finalization.cleanupReceipt); + }; const finishRecoveredRollback = ( journal: DockerBootstrapTransaction, sourcePhase: DockerBootstrapTransaction["phase"], @@ -2314,6 +2367,7 @@ export function createDockerManagedBootstrapAdapter( ); if (journal.phase === "staged") { + const stagedJournal: DockerBootstrapTransaction = journal; assertStableRunning(original, "staged original"); if (observedReplacement) { assertExplicitlyStopped(observedReplacement, "staged replacement"); @@ -2332,9 +2386,7 @@ export function createDockerManagedBootstrapAdapter( if (observedReplacement) { removeExactReplacement(journal, observedReplacement, deps); } - removeDockerBootstrapJournalDurably(journal, deps); - removeOwnedWorkload(handle.sandbox, deps, journal.originalRuntimeId); - return completedRollback(handle, false); + return completeRollbackTransaction(handle, stagedJournal); } if (journal.phase !== "cutover" && journal.phase !== "rollback-authorized") { @@ -2512,9 +2564,7 @@ export function createDockerManagedBootstrapAdapter( ) { throw new Error("Managed bootstrap Docker rollback did not restore its exact original."); } - removeDockerBootstrapJournalDurably(activeJournal, deps); - removeOwnedWorkload(handle.sandbox, deps, activeJournal.originalRuntimeId); - return completedRollback(handle, false); + return completeRollbackTransaction(handle, activeJournal); }; const commitBootstrapNow = ( handle: ManagedBootstrapHeldWorkloadHandle, @@ -2615,8 +2665,9 @@ export function createDockerManagedBootstrapAdapter( }); } } + const finalization = completedCommit(handle, receipt); removeDockerBootstrapJournalDurably(transaction, deps); - return completedCommit(handle, receipt); + return finalization; }; const finalizeBootstrap = async ( input: Parameters[0], @@ -2809,10 +2860,12 @@ export function createDockerManagedBootstrapAdapter( const receipts: ManagedBootstrapRecoveryReceipt[] = []; for (const journal of deps.journalStore.listUnfinished()) { const sourcePhase = journal.phase; + const finalized = compactRecoveredFinalization(journal, sourcePhase); receipts.push( - journal.phase === "shared-state-committed" - ? finishRecoveredCommit(journal, sourcePhase) - : finishRecoveredRollbackPhase(journal, sourcePhase), + finalized ?? + (journal.phase === "shared-state-committed" + ? finishRecoveredCommit(journal, sourcePhase) + : finishRecoveredRollbackPhase(journal, sourcePhase)), ); } return Object.freeze(receipts); From 310ebebc1585aa1d0a5c9906f6a70b181fe4c580 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 31 Jul 2026 23:50:10 -0700 Subject: [PATCH 082/117] test(onboard): fence create on restart recovery Signed-off-by: Aaron Erickson --- .../onboard/sandbox-gpu-create-flow.test.ts | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index 87979a668c7..a7362d037ce 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -166,7 +166,7 @@ beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); describe("runSandboxGpuCreateFlow provider-owned managed create", () => { - it("runs an MXC-style bundle without a Docker branch in central orchestration", async () => { + it("recovers before an MXC-style create without a Docker branch in central orchestration", async () => { const input = createInput(); input.sandboxGpuConfig = { mode: "0", @@ -201,12 +201,14 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { input.sandboxEnv = launch.sandboxEnv; input.sandboxStartupCommand = launch.sandboxStartupCommand; const patch = createPatch() as unknown as ManagedBootstrapRuntimePatch; + const recoverUnfinished = vi.fn(async () => []); + const prepareNetwork = vi.fn(async () => undefined); const createLifecycle = vi.fn( (lifecycleInput: ManagedBootstrapRuntimeCreateLifecycleInput) => ({ launchArgv: ["mxc-launch", ...lifecycleInput.launchArgv.slice(1)], patch, - recoverUnfinished: vi.fn(async () => []), - prepareNetwork: vi.fn(async () => undefined), + recoverUnfinished, + prepareNetwork, runCreate: async ( start: (held: { readonly heldWorkloadArgv: readonly string[]; @@ -285,6 +287,15 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { vi.mocked(deps.runCaptureOpenshell).mockImplementation((args) => args[1] === "get" ? "ID: mxc-alpha\n" : "alpha Ready", ); + recoverUnfinished.mockRejectedValueOnce(new Error("unfinished recovery failed")); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "unfinished recovery failed", + ); + expect(prepareNetwork).not.toHaveBeenCalled(); + expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); + recoverUnfinished.mockClear(); + createLifecycle.mockClear(); const result = await runSandboxGpuCreateFlow(input, deps); @@ -298,6 +309,12 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { input.sandboxEnv, expect.anything(), ); + expect(recoverUnfinished.mock.invocationCallOrder[0]).toBeLessThan( + prepareNetwork.mock.invocationCallOrder[0], + ); + expect(prepareNetwork.mock.invocationCallOrder[0]).toBeLessThan( + mocks.streamSandboxCreate.mock.invocationCallOrder[0], + ); expect(mocks.createDockerGpuSandboxCreatePatch).not.toHaveBeenCalled(); expect(mocks.queryOpenShellDockerSandboxContainers).not.toHaveBeenCalled(); expect(mocks.queryOpenShellDockerSandboxRuntimeSnapshot).not.toHaveBeenCalled(); From 0013772e013f00173373fa3c137f3f3a7a4988f6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 1 Aug 2026 00:01:12 -0700 Subject: [PATCH 083/117] test(messaging): keep staged channel fixture canonical Signed-off-by: Aaron Erickson --- src/lib/onboard/channel-state.test.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/lib/onboard/channel-state.test.ts b/src/lib/onboard/channel-state.test.ts index f32e6efcb30..aaffa4f373f 100644 --- a/src/lib/onboard/channel-state.test.ts +++ b/src/lib/onboard/channel-state.test.ts @@ -18,7 +18,17 @@ function sessionWithPlan( sandboxName, agent: "openclaw", workflow: "onboard", - channels: [], + channels: disabledChannels.map((channelId) => ({ + channelId, + displayName: channelId, + authMode: "none", + active: false, + selected: false, + configured: false, + disabled: true, + inputs: [], + hooks: [], + })), disabledChannels, credentialBindings: [], networkPolicy: { presets: [], entries: [] }, From ca85ca7840122ad0543bbc86d7feead5e8445b3d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 1 Aug 2026 06:04:05 -0700 Subject: [PATCH 084/117] fix(hermes): preserve broker write ownership Signed-off-by: Aaron Erickson --- agents/hermes/host/runtime-refresh-credentials.ts | 12 +++++++----- test/hermes-tool-gateway-broker.test.ts | 1 + test/hermes-tool-gateway-runtime-credentials.test.ts | 5 +++++ .../openshell-policy-boundary.test.ts | 1 + 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/agents/hermes/host/runtime-refresh-credentials.ts b/agents/hermes/host/runtime-refresh-credentials.ts index 34e4f38d4c6..a1a4908ecd9 100644 --- a/agents/hermes/host/runtime-refresh-credentials.ts +++ b/agents/hermes/host/runtime-refresh-credentials.ts @@ -21,20 +21,22 @@ class RuntimeRefreshCredentialStore { const normalized = String(refreshToken || "").trim(); if (!sandbox || !expectedHash || !normalized) return false; if (this.hashCredential(normalized) !== expectedHash) return false; - this.credentials.set(sandbox, normalized); + // Keep a distinct entry for every successful write. Rollback callbacks + // compare the entry identity so a later write of the same token still wins. + this.credentials.set(sandbox, { refreshToken: normalized }); return true; } resolve(state) { const sandbox = String(state?.sandbox || "").trim(); const expectedHash = String(state?.refresh_token_sha256 || "").trim(); - const refreshToken = this.credentials.get(sandbox); - if (!sandbox || !expectedHash || !refreshToken) return null; - if (this.hashCredential(refreshToken) !== expectedHash) { + const entry = this.credentials.get(sandbox); + if (!sandbox || !expectedHash || !entry) return null; + if (this.hashCredential(entry.refreshToken) !== expectedHash) { this.credentials.delete(sandbox); return null; } - return refreshToken; + return entry.refreshToken; } rotate(state, nextRefreshToken) { diff --git a/test/hermes-tool-gateway-broker.test.ts b/test/hermes-tool-gateway-broker.test.ts index 30246d1d210..bd69cda433c 100644 --- a/test/hermes-tool-gateway-broker.test.ts +++ b/test/hermes-tool-gateway-broker.test.ts @@ -920,6 +920,7 @@ describe("Hermes managed-tool gateway broker", () => { const openIncompleteControlRequest = async (): Promise => { const socket = net.createConnection(controlSocket); + socket.on("error", () => {}); await once(socket, "connect"); socket.write( [ diff --git a/test/hermes-tool-gateway-runtime-credentials.test.ts b/test/hermes-tool-gateway-runtime-credentials.test.ts index 49c17a8a24a..b46d25b0966 100644 --- a/test/hermes-tool-gateway-runtime-credentials.test.ts +++ b/test/hermes-tool-gateway-runtime-credentials.test.ts @@ -89,6 +89,11 @@ describe("Hermes tool-gateway runtime credentials", () => { expect(restoreWithoutClobber?.()).toBe(false); expect(store.resolve(concurrentState)).toBe(concurrentToken); + const restoreSameToken = store.replace(nextState, nextToken); + expect(store.register(nextState, nextToken)).toBe(true); + expect(restoreSameToken?.()).toBe(false); + expect(store.resolve(nextState)).toBe(nextToken); + const restoreAbsent = store.replace( { sandbox: "new-clone", refresh_token_sha256: sha256(nextToken) }, nextToken, diff --git a/test/package-contract/openshell-policy-boundary.test.ts b/test/package-contract/openshell-policy-boundary.test.ts index cde1952677c..edb6ea14a9d 100644 --- a/test/package-contract/openshell-policy-boundary.test.ts +++ b/test/package-contract/openshell-policy-boundary.test.ts @@ -195,6 +195,7 @@ describe("OpenShell policy boundary package contract", () => { isValidName: (value: unknown) => boolean; }; expect(controlContract.isValidName("packaged-hermes-sandbox")).toBe(true); + expect(controlContract.isValidName("../packaged-hermes-sandbox")).toBe(false); }); it("ships an out-of-tree runtime sandbox-policy schema validator", { timeout: 90_000 }, () => { From f2a580695bc5e865ff6f02d1bc227ec1dc1a4e7c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 1 Aug 2026 06:13:29 -0700 Subject: [PATCH 085/117] test(onboard): linearize transaction recovery cases Signed-off-by: Aaron Erickson --- ...d-startup-shared-state-transaction.test.ts | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts index 3358a070eed..b913d21983e 100644 --- a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts +++ b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts @@ -387,12 +387,12 @@ describe("managed startup shared-state transaction", () => { const rm = vi.spyOn(fs, "rmSync").mockImplementation((( target: fs.PathLike, removeOptions?: fs.RmDirOptions, - ) => { - if (String(target).endsWith(`${path.sep}backups`)) { - throw new Error("injected post-rename cleanup interruption"); - } - return originalRmSync(target, removeOptions); - }) as typeof fs.rmSync); + ) => + String(target).endsWith(`${path.sep}backups`) + ? (() => { + throw new Error("injected post-rename cleanup interruption"); + })() + : originalRmSync(target, removeOptions)) as typeof fs.rmSync); expect(() => commitManagedStartupSharedStateTransaction("openclaw", boundOptions)).toThrow( /injected post-rename cleanup interruption/u, ); @@ -402,20 +402,22 @@ describe("managed startup shared-state transaction", () => { const committedDirectory = commitReceiptDirectory(); const backups = path.join(committedDirectory, "backups"); const manifest = path.join(committedDirectory, "manifest.json"); - if (interruption === "during-compact-receipt-write") { - fs.renameSync( - path.join(committedDirectory, "receipt.json"), - path.join(committedDirectory, ".receipt.json.1234567890abcdef12345678"), - ); - } else if (interruption === "during-backup-removal") { - const [firstBackup] = fs.readdirSync(backups); - expect(firstBackup).toBeTruthy(); - fs.unlinkSync(path.join(backups, firstBackup!)); - } else if (interruption === "after-backup-removal") { - originalRmSync(backups, { force: false, recursive: true }); - } else if (interruption === "after-manifest-removal") { - fs.unlinkSync(manifest); - } + const applyInterruption: Record void> = { + "during-compact-receipt-write": () => + fs.renameSync( + path.join(committedDirectory, "receipt.json"), + path.join(committedDirectory, ".receipt.json.1234567890abcdef12345678"), + ), + "before-backup-removal": () => undefined, + "during-backup-removal": () => { + const [firstBackup] = fs.readdirSync(backups); + expect(firstBackup).toBeTruthy(); + fs.unlinkSync(path.join(backups, firstBackup!)); + }, + "after-backup-removal": () => originalRmSync(backups, { force: false, recursive: true }), + "after-manifest-removal": () => fs.unlinkSync(manifest), + }; + applyInterruption[interruption](); expect( getManagedStartupSharedStateTransactionStatus( { @@ -464,6 +466,10 @@ describe("managed startup shared-state transaction", () => { recursive: true, preserveTimestamps: true, }); + // Node 22.23 normalizes copied directory modes to 0755. Recreate the + // protected modes that the container-copy fixture is intended to model. + fs.chmodSync(copiedReceipt, 0o700); + fs.chmodSync(path.join(copiedReceipt, "backups"), 0o700); expect( getManagedStartupSharedStateTransactionStatus( From 5aaf98810c28cf9dee7824a061a76bad53f96634 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 1 Aug 2026 06:45:49 -0700 Subject: [PATCH 086/117] chore(stack): refresh E2E qualification Signed-off-by: Aaron Erickson From 7097017487a780ca5309e41a7de4d993a6449d14 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 06:56:56 -0700 Subject: [PATCH 087/117] test(hermes): assert clone cleanup writes no state Signed-off-by: Aaron Erickson --- test/hermes-tool-gateway-broker.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/hermes-tool-gateway-broker.test.ts b/test/hermes-tool-gateway-broker.test.ts index bd69cda433c..c762731829f 100644 --- a/test/hermes-tool-gateway-broker.test.ts +++ b/test/hermes-tool-gateway-broker.test.ts @@ -311,6 +311,7 @@ describe("Hermes managed-tool gateway broker", () => { ); expect(removeState).not.toHaveBeenCalled(); + const cleanupWriteState = vi.fn(); expect(() => broker.activateHermesToolGatewayCloneBinding( "new-destination", @@ -324,9 +325,11 @@ describe("Hermes managed-tool gateway broker", () => { }), controlRequest: () => ({ state: "discarded" }), removeState: () => false, + writeState: cleanupWriteState, }, ), ).toThrow("activation cleanup failed"); + expect(cleanupWriteState).not.toHaveBeenCalled(); }); it("removes broker state only for the exact registry identity", () => { From 8628b8a1bd7d1957e983e45e342a3db4027685fa Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 07:37:20 -0700 Subject: [PATCH 088/117] fix(hermes): harden broker control recovery Signed-off-by: Aaron Erickson --- src/lib/hermes-tool-gateway-broker.ts | 87 +++++++++++++++---- test/hermes-tool-gateway-broker.test.ts | 106 ++++++++++++++++++++++++ 2 files changed, 175 insertions(+), 18 deletions(-) diff --git a/src/lib/hermes-tool-gateway-broker.ts b/src/lib/hermes-tool-gateway-broker.ts index 0cad0463661..2dc48ba47a2 100644 --- a/src/lib/hermes-tool-gateway-broker.ts +++ b/src/lib/hermes-tool-gateway-broker.ts @@ -61,6 +61,62 @@ const HERMES_TOOL_GATEWAY_CONTROL_CONTRACT_PATH = path.join( "host", "tool-gateway-control-contract.ts", ); +const HERMES_TOOL_GATEWAY_RUNTIME_MISMATCH_RECOVERY = + "Reauthorize every managed-tool Hermes sandbox, then retry."; +const HERMES_TOOL_GATEWAY_CONTROL_CLIENT_SOURCE = [ + 'const http = require("node:http");', + "const [socketPath, route, timeoutValue] = process.argv.slice(1);", + "const timeoutMs = Number(timeoutValue);", + 'let requestBody = "";', + "let failed = false;", + "const fail = () => { failed = true; process.exitCode = 1; };", + 'process.stdin.setEncoding("utf8");', + 'process.stdin.on("data", (chunk) => {', + " requestBody += chunk;", + " if (Buffer.byteLength(requestBody) > 1024 * 1024) {", + " fail();", + " process.stdin.destroy();", + " }", + "});", + 'process.stdin.once("error", fail);', + 'process.stdin.once("end", () => {', + " if (failed) return;", + " const request = http.request(", + " {", + " socketPath,", + " path: `/${route}`,", + ' method: "POST",', + " headers: {", + ' "content-type": "application/json",', + ' "content-length": Buffer.byteLength(requestBody),', + " },", + " },", + " (response) => {", + " if ((response.statusCode ?? 500) < 200 || (response.statusCode ?? 500) >= 300) {", + " response.resume();", + " fail();", + " return;", + " }", + ' response.setEncoding("utf8");', + ' let responseBody = "";', + ' response.on("data", (chunk) => {', + " responseBody += chunk;", + " if (Buffer.byteLength(responseBody) > 1024 * 1024) {", + " fail();", + " response.destroy();", + " }", + " });", + ' response.once("error", fail);', + ' response.once("end", () => {', + " if (!failed) process.stdout.write(responseBody);", + " });", + " },", + " );", + ' request.setTimeout(timeoutMs, () => request.destroy(new Error("timeout")));', + ' request.once("error", fail);', + " request.end(requestBody);", + "});", +].join("\n"); let brokerStartedThisRun = false; @@ -178,32 +234,27 @@ function persistHermesToolGatewayProviderState( function brokerControlJsonRequest(route, payload, options = {}) { if (!fs.existsSync(HERMES_TOOL_GATEWAY_CONTROL_SOCKET_PATH)) return null; + if (!/^credentials\/(?:activate|discard|register|stage|status|unregister)$/u.test(route)) { + return null; + } const timeoutMs = options.timeoutMs ?? HERMES_CLONE_CONTROL_CLIENT_TIMEOUT_MS; const result = spawnSync( - "curl", + process.execPath, [ - "--silent", - "--show-error", - "--fail", - "--unix-socket", + "--input-type=commonjs", + "--eval", + HERMES_TOOL_GATEWAY_CONTROL_CLIENT_SOURCE, HERMES_TOOL_GATEWAY_CONTROL_SOCKET_PATH, - "--connect-timeout", - "3", - "--max-time", - (timeoutMs / 1000).toFixed(3), - "--request", - "POST", - "--header", - "Content-Type: application/json", - "--data-binary", - "@-", - `http://localhost/${route}`, + route, + String(timeoutMs), ], { encoding: "utf8", input: JSON.stringify(payload), + maxBuffer: 1024 * 1024, stdio: ["pipe", "pipe", "ignore"], timeout: timeoutMs + 1_000, + windowsHide: true, }, ); if (result.status !== 0) return null; @@ -516,7 +567,7 @@ function preflightHermesToolGatewayCloneBinding(sandboxName) { if (readBrokerHash() !== brokerRuntimeHash()) { throw new Error( "Hermes managed-tool broker runtime changed while an existing broker is active; " + - "reauthorize every managed-tool Hermes sandbox before retrying", + HERMES_TOOL_GATEWAY_RUNTIME_MISMATCH_RECOVERY, ); } if (!fs.existsSync(HERMES_TOOL_GATEWAY_CONTROL_SOCKET_PATH)) { @@ -719,7 +770,7 @@ function ensureHermesToolGatewayBroker(options = {}) { console.error( "Hermes managed-tool broker runtime changed while an existing broker is active; " + "refusing to restart it and discard other in-memory sandbox credentials. " + - "Reauthorize every managed-tool Hermes sandbox using the documented broker recovery flow.", + HERMES_TOOL_GATEWAY_RUNTIME_MISMATCH_RECOVERY, ); return false; } diff --git a/test/hermes-tool-gateway-broker.test.ts b/test/hermes-tool-gateway-broker.test.ts index c762731829f..c2e6d579191 100644 --- a/test/hermes-tool-gateway-broker.test.ts +++ b/test/hermes-tool-gateway-broker.test.ts @@ -270,6 +270,112 @@ describe("Hermes managed-tool gateway broker", () => { } }); + it("uses the current Node runtime for private control requests without a curl dependency", { + timeout: BROKER_TEST_TIMEOUT_MS, + }, async ({ resources }) => { + const previousHome = process.env.HOME; + const previousPath = process.env.PATH; + const home = resources.ownDirectory(fs.mkdtempSync("/tmp/nc-hermes-node-control-")); + let server: ChildProcess | undefined; + try { + process.env.HOME = home; + delete require.cache[require.resolve(BROKER_WRAPPER)]; + const broker = require(BROKER_WRAPPER); + broker.persistHermesToolGatewayProviderState( + "sandbox", + "test-only-refresh", + "test-only-broker", + "sandbox-hermes-inference", + ); + const capturePath = path.join(home, "control-request.json"); + const serverSource = [ + 'const fs = require("node:fs");', + 'const http = require("node:http");', + "const [socketPath, capturePath] = process.argv.slice(1);", + "const server = http.createServer((request, response) => {", + " const chunks = [];", + ' request.on("data", (chunk) => chunks.push(chunk));', + ' request.on("end", () => {', + ' const body = Buffer.concat(chunks).toString("utf8");', + " fs.writeFileSync(capturePath, JSON.stringify({", + " path: request.url,", + " body,", + " }));", + ' response.writeHead(body.includes("reject-refresh") ? 503 : 200, {', + ' "content-type": "application/json",', + " });", + ' response.end("{\\\"registered\\\":true}");', + " });", + "});", + "server.listen(socketPath, () => {", + " fs.chmodSync(socketPath, 0o600);", + ' process.stdout.write("ready\\n");', + "});", + 'process.once("SIGTERM", () => server.close(() => process.exit(0)));', + ].join("\n"); + server = resources.ownChild( + spawn( + process.execPath, + [ + "--input-type=commonjs", + "--eval", + serverSource, + broker.HERMES_TOOL_GATEWAY_CONTROL_SOCKET_PATH, + capturePath, + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ), + ); + let output = ""; + server.stdout?.on("data", (chunk) => { + output += chunk.toString(); + }); + server.stderr?.on("data", (chunk) => { + output += chunk.toString(); + }); + await waitForBrokerCondition( + "native Node control server", + server, + () => output, + () => output.includes("ready"), + ); + + process.env.PATH = "/path-with-no-curl"; + expect( + broker.registerHermesToolGatewayRuntimeCredential("test-only-refresh", "sandbox"), + ).toBe(true); + expect(JSON.parse(fs.readFileSync(capturePath, "utf8"))).toEqual({ + path: "/credentials/register", + body: JSON.stringify({ + sandbox: "sandbox", + refresh_token: "test-only-refresh", + }), + }); + broker.persistHermesToolGatewayProviderState( + "sandbox", + "reject-refresh", + "test-only-broker", + "sandbox-hermes-inference", + ); + expect(broker.registerHermesToolGatewayRuntimeCredential("reject-refresh", "sandbox")).toBe( + false, + ); + } finally { + if (server && server.exitCode === null && server.signalCode === null) { + const exited = once(server, "exit"); + server.kill("SIGTERM"); + await exited; + } + previousHome === undefined + ? Reflect.deleteProperty(process.env, "HOME") + : Reflect.set(process.env, "HOME", previousHome); + previousPath === undefined + ? Reflect.deleteProperty(process.env, "PATH") + : Reflect.set(process.env, "PATH", previousPath); + delete require.cache[require.resolve(BROKER_WRAPPER)]; + } + }); + it("restores a prior destination broker binding and reports cleanup failure", () => { delete require.cache[require.resolve(BROKER_WRAPPER)]; const broker = require(BROKER_WRAPPER); From d7cb7f76cceb69ec2787572e06fd65889b432eb6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 07:51:53 -0700 Subject: [PATCH 089/117] test(hermes): rely on owned child cleanup Signed-off-by: Aaron Erickson --- test/hermes-tool-gateway-broker.test.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/hermes-tool-gateway-broker.test.ts b/test/hermes-tool-gateway-broker.test.ts index c2e6d579191..0c182ebb937 100644 --- a/test/hermes-tool-gateway-broker.test.ts +++ b/test/hermes-tool-gateway-broker.test.ts @@ -276,7 +276,6 @@ describe("Hermes managed-tool gateway broker", () => { const previousHome = process.env.HOME; const previousPath = process.env.PATH; const home = resources.ownDirectory(fs.mkdtempSync("/tmp/nc-hermes-node-control-")); - let server: ChildProcess | undefined; try { process.env.HOME = home; delete require.cache[require.resolve(BROKER_WRAPPER)]; @@ -313,7 +312,7 @@ describe("Hermes managed-tool gateway broker", () => { "});", 'process.once("SIGTERM", () => server.close(() => process.exit(0)));', ].join("\n"); - server = resources.ownChild( + const server = resources.ownChild( spawn( process.execPath, [ @@ -361,11 +360,6 @@ describe("Hermes managed-tool gateway broker", () => { false, ); } finally { - if (server && server.exitCode === null && server.signalCode === null) { - const exited = once(server, "exit"); - server.kill("SIGTERM"); - await exited; - } previousHome === undefined ? Reflect.deleteProperty(process.env, "HOME") : Reflect.set(process.env, "HOME", previousHome); From c05a422f246d800b66af28afc48e724f0b7cab3e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 05:24:20 -0700 Subject: [PATCH 090/117] feat(onboard): add Docker bootstrap transaction primitives Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/README.md | 5 + .../managed-bootstrap/docker-journal.test.ts | 175 +++++++ .../managed-bootstrap/docker-journal.ts | 448 ++++++++++++++++++ .../managed-bootstrap/docker-spec.test.ts | 84 ++++ .../onboard/managed-bootstrap/docker-spec.ts | 316 ++++++++++++ test/runtime-provider-source-shape.test.ts | 2 + 6 files changed, 1030 insertions(+) create mode 100644 src/lib/onboard/managed-bootstrap/docker-journal.test.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-journal.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-spec.test.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-spec.ts diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 32213dd1249..adde636fa56 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -66,6 +66,11 @@ including its supervisor environment, to immutable prepared authority before activation. The native boundary introduces no driver-specific environment policy. +The first Docker-specific groundwork defines a private, monotonic cutover +journal and a canonical launch-spec normalizer. Each surface is independently +validated and remains dormant: no registered runtime provider imports either +module, and neither changes sandbox creation or lifecycle behavior. + ## Architectural disposition The coordinator deliberately lands as a dormant trust-boundary slice before a diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts new file mode 100644 index 00000000000..7ed0d8bba6a --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -0,0 +1,175 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapJournal, + parseDockerManagedBootstrapJournal, + serializeDockerManagedBootstrapJournal, +} from "./docker-journal"; + +const roots: string[] = []; +const IDENTITY = "1".repeat(64); +const journal = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + phase: "staged", + bootstrapIdentity: IDENTITY, + sandbox: { + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + driverId: "docker", + }, + profileFingerprint: "2".repeat(64), + imageReference: `registry.example/image@sha256:${"3".repeat(64)}`, + runtimeImageContentId: `sha256:${"4".repeat(64)}`, + originalRuntimeId: "5".repeat(64), + replacementRuntimeId: "6".repeat(64), + originalName: "openshell-alpha", + replacementStagingName: "openshell-alpha-staged", + backupName: "openshell-alpha-backup", + originalSpecHash: "7".repeat(64), + replacementSpecHash: "8".repeat(64), +} satisfies DockerManagedBootstrapJournal); + +function readPinnedPrivateFile(target: string): { readonly mode: number; readonly text: string } { + const descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + const text = fs.readFileSync(descriptor, "utf8"); + const after = fs.fstatSync(descriptor, { bigint: true }); + expect(after.dev).toBe(before.dev); + expect(after.ino).toBe(before.ino); + expect(after.size).toBe(before.size); + expect(after.mtimeNs).toBe(before.mtimeNs); + expect(after.ctimeNs).toBe(before.ctimeNs); + return { mode: Number(before.mode & 0o777n), text }; + } finally { + fs.closeSync(descriptor); + } +} + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +describe("Docker managed bootstrap journal", () => { + it("publishes private canonical state through only monotonic phases", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + const directory = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); + const file = path.join(directory, `${IDENTITY}.json`); + expect(fs.statSync(directory).mode & 0o777).toBe(0o700); + const persisted = readPinnedPrivateFile(file); + expect(persisted.mode).toBe(0o600); + expect(parseDockerManagedBootstrapJournal(persisted.text)).toEqual(journal); + expect(() => store.create(journal)).toThrow("already exists"); + expect(() => store.transition(IDENTITY, "staged", "shared-state-committed")).toThrow( + "unsupported", + ); + + expect(store.transition(IDENTITY, "staged", "cutover").phase).toBe("cutover"); + expect(store.transition(IDENTITY, "cutover", "shared-state-committed").phase).toBe( + "shared-state-committed", + ); + store.remove(IDENTITY, ["shared-state-committed"]); + expect(store.load(IDENTITY)).toBeNull(); + }); + + it("recovers one durable cutover decision before journal replacement", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + store.transition(IDENTITY, "staged", "cutover"); + const file = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, `${IDENTITY}.json`); + fs.writeFileSync(`${file}.decision`, "rollback-authorized\n", { mode: 0o600 }); + + expect(store.load(IDENTITY)?.phase).toBe("rollback-authorized"); + expect(parseDockerManagedBootstrapJournal(readPinnedPrivateFile(file).text).phase).toBe( + "rollback-authorized", + ); + fs.unlinkSync(`${file}.decision`); + expect(store.load(IDENTITY)?.phase).toBe("rollback-authorized"); + expect(() => store.transition(IDENTITY, "cutover", "shared-state-committed")).toThrow( + "expected phase cutover", + ); + store.remove(IDENTITY, ["rollback-authorized"]); + }); + + it("reconciles an exclusive decision collision by typed durable authority", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + store.transition(IDENTITY, "staged", "cutover"); + const target = path.join( + root, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, + `${IDENTITY}.json.decision`, + ); + const link = vi.spyOn(fs, "linkSync").mockImplementationOnce(() => { + fs.writeFileSync(target, "rollback-authorized\n", { flag: "wx", mode: 0o600 }); + throw Object.assign(new Error("exclusive decision collision"), { code: "EEXIST" }); + }); + try { + expect(store.transition(IDENTITY, "cutover", "rollback-authorized").phase).toBe( + "rollback-authorized", + ); + } finally { + link.mockRestore(); + } + }); + + it("preserves a primary journal write failure when temporary cleanup also fails", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + const rename = vi.spyOn(fs, "renameSync").mockImplementationOnce(() => { + throw new Error("primary journal rename failure"); + }); + const unlink = vi.spyOn(fs, "unlinkSync").mockImplementationOnce(() => { + throw new Error("temporary cleanup failure"); + }); + try { + expect(() => store.transition(IDENTITY, "staged", "cutover")).toThrow( + "primary journal rename failure", + ); + } finally { + rename.mockRestore(); + unlink.mockRestore(); + } + }); + + it.skipIf(process.platform === "win32")("refuses a symlink in place of journal authority", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + const file = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, `${IDENTITY}.json`); + const moved = `${file}.moved`; + fs.renameSync(file, moved); + fs.symlinkSync(moved, file); + + expect(() => store.load(IDENTITY)).toThrow("journal file ownership boundary is invalid"); + }); + + it("rejects non-canonical authority", () => { + expect(() => + parseDockerManagedBootstrapJournal(`${JSON.stringify({ ...journal, phase: "unknown" })}\n`), + ).toThrow("phase is unsupported"); + expect( + serializeDockerManagedBootstrapJournal(Object.freeze({ ...journal, phase: "staged" })), + ).toBe(`${JSON.stringify(journal)}\n`); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts new file mode 100644 index 00000000000..c6043409d7d --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -0,0 +1,448 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import type { ManagedBootstrapSandboxIdentity } from "./adapter"; + +export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 1 as const; +export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY = "managed-bootstrap"; + +const SHA256_RE = /^[a-f0-9]{64}$/u; +const MAX_JOURNAL_BYTES = 32 * 1024; +const JOURNAL_DIRECTORY_MODE = 0o700; +const JOURNAL_FILE_MODE = 0o600; +const DECISION_PHASES = new Set([ + "rollback-authorized", + "shared-state-committed", +]); + +export type DockerManagedBootstrapJournalPhase = + | "staged" + | "cutover" + | "rollback-authorized" + | "shared-state-committed"; + +export interface DockerManagedBootstrapJournal { + readonly schemaVersion: typeof DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION; + readonly phase: DockerManagedBootstrapJournalPhase; + readonly bootstrapIdentity: string; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly profileFingerprint: string; + readonly imageReference: string; + readonly runtimeImageContentId: string; + readonly originalRuntimeId: string; + readonly replacementRuntimeId: string; + readonly originalName: string; + readonly replacementStagingName: string; + readonly backupName: string; + readonly originalSpecHash: string; + readonly replacementSpecHash: string; +} + +export interface DockerManagedBootstrapJournalStore { + create(journal: DockerManagedBootstrapJournal): void; + load(bootstrapIdentity: string): DockerManagedBootstrapJournal | null; + transition( + bootstrapIdentity: string, + expected: DockerManagedBootstrapJournalPhase, + next: DockerManagedBootstrapJournalPhase, + ): DockerManagedBootstrapJournal; + remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]): void; +} + +/** + * Alternate stores may use this only when the durable mutation completed and + * the caller lost its acknowledgement. Ordinary I/O and fsync failures must + * retain their original error type and are never reconciled as success. + */ +export class DockerManagedBootstrapJournalAcknowledgementLostError extends Error { + constructor(message: string) { + super(message); + this.name = "DockerManagedBootstrapJournalAcknowledgementLostError"; + } +} + +class DockerManagedBootstrapJournalExistsError extends Error { + constructor() { + super( + "Managed bootstrap Docker journal is invalid: journal already exists for this bootstrap identity", + ); + this.name = "DockerManagedBootstrapJournalExistsError"; + } +} + +const ALLOWED_TRANSITIONS = new Set([ + "staged->cutover", + "cutover->rollback-authorized", + "cutover->shared-state-committed", +]); + +function fail(message: string): never { + throw new Error(`Managed bootstrap Docker journal is invalid: ${message}`); +} + +function exactString(value: unknown, label: string, maxBytes = 4096): string { + if ( + typeof value !== "string" || + value.length === 0 || + value !== value.trim() || + value.includes("\0") || + Buffer.byteLength(value, "utf8") > maxBytes + ) { + fail(`${label} must be one bounded exact string`); + } + return value; +} + +function exactSha256(value: unknown, label: string): string { + if (typeof value !== "string" || !SHA256_RE.test(value)) { + fail(`${label} must be lowercase SHA-256`); + } + return value; +} + +function exactPhase(value: unknown): DockerManagedBootstrapJournalPhase { + if ( + !["staged", "cutover", "rollback-authorized", "shared-state-committed"].includes(String(value)) + ) { + fail("phase is unsupported"); + } + return value as DockerManagedBootstrapJournalPhase; +} + +function exactSandbox(value: unknown): ManagedBootstrapSandboxIdentity { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("sandbox identity must be an object"); + } + const sandbox = value as Record; + if (Object.keys(sandbox).sort().join(",") !== "driverId,sandboxId,sandboxName") { + fail("sandbox identity schema is invalid"); + } + return Object.freeze({ + sandboxName: exactString(sandbox.sandboxName, "sandbox name"), + sandboxId: exactString(sandbox.sandboxId, "sandbox ID"), + driverId: exactString(sandbox.driverId, "driver ID"), + }); +} + +export function normalizeDockerManagedBootstrapJournal( + value: unknown, +): DockerManagedBootstrapJournal { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("journal must be an object"); + } + const journal = value as Record; + const expectedKeys = [ + "backupName", + "bootstrapIdentity", + "imageReference", + "originalName", + "originalRuntimeId", + "originalSpecHash", + "phase", + "profileFingerprint", + "replacementRuntimeId", + "replacementSpecHash", + "replacementStagingName", + "runtimeImageContentId", + "sandbox", + "schemaVersion", + ]; + if ( + Object.keys(journal).sort().join(",") !== expectedKeys.sort().join(",") || + journal.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION + ) { + fail("journal schema is invalid"); + } + const normalized = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + phase: exactPhase(journal.phase), + bootstrapIdentity: exactSha256(journal.bootstrapIdentity, "bootstrap identity"), + sandbox: exactSandbox(journal.sandbox), + profileFingerprint: exactSha256(journal.profileFingerprint, "profile fingerprint"), + imageReference: exactString(journal.imageReference, "image reference"), + runtimeImageContentId: exactString(journal.runtimeImageContentId, "runtime image content ID"), + originalRuntimeId: exactSha256(journal.originalRuntimeId, "original runtime ID"), + replacementRuntimeId: exactSha256(journal.replacementRuntimeId, "replacement runtime ID"), + originalName: exactString(journal.originalName, "original name", 253), + replacementStagingName: exactString( + journal.replacementStagingName, + "replacement staging name", + 253, + ), + backupName: exactString(journal.backupName, "backup name", 253), + originalSpecHash: exactSha256(journal.originalSpecHash, "original spec hash"), + replacementSpecHash: exactSha256(journal.replacementSpecHash, "replacement spec hash"), + } satisfies DockerManagedBootstrapJournal); + if (normalized.originalRuntimeId === normalized.replacementRuntimeId) { + fail("original and replacement runtime IDs must differ"); + } + if ( + new Set([normalized.originalName, normalized.replacementStagingName, normalized.backupName]) + .size !== 3 + ) { + fail("original, staging, and backup names must be distinct"); + } + return normalized; +} + +export function serializeDockerManagedBootstrapJournal( + journal: DockerManagedBootstrapJournal, +): string { + const normalized = normalizeDockerManagedBootstrapJournal(journal); + const serialized = `${JSON.stringify(normalized)}\n`; + if (Buffer.byteLength(serialized, "utf8") > MAX_JOURNAL_BYTES) { + fail("serialized journal exceeds its bounded transport"); + } + return serialized; +} + +export function parseDockerManagedBootstrapJournal(text: string): DockerManagedBootstrapJournal { + if ( + text.length === 0 || + text.includes("\0") || + Buffer.byteLength(text, "utf8") > MAX_JOURNAL_BYTES + ) { + fail("serialized journal is empty or too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("serialized journal is not valid JSON"); + } + const journal = normalizeDockerManagedBootstrapJournal(parsed); + if (serializeDockerManagedBootstrapJournal(journal) !== text) { + fail("serialized journal is not canonical"); + } + return journal; +} + +function assertDirectory(directory: string): void { + fs.mkdirSync(directory, { recursive: true, mode: JOURNAL_DIRECTORY_MODE }); + const stat = fs.lstatSync(directory); + if (!stat.isDirectory() || stat.isSymbolicLink() || (stat.mode & 0o077) !== 0) { + fail("journal directory must be a private real directory"); + } +} + +function journalPath(directory: string, bootstrapIdentity: string): string { + exactSha256(bootstrapIdentity, "bootstrap identity"); + return path.join(directory, `${bootstrapIdentity}.json`); +} + +function decisionPath(target: string): string { + return `${target}.decision`; +} + +function sameStableMetadata(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.mode === right.mode && + left.nlink === right.nlink && + left.uid === right.uid && + left.gid === right.gid && + left.size === right.size && + left.mtimeNs === right.mtimeNs && + left.ctimeNs === right.ctimeNs + ); +} + +function readPrivateFile(target: string, label: string): string | null { + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") { + fail(`cannot safely open ${label} because O_NOFOLLOW is unavailable`); + } + const nonblock = typeof fs.constants.O_NONBLOCK === "number" ? fs.constants.O_NONBLOCK : 0; + let descriptor: number; + try { + descriptor = fs.openSync(target, fs.constants.O_RDONLY | noFollow | nonblock); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + if ((error as NodeJS.ErrnoException).code === "ELOOP") { + fail(`${label} file ownership boundary is invalid`); + } + throw error; + } + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + (before.mode & 0o077n) !== 0n || + before.size <= 0n || + before.size > BigInt(MAX_JOURNAL_BYTES) + ) { + fail(`${label} file ownership boundary is invalid`); + } + const contents = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < contents.length) { + const count = fs.readSync(descriptor, contents, offset, contents.length - offset, offset); + if (count === 0) break; + offset += count; + } + const overflow = Buffer.alloc(1); + const overflowCount = fs.readSync(descriptor, overflow, 0, 1, offset); + const after = fs.fstatSync(descriptor, { bigint: true }); + if (offset !== contents.length || overflowCount !== 0 || !sameStableMetadata(before, after)) { + fail(`${label} file changed during its stable read`); + } + return contents.toString("utf8"); + } finally { + fs.closeSync(descriptor); + } +} + +function fsyncDirectory(directory: string): void { + const descriptor = fs.openSync(directory, "r"); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +function atomicWrite( + directory: string, + target: string, + contents: string, + exclusive: boolean, +): void { + const temporary = path.join( + directory, + `.${path.basename(target)}.${process.pid}.${Date.now().toString(16)}.tmp`, + ); + let descriptor: number | null = null; + let primaryFailure: { readonly error: unknown } | null = null; + try { + descriptor = fs.openSync(temporary, "wx", JOURNAL_FILE_MODE); + fs.writeFileSync(descriptor, contents, "utf8"); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = null; + if (exclusive) { + try { + fs.linkSync(temporary, target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + throw new DockerManagedBootstrapJournalExistsError(); + } + throw error; + } + fs.unlinkSync(temporary); + } else { + fs.renameSync(temporary, target); + } + fs.chmodSync(target, JOURNAL_FILE_MODE); + fsyncDirectory(directory); + } catch (error) { + primaryFailure = { error }; + } + let cleanupFailure: { readonly error: unknown } | null = null; + if (descriptor !== null) { + try { + fs.closeSync(descriptor); + } catch (error) { + cleanupFailure = { error }; + } + } + try { + fs.unlinkSync(temporary); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT" && cleanupFailure === null) { + cleanupFailure = { error }; + } + } + if (primaryFailure !== null) throw primaryFailure.error; + if (cleanupFailure !== null) throw cleanupFailure.error; +} + +export function createFileDockerManagedBootstrapJournalStore( + stateRoot: string, +): DockerManagedBootstrapJournalStore { + const directory = path.join(stateRoot, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); + const load = (bootstrapIdentity: string): DockerManagedBootstrapJournal | null => { + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const contents = readPrivateFile(target, "journal"); + if (contents === null) return null; + const journal = parseDockerManagedBootstrapJournal(contents); + const decision = readPrivateFile(decisionPath(target), "decision"); + if (decision === null) return journal; + const phase = decision.endsWith("\n") ? decision.slice(0, -1) : ""; + if ( + !DECISION_PHASES.has(phase as DockerManagedBootstrapJournalPhase) || + (journal.phase !== "cutover" && journal.phase !== phase) + ) { + fail("decision does not match its cutover journal"); + } + const decided = normalizeDockerManagedBootstrapJournal({ ...journal, phase }); + if (journal.phase === "cutover") { + atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(decided), false); + } + return decided; + }; + return Object.freeze({ + create(journal: DockerManagedBootstrapJournal) { + const normalized = normalizeDockerManagedBootstrapJournal(journal); + assertDirectory(directory); + const target = journalPath(directory, normalized.bootstrapIdentity); + if (readPrivateFile(decisionPath(target), "decision") !== null) { + fail("stale decision exists for this bootstrap identity"); + } + atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(normalized), true); + }, + load, + transition( + bootstrapIdentity: string, + expected: DockerManagedBootstrapJournalPhase, + next: DockerManagedBootstrapJournalPhase, + ) { + if (!ALLOWED_TRANSITIONS.has(`${expected}->${next}`)) { + fail(`transition ${expected} to ${next} is unsupported`); + } + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const current = load(bootstrapIdentity); + if (current?.phase === next) return current; + if (!current || current.phase !== expected) { + fail(`expected phase ${expected} before transition to ${next}`); + } + const updated = normalizeDockerManagedBootstrapJournal({ ...current, phase: next }); + if (expected === "cutover") { + const decision = decisionPath(target); + try { + atomicWrite(directory, decision, `${next}\n`, true); + } catch (error) { + if ( + !(error instanceof DockerManagedBootstrapJournalExistsError) || + readPrivateFile(decision, "decision") !== `${next}\n` + ) { + throw error; + } + } + } + atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(updated), false); + return updated; + }, + remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]) { + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const current = load(bootstrapIdentity); + if (!current || !expected.includes(current.phase)) { + fail(`journal removal is not authorized from phase ${current?.phase ?? "absent"}`); + } + const decision = decisionPath(target); + if (readPrivateFile(decision, "decision") !== null) { + fs.unlinkSync(decision); + fsyncDirectory(directory); + } + fs.unlinkSync(target); + fsyncDirectory(directory); + }, + }); +} diff --git a/src/lib/onboard/managed-bootstrap/docker-spec.test.ts b/src/lib/onboard/managed-bootstrap/docker-spec.test.ts new file mode 100644 index 00000000000..94124f0127b --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-spec.test.ts @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { createDockerGpuInspectFixture } from "../__test-helpers__/docker-gpu-patch-fixtures"; +import { + normalizeDockerManagedBootstrapLaunchSpec, + parseDockerManagedBootstrapLaunchSpec, +} from "./docker-spec"; + +describe("managed bootstrap Docker launch spec", () => { + it("hashes reproducible launch state while excluding runtime ID, phase, IP, and gateway", () => { + const first = createDockerGpuInspectFixture(); + const second = structuredClone(first); + second.Id = "another-runtime-id"; + Object.assign(second, { State: { Running: false, Dead: true } }); + second.NetworkSettings!.Networks!["openshell-docker"]!.IPAddress = "172.18.0.99"; + second.NetworkSettings!.Networks!["openshell-docker"]!.Gateway = "172.18.0.254"; + + const expected = normalizeDockerManagedBootstrapLaunchSpec(first); + const observed = normalizeDockerManagedBootstrapLaunchSpec(second); + + expect(observed.hash).toBe(expected.hash); + expect(observed.canonicalJson).toBe(expected.canonicalJson); + expect(parseDockerManagedBootstrapLaunchSpec(expected.canonicalJson)).toEqual(expected.spec); + }); + + it("changes the hash when a reproducible launch field changes", () => { + const first = createDockerGpuInspectFixture(); + const second = structuredClone(first); + Object.assign(second.Config!, { StopTimeout: 45 }); + + expect(normalizeDockerManagedBootstrapLaunchSpec(second).hash).not.toBe( + normalizeDockerManagedBootstrapLaunchSpec(first).hash, + ); + }); + + it("orders durable launch keys by code unit across host locale settings", () => { + const inspect = createDockerGpuInspectFixture(); + inspect.Config!.Labels = { + "com.nvidia.foo": "lower", + "com.nvidia.Foo": "upper", + "com.nvidia-foo": "punctuation", + }; + + const canonical = normalizeDockerManagedBootstrapLaunchSpec(inspect).canonicalJson; + + expect(canonical.indexOf('"com.nvidia-foo"')).toBeLessThan( + canonical.indexOf('"com.nvidia.Foo"'), + ); + expect(canonical.indexOf('"com.nvidia.Foo"')).toBeLessThan( + canonical.indexOf('"com.nvidia.foo"'), + ); + }); + + it.each([ + { + name: "anonymous Config.Volumes whose data source cannot be proven", + mutate: (inspect: ReturnType) => { + Object.assign(inspect.Config!, { Volumes: { "/var/lib/state": {} } }); + }, + error: /config fields it cannot reproduce exactly: Volumes\./u, + }, + { + name: "multiple attached networks", + mutate: (inspect: ReturnType) => { + inspect.NetworkSettings!.Networks!.secondary = { Aliases: ["alpha-secondary"] }; + }, + error: /multiple attached networks/u, + }, + { + name: "an unknown HostConfig field", + mutate: (inspect: ReturnType) => { + (inspect.HostConfig as Record).FutureRuntimeField = true; + }, + error: /unsupported fields: FutureRuntimeField/u, + }, + ])("fails closed for $name", ({ mutate, error }) => { + const inspect = createDockerGpuInspectFixture(); + mutate(inspect); + expect(() => normalizeDockerManagedBootstrapLaunchSpec(inspect)).toThrow(error); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker-spec.ts b/src/lib/onboard/managed-bootstrap/docker-spec.ts new file mode 100644 index 00000000000..c3cab4e3296 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-spec.ts @@ -0,0 +1,316 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import type { DockerContainerInspect } from "../docker-gpu-patch-types"; + +const CONFIG_KEYS = new Set([ + "ArgsEscaped", + "AttachStderr", + "AttachStdin", + "AttachStdout", + "Cmd", + "Domainname", + "Entrypoint", + "Env", + "ExposedPorts", + "Healthcheck", + "Hostname", + "Image", + "Labels", + "MacAddress", + "NetworkDisabled", + "OnBuild", + "OpenStdin", + "Shell", + "StdinOnce", + "StopSignal", + "StopTimeout", + "Tty", + "User", + "Volumes", + "WorkingDir", +]); + +const HOST_CONFIG_KEYS = new Set([ + "AutoRemove", + "Binds", + "BlkioDeviceReadBps", + "BlkioDeviceReadIOps", + "BlkioDeviceWriteBps", + "BlkioDeviceWriteIOps", + "BlkioWeight", + "BlkioWeightDevice", + "CapAdd", + "CapDrop", + "Cgroup", + "CgroupParent", + "CgroupnsMode", + "ConsoleSize", + "ContainerIDFile", + "CpuCount", + "CpuPercent", + "CpuPeriod", + "CpuQuota", + "CpuRealtimePeriod", + "CpuRealtimeRuntime", + "CpuShares", + "CpusetCpus", + "CpusetMems", + "DeviceCgroupRules", + "DeviceRequests", + "Devices", + "Dns", + "DnsOptions", + "DnsSearch", + "ExtraHosts", + "GroupAdd", + "IOMaximumBandwidth", + "IOMaximumIOps", + "Init", + "IpcMode", + "Isolation", + "Links", + "LogConfig", + "MaskedPaths", + "Memory", + "MemoryReservation", + "MemorySwap", + "MemorySwappiness", + "Mounts", + "NanoCpus", + "NetworkMode", + "OomKillDisable", + "OomScoreAdj", + "PidMode", + "PidsLimit", + "PortBindings", + "Privileged", + "PublishAllPorts", + "ReadonlyPaths", + "ReadonlyRootfs", + "RestartPolicy", + "Runtime", + "SecurityOpt", + "ShmSize", + "StorageOpt", + "Sysctls", + "Tmpfs", + "UTSMode", + "Ulimits", + "UsernsMode", + "VolumeDriver", + "VolumesFrom", +]); + +const UNSUPPORTED_CONFIG_KEYS = new Set([ + "ArgsEscaped", + "AttachStderr", + "AttachStdin", + "AttachStdout", + "MacAddress", + "OnBuild", + "Shell", + "Volumes", +]); + +const UNSUPPORTED_HOST_CONFIG_KEYS = new Set([ + "BlkioDeviceReadBps", + "BlkioDeviceReadIOps", + "BlkioDeviceWriteBps", + "BlkioDeviceWriteIOps", + "BlkioWeight", + "BlkioWeightDevice", + "Cgroup", + "ConsoleSize", + "ContainerIDFile", + "CpuCount", + "CpuPercent", + "CpuRealtimePeriod", + "CpuRealtimeRuntime", + "IOMaximumBandwidth", + "IOMaximumIOps", + "Isolation", + "Links", + "MaskedPaths", + "MemorySwappiness", + "ReadonlyPaths", + "StorageOpt", + "VolumeDriver", + "VolumesFrom", +]); + +export interface DockerManagedBootstrapLaunchSpec { + readonly schemaVersion: 1; + readonly inspect: Pick< + DockerContainerInspect, + "Name" | "Config" | "HostConfig" | "NetworkSettings" + > & { readonly Platform?: string }; +} + +function isEmptyDefault(value: unknown): boolean { + if (value === undefined || value === null || value === false || value === "" || value === 0) { + return true; + } + if (Array.isArray(value)) return value.length === 0; + if (typeof value === "object") return Object.keys(value as object).length === 0; + return false; +} + +function exactObject(value: unknown, label: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`Managed bootstrap Docker ${label} must be an object.`); + } + return value as Record; +} + +function assertKnownKeys( + record: Record, + allowed: ReadonlySet, + label: string, +): void { + const unknown = Object.keys(record).filter((key) => !allowed.has(key)); + if (unknown.length > 0) { + throw new Error( + `Managed bootstrap Docker ${label} contains unsupported fields: ${unknown.sort().join(", ")}.`, + ); + } +} + +function assertUnsupportedDefaults(host: Record): void { + const active = [...UNSUPPORTED_HOST_CONFIG_KEYS].filter((key) => !isEmptyDefault(host[key])); + if (active.length > 0) { + throw new Error( + `Managed bootstrap refuses Docker launch fields it cannot reproduce exactly: ${active + .sort() + .join(", ")}.`, + ); + } +} + +function byCodeUnit(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function normalizedNetworkSettings( + value: DockerContainerInspect["NetworkSettings"], +): DockerContainerInspect["NetworkSettings"] { + const networks = value?.Networks ?? {}; + return { + Networks: Object.fromEntries( + Object.entries(networks) + .sort(([left], [right]) => byCodeUnit(left, right)) + .map(([name, network]) => [ + name, + { + Aliases: [...(network.Aliases ?? [])].sort(), + }, + ]), + ), + }; +} + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalize); + if (typeof value !== "object" || value === null) return value; + return Object.fromEntries( + Object.entries(value as Record) + .sort(([left], [right]) => byCodeUnit(left, right)) + .map(([key, nested]) => [key, canonicalize(nested)]), + ); +} + +export function parseExactDockerContainerInspect(output: string): DockerContainerInspect { + let parsed: unknown; + try { + parsed = JSON.parse(output); + } catch { + throw new Error("Managed bootstrap Docker inspect output is malformed."); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("Managed bootstrap Docker inspect must return exactly one workload."); + } + return exactObject(parsed[0], "inspect") as DockerContainerInspect; +} + +export function normalizeDockerManagedBootstrapLaunchSpec(inspect: DockerContainerInspect): { + readonly canonicalJson: string; + readonly hash: string; + readonly spec: DockerManagedBootstrapLaunchSpec; +} { + const raw = inspect as DockerContainerInspect & Record; + const config = exactObject(raw.Config, "Config"); + const hostConfig = exactObject(raw.HostConfig, "HostConfig"); + assertKnownKeys(config, CONFIG_KEYS, "Config"); + assertKnownKeys(hostConfig, HOST_CONFIG_KEYS, "HostConfig"); + const unsupportedConfig = [...UNSUPPORTED_CONFIG_KEYS].filter( + (key) => !isEmptyDefault(config[key]), + ); + if (unsupportedConfig.length > 0) { + throw new Error( + `Managed bootstrap refuses Docker config fields it cannot reproduce exactly: ${unsupportedConfig + .sort() + .join(", ")}.`, + ); + } + assertUnsupportedDefaults(hostConfig); + + if (config.NetworkDisabled === true) { + throw new Error("Managed bootstrap does not support Config.NetworkDisabled."); + } + if (config.StdinOnce === true) { + throw new Error("Managed bootstrap does not support Config.StdinOnce."); + } + if (hostConfig.AutoRemove === true) { + throw new Error("Managed bootstrap cannot preserve an auto-remove held workload."); + } + if (hostConfig.PublishAllPorts === true) { + throw new Error("Managed bootstrap requires explicit Docker port bindings."); + } + if (Object.keys(inspect.NetworkSettings?.Networks ?? {}).length > 1) { + throw new Error("Managed bootstrap refuses a Docker workload with multiple attached networks."); + } + + const spec: DockerManagedBootstrapLaunchSpec = { + schemaVersion: 1, + inspect: { + Name: inspect.Name, + Config: config as DockerContainerInspect["Config"], + HostConfig: hostConfig as DockerContainerInspect["HostConfig"], + NetworkSettings: normalizedNetworkSettings(inspect.NetworkSettings), + ...("Platform" in raw && typeof raw.Platform === "string" ? { Platform: raw.Platform } : {}), + }, + }; + const canonicalJson = `${JSON.stringify(canonicalize(spec))}\n`; + return Object.freeze({ + canonicalJson, + hash: createHash("sha256").update(canonicalJson, "utf8").digest("hex"), + spec: Object.freeze(spec), + }); +} + +export function parseDockerManagedBootstrapLaunchSpec( + canonicalJson: string, +): DockerManagedBootstrapLaunchSpec { + let parsed: unknown; + try { + parsed = JSON.parse(canonicalJson); + } catch { + throw new Error("Managed bootstrap Docker launch snapshot is malformed."); + } + const record = exactObject(parsed, "launch snapshot"); + if ( + Object.keys(record).sort().join(",") !== ["inspect", "schemaVersion"].join(",") || + record.schemaVersion !== 1 + ) { + throw new Error("Managed bootstrap Docker launch snapshot schema is invalid."); + } + const normalized = normalizeDockerManagedBootstrapLaunchSpec( + exactObject(record.inspect, "launch snapshot inspect") as DockerContainerInspect, + ); + if (normalized.canonicalJson !== canonicalJson) { + throw new Error("Managed bootstrap Docker launch snapshot is not canonical."); + } + return normalized.spec; +} diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index fe6bd97d130..94d235cad52 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -121,6 +121,8 @@ describe("runtime provider central source boundary", () => { it("inventories every dormant managed-bootstrap protocol source", () => { expect(bootstrapProtocolPaths).toEqual([ "src/lib/onboard/managed-bootstrap/adapter.ts", + "src/lib/onboard/managed-bootstrap/docker-journal.ts", + "src/lib/onboard/managed-bootstrap/docker-spec.ts", "src/lib/onboard/managed-bootstrap/envelope.ts", "src/lib/onboard/managed-bootstrap/index.ts", ]); From a252baef3b174eb063a6f3f5144e1411558a7f18 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 05:34:10 -0700 Subject: [PATCH 091/117] feat(onboard): add transactional Docker bootstrap adapter Signed-off-by: Aaron Erickson --- .../snapshot-auto-create-failure.test.ts | 18 +- .../sandbox/snapshot-restore-test-fixture.ts | 5 + src/lib/actions/sandbox/snapshot.test.ts | 10 +- .../openshell/sandbox-identity.test.ts | 21 + .../adapters/openshell/sandbox-identity.ts | 16 + src/lib/onboard/docker-gpu-patch-clone.ts | 23 +- src/lib/onboard/docker-gpu-patch-types.ts | 17 + src/lib/onboard/managed-bootstrap/README.md | 48 +- src/lib/onboard/managed-bootstrap/adapter.ts | 4 +- .../managed-bootstrap/docker-shared-state.ts | 629 ++++ .../managed-bootstrap/docker-test-fixture.ts | 479 +++ .../onboard/managed-bootstrap/docker.test.ts | 320 ++ src/lib/onboard/managed-bootstrap/docker.ts | 2967 +++++++++++++++++ .../openshell-docker-sandbox-containers.ts | 1 + test/runtime-provider-source-shape.test.ts | 3 + tsconfig.src.json | 7 +- 16 files changed, 4538 insertions(+), 30 deletions(-) create mode 100644 src/lib/adapters/openshell/sandbox-identity.test.ts create mode 100644 src/lib/adapters/openshell/sandbox-identity.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-shared-state.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-test-fixture.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker.test.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker.ts diff --git a/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts b/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts index 65612e0ae3d..9dc200a96e1 100644 --- a/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts +++ b/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts @@ -28,13 +28,22 @@ const streamSandboxCreateMock = vi.fn(async () forcedReady: false, })); -vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => "") })); +vi.mock("../../adapters/docker", () => ({ + dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(), + dockerRunDetached: vi.fn(), +})); vi.mock("../../adapters/openshell/runtime", () => ({ captureOpenshell: captureOpenshellMock, getOpenshellBinary: vi.fn(() => "openshell"), runOpenshell: vi.fn(() => ({ status: 0, output: "" })), })); -vi.mock("../../credentials/store", () => ({ prompt: vi.fn() })); +vi.mock("../../credentials/store", () => ({ + deleteCredential: vi.fn(), + getCredential: vi.fn(() => null), + prompt: vi.fn(), + saveCredential: vi.fn(), +})); vi.mock("../../domain/sandbox/destroy", () => ({ getSandboxDeleteOutcome: vi.fn(() => ({ alreadyGone: false, gatewayUnreachable: false })), })); @@ -50,6 +59,11 @@ vi.mock("../../inference/nim", () => ({ stopNimContainerByName: vi.fn(), })); vi.mock("../../messaging/channels", () => ({ + BUILT_IN_CHANNEL_MANIFESTS: [], + getMessagingConfigEnvAliases: vi.fn(() => ({})), + getMessagingCredentialEnvKeysByChannel: vi.fn(() => ({})), + getMessagingProviderSuffixesByChannel: vi.fn(() => ({})), + listBuiltInMessagingChannelManifests: vi.fn(() => []), listMessagingProviderSuffixes: vi.fn(() => []), listMessagingCredentialMetadata: vi.fn(() => []), })); diff --git a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts index 66819eb0951..0d99c458141 100644 --- a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts +++ b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts @@ -213,7 +213,9 @@ export { lifecycleMock, shieldsMock }; vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(), dockerInspect: dockerInspectMock, + dockerRunDetached: vi.fn(), })); vi.mock("../../agent/defs", () => ({ @@ -227,7 +229,10 @@ vi.mock("../../adapters/openshell/runtime", () => ({ })); vi.mock("../../credentials/store", () => ({ + deleteCredential: vi.fn(), + getCredential: vi.fn(() => null), prompt: vi.fn(), + saveCredential: vi.fn(), })); vi.mock("../../domain/sandbox/destroy", () => ({ diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 977a2f49822..19cf6428de0 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -144,19 +144,21 @@ const latestBackupFixture = { vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(), dockerInspect: dockerInspectMock, + dockerRunDetached: vi.fn(), })); - vi.mock("../../adapters/openshell/runtime", () => ({ captureOpenshell: captureOpenshellMock, getOpenshellBinary: vi.fn(() => "openshell"), runOpenshell: runOpenshellMock, })); - vi.mock("../../credentials/store", () => ({ + deleteCredential: vi.fn(), + getCredential: vi.fn(() => null), prompt: vi.fn(), + saveCredential: vi.fn(), })); - vi.mock("../../domain/sandbox/destroy", () => ({ getSandboxDeleteOutcome: vi.fn(() => ({ alreadyGone: false, gatewayUnreachable: false })), })); @@ -165,7 +167,6 @@ vi.mock("../../inference/nim", () => ({ stopNimContainer: vi.fn(), stopNimContainerByName: vi.fn(), })); - vi.mock("../../policy", async (importOriginal) => ({ ...(await importOriginal()), applyPreset: applyPresetMock, @@ -176,7 +177,6 @@ vi.mock("../../policy", async (importOriginal) => ({ removePreset: removePresetMock, resolveAgentBaselinePolicy: resolveTestAgentBaselinePolicy, })); - vi.mock("../../runner", () => ({ ROOT: "/repo", run: vi.fn(() => ({ status: 0 })), diff --git a/src/lib/adapters/openshell/sandbox-identity.test.ts b/src/lib/adapters/openshell/sandbox-identity.test.ts new file mode 100644 index 00000000000..422bf71ccd8 --- /dev/null +++ b/src/lib/adapters/openshell/sandbox-identity.test.ts @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { parseOpenShellSandboxId } from "./sandbox-identity"; + +describe("OpenShell sandbox identity parsing", () => { + it("accepts one exact durable ID with optional terminal color", () => { + expect(parseOpenShellSandboxId("Name: alpha\nID: sandbox-alpha\n")).toBe("sandbox-alpha"); + expect(parseOpenShellSandboxId("\u001b[32mId: sandbox.alpha_2\u001b[0m\n")).toBe( + "sandbox.alpha_2", + ); + }); + + it("rejects ambiguous or non-canonical IDs", () => { + expect(parseOpenShellSandboxId("ID: first\nID: second\n")).toBeNull(); + expect(parseOpenShellSandboxId("ID: sandbox/alpha\n")).toBeNull(); + expect(parseOpenShellSandboxId("id: sandbox-alpha\n")).toBeNull(); + }); +}); diff --git a/src/lib/adapters/openshell/sandbox-identity.ts b/src/lib/adapters/openshell/sandbox-identity.ts new file mode 100644 index 00000000000..1820a8f8f7d --- /dev/null +++ b/src/lib/adapters/openshell/sandbox-identity.ts @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const ANSI_RE = /\x1b\[[0-9;]*m/gu; +const SANDBOX_ID_RE = /^[A-Za-z0-9._-]+$/u; + +export function parseOpenShellSandboxId(output: string): string | null { + const matches = [ + ...String(output) + .replace(ANSI_RE, "") + .matchAll(/^\s*(?:Id|ID):\s*(\S+)\s*$/gm), + ].map((match) => match[1] ?? ""); + return matches.length === 1 && SANDBOX_ID_RE.test(matches[0] as string) + ? (matches[0] as string) + : null; +} diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts index c92767adad0..828ce0c540b 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -329,7 +329,15 @@ export function buildDockerGpuCloneRunArgs( const image = String(options.image || config.Image || "").trim(); if (!image) throw new Error("Docker inspect output did not include Config.Image."); - const args: string[] = ["--name", dockerContainerName(inspect), ...mode.args]; + const containerName = String(options.containerName ?? dockerContainerName(inspect)).trim(); + if ( + containerName.length === 0 || + containerName.length > 253 || + !/^[A-Za-z0-9][A-Za-z0-9_.-]*$/u.test(containerName) + ) { + throw new Error("Docker clone container name is invalid."); + } + const args: string[] = ["--name", containerName, ...mode.args]; const gpuAugment = mode.kind !== "startup-command"; // Startup-command recreation must retain OpenShell's native CDI attachment. @@ -435,8 +443,17 @@ export function buildDockerGpuCloneRunArgs( if (host.Init) args.push("--init"); const entrypoint = stringArray(config.Entrypoint); - if (entrypoint.length > 0) args.push("--entrypoint", entrypoint[0]); - const commandArgs = sandboxCommand ? [] : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; + const replacementEntrypoint = String(options.containerEntrypoint ?? "").trim(); + if (replacementEntrypoint) { + args.push("--entrypoint", replacementEntrypoint); + } else if (entrypoint.length > 0) { + args.push("--entrypoint", entrypoint[0]); + } + const commandArgs = options.containerCommand + ? [...options.containerCommand] + : sandboxCommand + ? [] + : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; args.push(image, ...commandArgs); return args; } diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 32be0afe46f..d72046bd320 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -112,6 +112,14 @@ export type DockerGpuCloneRunOptions = { sandboxFallbackDns?: string | null; openshellSandboxCommand?: readonly string[] | null; requiredUlimits?: readonly DockerUlimit[] | null; + /** + * Exact replacement process boundary used only by dormant managed bootstrap. + * Ordinary recreation leaves both fields unset. + */ + containerEntrypoint?: string | null; + containerCommand?: readonly string[] | null; + /** Stopped staging name used before exact-name cutover. */ + containerName?: string | null; /** * Extra supplementary group IDs to add to the recreated container via * `--group-add`. On Jetson these are the host group(s) owning the Tegra GPU @@ -190,6 +198,14 @@ export type DockerContainerInspect = { Hostname?: string; Tty?: boolean; OpenStdin?: boolean; + StopTimeout?: number | null; + Volumes?: Record | null; + } | null; + State?: { + Running?: boolean; + Paused?: boolean; + Restarting?: boolean; + Dead?: boolean; } | null; HostConfig?: { Binds?: string[] | null; @@ -244,6 +260,7 @@ export type DockerContainerInspect = { DeviceIDs?: string[] | null; }> | null; ShmSize?: number; + ReadonlyRootfs?: boolean; ReadonlyPaths?: string[] | null; MaskedPaths?: string[] | null; } | null; diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index adde636fa56..717fcaa048b 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -3,9 +3,9 @@ # Managed bootstrap protocol -This directory defines a dormant, driver-neutral transaction contract. It does -not register a runtime provider or change sandbox creation, onboarding, -snapshot, clone, or restore behavior. +This directory defines a dormant, driver-neutral transaction contract and its +first driver adapter. It does not register a runtime provider or change sandbox +creation, onboarding, snapshot, clone, or restore behavior. The protocol binds one random bootstrap identity to: @@ -71,6 +71,21 @@ journal and a canonical launch-spec normalizer. Each surface is independently validated and remains dormant: no registered runtime provider imports either module, and neither changes sandbox creation or lifecycle behavior. +The Docker adapter creates and validates a stopped replacement under an +identity-derived staging name while the original remains running. It stages the +0400 envelope and returns exact cleanup authority without quiescing, renaming, +or otherwise mutating the original. Only after the coordinator durably records +that complete prepared authority may activation journal both full runtime IDs, +all three names, both launch-spec hashes, image identity, profile fingerprint, +and sandbox ID and then enter the destructive cutover. Rollback publishes +`rollback-authorized` before exact replacement deletion; commit publishes +`shared-state-committed` before exact backup deletion. Cleanup is bound to full +runtime IDs. Mutable OpenShell names are read only to detect ownership reuse, +and unsafe name-only deletion returns a typed retention error. The dormant +adapter assumes the protocol's single coordinator; multi-process +lease/arbitration remains an explicit production-activation gate. Activation +must also inject the selected gateway's canonical state root. + ## Architectural disposition The coordinator deliberately lands as a dormant trust-boundary slice before a @@ -91,17 +106,16 @@ activation slice must add a registered-provider contract test for the same transaction before removing those dormancy assertions. The native entrypoint source is intentionally not compiled into production -artifacts, and neither source is packaged or selected yet. No production -TypeScript module imports this protocol. The current image definitions do not -package `nemoclaw-managed-startup-hold` or -`managed-startup-image-runtime.cjs`. A later -provider integration must compile and verify the freestanding entrypoint -natively for amd64 and arm64 in every agent image. It must add those -prerequisites together with their image-runtime bootstrap modes, implement -driver-specific prepare, durable-record, activate, exact cleanup, and rollback, -and only then wire the coordinator into create. The same contract is exercised -for OpenClaw, Hermes, and Deep Agents Code without a provider-specific central -switch. The remaining integration and qualification work is tracked in -[epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) and its linked -implementation stack. Until that complete boundary lands, every registered -runtime provider keeps its bootstrap surface unsupported. +artifacts, and neither image-owned source is packaged or selected yet. No +production TypeScript module imports this protocol or the Docker adapter. The +current image definitions do not package `nemoclaw-managed-startup-hold`, +`managed-startup-image-runtime.cjs`, or the shared-state bootstrap modes consumed +by the adapter. A later provider integration must compile and verify the +freestanding entrypoint natively for amd64 and arm64 in every agent image. It +must add those prerequisites together with their image-runtime bootstrap modes +and wire the coordinator and Docker adapter into create as one boundary. The +same contract is exercised for OpenClaw, Hermes, and Deep Agents Code without a +provider-specific central switch. The remaining integration and qualification +work is tracked in [epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) +and its linked implementation stack. Until that complete boundary lands, every +registered runtime provider keeps its bootstrap surface unsupported. diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index c29fa7a01d7..b0edaa76627 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -931,7 +931,7 @@ function normalizePreparedReplacement( }); } -function createPreparedAuthority( +export function createManagedBootstrapPreparedAuthority( transaction: ManagedBootstrapPreparedTransaction, ): ManagedBootstrapPreparedAuthority { const { handle, snapshot, prepared } = transaction; @@ -1358,7 +1358,7 @@ export async function activateManagedBootstrapSequence( let durablePreparation: ManagedBootstrapDurablePreparationReceipt | null = null; let replacement: ManagedBootstrapReplacementHandle | null = null; try { - const authority = createPreparedAuthority(input.transaction); + const authority = createManagedBootstrapPreparedAuthority(input.transaction); durablePreparation = normalizeDurablePreparationReceipt( await input.authorityStore.recordPreparedAuthority(authority), authority, diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts new file mode 100644 index 00000000000..5953fc2b356 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts @@ -0,0 +1,629 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { + dockerRm as defaultDockerRm, + dockerStop as defaultDockerStop, +} from "../../adapters/docker/container"; +import { dockerRun as defaultDockerRun } from "../../adapters/docker/run"; +import { hasZeroDockerExitStatus } from "../docker-command-result"; +import { + DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + DOCKER_GPU_PATCH_TIMEOUT_MS, +} from "../docker-gpu-patch-constants"; +import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "../docker-gpu-patch-types"; +import { MANAGED_STARTUP_RUNTIME_EXECUTABLE } from "../managed-startup/image-runtime"; +import { MANAGED_STARTUP_AGENTS, type ManagedStartupAgent } from "../managed-startup/profile"; +import { + MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, +} from "../managed-startup/shared-state-transaction"; +import { isImmutableDockerImageId } from "../openshell-docker-sandbox-containers"; +import { cleanupTempDir, secureTempFile } from "../temp-files"; + +const RECEIPT_TEMP_PREFIX = "nemoclaw-managed-startup-receipt"; +const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY = + "/var/lib/nemoclaw/managed-startup-shared-state-commit-v1"; +const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; +const DURABLE_IDENTITY_RE = /^[a-f0-9]{64}$/u; +const NEUTRALIZED_PROCESS_INJECTION_ENV = [ + "--env", + "NODE_OPTIONS=", + "--env", + "NODE_PATH=", + "--env", + "BASH_ENV=", + "--env", + "ENV=", + "--env", + "LD_PRELOAD=", + "--env", + "LD_AUDIT=", + "--env", + "LD_LIBRARY_PATH=", + "--env", + "SHELLOPTS=", + "--env", + "PS4=", +] as const; + +export interface DockerManagedBootstrapSharedStateTransaction { + readonly agent: ManagedStartupAgent; + readonly bootstrapIdentity: string; + readonly containerId: string; + readonly image: string; + readonly profileFingerprint: string; +} + +export interface DockerManagedStartupSharedStateOutcome { + /** + * True only when the new supervisor is still eligible for successful + * container cutover. A commit failure forces shared-state rollback first. + */ + readonly supervisorReady: boolean; + /** Original commit failure after a successful shared-state rollback. */ + readonly failure: Error | null; +} + +export class DockerManagedStartupSharedStateCommitIndeterminateError extends Error { + constructor(detail: string, options?: ErrorOptions) { + super( + `Managed-startup shared-state commit may have completed, but immutable status is unavailable: ${detail}`, + options, + ); + this.name = "DockerManagedStartupSharedStateCommitIndeterminateError"; + } +} + +export function probeDockerManagedStartupSharedState( + input: { + readonly transaction: DockerManagedBootstrapSharedStateTransaction; + readonly profileFingerprint: string; + }, + deps: DockerGpuPatchDeps = {}, +): "committed" | "none" | "pending" { + const transaction = input.transaction; + assertValidManagedStartupTransaction(transaction); + if (input.profileFingerprint !== transaction.profileFingerprint) { + throw new Error("Managed bootstrap shared-state status fingerprint does not match."); + } + const committedReceiptPath = copyManagedStartupReceiptAt( + transaction, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, + deps, + true, + ); + if (committedReceiptPath) { + let verified = false; + try { + verifyCopiedManagedStartupReceipt( + transaction, + input.profileFingerprint, + committedReceiptPath, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, + "committed", + deps, + ); + verified = true; + return "committed"; + } finally { + if (verified) cleanupReceiptBestEffort(committedReceiptPath); + } + } + const receiptPath = copyManagedStartupReceipt(transaction, deps, true); + if (!receiptPath) return "none"; + let verified = false; + try { + verifyCopiedManagedStartupReceipt( + transaction, + input.profileFingerprint, + receiptPath, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + "pending", + deps, + ); + verified = true; + return "pending"; + } finally { + if (verified) cleanupReceiptBestEffort(receiptPath); + } +} + +function verifyCopiedManagedStartupReceipt( + transaction: DockerManagedBootstrapSharedStateTransaction, + profileFingerprint: string, + receiptPath: string, + receiptDirectory: string, + expectedStatus: "committed" | "pending", + deps: DockerGpuPatchDeps, +): void { + if (!transaction.bootstrapIdentity || !/^[a-f0-9]{64}$/u.test(profileFingerprint)) { + throw new Error("Managed bootstrap copied-receipt identity is incomplete."); + } + const dockerRun = deps.dockerRun ?? defaultDockerRun; + const result = dockerRun( + [ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + "--mount", + transactionReceiptMount(receiptPath, receiptDirectory), + "--entrypoint", + "/usr/local/bin/node", + transaction.image, + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + "--shared-state-transaction-status", + "--agent", + transaction.agent, + "--profile-fingerprint", + profileFingerprint, + "--bootstrap-identity", + transaction.bootstrapIdentity, + ], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(result)) { + throw new Error( + `Immutable managed-startup helper could not verify shared-state status: ${commandDetail(result)}. ` + + `Protected receipt retained at ${receiptPath}`, + ); + } + if (String(result.stdout ?? "").trim() !== expectedStatus) { + throw new Error( + `Immutable managed-startup helper returned an invalid copied transaction status. Protected receipt retained at ${receiptPath}`, + ); + } +} + +function commandDetail(result: { + readonly stderr?: string | Buffer | null; + readonly stdout?: string | Buffer | null; + readonly error?: Error | null; +}): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}` + .trim() + .slice(-800); +} + +function cleanupReceiptBestEffort(receiptPath: string): void { + try { + cleanupTempDir(receiptPath, RECEIPT_TEMP_PREFIX); + } catch (error) { + console.warn( + ` ⚠ Managed-startup shared state is finalized, but its protected host receipt could not be removed (${receiptPath}): ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +function assertValidManagedStartupTransaction( + transaction: DockerManagedBootstrapSharedStateTransaction, +): asserts transaction is DockerManagedBootstrapSharedStateTransaction & { + readonly bootstrapIdentity: string; + readonly profileFingerprint: string; +} { + if (!(MANAGED_STARTUP_AGENTS as readonly string[]).includes(transaction.agent)) { + throw new Error("Managed bootstrap shared-state transaction agent is invalid."); + } + if (!FULL_CONTAINER_ID_RE.test(transaction.containerId)) { + throw new Error("Managed bootstrap shared-state transaction container identity is invalid."); + } + if (!isImmutableDockerImageId(transaction.image)) { + throw new Error("Managed bootstrap shared-state transaction image identity is not immutable."); + } + if (!transaction.bootstrapIdentity || !DURABLE_IDENTITY_RE.test(transaction.bootstrapIdentity)) { + throw new Error("Managed bootstrap shared-state transaction identity is missing or invalid."); + } + if ( + !transaction.profileFingerprint || + !DURABLE_IDENTITY_RE.test(transaction.profileFingerprint) + ) { + throw new Error( + "Managed bootstrap shared-state transaction profile fingerprint is missing or invalid.", + ); + } +} + +function transactionCommand( + action: "clear-shared-state-commit-receipt" | "commit" | "rollback", + transaction: DockerManagedBootstrapSharedStateTransaction, +): string[] { + assertValidManagedStartupTransaction(transaction); + return [ + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + action === "clear-shared-state-commit-receipt" + ? "--clear-shared-state-commit-receipt" + : `--${action}-shared-state-transaction`, + "--agent", + transaction.agent, + "--bootstrap-identity", + transaction.bootstrapIdentity, + ]; +} + +export function clearDockerManagedStartupSharedStateCommitReceipt( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps = {}, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + assertValidManagedStartupTransaction(transaction); + const command = transactionCommand("clear-shared-state-commit-receipt", transaction); + const cleared = dockerRun( + [ + "exec", + "--user", + "0:0", + "--workdir", + "/", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + transaction.containerId, + "/usr/bin/env", + "-i", + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + ...command, + ], + DOCKER_MUTATION_OPTIONS, + ); + // Accept a lost Docker acknowledgement only when both exact image-owned + // receipt paths are independently proven absent by the immutable helper. + let status: "committed" | "none" | "pending"; + try { + status = probeDockerManagedStartupSharedState( + { + transaction, + profileFingerprint: transaction.profileFingerprint, + }, + deps, + ); + } catch (error) { + throw new DockerManagedStartupSharedStateCommitIndeterminateError( + error instanceof Error ? error.message : String(error), + { cause: error }, + ); + } + if (status === "none") return; + if (!hasZeroDockerExitStatus(cleared)) { + throw new Error( + `Managed-startup durable commit receipt cleanup failed and exact absence was not proven (status=${status}): ${commandDetail(cleared)}`, + ); + } + throw new Error( + `Managed-startup durable commit receipt cleanup returned success, but exact absence was not proven (status=${status}).`, + ); +} + +function commitManagedStartupSharedState( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + assertValidManagedStartupTransaction(transaction); + const command = transactionCommand("commit", transaction); + const commit = dockerRun( + [ + "exec", + "--user", + "0:0", + "--workdir", + "/", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + transaction.containerId, + "/usr/bin/env", + "-i", + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + ...command, + ], + DOCKER_MUTATION_OPTIONS, + ); + // The commit helper atomically renames the rollback receipt into a compact + // identity-bound commit receipt before Docker returns. Always probe it + // afterward so a lost daemon acknowledgement is accepted only when durable + // commit state is independently proven. + let status: "committed" | "none" | "pending"; + try { + status = probeDockerManagedStartupSharedState( + { + transaction, + profileFingerprint: transaction.profileFingerprint, + }, + deps, + ); + } catch (error) { + throw new DockerManagedStartupSharedStateCommitIndeterminateError( + error instanceof Error ? error.message : String(error), + { cause: error }, + ); + } + if (status === "committed") return; + if (!hasZeroDockerExitStatus(commit)) { + throw new Error( + `Managed-startup shared-state commit helper failed and durable commit was not proven (status=${status}): ${commandDetail(commit)}`, + ); + } + throw new Error( + `Managed-startup shared-state commit helper returned success, but durable commit was not proven (status=${status}).`, + ); +} + +const DOCKER_MUTATION_OPTIONS = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, +} as const; + +function quiesceManagedStartupContainer( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerStop = deps.dockerStop ?? defaultDockerStop; + const stopped = dockerStop(transaction.containerId, { + ...DOCKER_MUTATION_OPTIONS, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopped)) { + throw new Error( + `Could not quiesce the failed managed-startup container before shared-state rollback: ${commandDetail(stopped)}`, + ); + } +} + +function isExactMissingReceiptCopy( + transaction: DockerManagedBootstrapSharedStateTransaction, + sourcePath: string, + result: { + readonly stderr?: string | Buffer | null; + readonly stdout?: string | Buffer | null; + readonly error?: Error | null; + }, +): boolean { + const detail = commandDetail(result); + const escapedPath = sourcePath.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const escapedContainer = transaction.containerId.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + return [ + new RegExp( + `^(?:Error response from daemon: )?Could not find the file ${escapedPath} in container ${escapedContainer}$`, + "u", + ), + new RegExp(`^(?:lstat|stat) ${escapedPath}: no such file or directory$`, "u"), + ].some((pattern) => pattern.test(detail)); +} + +function transactionReceiptMount(receiptPath: string, receiptDirectory: string): string { + return `type=bind,src=${receiptPath},dst=${receiptDirectory},readonly`; +} + +function copyManagedStartupReceiptAt( + transaction: DockerManagedBootstrapSharedStateTransaction, + sourcePath: string, + deps: DockerGpuPatchDeps, + allowAbsent = false, +): string | null { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + const tempSeed = secureTempFile(RECEIPT_TEMP_PREFIX); + const receiptPath = path.join(path.dirname(tempSeed), path.basename(sourcePath)); + try { + const copy = dockerRun( + ["cp", "-a", `${transaction.containerId}:${sourcePath}`, receiptPath], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(copy)) { + if (allowAbsent && isExactMissingReceiptCopy(transaction, sourcePath, copy)) { + cleanupReceiptBestEffort(receiptPath); + return null; + } + throw new Error( + `Could not copy the managed-startup rollback receipt from the failed container: ${commandDetail(copy)}`, + ); + } + if (receiptPath.includes(",") || /[\r\n\0]/u.test(receiptPath)) { + throw new Error("Managed-startup rollback receipt path is unsafe for a Docker bind mount"); + } + return receiptPath; + } catch (error) { + cleanupReceiptBestEffort(receiptPath); + throw error; + } +} + +function copyManagedStartupReceipt( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, + allowAbsent = false, +): string | null { + return copyManagedStartupReceiptAt( + transaction, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + deps, + allowAbsent, + ); +} + +function rollbackManagedStartupSharedState( + transaction: DockerManagedBootstrapSharedStateTransaction, + receiptPath: string, + deps: DockerGpuPatchDeps, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + let restored = false; + try { + const helper = dockerRun( + [ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--cap-add", + "CHOWN", + "--cap-add", + "DAC_OVERRIDE", + "--cap-add", + "FOWNER", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + "--volumes-from", + transaction.containerId, + "--mount", + `type=bind,src=${receiptPath},dst=${MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY},readonly`, + "--entrypoint", + "/usr/local/bin/node", + transaction.image, + ...transactionCommand("rollback", transaction), + "--read-only-receipt", + ], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(helper)) { + throw new Error( + `Immutable managed-startup helper could not restore and verify shared state: ${commandDetail(helper)}. ` + + `Protected receipt retained at ${receiptPath}`, + ); + } + restored = true; + } finally { + if (restored) { + cleanupReceiptBestEffort(receiptPath); + } + } +} + +function removeFailedUnbackedContainer( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerRm = deps.dockerRm ?? defaultDockerRm; + const removed = dockerRm(transaction.containerId, DOCKER_MUTATION_OPTIONS); + if (!hasZeroDockerExitStatus(removed)) { + throw new Error( + `Could not remove the failed managed-startup container after shared-state rollback: ${commandDetail(removed)}`, + ); + } +} + +/** + * Finalize the shared-state half of managed container cutover before generic + * backup removal or rollback. A shared-state rollback failure deliberately + * throws so callers cannot remove the new container or restart the old one + * while `/sandbox` remains partially applied. + */ +export function finalizeDockerManagedStartupSharedState( + input: { + readonly transaction: DockerManagedBootstrapSharedStateTransaction | null; + readonly patchResult?: DockerGpuPatchResult | null; + /** + * The managed-bootstrap journal owns exact replacement removal. Retaining + * it lets the caller publish rollback authorization after shared-state + * restoration and before the first runtime deletion. + */ + readonly retainContainerAfterRollback?: boolean; + readonly supervisorReady: boolean; + }, + deps: DockerGpuPatchDeps = {}, +): DockerManagedStartupSharedStateOutcome { + const transaction = input.transaction; + if (!transaction) { + return { supervisorReady: input.supervisorReady, failure: null }; + } + assertValidManagedStartupTransaction(transaction); + if (input.supervisorReady) { + // Preserve and validate an explicit writable-layer receipt before logical + // commit. The helper receives the copy read-only and does not delete it; + // this keeps rollback possible when Docker loses the helper acknowledgement. + // --volumes-from exposes shared mounts only; it cannot expose this + // container-local transaction directory to an immutable helper. + let receiptPath: string; + try { + const copiedReceipt = copyManagedStartupReceipt(transaction, deps); + if (!copiedReceipt) { + throw new Error("Managed-startup pending receipt disappeared before commit."); + } + receiptPath = copiedReceipt; + } catch (error) { + try { + quiesceManagedStartupContainer(transaction, deps); + } catch (stopError) { + throw new Error( + `Managed-startup receipt preservation failed and the new workload could not be quiesced: ${ + error instanceof Error ? error.message : String(error) + }; ${stopError instanceof Error ? stopError.message : String(stopError)}`, + ); + } + throw error; + } + let commitFailure: Error | null = null; + try { + verifyCopiedManagedStartupReceipt( + transaction, + transaction.profileFingerprint, + receiptPath, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + "pending", + deps, + ); + commitManagedStartupSharedState(transaction, deps); + cleanupReceiptBestEffort(receiptPath); + return { supervisorReady: true, failure: null }; + } catch (error) { + if (error instanceof DockerManagedStartupSharedStateCommitIndeterminateError) { + throw error; + } + commitFailure = error instanceof Error ? error : new Error(String(error)); + } + const failure = new Error( + `OpenShell supervisor reconnected, but managed shared-state logical commit validation failed: ${commitFailure.message}`, + ); + quiesceManagedStartupContainer(transaction, deps); + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!input.patchResult && !input.retainContainerAfterRollback) { + removeFailedUnbackedContainer(transaction, deps); + } + return { supervisorReady: false, failure }; + } + + quiesceManagedStartupContainer(transaction, deps); + const receiptPath = copyManagedStartupReceipt(transaction, deps, true); + if (!receiptPath) { + if (!input.patchResult && !input.retainContainerAfterRollback) { + removeFailedUnbackedContainer(transaction, deps); + } + return { supervisorReady: false, failure: null }; + } + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!input.patchResult && !input.retainContainerAfterRollback) { + removeFailedUnbackedContainer(transaction, deps); + } + return { supervisorReady: false, failure: null }; +} diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts new file mode 100644 index 00000000000..1c2a6871b6a --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -0,0 +1,479 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { expect, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import type { DockerContainerInspect } from "../docker-gpu-patch-types"; +import { encodeManagedStartupProfile, type ManagedStartupAgent } from "../managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import { + createManagedBootstrapPreparedAuthority, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapCompletionReceipt, + type ManagedBootstrapDurablePreparationReceipt, + type ManagedBootstrapHeldWorkloadHandle, + type ManagedBootstrapObservedSnapshot, + type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapReplacementHandle, +} from "./adapter"; +import type { DockerManagedBootstrapDeps } from "./docker"; +import { + type DockerManagedBootstrapJournal, + DockerManagedBootstrapJournalAcknowledgementLostError, + type DockerManagedBootstrapJournalPhase, + type DockerManagedBootstrapJournalStore, +} from "./docker-journal"; +import { normalizeDockerManagedBootstrapLaunchSpec } from "./docker-spec"; +import { parseManagedBootstrapEnvelope } from "./envelope"; + +export const IDENTITY = "1".repeat(64); +export const OLD_ID = "2".repeat(64); +export const NEW_ID = "3".repeat(64); +const CONFIG_ID = `sha256:${"4".repeat(64)}`; +const MANIFEST = `sha256:${"5".repeat(64)}` as const; +const REPOSITORY = "registry.example/nemoclaw/hermes"; +const IMAGE = `${REPOSITORY}@${MANIFEST}`; +const SUPERVISOR = ["/opt/openshell/bin/openshell-sandbox", "supervise"] as const; +export const SUPPORTED_AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; + +type FixtureCommandResult = { + readonly status: number; + readonly stdout?: string; + readonly stderr?: string; +}; + +export type DockerFixtureAcknowledgement = + | "container:create" + | "container:remove" + | "container:rename" + | "container:start" + | "container:stop" + | "journal:create" + | "journal:cutover" + | "journal:remove" + | "journal:rollback-authorized" + | "journal:staged" + | "journal:shared-state-committed"; + +export type DockerFixtureOptions = { + readonly agent?: ManagedStartupAgent; + readonly dockerStartResults?: Readonly>; + readonly journalTransitionFailures?: Partial< + Readonly> + >; + readonly lostAcknowledgements?: readonly DockerFixtureAcknowledgement[]; + readonly ownerId?: string; + readonly sharedState?: "committed" | "none" | "pending"; +}; + +function agentInputs(agent: ManagedStartupAgent = "hermes") { + const request = createManagedStartupRootApplyRequest({ + agent, + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agent, false, false)), + }); + const heldArgv = [ + "env", + "A=1", + "/usr/local/bin/nemoclaw-managed-startup-hold", + "--agent", + agent, + "--profile-fingerprint", + request.profileFingerprint, + "--bootstrap-identity", + IDENTITY, + ] as const; + return { + request, + heldArgv, + metadata: { "nemoclaw.ai/managed-profile": request.profileFingerprint }, + }; +} + +export const { heldArgv } = agentInputs(); +export const sandbox = { + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + driverId: "docker", +}; + +function shellArgv(argv: readonly string[]): string { + return argv.join(" "); +} + +function originalInspect(inputs = agentInputs()): DockerContainerInspect { + return { + Id: OLD_ID, + Image: CONFIG_ID, + Name: "/openshell-alpha", + Config: { + Image: IMAGE, + Env: ["A=1", `OPENSHELL_SANDBOX_COMMAND=${shellArgv(inputs.heldArgv)}`], + Labels: { + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-name": "alpha", + "openshell.ai/sandbox-id": "sandbox-alpha", + ...inputs.metadata, + }, + Entrypoint: [SUPERVISOR[0]], + Cmd: SUPERVISOR.slice(1), + User: "root", + WorkingDir: "/sandbox", + Hostname: "alpha", + }, + State: { Running: true, Paused: false, Restarting: false, Dead: false }, + HostConfig: { + Binds: ["/host/workspace:/sandbox:rw"], + NetworkMode: "openshell", + RestartPolicy: { Name: "unless-stopped" }, + CapDrop: ["NET_RAW"], + SecurityOpt: ["no-new-privileges"], + Ulimits: [{ Name: "nofile", Soft: 65_536, Hard: 65_536 }], + }, + NetworkSettings: { Networks: { openshell: { Aliases: ["openshell-alpha"] } } }, + }; +} + +export function authority(agent: ManagedStartupAgent = "hermes") { + const inputs = agentInputs(agent); + const inspect = originalInspect(inputs); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(inspect); + const plan = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandboxName: "alpha", + driverId: "docker", + image: { repository: REPOSITORY, manifestDigest: MANIFEST }, + profile: { agent, fingerprint: inputs.request.profileFingerprint }, + agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + intendedWorkloadArgv: ["env", "A=1", "nemoclaw-start"], + expectedSupervisorArgv: SUPERVISOR, + metadata: inputs.metadata, + }; + const handle: ManagedBootstrapHeldWorkloadHandle = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: inputs.heldArgv, + intendedWorkloadArgv: plan.intendedWorkloadArgv, + plan, + createReceipt: { sandbox, ready: true, readyAt: "2026-07-31T12:00:00.000Z" }, + }; + const snapshot: ManagedBootstrapObservedSnapshot = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + runtimeId: OLD_ID, + bootstrapIdentity: IDENTITY, + image: plan.image, + runtimeImageContentId: CONFIG_ID, + specHash: normalized.hash, + specCanonicalJson: normalized.canonicalJson, + agentIdentity: plan.agentIdentity, + supervisorArgv: SUPERVISOR, + heldWorkloadArgv: inputs.heldArgv, + metadata: inputs.metadata, + }; + return { handle, plan, request: inputs.request, snapshot }; +} + +function failFixture(message: string): never { + throw new Error(message); +} + +function readProtectedEnvelope(source: string): ReturnType { + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") throw new Error("test requires O_NOFOLLOW"); + const descriptor = fs.openSync(source, fs.constants.O_RDONLY | noFollow); + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + expect(Number(before.mode & 0o777n)).toBe(0o400); + const parsed = parseManagedBootstrapEnvelope(fs.readFileSync(descriptor, "utf8")); + const after = fs.fstatSync(descriptor, { bigint: true }); + expect(after.dev).toBe(before.dev); + expect(after.ino).toBe(before.ino); + expect(after.size).toBe(before.size); + expect(after.mtimeNs).toBe(before.mtimeNs); + expect(after.ctimeNs).toBe(before.ctimeNs); + return parsed; + } finally { + fs.closeSync(descriptor); + } +} + +export function fixture(options: DockerFixtureOptions = {}) { + let original = originalInspect(agentInputs(options.agent)); + let replacement: DockerContainerInspect | null = null; + let journal: DockerManagedBootstrapJournal | null = null; + let sharedState: "committed" | "none" | "pending" = options.sharedState ?? "none"; + const events: string[] = []; + const lostAcknowledgements = new Set(options.lostAcknowledgements ?? []); + const losesAcknowledgement = (operation: DockerFixtureAcknowledgement) => + lostAcknowledgements.has(operation); + const ok = (stdout = ""): FixtureCommandResult => ({ status: 0, stdout, stderr: "" }); + const copyJournal = () => (journal ? structuredClone(journal) : null); + const store: DockerManagedBootstrapJournalStore = { + create(value) { + journal = structuredClone(value); + events.push("journal:staged"); + if (losesAcknowledgement("journal:create")) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal create acknowledgement", + ); + } + }, + load: () => copyJournal(), + transition(_identity, expected, next) { + const current = + journal !== null && journal.phase === expected + ? journal + : failFixture("stale journal transition"); + journal = { ...current, phase: next }; + events.push(`journal:${next}`); + const injectedFailure = options.journalTransitionFailures?.[next]; + if (injectedFailure) throw injectedFailure; + if (losesAcknowledgement(`journal:${next}`)) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal transition acknowledgement", + ); + } + return structuredClone(journal); + }, + remove(_identity, expected) { + const current = journal; + void (current !== null && expected.includes(current.phase) + ? current + : failFixture("stale journal remove")); + journal = null; + events.push("journal:removed"); + if (losesAcknowledgement("journal:remove")) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal remove acknowledgement", + ); + } + }, + }; + const inspect = (reference: string): DockerContainerInspect => { + const candidates = [original, replacement].filter( + (value): value is DockerContainerInspect => value !== null, + ); + const found = candidates.find( + (value) => + value.Id === reference || String(value.Name ?? "").replace(/^\/+/u, "") === reference, + ); + return found ? structuredClone(found) : failFixture(`No such container: ${reference}`); + }; + const dockerCapture: NonNullable = vi.fn((args) => { + switch (args[0]) { + case "image": + return JSON.stringify([{ Id: CONFIG_ID, RepoDigests: [IMAGE] }]); + default: + return JSON.stringify([inspect(String(args[3] ?? ""))]); + } + }); + const dockerRun: NonNullable = vi.fn( + (args: readonly string[]) => { + switch (args[0]) { + case "create": { + events.push("create:replacement"); + const name = String(args[args.indexOf("--name") + 1] ?? ""); + const entrypoint = String(args[args.indexOf("--entrypoint") + 1] ?? ""); + const imageIndex = args.indexOf(IMAGE); + const env = args.flatMap((value, index) => + value === "--env" ? [String(args[index + 1] ?? "")] : [], + ); + replacement = { + ...structuredClone(original), + Id: NEW_ID, + Name: `/${name}`, + Config: { + ...structuredClone(original.Config), + Image: IMAGE, + Env: env, + Entrypoint: [entrypoint], + Cmd: args.slice(imageIndex + 1), + }, + State: { Running: false, Paused: false, Restarting: false, Dead: false }, + }; + return losesAcknowledgement("container:create") + ? { status: 1, stdout: "", stderr: "lost create acknowledgement" } + : ok(NEW_ID); + } + case "ps": + return ok(original ? OLD_ID : ""); + case "inspect": { + const id = String(args[3] ?? ""); + try { + inspect(id); + return ok(`[{"Id":"${id}"}]`); + } catch { + return { status: 1, stderr: `Error response from daemon: No such container: ${id}` }; + } + } + case "cp": { + const sourceIndex = args[1] === "-a" ? 2 : 1; + const source = String(args[sourceIndex] ?? ""); + const destination = String(args[sourceIndex + 1] ?? ""); + const copyIntoContainer = () => { + events.push("stage:envelope"); + expect(readProtectedEnvelope(source).bootstrapIdentity).toBe(IDENTITY); + return ok(); + }; + const copyFromContainer = () => { + const receipt = source.split(":")[1]; + const expected = receipt?.includes("shared-state-commit") ? "committed" : "pending"; + return sharedState === expected + ? (() => { + fs.mkdirSync(destination, { recursive: true }); + return ok(); + })() + : { + status: 1, + stderr: `Error response from daemon: Could not find the file ${receipt} in container ${NEW_ID}`, + }; + }; + return source.includes(":") ? copyFromContainer() : copyIntoContainer(); + } + case "run": + switch (true) { + case args.includes("--shared-state-transaction-status"): + return ok(`${sharedState}\n`); + case args.includes("--rollback-shared-state-transaction"): + sharedState = "none"; + events.push("shared:rollback"); + return ok(); + } + break; + case "exec": + switch (true) { + case args.includes("--commit-shared-state-transaction"): + sharedState = "committed"; + events.push("shared:commit"); + return ok(); + case args.includes("--clear-shared-state-commit-receipt"): + sharedState = "none"; + events.push("shared:clear"); + return ok(); + } + break; + } + throw new Error(`unexpected Docker command: ${args.join(" ")}`); + }, + ); + const deps: DockerManagedBootstrapDeps = { + journalStore: store, + dockerCapture, + dockerRun, + dockerStop: vi.fn((id) => { + events.push(`stop:${id}`); + const target = id === OLD_ID ? original : replacement; + [target] + .filter((value): value is DockerContainerInspect => value?.State !== undefined) + .forEach((value) => { + value.State = { ...value.State, Running: false }; + }); + return losesAcknowledgement("container:stop") + ? { status: 1, stderr: "lost stop acknowledgement" } + : ok(); + }), + dockerRename: vi.fn((id, name) => { + events.push(`rename:${id}:${name}`); + const target = id === OLD_ID ? original : replacement; + [target] + .filter((value): value is DockerContainerInspect => value !== null) + .forEach((value) => { + value.Name = `/${name}`; + }); + return losesAcknowledgement("container:rename") + ? { status: 1, stderr: "lost rename acknowledgement" } + : ok(); + }), + dockerStart: vi.fn((id) => { + events.push(`start:${id}`); + const result = options.dockerStartResults?.[id] ?? ok(); + const target = id === OLD_ID ? original : replacement; + [target] + .filter( + (value): value is DockerContainerInspect => + value?.State !== undefined && result.status === 0, + ) + .forEach((value) => { + value.State = { ...value.State, Running: true }; + }); + return losesAcknowledgement("container:start") + ? { status: 1, stderr: "lost start acknowledgement" } + : result; + }), + dockerRm: vi.fn((id) => { + events.push(`rm:${id}`); + switch (id) { + case OLD_ID: + original = null as unknown as DockerContainerInspect; + break; + case NEW_ID: + replacement = null; + break; + } + return losesAcknowledgement("container:remove") + ? { status: 1, stderr: "lost rm acknowledgement" } + : ok(); + }), + runCaptureOpenshell: vi.fn(() => `Name: alpha\nID: ${options.ownerId ?? "sandbox-alpha"}\n`), + runOpenshell: vi.fn(() => ok()), + now: () => new Date("2026-07-31T12:30:00.000Z"), + }; + return { + deps, + events, + get journal() { + return journal; + }, + get original() { + return original; + }, + get replacement() { + return replacement; + }, + get sharedState() { + return sharedState; + }, + }; +} + +export function completion( + replacement: ManagedBootstrapReplacementHandle, +): ManagedBootstrapCompletionReceipt { + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + runtimeId: replacement.replacementRuntimeId, + image: replacement.image, + runtimeImageContentId: replacement.runtimeImageContentId, + originalSpecHash: replacement.originalSpecHash, + replacementSpecHash: replacement.replacementSpecHash, + profileFingerprint: replacement.profileFingerprint, + bootstrapIdentity: replacement.bootstrapIdentity, + transactionPending: true, + completedAt: "2026-07-31T12:15:00.000Z", + }; +} + +export function durablePreparation( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, +): ManagedBootstrapDurablePreparationReceipt { + const preparedAuthority = createManagedBootstrapPreparedAuthority({ + handle, + snapshot, + prepared, + }); + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + authorityFingerprint: preparedAuthority.authorityFingerprint, + recordId: `test-authority-${handle.plan.profile.agent}`, + recordedAt: "2026-07-31T12:10:00.000Z", + }; +} diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts new file mode 100644 index 00000000000..0f848d86c18 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -0,0 +1,320 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { ManagedBootstrapOwnerCleanupRequiredError } from "./adapter"; +import { createDockerManagedBootstrapAdapter } from "./docker"; +import { + authority, + completion, + durablePreparation, + fixture, + heldArgv, + IDENTITY, + NEW_ID, + OLD_ID, + SUPPORTED_AGENTS, +} from "./docker-test-fixture"; + +describe("Docker managed bootstrap adapter", () => { + it("publishes durable commit authority before deleting the rollback backup after lost acknowledgements", async () => { + const fake = fixture({ + lostAcknowledgements: [ + "container:create", + "container:remove", + "container:rename", + "container:start", + "container:stop", + "journal:create", + "journal:cutover", + "journal:remove", + "journal:shared-state-committed", + ], + sharedState: "pending", + }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + expect(fake.journal).toBeNull(); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + fake.events.push("authority:recorded"); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + const order = fake.events; + expect(order.indexOf("journal:staged")).toBeGreaterThan(order.indexOf("authority:recorded")); + expect(order.indexOf("journal:cutover")).toBeLessThan(order.indexOf(`stop:${OLD_ID}`)); + expect(fake.journal).toMatchObject({ + phase: "cutover", + originalRuntimeId: OLD_ID, + replacementRuntimeId: NEW_ID, + }); + + await expect( + adapter.finalizeBootstrap({ + outcome: "commit", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: completion(replacement), + }), + ).resolves.toMatchObject({ outcome: "committed" }); + expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( + fake.events.indexOf(`rm:${OLD_ID}`), + ); + expect(fake.journal).toBeNull(); + expect(fake.sharedState).toBe("none"); + expect(fake.replacement?.Id).toBe(NEW_ID); + }); + + it("publishes durable rollback authority before deleting the replacement after restart", async () => { + const fake = fixture({ + dockerStartResults: { + [NEW_ID]: { status: 1, stderr: "injected start failure" }, + }, + }); + const first = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await first.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + await expect( + first.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }), + ).rejects.toThrow("could not prove its exact replacement running"); + expect(fake.journal?.phase).toBe("cutover"); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect( + restarted.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement: null, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( + fake.events.indexOf(`rm:${NEW_ID}`), + ); + expect(fake.journal).toBeNull(); + expect(fake.replacement).toBeNull(); + expect(fake.original.Name).toBe("/openshell-alpha"); + expect(fake.original.State?.Running).toBe(false); + }); + + it("recovers the pre-stop cutover crash state after adapter restart", async () => { + const fake = fixture({ + journalTransitionFailures: { + cutover: new Error("injected crash after durable cutover fence"), + }, + }); + const { handle, request: rootRequest, snapshot } = authority(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + await expect( + adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }), + ).rejects.toThrow("crash after durable cutover fence"); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + await expect( + createDockerManagedBootstrapAdapter(fake.deps).finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement: null, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( + fake.events.indexOf(`rm:${NEW_ID}`), + ); + expect(fake.journal).toBeNull(); + }); + + it("fences rollback when image-owned shared state is already committed", async () => { + const fake = fixture({ sharedState: "committed" }); + const { handle, request: rootRequest, snapshot } = authority(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + const eventCount = fake.events.length; + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: null, + }), + ).rejects.toMatchObject({ name: "ManagedBootstrapDurableCommitCleanupPendingError" }); + expect(fake.journal?.phase).toBe("shared-state-committed"); + expect(fake.events.slice(eventCount)).toEqual(["journal:shared-state-committed"]); + }); + + it("rejects cutover before the exact durable authority receipt", async () => { + const fake = fixture(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const invalid = { + ...durablePreparation(handle, snapshot, prepared), + authorityFingerprint: "f".repeat(64), + }; + await expect( + adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: invalid, + }), + ).rejects.toThrow("exact durable prepared-authority receipt"); + expect(fake.journal).toBeNull(); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: null, + replacement: null, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.replacement).toBeNull(); + }); + + it.each( + SUPPORTED_AGENTS, + )("prepares, activates, and exactly rolls back the %s agent without a central switch", async (agent) => { + const fake = fixture({ agent, sharedState: "pending" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(agent); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.journal).toBeNull(); + expect(fake.replacement).toBeNull(); + expect( + vi.mocked(fake.deps.dockerRun!).mock.calls.some(([args]) => { + const agentIndex = args.indexOf("--agent"); + return args.includes("--shared-state-transaction-status") && args[agentIndex + 1] === agent; + }), + ).toBe(true); + }); + + it("quiesces and retains an exact incomplete create when its mutable name is reused", async () => { + const fake = fixture({ ownerId: "sandbox-alpha-recreated" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, plan } = authority(); + await expect( + adapter.cleanupIncompleteCreate({ + plan, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: heldArgv, + createReceipt: handle.createReceipt, + }), + ).rejects.toMatchObject({ + name: "ManagedBootstrapOwnerCleanupRequiredError", + sandboxId: "sandbox-alpha", + runtimeId: OLD_ID, + }); + expect(fake.original.State?.Running).toBe(false); + expect(fake.events).not.toContain(`rm:${OLD_ID}`); + expect(vi.mocked(fake.deps.runOpenshell!)).not.toHaveBeenCalled(); + }); + + it("retains a same-name workload that differs from the validated create receipt", async () => { + const replacementSandboxId = "sandbox-alpha-recreated"; + const fake = fixture({ ownerId: replacementSandboxId }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, plan } = authority(); + if (!fake.original.Config?.Labels) throw new Error("fixture labels are required"); + fake.original.Config.Labels["openshell.ai/sandbox-id"] = replacementSandboxId; + + await expect( + adapter.cleanupIncompleteCreate({ + plan, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: heldArgv, + createReceipt: handle.createReceipt, + }), + ).rejects.toThrow(/does not match the exact validated create receipt/u); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + expect(fake.events).not.toContain(`rm:${OLD_ID}`); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts new file mode 100644 index 00000000000..b463a820051 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -0,0 +1,2967 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { + dockerRename as defaultDockerRename, + dockerRm as defaultDockerRm, + dockerStart as defaultDockerStart, + dockerStop as defaultDockerStop, +} from "../../adapters/docker/container"; +import { + dockerCapture as defaultDockerCapture, + dockerRun as defaultDockerRun, +} from "../../adapters/docker/run"; +import { parseOpenShellSandboxId } from "../../adapters/openshell/sandbox-identity"; +import { hasZeroDockerExitStatus } from "../docker-command-result"; +import { buildDockerGpuCloneRunArgs, dockerContainerName } from "../docker-gpu-patch-clone"; +import { + DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + DOCKER_GPU_PATCH_TIMEOUT_MS, +} from "../docker-gpu-patch-constants"; +import type { + DockerContainerInspect, + DockerGpuPatchDeps, + DockerGpuPatchMode, + DockerGpuPatchModeKind, + DockerUlimit, +} from "../docker-gpu-patch-types"; +import { waitForOpenShellSupervisorReconnect } from "../docker-gpu-supervisor-reconnect"; +import { openshellSandboxCommandEnvValue } from "../docker-startup-command-env"; +import { + isImmutableDockerImageId, + OPENSHELL_MANAGED_BY_LABEL, + OPENSHELL_MANAGED_BY_VALUE, + OPENSHELL_SANDBOX_ID_LABEL, + OPENSHELL_SANDBOX_NAME_LABEL, + queryOpenShellDockerSandboxContainers, +} from "../openshell-docker-sandbox-containers"; +import { cleanupTempDir, secureTempFile } from "../temp-files"; +import { + assertManagedBootstrapIdentity, + assertManagedBootstrapSafeProcessEnvironmentKey, + attachManagedBootstrapRollbackError, + createManagedBootstrapIdentity, + createManagedBootstrapPreparedAuthority, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapAdapter, + ManagedBootstrapCommitStateIndeterminateError, + type ManagedBootstrapCompletionReceipt, + type ManagedBootstrapDiscoveredWorkload, + type ManagedBootstrapDiscoveryInput, + ManagedBootstrapDurableCommitCleanupPendingError, + type ManagedBootstrapDurablePreparationReceipt, + type ManagedBootstrapFinalizationReceipt, + type ManagedBootstrapHeldWorkloadHandle, + type ManagedBootstrapIncompleteCreateCleanupInput, + type ManagedBootstrapObservedSnapshot, + ManagedBootstrapOwnerCleanupRequiredError, + type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapReplacementHandle, + type ManagedBootstrapReplacementOptions, + type ManagedBootstrapSandboxIdentity, + renderManagedBootstrapHeldCommand, +} from "./adapter"; +import { + createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapJournal, + DockerManagedBootstrapJournalAcknowledgementLostError, + type DockerManagedBootstrapJournalStore, + parseDockerManagedBootstrapJournal, + serializeDockerManagedBootstrapJournal, +} from "./docker-journal"; +import { + clearDockerManagedStartupSharedStateCommitReceipt, + DockerManagedStartupSharedStateCommitIndeterminateError, + finalizeDockerManagedStartupSharedState, + probeDockerManagedStartupSharedState, +} from "./docker-shared-state"; +import { + normalizeDockerManagedBootstrapLaunchSpec, + parseDockerManagedBootstrapLaunchSpec, + parseExactDockerContainerInspect, +} from "./docker-spec"; +import { + MANAGED_BOOTSTRAP_COMPLETION_FILE, + MANAGED_BOOTSTRAP_REQUEST_FILE, + parseManagedBootstrapImageCompletion, + serializeManagedBootstrapEnvelope, +} from "./envelope"; + +const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; +const FULL_SHA256_RE = /^sha256:[a-f0-9]{64}$/u; +const MAX_ARGV_BYTES = 128 * 1024; +const MAX_CONTAINER_NAME_LENGTH = 253; +const REQUEST_TEMP_PREFIX = "nemoclaw-managed-bootstrap-request"; +const COMPLETION_TEMP_PREFIX = "nemoclaw-managed-bootstrap-completion"; +const COMPLETION_MAX_BYTES = 4096; +const DOCKER_DRIVER_ID = "docker"; + +export const MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE = "/usr/local/bin/nemoclaw-managed-bootstrap"; + +type DockerCommandResult = { + readonly status?: number | null; + readonly stdout?: string | Buffer | null; + readonly stderr?: string | Buffer | null; + readonly error?: Error | null; +}; + +export type DockerManagedBootstrapDeps = Pick< + DockerGpuPatchDeps, + | "dockerCapture" + | "dockerRename" + | "dockerRm" + | "dockerRun" + | "dockerStart" + | "dockerStop" + | "runCaptureOpenshell" + | "runOpenshell" + | "sleep" + | "now" +> & { + readonly createBootstrapIdentity?: () => string; + readonly journalStore?: DockerManagedBootstrapJournalStore; + /** Canonical gateway-scoped state root; required when no store is injected. */ + readonly stateRoot?: string; +}; + +type ResolvedDeps = Required< + Pick< + DockerManagedBootstrapDeps, + | "dockerCapture" + | "dockerRename" + | "dockerRm" + | "dockerRun" + | "dockerStart" + | "dockerStop" + | "journalStore" + | "now" + | "createBootstrapIdentity" + > +> & + DockerManagedBootstrapDeps; + +type DockerBootstrapTransaction = DockerManagedBootstrapJournal; + +interface DockerBootstrapRollbackTombstone { + readonly profileFingerprint: string; + readonly imageReference: string; + readonly receipt: ManagedBootstrapFinalizationReceipt; +} + +export interface DockerManagedBootstrapAdapter extends ManagedBootstrapAdapter {} + +function resolveDeps(deps: DockerManagedBootstrapDeps): ResolvedDeps { + const journalStore = + deps.journalStore ?? + (deps.stateRoot ? createFileDockerManagedBootstrapJournalStore(deps.stateRoot) : null); + if (!journalStore) { + throw new Error( + "Managed bootstrap Docker requires its canonical state root or an injected journal store.", + ); + } + return { + dockerCapture: defaultDockerCapture, + dockerRename: defaultDockerRename, + dockerRm: defaultDockerRm, + dockerRun: defaultDockerRun, + dockerStart: defaultDockerStart, + dockerStop: defaultDockerStop, + journalStore, + now: () => new Date(), + createBootstrapIdentity: createManagedBootstrapIdentity, + ...deps, + }; +} + +function commandDetail(result: DockerCommandResult): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}` + .trim() + .slice(-1200); +} + +function isExactMissingDockerContainer(containerId: string, result: DockerCommandResult): boolean { + const escapedContainerId = containerId.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const patterns = [ + new RegExp( + `^(?:Error response from daemon: )?No such (?:container|object): ${escapedContainerId}$`, + "u", + ), + new RegExp(`^Error: No such (?:container|object): ${escapedContainerId}$`, "u"), + ]; + return [result.stderr, result.stdout, result.error?.message] + .map((value) => String(value ?? "").trim()) + .filter(Boolean) + .some((detail) => patterns.some((pattern) => pattern.test(detail))); +} + +function probeExactDockerContainerAbsence( + containerId: string, + deps: ResolvedDeps, +): "absent" | "present" | "unknown" { + let result: DockerCommandResult; + try { + result = deps.dockerRun(["inspect", "--type", "container", containerId], { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + } catch { + return "unknown"; + } + if (hasZeroDockerExitStatus(result)) return "present"; + return isExactMissingDockerContainer(containerId, result) ? "absent" : "unknown"; +} + +function assertZero(result: DockerCommandResult, message: string): void { + if (!hasZeroDockerExitStatus(result)) { + throw new Error(`${message}: ${commandDetail(result) || "Docker command failed"}`); + } +} + +function exactStringArray(value: unknown, label: string): string[] { + if (value === null || value === undefined) return []; + const values = typeof value === "string" ? [value] : value; + if ( + !Array.isArray(values) || + values.some( + (item) => + typeof item !== "string" || + item.length === 0 || + item.includes("\0") || + Buffer.byteLength(item, "utf8") > 64 * 1024, + ) + ) { + throw new Error(`Managed bootstrap Docker ${label} is not an exact bounded argv.`); + } + const result = [...values]; + if (Buffer.byteLength(JSON.stringify(result), "utf8") > MAX_ARGV_BYTES) { + throw new Error(`Managed bootstrap Docker ${label} exceeds its bounded argv transport.`); + } + return result; +} + +function exactSupervisorArgv(inspect: DockerContainerInspect): readonly string[] { + const argv = [ + ...exactStringArray(inspect.Config?.Entrypoint, "entrypoint"), + ...exactStringArray(inspect.Config?.Cmd, "command"), + ]; + if (argv.length === 0 || !argv[0]?.startsWith("/")) { + throw new Error( + "Managed bootstrap requires one bounded absolute supervisor argv from Docker inspect.", + ); + } + return Object.freeze(argv); +} + +function exactArrayEqual(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + +function envValue(env: readonly string[] | null | undefined, key: string): string | null { + const prefix = `${key}=`; + const matches = (env ?? []).filter((value) => value.startsWith(prefix)); + return matches.length === 1 ? (matches[0]?.slice(prefix.length) ?? null) : null; +} + +function assertNoRootProcessInjectionEnvironment(env: readonly string[] | null | undefined): void { + for (const entry of env ?? []) { + const separator = entry.indexOf("="); + const key = separator < 0 ? entry : entry.slice(0, separator); + try { + assertManagedBootstrapSafeProcessEnvironmentKey(key); + } catch { + throw new Error(`Managed bootstrap refuses root-process injection environment '${key}'.`); + } + } +} + +function assertRootSupervisor(inspect: DockerContainerInspect): void { + const user = String(inspect.Config?.User ?? "") + .trim() + .toLowerCase(); + if (!["", "0", "0:0", "root", "root:root"].includes(user)) { + throw new Error("Managed bootstrap Docker workload must retain a root supervisor user."); + } +} + +function isStableRunning(inspect: DockerContainerInspect): boolean { + return inspect.State?.Running !== true || + inspect.State.Paused === true || + inspect.State.Restarting === true || + inspect.State.Dead === true + ? false + : true; +} + +function assertStableRunning(inspect: DockerContainerInspect, label: string): void { + if (!isStableRunning(inspect)) { + throw new Error(`Managed bootstrap Docker ${label} is not stably running.`); + } +} + +function isExplicitlyStopped(inspect: DockerContainerInspect): boolean { + return ( + inspect.State?.Running === false && + inspect.State.Paused === false && + inspect.State.Restarting === false && + inspect.State.Dead === false + ); +} + +function assertExplicitlyStopped(inspect: DockerContainerInspect, label: string): void { + if (!isExplicitlyStopped(inspect)) { + throw new Error(`Managed bootstrap Docker ${label} is not explicitly stopped.`); + } +} + +function expectedImageReference(repository: string, manifestDigest: string): string { + if ( + repository.length === 0 || + repository !== repository.trim() || + repository.includes("@") || + repository.includes("\0") || + !FULL_SHA256_RE.test(manifestDigest) + ) { + throw new Error("Managed bootstrap image repository/manifest identity is invalid."); + } + return `${repository}@${manifestDigest}`; +} + +function assertImage( + inspect: DockerContainerInspect, + image: ManagedBootstrapHeldWorkloadHandle["plan"]["image"], + deps: ResolvedDeps, +): string { + const runtimeContentId = String(inspect.Image ?? "").toLowerCase(); + if (!FULL_SHA256_RE.test(runtimeContentId)) { + throw new Error("Managed bootstrap Docker image does not have an immutable local content ID."); + } + const expectedReference = expectedImageReference(image.repository, image.manifestDigest); + const configuredImage = String(inspect.Config?.Image ?? "").trim(); + if (configuredImage !== expectedReference) { + throw new Error( + "Managed bootstrap Docker configured image is not the exact repository@manifestDigest.", + ); + } + const imageOutput = deps.dockerCapture(["image", "inspect", expectedReference], { + ignoreError: false, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + let parsed: unknown; + try { + parsed = JSON.parse(imageOutput); + } catch { + throw new Error("Managed bootstrap Docker image evidence is malformed."); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("Managed bootstrap Docker image evidence is not exact."); + } + const evidence = parsed[0] as { + readonly Id?: unknown; + readonly RepoDigests?: unknown; + }; + const evidenceId = String(evidence.Id ?? "").toLowerCase(); + const repoDigests = Array.isArray(evidence.RepoDigests) + ? evidence.RepoDigests.filter((value): value is string => typeof value === "string") + : []; + if (evidenceId !== runtimeContentId || !repoDigests.includes(expectedReference)) { + throw new Error( + "Managed bootstrap Docker image manifest evidence does not match its local content ID.", + ); + } + return runtimeContentId; +} + +function assertMetadata( + inspect: DockerContainerInspect, + sandbox: ManagedBootstrapHeldWorkloadHandle["sandbox"], + metadata: Readonly>, +): void { + const labels = inspect.Config?.Labels ?? {}; + if ( + labels[OPENSHELL_MANAGED_BY_LABEL] !== OPENSHELL_MANAGED_BY_VALUE || + labels[OPENSHELL_SANDBOX_NAME_LABEL] !== sandbox.sandboxName || + labels[OPENSHELL_SANDBOX_ID_LABEL] !== sandbox.sandboxId + ) { + throw new Error( + "Managed bootstrap Docker workload does not match the durable OpenShell sandbox identity.", + ); + } + for (const [key, value] of Object.entries(metadata)) { + if (labels[key] !== value) { + throw new Error(`Managed bootstrap Docker metadata label '${key}' changed.`); + } + } +} + +function assertHeldCommand( + inspect: DockerContainerInspect, + heldWorkloadArgv: readonly string[], + bootstrapIdentity: string, +): void { + assertManagedBootstrapIdentity(bootstrapIdentity); + const expected = openshellSandboxCommandEnvValue(heldWorkloadArgv); + const observed = envValue(inspect.Config?.Env, "OPENSHELL_SANDBOX_COMMAND"); + if (!expected || observed !== expected) { + throw new Error( + "Managed bootstrap Docker workload does not contain the exact identity-bound hold.", + ); + } + const identityIndexes = heldWorkloadArgv + .map((value, index) => (value === bootstrapIdentity ? index : -1)) + .filter((index) => index >= 0); + if (identityIndexes.length !== 1) { + throw new Error("Managed bootstrap hold does not contain exactly one bootstrap identity."); + } +} + +function assertBootstrapIdentityInObservedHold( + inspect: DockerContainerInspect, + bootstrapIdentity: string, +): void { + assertManagedBootstrapIdentity(bootstrapIdentity); + const observed = envValue(inspect.Config?.Env, "OPENSHELL_SANDBOX_COMMAND"); + if (!observed) { + throw new Error("Managed bootstrap Docker workload is missing its held command."); + } + const occurrences = observed.split(bootstrapIdentity).length - 1; + if (occurrences !== 1) { + throw new Error( + "Managed bootstrap Docker held command does not contain one exact bootstrap identity.", + ); + } +} + +function inspectExact(containerId: string, deps: ResolvedDeps): DockerContainerInspect { + if (!FULL_CONTAINER_ID_RE.test(containerId)) { + throw new Error("Managed bootstrap requires one full lowercase Docker container ID."); + } + const output = deps.dockerCapture(["inspect", "--type", "container", containerId], { + ignoreError: false, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + const inspect = parseExactDockerContainerInspect(output); + if (String(inspect.Id ?? "").toLowerCase() !== containerId) { + throw new Error("Managed bootstrap Docker workload identity changed during inspection."); + } + return inspect; +} + +function inspectDockerContainerReference( + reference: string, + deps: ResolvedDeps, +): DockerContainerInspect { + if ( + reference.length === 0 || + reference !== reference.trim() || + reference.includes("\0") || + Buffer.byteLength(reference, "utf8") > MAX_CONTAINER_NAME_LENGTH + ) { + throw new Error("Managed bootstrap Docker lookup reference is invalid."); + } + const output = deps.dockerCapture(["inspect", "--type", "container", reference], { + ignoreError: false, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + const inspect = parseExactDockerContainerInspect(output); + const runtimeId = String(inspect.Id ?? "").toLowerCase(); + if (!FULL_CONTAINER_ID_RE.test(runtimeId)) { + throw new Error("Managed bootstrap Docker lookup did not resolve one full runtime ID."); + } + return inspect; +} + +function tryInspectExact(containerId: string, deps: ResolvedDeps): DockerContainerInspect | null { + try { + return inspectExact(containerId, deps); + } catch { + return null; + } +} + +function backupName(originalName: string, bootstrapIdentity: string): string { + const suffix = `-nemoclaw-bootstrap-${bootstrapIdentity.slice(0, 20)}`; + return `${originalName.slice(0, Math.max(1, MAX_CONTAINER_NAME_LENGTH - suffix.length))}${suffix}`; +} + +function replacementStagingName(originalName: string, bootstrapIdentity: string): string { + const suffix = `-nemoclaw-staged-${bootstrapIdentity.slice(0, 20)}`; + return `${originalName.slice(0, Math.max(1, MAX_CONTAINER_NAME_LENGTH - suffix.length))}${suffix}`; +} + +function writeProtectedEnvelope( + bootstrapIdentity: string, + request: Parameters[0]["rootApplyRequest"], +): string { + const file = secureTempFile(REQUEST_TEMP_PREFIX, ".json"); + try { + fs.writeFileSync( + file, + serializeManagedBootstrapEnvelope({ bootstrapIdentity, rootApplyRequest: request }), + { encoding: "utf8", flag: "wx", mode: 0o400 }, + ); + fs.chmodSync(file, 0o400); + const stat = fs.lstatSync(file); + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.nlink !== 1 || + (stat.mode & 0o777) !== 0o400 + ) { + throw new Error("Managed bootstrap request source is not one protected 0400 file."); + } + return file; + } catch (error) { + cleanupTempDir(file, REQUEST_TEMP_PREFIX); + throw error; + } +} + +function readProtectedImageCompletion( + replacementRuntimeId: string, + deps: ResolvedDeps, +): ReturnType { + const file = secureTempFile(COMPLETION_TEMP_PREFIX, ".json"); + let descriptor: number | undefined; + try { + const copied = deps.dockerRun( + ["cp", `${replacementRuntimeId}:${MANAGED_BOOTSTRAP_COMPLETION_FILE}`, file], + { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }, + ); + assertZero(copied, "Managed bootstrap could not retrieve its image completion receipt"); + descriptor = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + const before = fs.fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.isSymbolicLink() || + before.nlink !== 1n || + Number(before.mode & 0o777n) !== 0o444 || + before.size < 1n || + before.size > BigInt(COMPLETION_MAX_BYTES) + ) { + throw new Error("Managed bootstrap image completion is not one protected bounded 0444 file."); + } + const bytes = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < bytes.length) { + const count = fs.readSync(descriptor, bytes, offset, bytes.length - offset, offset); + if (count === 0) break; + offset += count; + } + const after = fs.fstatSync(descriptor, { bigint: true }); + if ( + offset !== bytes.length || + after.dev !== before.dev || + after.ino !== before.ino || + after.size !== before.size || + after.mtimeNs !== before.mtimeNs || + after.ctimeNs !== before.ctimeNs || + after.mode !== before.mode || + after.nlink !== before.nlink + ) { + throw new Error("Managed bootstrap image completion changed during stable read."); + } + return parseManagedBootstrapImageCompletion(bytes.toString("utf8")); + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + cleanupTempDir(file, COMPLETION_TEMP_PREFIX); + } +} + +function parseRequiredUlimits(value: unknown): DockerUlimit[] { + if (value === undefined) return []; + if ( + !Array.isArray(value) || + value.some((entry) => typeof entry !== "string" || entry.includes("\0")) + ) { + throw new Error("Managed bootstrap Docker requiredUlimits must be string entries."); + } + return value.map((entry) => { + const match = /^([a-z][a-z0-9_]*)=(\d+):(\d+)$/u.exec(entry); + if (!match) { + throw new Error(`Managed bootstrap Docker ulimit '${entry}' is invalid.`); + } + const soft = Number(match[2]); + const hard = Number(match[3]); + if (!Number.isSafeInteger(soft) || !Number.isSafeInteger(hard) || hard < soft) { + throw new Error(`Managed bootstrap Docker ulimit '${entry}' is invalid.`); + } + return { name: match[1] as string, soft, hard }; + }); +} + +function replacementPlan(options: ManagedBootstrapReplacementOptions): { + readonly mode: DockerGpuPatchMode; + readonly requiredUlimits: readonly DockerUlimit[]; + readonly extraGroupGids: readonly string[]; +} { + const allowed = new Set([ + "gpuModeArgs", + "gpuModeDevice", + "gpuModeKind", + "gpuModeLabel", + "extraGroupGids", + "requiredUlimits", + ]); + const unknown = Object.keys(options.values).filter((key) => !allowed.has(key)); + if (unknown.length > 0) { + throw new Error( + `Managed bootstrap Docker replacement options are unsupported: ${unknown.sort().join(", ")}.`, + ); + } + const kind = String(options.values.gpuModeKind ?? "startup-command") as DockerGpuPatchModeKind; + if (!["gpus", "nvidia-runtime", "cdi", "startup-command"].includes(kind)) { + throw new Error(`Managed bootstrap Docker GPU mode '${kind}' is invalid.`); + } + const args = exactStringArray(options.values.gpuModeArgs ?? [], "GPU mode arguments"); + return { + mode: { + kind, + label: String(options.values.gpuModeLabel ?? "managed bootstrap"), + device: String(options.values.gpuModeDevice ?? ""), + args, + }, + extraGroupGids: exactStringArray(options.values.extraGroupGids ?? [], "extra group GIDs").map( + (value) => { + if (!/^\d+$/u.test(value)) { + throw new Error(`Managed bootstrap Docker supplementary group '${value}' is invalid.`); + } + return value; + }, + ), + requiredUlimits: parseRequiredUlimits(options.values.requiredUlimits), + }; +} + +function replacementCommand( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, +): readonly string[] { + return Object.freeze([ + "--agent", + handle.plan.profile.agent, + "--profile-fingerprint", + handle.plan.profile.fingerprint, + "--bootstrap-identity", + handle.bootstrapIdentity, + "--agent-uid", + String(snapshot.agentIdentity.uid), + "--agent-gid", + String(snapshot.agentIdentity.gid), + "--agent-workdir", + snapshot.agentIdentity.workdir, + "--request-file", + MANAGED_BOOTSTRAP_REQUEST_FILE, + "--", + ...snapshot.supervisorArgv, + ]); +} + +function assertReplacementBoundary( + inspect: DockerContainerInspect, + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, +): void { + const entrypoint = exactStringArray(inspect.Config?.Entrypoint, "replacement entrypoint"); + const command = exactStringArray(inspect.Config?.Cmd, "replacement command"); + if ( + !exactArrayEqual(entrypoint, [MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE]) || + !exactArrayEqual(command, replacementCommand(handle, snapshot)) + ) { + throw new Error("Managed bootstrap Docker replacement process boundary changed."); + } + const intended = openshellSandboxCommandEnvValue(handle.intendedWorkloadArgv); + if (envValue(inspect.Config?.Env, "OPENSHELL_SANDBOX_COMMAND") !== intended) { + throw new Error( + "Managed bootstrap Docker replacement did not restore the intended sandbox command.", + ); + } +} + +const REPLACED_GPU_ENV_KEYS = new Set([ + "NVIDIA_DISABLE_REQUIRE", + "NVIDIA_DRIVER_CAPABILITIES", + "NVIDIA_REQUIRE_CUDA", + "NVIDIA_VISIBLE_DEVICES", +]); + +function canonicalObject(text: string): Record { + const value = JSON.parse(text) as unknown; + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("Managed bootstrap normalized Docker spec is not an object."); + } + return value as Record; +} + +function objectField(record: Record, key: string): Record { + const value = record[key]; + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`Managed bootstrap normalized Docker spec is missing ${key}.`); + } + return value as Record; +} + +function exactJson(value: unknown): string { + return JSON.stringify(value ?? null); +} + +function stringSet(value: unknown, label: string): string[] { + const values = exactStringArray(value ?? [], label); + if (new Set(values).size !== values.length) { + throw new Error(`Managed bootstrap Docker ${label} contains duplicate entries.`); + } + return values.sort(); +} + +function assertExactStringSet(observed: unknown, expected: readonly string[], label: string): void { + if (!exactArrayEqual(stringSet(observed, label), [...expected].sort())) { + throw new Error(`Managed bootstrap Docker ${label} changed outside declared deltas.`); + } +} + +function modeEnvironment(mode: DockerGpuPatchMode): string[] { + const values: string[] = []; + for (let index = 0; index < mode.args.length; index += 1) { + if (mode.args[index] === "--env") { + const value = mode.args[index + 1]; + if (!value || !value.includes("=")) { + throw new Error("Managed bootstrap Docker GPU mode has an invalid environment delta."); + } + values.push(value); + index += 1; + } + } + return values; +} + +function assertExactEnvironmentDelta( + original: Record, + replacement: Record, + mode: DockerGpuPatchMode, + intendedSandboxCommand: string, +): void { + const gpuAugment = mode.kind !== "startup-command"; + const originalEnv = exactStringArray(original.Env ?? [], "original environment"); + const expected = [ + ...modeEnvironment(mode), + ...originalEnv + .filter((entry) => !gpuAugment || !REPLACED_GPU_ENV_KEYS.has(entry.split("=", 1)[0] ?? "")) + .map((entry) => + entry.startsWith("OPENSHELL_SANDBOX_COMMAND=") + ? `OPENSHELL_SANDBOX_COMMAND=${intendedSandboxCommand}` + : entry, + ), + ]; + const observed = exactStringArray(replacement.Env ?? [], "replacement environment"); + if (!exactArrayEqual(observed, expected)) { + throw new Error( + "Managed bootstrap Docker replacement environment changed outside declared deltas.", + ); + } +} + +function canonicalUlimits(value: unknown, label: string): string { + if (!Array.isArray(value)) { + if (value === undefined || value === null) return "[]"; + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + const normalized = value.map((entry) => { + if (typeof entry !== "object" || entry === null || Array.isArray(entry)) { + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + const record = entry as Record; + const name = String(record.Name ?? ""); + const soft = record.Soft; + const hard = record.Hard; + if (!name || !Number.isSafeInteger(soft) || !Number.isSafeInteger(hard)) { + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + return { Hard: hard, Name: name, Soft: soft }; + }); + if (new Set(normalized.map((entry) => entry.Name)).size !== normalized.length) { + throw new Error(`Managed bootstrap Docker ${label} contains duplicate entries.`); + } + return JSON.stringify(normalized.sort((left, right) => left.Name.localeCompare(right.Name))); +} + +function expectedUlimits(original: unknown, required: readonly DockerUlimit[]): string { + const existing = JSON.parse(canonicalUlimits(original, "original ulimits")) as Array<{ + Hard: number; + Name: string; + Soft: number; + }>; + const merged = new Map(existing.map((entry) => [entry.Name, entry])); + for (const requiredEntry of required) { + merged.set(requiredEntry.name, { + Name: requiredEntry.name, + Soft: requiredEntry.soft, + Hard: requiredEntry.hard, + }); + } + return JSON.stringify( + [...merged.values()].sort((left, right) => left.Name.localeCompare(right.Name)), + ); +} + +function assertExactDeviceRequests( + original: unknown, + observed: unknown, + mode: DockerGpuPatchMode, +): void { + if (mode.kind === "startup-command") { + if (exactJson(observed) !== exactJson(original)) { + throw new Error("Managed bootstrap Docker device requests were not preserved exactly."); + } + return; + } + if (Array.isArray(original) && original.length > 0) { + throw new Error( + "Managed bootstrap Docker GPU augmentation cannot replace an existing device request.", + ); + } + const requests = Array.isArray(observed) ? observed : []; + if (mode.kind === "nvidia-runtime") { + if (requests.length !== 0) { + throw new Error( + "Managed bootstrap Docker NVIDIA runtime added an undeclared device request.", + ); + } + return; + } + if (requests.length !== 1 || typeof requests[0] !== "object" || requests[0] === null) { + throw new Error("Managed bootstrap Docker GPU mode did not add one exact device request."); + } + const request = requests[0] as Record; + if (mode.kind === "gpus") { + const all = mode.device === "all"; + const expectedIds = all ? [] : [mode.device]; + const ids = Array.isArray(request.DeviceIDs) ? request.DeviceIDs : []; + if ( + String(request.Driver ?? "") !== "" || + Number(request.Count) !== (all ? -1 : 0) || + !exactArrayEqual(ids.map(String), expectedIds) || + exactJson(request.Capabilities) !== JSON.stringify([["gpu"]]) || + exactJson(request.Options ?? {}) !== "{}" + ) { + throw new Error("Managed bootstrap Docker --gpus request changed outside its exact delta."); + } + return; + } + const ids = Array.isArray(request.DeviceIDs) ? request.DeviceIDs.map(String) : []; + if ( + request.Driver !== "cdi" || + ![-1, 0].includes(Number(request.Count ?? 0)) || + !exactArrayEqual(ids, [mode.device]) || + (request.Capabilities != null && + (!Array.isArray(request.Capabilities) || request.Capabilities.length > 0)) || + exactJson(request.Options ?? {}) !== "{}" + ) { + throw new Error("Managed bootstrap Docker CDI request changed outside its exact delta."); + } +} + +function scrubVerifiedReplacementDeltas(canonicalJson: string): string { + const root = canonicalObject(canonicalJson); + const inspect = objectField(root, "inspect"); + const config = objectField(inspect, "Config"); + const host = objectField(inspect, "HostConfig"); + config.Image = ""; + config.Entrypoint = [""]; + config.Cmd = [""]; + config.Env = ""; + for (const key of [ + "CapAdd", + "DeviceRequests", + "Devices", + "GroupAdd", + "Runtime", + "SecurityOpt", + "Ulimits", + ]) { + host[key] = ``; + } + return JSON.stringify(root); +} + +function assertReplacementMatchesIntent( + originalCanonicalJson: string, + replacement: DockerContainerInspect, + authoritativeName: string, + plan: { + readonly mode: DockerGpuPatchMode; + readonly requiredUlimits: readonly DockerUlimit[]; + readonly extraGroupGids: readonly string[]; + }, + intendedSandboxCommand: string, +): string { + const original = canonicalObject(originalCanonicalJson); + const originalInspect = objectField(original, "inspect"); + const originalConfig = objectField(originalInspect, "Config"); + const originalHost = objectField(originalInspect, "HostConfig"); + const replacementSpec = normalizeDockerManagedBootstrapLaunchSpec({ + ...replacement, + Name: `/${authoritativeName}`, + }); + const observed = canonicalObject(replacementSpec.canonicalJson); + const observedInspect = objectField(observed, "inspect"); + const observedConfig = objectField(observedInspect, "Config"); + const observedHost = objectField(observedInspect, "HostConfig"); + const gpuAugment = plan.mode.kind !== "startup-command"; + assertExactEnvironmentDelta(originalConfig, observedConfig, plan.mode, intendedSandboxCommand); + assertExactStringSet( + observedHost.CapAdd, + [ + ...stringSet(originalHost.CapAdd, "original capability additions"), + ...(gpuAugment ? ["SYS_PTRACE"] : []), + ].filter((value, index, values) => values.indexOf(value) === index), + "capability additions", + ); + const originalSecurity = stringSet(originalHost.SecurityOpt, "original security options"); + assertExactStringSet( + observedHost.SecurityOpt, + [ + ...originalSecurity, + ...(gpuAugment && !originalSecurity.some((value) => value.startsWith("apparmor")) + ? ["apparmor=unconfined"] + : []), + ], + "security options", + ); + if (exactJson(observedHost.Devices) !== exactJson(originalHost.Devices)) { + throw new Error("Managed bootstrap Docker non-GPU devices were not preserved exactly."); + } + assertExactDeviceRequests(originalHost.DeviceRequests, observedHost.DeviceRequests, plan.mode); + const expectedRuntime = plan.mode.kind === "nvidia-runtime" ? "nvidia" : originalHost.Runtime; + if (exactJson(observedHost.Runtime) !== exactJson(expectedRuntime)) { + throw new Error("Managed bootstrap Docker runtime changed outside its selected GPU delta."); + } + assertExactStringSet( + observedHost.GroupAdd, + [ + ...stringSet(originalHost.GroupAdd, "original supplementary groups"), + ...plan.extraGroupGids, + ].filter((value, index, values) => values.indexOf(value) === index), + "supplementary groups", + ); + if ( + canonicalUlimits(observedHost.Ulimits, "replacement ulimits") !== + expectedUlimits(originalHost.Ulimits, plan.requiredUlimits) + ) { + throw new Error("Managed bootstrap Docker ulimits changed outside declared requirements."); + } + const expectedPreserved = scrubVerifiedReplacementDeltas(originalCanonicalJson); + const observedPreserved = scrubVerifiedReplacementDeltas(replacementSpec.canonicalJson); + if (observedPreserved !== expectedPreserved) { + throw new Error( + "Managed bootstrap Docker replacement normalized spec changed outside declared deltas.", + ); + } + return replacementSpec.hash; +} + +function inspectTransactionRuntime( + transaction: DockerBootstrapTransaction, + runtimeId: string, + deps: ResolvedDeps, +): DockerContainerInspect | null { + const presence = probeExactDockerContainerAbsence(runtimeId, deps); + if (presence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId, + detail: "exact Docker runtime presence could not be proven before mutation", + }); + } + if (presence === "absent") return null; + try { + return inspectExact(runtimeId, deps); + } catch (error) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId, + detail: `exact Docker runtime inspection became unavailable: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + } +} + +function assertTransactionOriginal( + transaction: DockerBootstrapTransaction, + inspect: DockerContainerInspect, +): void { + const name = dockerContainerName(inspect); + if (name !== transaction.originalName && name !== transaction.backupName) { + throw new Error("Managed bootstrap original container has an unexpected transaction name."); + } + const normalized = normalizeDockerManagedBootstrapLaunchSpec({ + ...inspect, + Name: `/${transaction.originalName}`, + }); + if (normalized.hash !== transaction.originalSpecHash) { + throw new Error( + "Managed bootstrap refused mutation because the exact original launch spec changed.", + ); + } +} + +function assertTransactionReplacement( + transaction: DockerBootstrapTransaction, + inspect: DockerContainerInspect, +): void { + const name = dockerContainerName(inspect); + if (name !== transaction.replacementStagingName && name !== transaction.originalName) { + throw new Error("Managed bootstrap replacement container has an unexpected transaction name."); + } + const normalized = normalizeDockerManagedBootstrapLaunchSpec({ + ...inspect, + Name: `/${transaction.originalName}`, + }); + if (normalized.hash !== transaction.replacementSpecHash) { + throw new Error( + "Managed bootstrap refused mutation because the exact replacement launch spec changed.", + ); + } +} + +function assertCompletedCutoverRuntimeState( + transaction: DockerBootstrapTransaction, + deps: ResolvedDeps, +): void { + const original = inspectTransactionRuntime(transaction, transaction.originalRuntimeId, deps); + const replacement = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (!original || !replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: original ? transaction.replacementRuntimeId : transaction.originalRuntimeId, + detail: "completed cutover requires both exact transaction runtimes", + }); + } + assertTransactionOriginal(transaction, original); + assertTransactionReplacement(transaction, replacement); + assertExplicitlyStopped(original, "rollback backup"); + assertStableRunning(replacement, "replacement"); + if ( + dockerContainerName(original) !== transaction.backupName || + dockerContainerName(replacement) !== transaction.originalName + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.replacementRuntimeId, + detail: "completed cutover runtime names do not match durable authority", + }); + } +} + +function removeExactReplacement( + transaction: DockerBootstrapTransaction, + replacement: DockerContainerInspect, + deps: ResolvedDeps, +): void { + assertTransactionReplacement(transaction, replacement); + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + if (replacement.State?.Running === true) { + const stopped = deps.dockerStop(transaction.replacementRuntimeId, { + ...options, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopped)) { + const afterStop = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (!afterStop || afterStop.State?.Running === true) { + throw new Error( + `Managed bootstrap could not quiesce its exact replacement: ${ + commandDetail(stopped) || "Docker stop failed" + }`, + ); + } + assertTransactionReplacement(transaction, afterStop); + } + } + const removed = deps.dockerRm(transaction.replacementRuntimeId, options); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(transaction.replacementRuntimeId, deps) !== "absent" + ) { + throw new Error( + `Managed bootstrap could not remove its exact replacement: ${ + commandDetail(removed) || "Docker removal failed" + }`, + ); + } +} + +function restoreOriginal(transaction: DockerBootstrapTransaction, deps: ResolvedDeps): void { + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + const originalBeforeReplacementRemoval = inspectTransactionRuntime( + transaction, + transaction.originalRuntimeId, + deps, + ); + if (!originalBeforeReplacementRemoval) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "the exact rollback original is absent", + }); + } + assertTransactionOriginal(transaction, originalBeforeReplacementRemoval); + const replacement = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (replacement) { + removeExactReplacement(transaction, replacement, deps); + } + const original = inspectExact(transaction.originalRuntimeId, deps); + assertTransactionOriginal(transaction, original); + const currentName = dockerContainerName(original); + if (currentName !== transaction.originalName) { + if (currentName !== transaction.backupName) { + throw new Error("Managed bootstrap original container has an unexpected rollback name."); + } + const renamed = deps.dockerRename( + transaction.originalRuntimeId, + transaction.originalName, + options, + ); + if (!hasZeroDockerExitStatus(renamed)) { + const afterRename = inspectTransactionRuntime( + transaction, + transaction.originalRuntimeId, + deps, + ); + if (!afterRename || dockerContainerName(afterRename) !== transaction.originalName) { + throw new Error( + `Managed bootstrap could not restore the original container name: ${ + commandDetail(renamed) || "Docker rename failed" + }`, + ); + } + assertTransactionOriginal(transaction, afterRename); + } + } + const restoredBeforeStart = inspectExact(transaction.originalRuntimeId, deps); + if (restoredBeforeStart.State?.Running !== true) { + const started = deps.dockerStart(transaction.originalRuntimeId, options); + if (!hasZeroDockerExitStatus(started)) { + const afterStart = inspectTransactionRuntime( + transaction, + transaction.originalRuntimeId, + deps, + ); + if (!afterStart || afterStart.State?.Running !== true) { + throw new Error( + `Managed bootstrap could not restart the original container: ${ + commandDetail(started) || "Docker start failed" + }`, + ); + } + assertTransactionOriginal(transaction, afterStart); + } + } + const restored = inspectExact(transaction.originalRuntimeId, deps); + assertStableRunning(restored, "restored workload"); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(restored); + if (normalized.hash !== transaction.originalSpecHash) { + throw new Error("Managed bootstrap rollback did not restore the exact launch spec."); + } +} + +function removeOwnedWorkload( + sandbox: ManagedBootstrapSandboxIdentity, + deps: ResolvedDeps, + expectedRuntimeId?: string, +): never { + const expectedIdentity = + expectedRuntimeId === undefined + ? `sandbox ${sandbox.sandboxId} with no previously resolved runtime ID` + : `sandbox ${sandbox.sandboxId} expected runtime ${expectedRuntimeId}`; + let containers: DockerCommandResult; + try { + containers = deps.dockerRun( + [ + "ps", + "-a", + "--no-trunc", + "--filter", + `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, + "--filter", + `label=${OPENSHELL_SANDBOX_ID_LABEL}=${sandbox.sandboxId}`, + "--format", + "{{.ID}}", + ], + { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }, + ); + } catch (error) { + throw new Error( + `Managed bootstrap owner cleanup could not enumerate the exact held runtime for ${expectedIdentity}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + if (Number(containers.status ?? 1) !== 0) { + throw new Error( + `Managed bootstrap owner cleanup could not verify the exact held runtime for ${expectedIdentity}: ${ + commandDetail(containers) || "Docker enumeration failed" + }`, + ); + } + const runtimeIds = String(containers.stdout ?? "") + .trim() + .split(/\r?\n/u) + .map((value) => value.trim().toLowerCase()) + .filter(Boolean); + if ( + runtimeIds.length !== 1 || + !FULL_CONTAINER_ID_RE.test(runtimeIds[0] ?? "") || + (expectedRuntimeId !== undefined && runtimeIds[0] !== expectedRuntimeId) + ) { + throw new Error( + `Managed bootstrap owner cleanup could not bind retention for ${expectedIdentity}; resolved runtime IDs: ${ + runtimeIds.length === 0 ? "none" : runtimeIds.join(", ") + }.`, + ); + } + const runtimeId = runtimeIds[0] as string; + let inspect: DockerContainerInspect; + try { + inspect = inspectExact(runtimeId, deps); + } catch (error) { + throw new Error( + `Managed bootstrap could not inspect retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + const labels = inspect.Config?.Labels ?? {}; + if ( + labels[OPENSHELL_MANAGED_BY_LABEL] !== OPENSHELL_MANAGED_BY_VALUE || + labels[OPENSHELL_SANDBOX_NAME_LABEL] !== sandbox.sandboxName || + labels[OPENSHELL_SANDBOX_ID_LABEL] !== sandbox.sandboxId + ) { + throw new Error( + `Managed bootstrap owner cleanup refused retention after exact runtime ${runtimeId} ownership changed for sandbox ${sandbox.sandboxId}.`, + ); + } + let stopped: DockerCommandResult; + try { + stopped = deps.dockerStop(runtimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + } catch (error) { + throw new Error( + `Managed bootstrap could not quiesce retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + assertZero( + stopped, + `Managed bootstrap could not quiesce retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}`, + ); + let retained: DockerContainerInspect; + try { + retained = inspectExact(runtimeId, deps); + } catch (error) { + throw new Error( + `Managed bootstrap could not re-inspect quiesced sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + if ( + retained.State?.Running !== false || + retained.State.Paused !== false || + retained.State.Restarting !== false + ) { + throw new Error( + `Managed bootstrap retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId} did not prove an explicitly quiescent state.`, + ); + } + if (!deps.runCaptureOpenshell) { + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + }); + } + let getBeforeDelete: string; + try { + getBeforeDelete = deps.runCaptureOpenshell(["sandbox", "get", sandbox.sandboxName], { + ignoreError: false, + }); + } catch (error) { + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + detail: `OpenShell owner lookup also failed: ${ + error instanceof Error ? error.message : String(error) + }.`, + }); + } + const sandboxIdBeforeDelete = parseOpenShellSandboxId(getBeforeDelete); + if (sandboxIdBeforeDelete !== sandbox.sandboxId) { + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + detail: `The same mutable name now resolves to durable sandbox ID ${ + sandboxIdBeforeDelete ?? "unknown" + } instead of ${sandbox.sandboxId}.`, + }); + } + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + }); +} + +function resolveIncompleteCreateSandbox( + input: ManagedBootstrapIncompleteCreateCleanupInput, + deps: ResolvedDeps, +): { + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly runtimeId: string; +} { + if ( + input.plan.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + input.plan.driverId !== DOCKER_DRIVER_ID + ) { + throw new Error("Managed bootstrap Docker incomplete-create cleanup received another driver."); + } + assertManagedBootstrapIdentity(input.bootstrapIdentity); + const query = queryOpenShellDockerSandboxContainers(input.plan.sandboxName, deps); + if (!query.ok) { + throw new Error(`Managed bootstrap Docker incomplete-create discovery failed: ${query.error}`); + } + if (query.ids.length !== 1) { + throw new Error( + `Managed bootstrap incomplete-create cleanup requires exactly one labeled Docker workload; found ${String( + query.ids.length, + )}.`, + ); + } + const runtimeId = String(query.ids[0] ?? "").toLowerCase(); + const inspect = inspectExact(runtimeId, deps); + const sandboxId = String(inspect.Config?.Labels?.[OPENSHELL_SANDBOX_ID_LABEL] ?? ""); + if (parseOpenShellSandboxId(`ID: ${sandboxId}\n`) !== sandboxId) { + throw new Error( + "Managed bootstrap Docker incomplete-create workload has no exact durable sandbox ID.", + ); + } + const sandbox = Object.freeze({ + sandboxName: input.plan.sandboxName, + sandboxId, + driverId: input.plan.driverId, + }); + if ( + input.createReceipt.ready !== true || + input.createReceipt.sandbox.sandboxName !== sandbox.sandboxName || + input.createReceipt.sandbox.sandboxId !== sandbox.sandboxId || + input.createReceipt.sandbox.driverId !== sandbox.driverId + ) { + throw new Error( + "Managed bootstrap Docker incomplete-create workload does not match the exact validated create receipt.", + ); + } + assertImage(inspect, input.plan.image, deps); + assertMetadata(inspect, sandbox, input.plan.metadata); + assertHeldCommand(inspect, input.heldWorkloadArgv, input.bootstrapIdentity); + return { sandbox, runtimeId }; +} + +function managedSharedStateTransaction( + handle: ManagedBootstrapHeldWorkloadHandle, + containerId: string, + image: string, +) { + return { + agent: handle.plan.profile.agent, + bootstrapIdentity: handle.bootstrapIdentity, + containerId, + image, + profileFingerprint: handle.plan.profile.fingerprint, + } as const; +} + +function sameDockerBootstrapJournal( + left: DockerBootstrapTransaction, + right: DockerBootstrapTransaction, +): boolean { + return ( + serializeDockerManagedBootstrapJournal(left) === serializeDockerManagedBootstrapJournal(right) + ); +} + +function createDockerBootstrapJournalDurably( + journal: DockerBootstrapTransaction, + deps: ResolvedDeps, +): DockerBootstrapTransaction { + try { + deps.journalStore.create(journal); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + if (!recovered || !sameDockerBootstrapJournal(recovered, journal)) throw error; + return recovered; + } + const persisted = deps.journalStore.load(journal.bootstrapIdentity); + if (!persisted || !sameDockerBootstrapJournal(persisted, journal)) { + throw new Error("Managed bootstrap Docker staged journal was not durably re-readable."); + } + return persisted; +} + +function transitionDockerBootstrapJournalDurably( + journal: DockerBootstrapTransaction, + next: "cutover" | "rollback-authorized" | "shared-state-committed", + deps: ResolvedDeps, +): DockerBootstrapTransaction { + try { + deps.journalStore.transition(journal.bootstrapIdentity, journal.phase, next); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + const expected = Object.freeze({ ...journal, phase: next }); + if (!recovered || !sameDockerBootstrapJournal(recovered, expected)) throw error; + return recovered; + } + const persisted = deps.journalStore.load(journal.bootstrapIdentity); + const expected = Object.freeze({ ...journal, phase: next }); + if (!persisted || !sameDockerBootstrapJournal(persisted, expected)) { + throw new Error(`Managed bootstrap Docker journal transition to ${next} was not durable.`); + } + return persisted; +} + +function removeDockerBootstrapJournalDurably( + journal: DockerBootstrapTransaction, + deps: ResolvedDeps, +): void { + try { + deps.journalStore.remove(journal.bootstrapIdentity, [journal.phase]); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + if (recovered !== null) throw error; + return; + } + if (deps.journalStore.load(journal.bootstrapIdentity) !== null) { + throw new Error("Managed bootstrap Docker journal removal was not durable."); + } +} + +function assertDockerBootstrapTransactionAuthority( + transaction: DockerBootstrapTransaction, + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared?: ManagedBootstrapPreparedReplacementHandle | null, + replacement?: ManagedBootstrapReplacementHandle | null, +): void { + const originalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, + ); + const expectedSandbox = handle.sandbox; + if ( + transaction.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION || + transaction.bootstrapIdentity !== handle.bootstrapIdentity || + transaction.sandbox.sandboxName !== expectedSandbox.sandboxName || + transaction.sandbox.sandboxId !== expectedSandbox.sandboxId || + transaction.sandbox.driverId !== expectedSandbox.driverId || + transaction.profileFingerprint !== handle.plan.profile.fingerprint || + transaction.imageReference !== + expectedImageReference(snapshot.image.repository, snapshot.image.manifestDigest) || + transaction.runtimeImageContentId !== snapshot.runtimeImageContentId || + transaction.originalRuntimeId !== snapshot.runtimeId || + transaction.originalName !== originalName || + transaction.replacementStagingName !== + replacementStagingName(originalName, handle.bootstrapIdentity) || + transaction.backupName !== backupName(originalName, handle.bootstrapIdentity) || + transaction.originalSpecHash !== snapshot.specHash || + (prepared !== undefined && + prepared !== null && + (transaction.originalRuntimeId !== prepared.originalRuntimeId || + transaction.replacementRuntimeId !== prepared.preparedRuntimeId || + transaction.replacementSpecHash !== prepared.expectedActivatedSpecHash)) || + (replacement !== undefined && + replacement !== null && + (transaction.originalRuntimeId !== replacement.originalRuntimeId || + transaction.replacementRuntimeId !== replacement.replacementRuntimeId || + transaction.replacementSpecHash !== replacement.replacementSpecHash)) + ) { + throw new Error( + "Managed bootstrap receipts do not match the durable Docker transaction authority.", + ); + } +} + +function transactionFromPreparedAuthority( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, +): DockerBootstrapTransaction { + const transaction = parseDockerManagedBootstrapJournal(prepared.rollbackAuthority); + if (transaction.phase !== "staged") { + throw new Error("Managed bootstrap Docker prepared authority must describe a staged runtime."); + } + assertDockerBootstrapTransactionAuthority(transaction, handle, snapshot, prepared); + return transaction; +} + +function assertDurablePreparationAuthority( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, + receipt: ManagedBootstrapDurablePreparationReceipt, +): void { + const authority = createManagedBootstrapPreparedAuthority({ handle, snapshot, prepared }); + const recordedAt = new Date(receipt.recordedAt); + if ( + receipt.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + receipt.sandbox.sandboxName !== authority.sandbox.sandboxName || + receipt.sandbox.sandboxId !== authority.sandbox.sandboxId || + receipt.sandbox.driverId !== authority.sandbox.driverId || + receipt.bootstrapIdentity !== authority.bootstrapIdentity || + receipt.authorityFingerprint !== authority.authorityFingerprint || + typeof receipt.recordId !== "string" || + receipt.recordId.length === 0 || + receipt.recordId.includes("\0") || + typeof receipt.recordedAt !== "string" || + !Number.isFinite(recordedAt.getTime()) || + recordedAt.toISOString() !== receipt.recordedAt + ) { + throw new Error( + "Managed bootstrap Docker activation requires the exact durable prepared-authority receipt.", + ); + } +} + +function reconstructDockerBootstrapTransaction( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + replacement: ManagedBootstrapReplacementHandle, + deps: ResolvedDeps, +): DockerBootstrapTransaction { + if ( + replacement.bootstrapIdentity !== handle.bootstrapIdentity || + replacement.originalRuntimeId !== snapshot.runtimeId || + replacement.originalSpecHash !== snapshot.specHash || + replacement.replacementRuntimeId === replacement.originalRuntimeId + ) { + throw new Error( + "Managed bootstrap finalization receipts do not reconstruct one exact Docker transaction.", + ); + } + const transaction = deps.journalStore.load(handle.bootstrapIdentity); + if (!transaction) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: "the durable Docker cutover journal is absent", + }); + } + assertDockerBootstrapTransactionAuthority(transaction, handle, snapshot, null, replacement); + return transaction; +} + +function rollbackReplacementSharedStateIfPending( + input: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly replacementRuntimeId: string; + readonly runtimeImageContentId: string; + }, + deps: ResolvedDeps, +): void { + if (!tryInspectExact(input.replacementRuntimeId, deps)) { + throw new Error( + "Managed bootstrap replacement disappeared before shared-state rollback could be proven; the preserved original remains stopped.", + ); + } + const transaction = managedSharedStateTransaction( + input.handle, + input.replacementRuntimeId, + input.runtimeImageContentId, + ); + finalizeDockerManagedStartupSharedState( + { transaction, supervisorReady: false, retainContainerAfterRollback: true }, + deps, + ); +} + +function cleanupUnjournaledPreparedContainer( + input: { + readonly snapshot: ManagedBootstrapObservedSnapshot; + readonly preparedRuntimeId: string; + readonly stagingName: string; + }, + deps: ResolvedDeps, +): void { + if (!FULL_CONTAINER_ID_RE.test(input.preparedRuntimeId)) return; + const original = inspectExact(input.snapshot.runtimeId, deps); + const originalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(input.snapshot.specCanonicalJson).inspect, + ); + if ( + !isStableRunning(original) || + dockerContainerName(original) !== originalName || + normalizeDockerManagedBootstrapLaunchSpec(original).hash !== input.snapshot.specHash + ) { + throw new Error( + "Managed bootstrap cannot clean an unjournaled replacement after original drift.", + ); + } + const prepared = tryInspectExact(input.preparedRuntimeId, deps); + if (!prepared) return; + if ( + String(prepared.Id ?? "").toLowerCase() !== input.preparedRuntimeId || + dockerContainerName(prepared) !== input.stagingName || + !isExplicitlyStopped(prepared) + ) { + throw new Error( + "Managed bootstrap refused cleanup because the unjournaled prepared runtime changed.", + ); + } + const removed = deps.dockerRm(input.preparedRuntimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(input.preparedRuntimeId, deps) !== "absent" + ) { + throw new Error( + `Managed bootstrap could not remove its unjournaled prepared runtime: ${ + commandDetail(removed) || "Docker removal failed" + }`, + ); + } +} + +function resolvePreparedRollbackAuthority(input: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot; + readonly prepared: ManagedBootstrapPreparedReplacementHandle | null; + readonly durablePreparation: ManagedBootstrapDurablePreparationReceipt | null; +}): DockerBootstrapTransaction | null { + if (input.durablePreparation && !input.prepared) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: input.handle.bootstrapIdentity, + runtimeId: input.snapshot.runtimeId, + detail: "durable prepared authority is present without its exact prepared handle", + }); + } + if (!input.prepared) return null; + const authority = transactionFromPreparedAuthority(input.handle, input.snapshot, input.prepared); + if (input.durablePreparation) { + assertDurablePreparationAuthority( + input.handle, + input.snapshot, + input.prepared, + input.durablePreparation, + ); + } + return authority; +} + +export function createDockerManagedBootstrapAdapter( + dependencies: DockerManagedBootstrapDeps = {}, +): DockerManagedBootstrapAdapter { + const deps = resolveDeps(dependencies); + const committedTransactions = new Set(); + const rollbackTombstones = new Map(); + const completedRollback = ( + handle: ManagedBootstrapHeldWorkloadHandle, + alreadyRolledBack: boolean, + ): ManagedBootstrapFinalizationReceipt => { + const receipt = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: true, + alreadyRolledBack, + finalizedAt: deps.now().toISOString(), + } satisfies ManagedBootstrapFinalizationReceipt); + rollbackTombstones.set(handle.bootstrapIdentity, { + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + handle.plan.image.repository, + handle.plan.image.manifestDigest, + ), + receipt, + }); + return receipt; + }; + const priorRollback = ( + handle: ManagedBootstrapHeldWorkloadHandle, + ): ManagedBootstrapFinalizationReceipt | null => { + const tombstone = rollbackTombstones.get(handle.bootstrapIdentity); + if (!tombstone) return null; + const receipt = tombstone.receipt; + if ( + receipt.sandbox.sandboxName !== handle.sandbox.sandboxName || + receipt.sandbox.sandboxId !== handle.sandbox.sandboxId || + receipt.sandbox.driverId !== handle.sandbox.driverId || + tombstone.profileFingerprint !== handle.plan.profile.fingerprint || + tombstone.imageReference !== + expectedImageReference(handle.plan.image.repository, handle.plan.image.manifestDigest) + ) { + throw new Error("Managed bootstrap rollback tombstone does not match its durable identity."); + } + return Object.freeze({ + ...receipt, + alreadyRolledBack: true, + }); + }; + const rollbackBootstrapNow = ({ + handle, + snapshot, + prepared, + durablePreparation, + replacement, + sharedStateAlreadyRolledBack = false, + }: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot | null; + readonly prepared: ManagedBootstrapPreparedReplacementHandle | null; + readonly durablePreparation: ManagedBootstrapDurablePreparationReceipt | null; + readonly replacement: ManagedBootstrapReplacementHandle | null; + readonly sharedStateAlreadyRolledBack?: boolean; + }): ManagedBootstrapFinalizationReceipt => { + const finalized = priorRollback(handle); + if (finalized) return finalized; + const journal = deps.journalStore.load(handle.bootstrapIdentity); + if ( + committedTransactions.has(handle.bootstrapIdentity) || + journal?.phase === "shared-state-committed" + ) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: handle.bootstrapIdentity, + cleanupRuntimeId: journal?.originalRuntimeId ?? snapshot?.runtimeId ?? "unknown", + detail: + "rollback is no longer legal after the durable Docker commit fence; retry commit finalization", + }); + } + if (!snapshot) { + if (journal || prepared || durablePreparation || replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal?.originalRuntimeId ?? prepared?.originalRuntimeId ?? "unknown", + detail: "Docker replacement authority exists without its observed snapshot", + }); + } + removeOwnedWorkload(handle.sandbox, deps); + return completedRollback(handle, false); + } + + const preparedAuthority = resolvePreparedRollbackAuthority({ + handle, + snapshot, + prepared, + durablePreparation, + }); + + if (!journal) { + const originalPresence = probeExactDockerContainerAbsence(snapshot.runtimeId, deps); + if (originalPresence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: snapshot.runtimeId, + detail: "the original runtime presence is unknown and no durable journal is available", + }); + } + if (originalPresence === "absent") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: handle.bootstrapIdentity, + cleanupRuntimeId: snapshot.runtimeId, + detail: + "rollback is forbidden because the exact original is absent after journal retirement", + }); + } + const original = inspectExact(snapshot.runtimeId, deps); + const expectedOriginalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, + ); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(original); + if ( + dockerContainerName(original) !== expectedOriginalName || + original.State?.Running !== true || + normalized.hash !== snapshot.specHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: snapshot.runtimeId, + detail: "the journal is absent and the exact original is not a proven restored workload", + }); + } + if (preparedAuthority) { + const observedPrepared = inspectTransactionRuntime( + preparedAuthority, + preparedAuthority.replacementRuntimeId, + deps, + ); + if (observedPrepared) { + assertExplicitlyStopped(observedPrepared, "prepared replacement"); + if ( + dockerContainerName(observedPrepared) !== preparedAuthority.replacementStagingName || + normalizeDockerManagedBootstrapLaunchSpec(observedPrepared).canonicalJson !== + prepared?.preparedSpecCanonicalJson + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: preparedAuthority.replacementRuntimeId, + detail: "the unjournaled prepared runtime changed before exact cleanup", + }); + } + removeExactReplacement(preparedAuthority, observedPrepared, deps); + } + } else if (replacement) { + const replacementPresence = probeExactDockerContainerAbsence( + replacement.replacementRuntimeId, + deps, + ); + if (replacementPresence !== "absent") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: + replacementPresence === "present" + ? "the replacement still exists without durable journal authority" + : "replacement absence is unknown without durable journal authority", + }); + } + } + removeOwnedWorkload(handle.sandbox, deps, snapshot.runtimeId); + return completedRollback(handle, true); + } + + if (!preparedAuthority || !durablePreparation) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker cutover lacks its coordinator-recorded prepared authority", + }); + } + const stagedJournal = Object.freeze({ ...journal, phase: "staged" as const }); + if (!sameDockerBootstrapJournal(stagedJournal, preparedAuthority)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker cutover changed its prepared rollback authority", + }); + } + assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); + const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); + if (!original) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "the exact rollback original is absent", + }); + } + assertTransactionOriginal(journal, original); + const observedReplacement = inspectTransactionRuntime( + journal, + journal.replacementRuntimeId, + deps, + ); + + if (journal.phase === "staged") { + assertStableRunning(original, "staged original"); + if (observedReplacement) { + assertExplicitlyStopped(observedReplacement, "staged replacement"); + } + if ( + dockerContainerName(original) !== journal.originalName || + (observedReplacement !== null && + dockerContainerName(observedReplacement) !== journal.replacementStagingName) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "staged transaction runtime state does not match its pre-cutover fence", + }); + } + if (observedReplacement) { + removeExactReplacement(journal, observedReplacement, deps); + } + removeDockerBootstrapJournalDurably(journal, deps); + removeOwnedWorkload(handle.sandbox, deps, journal.originalRuntimeId); + return completedRollback(handle, false); + } + + if (journal.phase !== "cutover" && journal.phase !== "rollback-authorized") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: journal.bootstrapIdentity, + cleanupRuntimeId: journal.originalRuntimeId, + detail: "rollback is forbidden by the durable Docker commit phase", + }); + } + + const originalNameNow = dockerContainerName(original); + const replacementNameNow = observedReplacement + ? dockerContainerName(observedReplacement) + : null; + const originalAtTargetRecoverable = + originalNameNow === journal.originalName && + (isStableRunning(original) || isExplicitlyStopped(original)); + const originalAtBackupRecoverable = + originalNameNow === journal.backupName && isExplicitlyStopped(original); + const replacementAtStagingRecoverable = + replacementNameNow === journal.replacementStagingName && + observedReplacement !== null && + isExplicitlyStopped(observedReplacement); + const replacementAtTargetRecoverable = + replacementNameNow === journal.originalName && + observedReplacement !== null && + (isStableRunning(observedReplacement) || isExplicitlyStopped(observedReplacement)); + const validCutoverState = + (originalAtTargetRecoverable && replacementAtStagingRecoverable) || + (originalAtBackupRecoverable && replacementAtStagingRecoverable) || + (originalAtBackupRecoverable && replacementAtTargetRecoverable); + let activeJournal = journal; + + if (journal.phase === "cutover") { + if ( + (!sharedStateAlreadyRolledBack && (!observedReplacement || !validCutoverState)) || + (sharedStateAlreadyRolledBack && + (observedReplacement !== null || + !( + (originalNameNow === journal.backupName && isExplicitlyStopped(original)) || + originalAtTargetRecoverable + ))) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: + observedReplacement === null && !sharedStateAlreadyRolledBack + ? "the exact replacement disappeared before rollback authorization was durable" + : "cutover runtime names or states do not match a recoverable phase", + }); + } + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable transaction authority changed before rollback authorization", + }); + } + + let sharedStatus: "committed" | "none" | "pending" = "none"; + const sharedTransaction = managedSharedStateTransaction( + handle, + journal.replacementRuntimeId, + journal.runtimeImageContentId, + ); + if (!sharedStateAlreadyRolledBack) { + sharedStatus = probeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + profileFingerprint: journal.profileFingerprint, + }, + deps, + ); + if (sharedStatus === "committed") { + const committedJournal = transitionDockerBootstrapJournalDurably( + journal, + "shared-state-committed", + deps, + ); + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: committedJournal.bootstrapIdentity, + cleanupRuntimeId: committedJournal.originalRuntimeId, + detail: "image-owned shared state is durably committed; rollback is no longer legal", + }); + } + } + activeJournal = transitionDockerBootstrapJournalDurably(journal, "rollback-authorized", deps); + if (!sharedStateAlreadyRolledBack && sharedStatus === "pending") { + finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: false, + retainContainerAfterRollback: true, + }, + deps, + ); + } + } else { + if (!originalAtTargetRecoverable && !originalAtBackupRecoverable) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "rollback-authorized original runtime state is not recoverable", + }); + } + if ( + observedReplacement && + originalNameNow === journal.originalName && + replacementNameNow === journal.originalName + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "both transaction runtimes claim the authoritative workload name", + }); + } + if (observedReplacement) { + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "rollback authorization changed before replacement cleanup", + }); + } + const sharedTransaction = managedSharedStateTransaction( + handle, + journal.replacementRuntimeId, + journal.runtimeImageContentId, + ); + const sharedStatus = probeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + profileFingerprint: journal.profileFingerprint, + }, + deps, + ); + if (sharedStatus === "committed") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: + "shared state became committed after rollback authorization; no mutation was attempted", + }); + } + if (sharedStatus === "pending") { + finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: false, + retainContainerAfterRollback: true, + }, + deps, + ); + } + } + } + + const beforeRestore = deps.journalStore.load(activeJournal.bootstrapIdentity); + if (!beforeRestore || !sameDockerBootstrapJournal(beforeRestore, activeJournal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "durable transaction authority changed before original restoration", + }); + } + restoreOriginal(activeJournal, deps); + const restored = inspectExact(activeJournal.originalRuntimeId, deps); + assertStableRunning(restored, "restored workload"); + if ( + dockerContainerName(restored) !== activeJournal.originalName || + normalizeDockerManagedBootstrapLaunchSpec(restored).hash !== activeJournal.originalSpecHash + ) { + throw new Error("Managed bootstrap Docker rollback did not restore its exact original."); + } + removeDockerBootstrapJournalDurably(activeJournal, deps); + removeOwnedWorkload(handle.sandbox, deps, activeJournal.originalRuntimeId); + return completedRollback(handle, false); + }; + const commitBootstrapNow = ( + receipt: ManagedBootstrapCompletionReceipt, + transaction: DockerBootstrapTransaction, + input: { + readonly sharedStateStatus: "committed" | "none"; + readonly sharedStateTransaction: ReturnType; + }, + ): void => { + if (committedTransactions.has(receipt.bootstrapIdentity)) return; + if ( + transaction.phase !== "shared-state-committed" || + transaction.replacementRuntimeId !== receipt.runtimeId || + transaction.originalSpecHash !== receipt.originalSpecHash || + transaction.replacementSpecHash !== receipt.replacementSpecHash + ) { + throw new Error("Managed bootstrap Docker commit receipt does not match its commit fence."); + } + const current = deps.journalStore.load(transaction.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, transaction)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "durable commit authority changed before exact cleanup", + }); + } + + const replacement = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (!replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.replacementRuntimeId, + detail: "the exact committed replacement is absent", + }); + } + assertTransactionReplacement(transaction, replacement); + if ( + dockerContainerName(replacement) !== transaction.originalName || + replacement.State?.Running !== true + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.replacementRuntimeId, + detail: "the exact replacement is not running under the authoritative workload name", + }); + } + + const original = inspectTransactionRuntime(transaction, transaction.originalRuntimeId, deps); + if (original) { + assertTransactionOriginal(transaction, original); + if (dockerContainerName(original) !== transaction.backupName) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "the exact rollback backup is not quiescent under its durable backup name", + }); + } + assertExplicitlyStopped(original, "commit rollback backup"); + const beforeRemove = deps.journalStore.load(transaction.bootstrapIdentity); + if (!beforeRemove || !sameDockerBootstrapJournal(beforeRemove, transaction)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "durable commit authority changed before exact rollback-backup removal", + }); + } + const removed = deps.dockerRm(transaction.originalRuntimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(transaction.originalRuntimeId, deps) !== "absent" + ) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: receipt.bootstrapIdentity, + cleanupRuntimeId: transaction.originalRuntimeId, + detail: `${commandDetail(removed) || "Docker removal failed"}; exact backup absence was not proven`, + }); + } + } + + if (input.sharedStateStatus === "committed") { + try { + clearDockerManagedStartupSharedStateCommitReceipt(input.sharedStateTransaction, deps); + } catch (error) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: receipt.bootstrapIdentity, + cleanupRuntimeId: transaction.replacementRuntimeId, + detail: `exact rollback backup is absent, but its image-owned commit receipt could not be retired: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + } + } + removeDockerBootstrapJournalDurably(transaction, deps); + committedTransactions.add(receipt.bootstrapIdentity); + }; + const finalizeBootstrap = async ( + input: Parameters[0], + ): Promise => { + if (input.outcome === "rollback") { + return rollbackBootstrapNow(input); + } + const { completion, durablePreparation, handle, prepared, replacement, snapshot } = input; + if (!completion || !snapshot || !prepared || !durablePreparation || !replacement) { + throw new Error("Managed bootstrap commit requires one complete cutover receipt."); + } + const preparedAuthority = transactionFromPreparedAuthority(handle, snapshot, prepared); + assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); + const sharedTransaction = managedSharedStateTransaction( + handle, + replacement.replacementRuntimeId, + replacement.runtimeImageContentId, + ); + let sharedStatus = probeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + profileFingerprint: completion.profileFingerprint, + }, + deps, + ); + let journal = deps.journalStore.load(handle.bootstrapIdentity); + + if (!journal) { + if (committedTransactions.has(completion.bootstrapIdentity)) { + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + } + const originalPresence = probeExactDockerContainerAbsence(snapshot.runtimeId, deps); + if (originalPresence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: snapshot.runtimeId, + detail: "the retired-journal commit cannot prove exact backup absence", + }); + } + if (originalPresence !== "absent" || sharedStatus !== "none") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: + originalPresence === "absent" ? replacement.replacementRuntimeId : snapshot.runtimeId, + detail: + "the durable journal is absent before both exact backup and shared commit receipt retirement were proven", + }); + } + const committedReplacement = inspectExact(replacement.replacementRuntimeId, deps); + assertStableRunning(committedReplacement, "committed replacement"); + const originalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, + ); + if ( + dockerContainerName(committedReplacement) !== originalName || + normalizeDockerManagedBootstrapLaunchSpec(committedReplacement).hash !== + replacement.replacementSpecHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: "the retired-journal replacement does not match the exact completion receipt", + }); + } + committedTransactions.add(completion.bootstrapIdentity); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + } + + if ( + !sameDockerBootstrapJournal( + Object.freeze({ ...journal, phase: "staged" as const }), + preparedAuthority, + ) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker commit changed its prepared rollback authority", + }); + } + assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); + if (journal.phase === "staged" || journal.phase === "rollback-authorized") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: `commit is forbidden from durable journal phase ${journal.phase}`, + }); + } + if (!completion.transactionPending && sharedStatus !== "none") { + throw new Error( + "Managed bootstrap image completion disagrees with shared-state transaction status.", + ); + } + + if (journal.phase === "cutover") { + if (completion.transactionPending && sharedStatus === "none") { + throw new Error( + "Managed bootstrap image completion lost its shared-state receipt before the durable commit fence.", + ); + } + if (sharedStatus === "pending") { + let outcome; + try { + outcome = finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: true, + retainContainerAfterRollback: true, + }, + deps, + ); + } catch (error) { + if (error instanceof DockerManagedStartupSharedStateCommitIndeterminateError) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: error.message, + }); + } + throw error; + } + if (!outcome.supervisorReady) { + const failure = + outcome.failure ?? new Error("Managed bootstrap shared-state commit did not complete."); + try { + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: + "durable authority changed after shared-state rollback and before restoration", + }); + } + journal = transitionDockerBootstrapJournalDurably(journal, "rollback-authorized", deps); + await rollbackBootstrapNow({ + handle, + snapshot, + prepared, + durablePreparation, + replacement, + sharedStateAlreadyRolledBack: true, + }); + } catch (rollbackError) { + attachManagedBootstrapRollbackError(failure, rollbackError); + } + throw failure; + } + sharedStatus = "committed"; + } + journal = transitionDockerBootstrapJournalDurably(journal, "shared-state-committed", deps); + } else if (completion.transactionPending && sharedStatus === "pending") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "shared state is pending after the durable Docker commit fence", + }); + } + + commitBootstrapNow(completion, journal, { + sharedStateStatus: sharedStatus === "committed" ? "committed" : "none", + sharedStateTransaction: sharedTransaction, + }); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + }; + return { + async createHeldWorkload(input) { + if ( + input.plan.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + input.plan.driverId !== DOCKER_DRIVER_ID || + input.request.agent !== input.plan.profile.agent || + input.request.profileFingerprint !== input.plan.profile.fingerprint + ) { + throw new Error("Managed bootstrap Docker create plan does not match its root request."); + } + const bootstrapIdentity = input.bootstrapIdentity ?? deps.createBootstrapIdentity(); + assertManagedBootstrapIdentity(bootstrapIdentity); + const heldWorkloadArgv = renderManagedBootstrapHeldCommand( + input.request, + bootstrapIdentity, + input.plan.intendedWorkloadArgv, + ); + const createReceipt = await input.launch({ heldWorkloadArgv, bootstrapIdentity }); + if ( + createReceipt.ready !== true || + createReceipt.sandbox.sandboxName !== input.plan.sandboxName || + createReceipt.sandbox.driverId !== input.plan.driverId || + !createReceipt.sandbox.sandboxId + ) { + throw new Error( + "Managed bootstrap Docker create did not return one Ready durable sandbox identity.", + ); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: Object.freeze({ ...createReceipt.sandbox }), + bootstrapIdentity, + heldWorkloadArgv, + intendedWorkloadArgv: Object.freeze([...input.plan.intendedWorkloadArgv]), + plan: input.plan, + createReceipt, + }); + }, + + async cleanupIncompleteCreate(input) { + const { sandbox, runtimeId } = resolveIncompleteCreateSandbox(input, deps); + removeOwnedWorkload(sandbox, deps, runtimeId); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + bootstrapIdentity: input.bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: true, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + }, + + async discoverHeldWorkload( + input: ManagedBootstrapDiscoveryInput, + ): Promise { + if (input.sandbox.driverId !== DOCKER_DRIVER_ID) { + throw new Error("Managed bootstrap Docker adapter received another runtime driver."); + } + const query = queryOpenShellDockerSandboxContainers(input.sandbox.sandboxName, deps); + if (!query.ok) { + throw new Error(`Managed bootstrap Docker discovery failed: ${query.error}`); + } + if (query.ids.length !== 1) { + throw new Error( + `Managed bootstrap requires exactly one labeled Docker workload after Ready; found ${String( + query.ids.length, + )}.`, + ); + } + const runtimeId = String(query.ids[0] ?? "").toLowerCase(); + const inspect = inspectExact(runtimeId, deps); + assertStableRunning(inspect, "held workload"); + assertRootSupervisor(inspect); + assertImage(inspect, input.expectedImage, deps); + assertMetadata(inspect, input.sandbox, input.metadata); + assertBootstrapIdentityInObservedHold(inspect, input.bootstrapIdentity); + return Object.freeze({ + sandbox: input.sandbox, + runtimeId, + bootstrapIdentity: input.bootstrapIdentity, + }); + }, + + async inspectHeldWorkload({ handle, discovered }) { + if ( + discovered.bootstrapIdentity !== handle.bootstrapIdentity || + discovered.sandbox.sandboxId !== handle.sandbox.sandboxId || + discovered.sandbox.driverId !== handle.sandbox.driverId + ) { + throw new Error("Managed bootstrap Docker identity changed before inspection."); + } + const first = inspectExact(discovered.runtimeId, deps); + assertStableRunning(first, "held workload"); + assertRootSupervisor(first); + assertNoRootProcessInjectionEnvironment(first.Config?.Env); + const runtimeImageContentId = assertImage(first, handle.plan.image, deps); + assertMetadata(first, handle.sandbox, handle.plan.metadata); + assertHeldCommand(first, handle.heldWorkloadArgv, handle.bootstrapIdentity); + const firstNormalized = normalizeDockerManagedBootstrapLaunchSpec(first); + const inspect = inspectExact(discovered.runtimeId, deps); + assertStableRunning(inspect, "held workload"); + assertRootSupervisor(inspect); + assertNoRootProcessInjectionEnvironment(inspect.Config?.Env); + if (assertImage(inspect, handle.plan.image, deps) !== runtimeImageContentId) { + throw new Error("Managed bootstrap Docker image content changed during stable capture."); + } + assertMetadata(inspect, handle.sandbox, handle.plan.metadata); + assertHeldCommand(inspect, handle.heldWorkloadArgv, handle.bootstrapIdentity); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(inspect); + if ( + normalized.hash !== firstNormalized.hash || + normalized.canonicalJson !== firstNormalized.canonicalJson + ) { + throw new Error("Managed bootstrap Docker launch spec changed during stable capture."); + } + const supervisorArgv = exactSupervisorArgv(inspect); + if (!exactArrayEqual(supervisorArgv, handle.plan.expectedSupervisorArgv)) { + throw new Error("Managed bootstrap Docker supervisor argv changed before replacement."); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: discovered.runtimeId, + bootstrapIdentity: handle.bootstrapIdentity, + image: handle.plan.image, + runtimeImageContentId, + specHash: normalized.hash, + specCanonicalJson: normalized.canonicalJson, + agentIdentity: Object.freeze({ ...handle.plan.agentIdentity }), + supervisorArgv, + heldWorkloadArgv: handle.heldWorkloadArgv, + metadata: handle.plan.metadata, + }); + }, + + async prepareBootstrapReplacement({ handle, snapshot, request, replacementOptions }) { + if ( + snapshot.bootstrapIdentity !== handle.bootstrapIdentity || + !FULL_CONTAINER_ID_RE.test(snapshot.runtimeId) || + request.agent !== handle.plan.profile.agent || + request.profileFingerprint !== handle.plan.profile.fingerprint + ) { + throw new Error("Managed bootstrap Docker replacement identities do not match."); + } + const parsed = parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson); + const normalizedOriginal = normalizeDockerManagedBootstrapLaunchSpec(parsed.inspect); + if (normalizedOriginal.hash !== snapshot.specHash) { + throw new Error("Managed bootstrap Docker replacement snapshot is not exact."); + } + if (parsed.inspect.HostConfig?.ReadonlyRootfs === true) { + throw new Error( + "Managed bootstrap cannot stage its root-owned request in a read-only root filesystem.", + ); + } + const plan = replacementPlan(replacementOptions); + const originalName = dockerContainerName(parsed.inspect); + const backupContainerName = backupName(originalName, handle.bootstrapIdentity); + const stagingName = replacementStagingName(originalName, handle.bootstrapIdentity); + const existingJournal = deps.journalStore.load(handle.bootstrapIdentity); + if (existingJournal) { + assertDockerBootstrapTransactionAuthority(existingJournal, handle, snapshot); + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: existingJournal.bootstrapIdentity, + runtimeId: existingJournal.replacementRuntimeId, + detail: `preparation requires rollback or commit from durable phase ${existingJournal.phase}`, + }); + } + const trampolineCommand = replacementCommand(handle, snapshot); + const cloneArgs = buildDockerGpuCloneRunArgs(parsed.inspect, plan.mode, { + image: expectedImageReference(snapshot.image.repository, snapshot.image.manifestDigest), + openshellSandboxCommand: handle.intendedWorkloadArgv, + requiredUlimits: plan.requiredUlimits, + extraGroupGids: plan.extraGroupGids, + containerEntrypoint: MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, + containerCommand: trampolineCommand, + containerName: stagingName, + }); + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + + let requestFile = ""; + let replacementRuntimeId = ""; + let stagedAuthority: DockerBootstrapTransaction | null = null; + try { + const created = deps.dockerRun(["create", ...cloneArgs], options); + const returnedRuntimeId = String(created.stdout ?? "") + .trim() + .toLowerCase(); + let createdInspect: DockerContainerInspect; + if (FULL_CONTAINER_ID_RE.test(returnedRuntimeId)) { + replacementRuntimeId = returnedRuntimeId; + createdInspect = inspectExact(replacementRuntimeId, deps); + } else { + try { + createdInspect = inspectDockerContainerReference(stagingName, deps); + } catch (lookupError) { + throw new Error( + "Managed bootstrap could not prove a stopped Docker replacement after create: " + + (commandDetail(created) || + (lookupError instanceof Error ? lookupError.message : String(lookupError))), + ); + } + replacementRuntimeId = String(createdInspect.Id ?? "").toLowerCase(); + } + if ( + !FULL_CONTAINER_ID_RE.test(replacementRuntimeId) || + dockerContainerName(createdInspect) !== stagingName + ) { + throw new Error( + "Managed bootstrap Docker create did not resolve one stopped identity-bound staging container.", + ); + } + assertExplicitlyStopped(createdInspect, "created replacement"); + const createdImageContentId = assertImage(createdInspect, snapshot.image, deps); + if (createdImageContentId !== snapshot.runtimeImageContentId) { + throw new Error( + "Managed bootstrap Docker replacement resolved a different image content ID.", + ); + } + assertMetadata(createdInspect, handle.sandbox, snapshot.metadata); + assertRootSupervisor(createdInspect); + assertReplacementBoundary(createdInspect, handle, snapshot); + const expectedActivatedSpecHash = assertReplacementMatchesIntent( + snapshot.specCanonicalJson, + createdInspect, + originalName, + plan, + openshellSandboxCommandEnvValue(handle.intendedWorkloadArgv) as string, + ); + const preparedSpec = normalizeDockerManagedBootstrapLaunchSpec(createdInspect); + const expectedActivatedSpec = normalizeDockerManagedBootstrapLaunchSpec({ + ...createdInspect, + Name: `/${originalName}`, + }); + if (expectedActivatedSpec.hash !== expectedActivatedSpecHash) { + throw new Error("Managed bootstrap Docker expected activation spec is inconsistent."); + } + stagedAuthority = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + phase: "staged", + bootstrapIdentity: handle.bootstrapIdentity, + sandbox: Object.freeze({ ...handle.sandbox }), + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + snapshot.image.repository, + snapshot.image.manifestDigest, + ), + runtimeImageContentId: snapshot.runtimeImageContentId, + originalRuntimeId: snapshot.runtimeId, + replacementRuntimeId, + originalName, + replacementStagingName: stagingName, + backupName: backupContainerName, + originalSpecHash: snapshot.specHash, + replacementSpecHash: expectedActivatedSpecHash, + }); + + requestFile = writeProtectedEnvelope(handle.bootstrapIdentity, request); + const copied = deps.dockerRun( + ["cp", requestFile, replacementRuntimeId + ":" + MANAGED_BOOTSTRAP_REQUEST_FILE], + options, + ); + assertZero( + copied, + "Managed bootstrap could not stage its protected root-owned 0400 envelope", + ); + + const originalBeforeJournal = inspectExact(snapshot.runtimeId, deps); + assertStableRunning(originalBeforeJournal, "pre-journal original"); + if ( + dockerContainerName(originalBeforeJournal) !== originalName || + normalizeDockerManagedBootstrapLaunchSpec(originalBeforeJournal).hash !== + snapshot.specHash + ) { + throw new Error( + "Managed bootstrap Docker original changed while the replacement was staged.", + ); + } + const replacementBeforeJournal = inspectExact(replacementRuntimeId, deps); + assertTransactionReplacement(stagedAuthority, replacementBeforeJournal); + const observedPreparedSpec = + normalizeDockerManagedBootstrapLaunchSpec(replacementBeforeJournal); + const observedActivatedSpec = normalizeDockerManagedBootstrapLaunchSpec({ + ...replacementBeforeJournal, + Name: `/${originalName}`, + }); + if ( + dockerContainerName(replacementBeforeJournal) !== stagingName || + observedPreparedSpec.canonicalJson !== preparedSpec.canonicalJson || + observedActivatedSpec.canonicalJson !== expectedActivatedSpec.canonicalJson + ) { + throw new Error("Managed bootstrap Docker replacement changed before durable staging."); + } + assertExplicitlyStopped(replacementBeforeJournal, "pre-journal replacement"); + + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + originalRuntimeId: snapshot.runtimeId, + preparedRuntimeId: replacementRuntimeId, + image: snapshot.image, + runtimeImageContentId: snapshot.runtimeImageContentId, + originalSpecHash: snapshot.specHash, + preparedSpecHash: preparedSpec.hash, + preparedSpecCanonicalJson: preparedSpec.canonicalJson, + expectedActivatedSpecHash, + expectedActivatedSpecCanonicalJson: expectedActivatedSpec.canonicalJson, + profileFingerprint: handle.plan.profile.fingerprint, + rollbackAuthority: serializeDockerManagedBootstrapJournal(stagedAuthority), + }); + } catch (error) { + let rollbackError: unknown = null; + try { + const durable = deps.journalStore.load(handle.bootstrapIdentity); + if (!durable) { + cleanupUnjournaledPreparedContainer( + { snapshot, preparedRuntimeId: replacementRuntimeId, stagingName }, + deps, + ); + } + } catch (cleanupError) { + rollbackError = cleanupError; + } + const failure = error instanceof Error ? error : new Error(String(error)); + if (rollbackError) attachManagedBootstrapRollbackError(failure, rollbackError); + throw failure; + } finally { + if (requestFile) cleanupTempDir(requestFile, REQUEST_TEMP_PREFIX); + } + }, + async activateBootstrapReplacement({ handle, snapshot, prepared, durablePreparation }) { + const authority = transactionFromPreparedAuthority(handle, snapshot, prepared); + assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); + const existingJournal = deps.journalStore.load(handle.bootstrapIdentity); + if (existingJournal) { + assertDockerBootstrapTransactionAuthority(existingJournal, handle, snapshot, prepared); + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: existingJournal.bootstrapIdentity, + runtimeId: existingJournal.replacementRuntimeId, + detail: `activation requires rollback or commit from durable phase ${existingJournal.phase}`, + }); + } + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + try { + const originalBeforeJournal = inspectExact(snapshot.runtimeId, deps); + const preparedBeforeJournal = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionOriginal(authority, originalBeforeJournal); + assertTransactionReplacement(authority, preparedBeforeJournal); + assertStableRunning(originalBeforeJournal, "pre-activation original"); + assertExplicitlyStopped(preparedBeforeJournal, "pre-activation replacement"); + if ( + dockerContainerName(originalBeforeJournal) !== authority.originalName || + dockerContainerName(preparedBeforeJournal) !== authority.replacementStagingName || + normalizeDockerManagedBootstrapLaunchSpec(preparedBeforeJournal).canonicalJson !== + prepared.preparedSpecCanonicalJson + ) { + throw new Error( + "Managed bootstrap Docker prepared runtimes changed before durable activation.", + ); + } + + let journal = createDockerBootstrapJournalDurably(authority, deps); + const originalAtFence = inspectExact(snapshot.runtimeId, deps); + const replacementAtFence = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionOriginal(journal, originalAtFence); + assertTransactionReplacement(journal, replacementAtFence); + if ( + dockerContainerName(originalAtFence) !== journal.originalName || + originalAtFence.State?.Running !== true || + dockerContainerName(replacementAtFence) !== journal.replacementStagingName || + normalizeDockerManagedBootstrapLaunchSpec(replacementAtFence).canonicalJson !== + prepared.preparedSpecCanonicalJson + ) { + throw new Error( + "Managed bootstrap Docker staged runtimes changed before the cutover fence.", + ); + } + assertExplicitlyStopped(replacementAtFence, "staged replacement"); + journal = transitionDockerBootstrapJournalDurably(journal, "cutover", deps); + + const stopped = deps.dockerStop(snapshot.runtimeId, { + ...options, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + const afterStop = inspectExact(snapshot.runtimeId, deps); + assertTransactionOriginal(journal, afterStop); + if (dockerContainerName(afterStop) !== journal.originalName) { + throw new Error( + "Managed bootstrap could not prove its exact original stopped after Docker stop: " + + (commandDetail(stopped) || "state did not reach stopped"), + ); + } + assertExplicitlyStopped(afterStop, "stopped original"); + + const renamedOriginal = deps.dockerRename(snapshot.runtimeId, journal.backupName, options); + const afterOriginalRename = inspectExact(snapshot.runtimeId, deps); + assertTransactionOriginal(journal, afterOriginalRename); + if (dockerContainerName(afterOriginalRename) !== journal.backupName) { + throw new Error( + "Managed bootstrap could not prove its exact original backup rename: " + + (commandDetail(renamedOriginal) || "name did not reach backup"), + ); + } + assertExplicitlyStopped(afterOriginalRename, "renamed rollback backup"); + + const renamedReplacement = deps.dockerRename( + prepared.preparedRuntimeId, + journal.originalName, + options, + ); + const afterReplacementRename = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionReplacement(journal, afterReplacementRename); + if (dockerContainerName(afterReplacementRename) !== journal.originalName) { + throw new Error( + "Managed bootstrap could not prove its exact replacement cutover rename: " + + (commandDetail(renamedReplacement) || "name did not reach target"), + ); + } + + const started = deps.dockerStart(prepared.preparedRuntimeId, options); + const running = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionReplacement(journal, running); + const runningSpec = normalizeDockerManagedBootstrapLaunchSpec(running); + if ( + dockerContainerName(running) !== journal.originalName || + running.State?.Running !== true || + running.State.Paused === true || + running.State.Restarting === true || + running.State.Dead === true || + runningSpec.canonicalJson !== prepared.expectedActivatedSpecCanonicalJson + ) { + throw new Error( + "Managed bootstrap could not prove its exact replacement running after Docker start: " + + (commandDetail(started) || "state did not reach running"), + ); + } + assertReplacementBoundary(running, handle, snapshot); + const preservedOriginal = inspectExact(snapshot.runtimeId, deps); + assertTransactionOriginal(journal, preservedOriginal); + if (dockerContainerName(preservedOriginal) !== journal.backupName) { + throw new Error("Managed bootstrap Docker rollback backup changed during cutover."); + } + assertExplicitlyStopped(preservedOriginal, "preserved rollback backup"); + + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + originalRuntimeId: snapshot.runtimeId, + replacementRuntimeId: prepared.preparedRuntimeId, + image: snapshot.image, + runtimeImageContentId: snapshot.runtimeImageContentId, + originalSpecHash: snapshot.specHash, + replacementSpecHash: prepared.expectedActivatedSpecHash, + replacementSpecCanonicalJson: prepared.expectedActivatedSpecCanonicalJson, + profileFingerprint: handle.plan.profile.fingerprint, + }); + } catch (error) { + let rollbackError: unknown = null; + try { + if (!deps.journalStore.load(handle.bootstrapIdentity)) { + cleanupUnjournaledPreparedContainer( + { + snapshot, + preparedRuntimeId: prepared.preparedRuntimeId, + stagingName: authority.replacementStagingName, + }, + deps, + ); + } + } catch (cleanupError) { + rollbackError = cleanupError; + } + const failure = error instanceof Error ? error : new Error(String(error)); + if (rollbackError) attachManagedBootstrapRollbackError(failure, rollbackError); + throw failure; + } + }, + async awaitBootstrap({ handle, snapshot, replacement, timeoutSecs }) { + if ( + replacement.bootstrapIdentity !== handle.bootstrapIdentity || + replacement.originalRuntimeId !== snapshot.runtimeId || + replacement.profileFingerprint !== handle.plan.profile.fingerprint + ) { + throw new Error("Managed bootstrap Docker completion identities do not match."); + } + const journal = reconstructDockerBootstrapTransaction(handle, snapshot, replacement, deps); + if (journal.phase !== "cutover") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: `bootstrap completion is invalid from durable journal phase ${journal.phase}`, + }); + } + assertCompletedCutoverRuntimeState(journal, deps); + const before = inspectExact(replacement.replacementRuntimeId, deps); + assertStableRunning(before, "replacement"); + const beforeImageContentId = assertImage(before, replacement.image, deps); + if (beforeImageContentId !== replacement.runtimeImageContentId) { + throw new Error("Managed bootstrap Docker replacement image content changed."); + } + assertReplacementBoundary(before, handle, snapshot); + if (!waitForOpenShellSupervisorReconnect(handle.sandbox.sandboxName, timeoutSecs, deps)) { + throw new Error("Managed bootstrap Docker supervisor did not reconnect."); + } + const afterWaitJournal = deps.journalStore.load(journal.bootstrapIdentity); + if (!afterWaitJournal || !sameDockerBootstrapJournal(afterWaitJournal, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable transaction authority changed while awaiting bootstrap", + }); + } + assertCompletedCutoverRuntimeState(afterWaitJournal, deps); + const after = inspectExact(replacement.replacementRuntimeId, deps); + assertStableRunning(after, "completed replacement"); + if (assertImage(after, replacement.image, deps) !== replacement.runtimeImageContentId) { + throw new Error("Managed bootstrap Docker completed image content changed."); + } + assertReplacementBoundary(after, handle, snapshot); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(after); + if (normalized.hash !== replacement.replacementSpecHash) { + throw new Error("Managed bootstrap Docker replacement changed during bootstrap."); + } + const imageCompletion = readProtectedImageCompletion(replacement.replacementRuntimeId, deps); + if ( + imageCompletion.bootstrapIdentity !== replacement.bootstrapIdentity || + imageCompletion.agent !== handle.plan.profile.agent || + imageCompletion.profileFingerprint !== replacement.profileFingerprint + ) { + throw new Error( + "Managed bootstrap Docker image completion identities do not match the transaction.", + ); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: replacement.replacementRuntimeId, + image: replacement.image, + runtimeImageContentId: replacement.runtimeImageContentId, + originalSpecHash: replacement.originalSpecHash, + replacementSpecHash: replacement.replacementSpecHash, + profileFingerprint: replacement.profileFingerprint, + bootstrapIdentity: replacement.bootstrapIdentity, + transactionPending: imageCompletion.transactionPending, + completedAt: deps.now().toISOString(), + }); + }, + + finalizeBootstrap, + }; +} diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.ts b/src/lib/onboard/openshell-docker-sandbox-containers.ts index a8934027fd6..94c067f027c 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.ts @@ -7,6 +7,7 @@ import type { DockerGpuPatchDeps } from "./docker-gpu-patch-types"; export const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; export const OPENSHELL_MANAGED_BY_VALUE = "openshell"; export const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; +export const OPENSHELL_SANDBOX_ID_LABEL = "openshell.ai/sandbox-id"; const DOCKER_SANDBOX_QUERY_TIMEOUT_MS = 30_000; diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 94d235cad52..48e889d78c1 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -122,7 +122,10 @@ describe("runtime provider central source boundary", () => { expect(bootstrapProtocolPaths).toEqual([ "src/lib/onboard/managed-bootstrap/adapter.ts", "src/lib/onboard/managed-bootstrap/docker-journal.ts", + "src/lib/onboard/managed-bootstrap/docker-shared-state.ts", "src/lib/onboard/managed-bootstrap/docker-spec.ts", + "src/lib/onboard/managed-bootstrap/docker-test-fixture.ts", + "src/lib/onboard/managed-bootstrap/docker.ts", "src/lib/onboard/managed-bootstrap/envelope.ts", "src/lib/onboard/managed-bootstrap/index.ts", ]); diff --git a/tsconfig.src.json b/tsconfig.src.json index da1287ae436..13c12fb1310 100644 --- a/tsconfig.src.json +++ b/tsconfig.src.json @@ -16,5 +16,10 @@ "types": ["node"] }, "include": ["src/**/*.ts"], - "exclude": ["node_modules", "nemoclaw", "src/**/*.test.ts"] + "exclude": [ + "node_modules", + "nemoclaw", + "src/**/*.test.ts", + "src/**/*-test-fixture.ts" + ] } From 4be7221917a1597cdbf2a1d65524c84084a662b6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 06:13:36 -0700 Subject: [PATCH 092/117] fix(onboard): validate managed bootstrap clone inputs Signed-off-by: Aaron Erickson --- .../onboard/docker-gpu-patch-clone.test.ts | 35 ++++++++++++++++++ src/lib/onboard/managed-bootstrap/adapter.ts | 1 + .../onboard/managed-bootstrap/docker.test.ts | 37 ++++++++++++++++++- src/lib/onboard/managed-bootstrap/docker.ts | 1 + 4 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard/docker-gpu-patch-clone.test.ts b/src/lib/onboard/docker-gpu-patch-clone.test.ts index 8ad39b30162..cbdbfb1e252 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.test.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.test.ts @@ -122,6 +122,41 @@ describe("Docker GPU clone envelope", () => { expect(args).not.toContain("nofile=1024:1024"); }); + it("uses exact managed-bootstrap container, entrypoint, and command overrides", () => { + const args = buildDockerGpuCloneRunArgs( + inspectFixture(), + buildDockerGpuMode("startup-command"), + { + containerName: "openshell-alpha-bootstrap-stage", + containerEntrypoint: "/usr/local/bin/nemoclaw-managed-bootstrap", + containerCommand: ["--request", "/run/nemoclaw/bootstrap-request.json"], + }, + ); + + expect(args.slice(0, 2)).toEqual(["--name", "openshell-alpha-bootstrap-stage"]); + expect(args).toEqual( + expect.arrayContaining(["--entrypoint", "/usr/local/bin/nemoclaw-managed-bootstrap"]), + ); + expect(args.slice(args.indexOf("openshell/sandbox:abc"))).toEqual([ + "openshell/sandbox:abc", + "--request", + "/run/nemoclaw/bootstrap-request.json", + ]); + }); + + it.each([ + "", + "-starts-with-dash", + "contains/slash", + "a".repeat(254), + ])("rejects invalid managed-bootstrap container name %j", (containerName) => { + expect(() => + buildDockerGpuCloneRunArgs(inspectFixture(), buildDockerGpuMode("startup-command"), { + containerName, + }), + ).toThrow("Docker clone container name is invalid."); + }); + it("adds SYS_PTRACE to the GPU clone when the baseline container lacks it", () => { const inspect = inspectFixture(); inspect.HostConfig!.CapAdd = ["SYS_ADMIN", "NET_ADMIN"]; diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index b0edaa76627..4721d059ef1 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -25,6 +25,7 @@ const PROCESS_INJECTION_ENV_KEYS = new Set([ "LD_AUDIT", "LD_LIBRARY_PATH", "LD_PRELOAD", + "NODE_OPTIONS", "PS4", "SHELLOPTS", ]); diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 0f848d86c18..5c534649575 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -5,6 +5,10 @@ import { describe, expect, it, vi } from "vitest"; import { ManagedBootstrapOwnerCleanupRequiredError } from "./adapter"; import { createDockerManagedBootstrapAdapter } from "./docker"; +import { + normalizeDockerManagedBootstrapLaunchSpec, + parseDockerManagedBootstrapLaunchSpec, +} from "./docker-spec"; import { authority, completion, @@ -277,6 +281,35 @@ describe("Docker managed bootstrap adapter", () => { ).toBe(true); }); + it.each([ + "NODE_OPTIONS", + "LD_PRELOAD", + "BASH_ENV", + ])("rejects hostile %s from the launch snapshot before replacement creation", async (key) => { + const fake = fixture(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority(); + const parsed = parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson); + const hostileInspect = structuredClone(parsed.inspect); + hostileInspect.Config!.Env = [...(hostileInspect.Config!.Env ?? []), `${key}=/tmp/hostile`]; + const hostileSpec = normalizeDockerManagedBootstrapLaunchSpec(hostileInspect); + + await expect( + adapter.prepareBootstrapReplacement({ + handle, + snapshot: { + ...snapshot, + specHash: hostileSpec.hash, + specCanonicalJson: hostileSpec.canonicalJson, + }, + request, + replacementOptions: { values: {} }, + }), + ).rejects.toThrow(`Managed bootstrap refuses root-process injection environment '${key}'.`); + expect(fake.events).not.toContain("create:replacement"); + expect(fake.replacement).toBeNull(); + }); + it("quiesces and retains an exact incomplete create when its mutable name is reused", async () => { const fake = fixture({ ownerId: "sandbox-alpha-recreated" }); const adapter = createDockerManagedBootstrapAdapter(fake.deps); @@ -303,8 +336,8 @@ describe("Docker managed bootstrap adapter", () => { const fake = fixture({ ownerId: replacementSandboxId }); const adapter = createDockerManagedBootstrapAdapter(fake.deps); const { handle, plan } = authority(); - if (!fake.original.Config?.Labels) throw new Error("fixture labels are required"); - fake.original.Config.Labels["openshell.ai/sandbox-id"] = replacementSandboxId; + expect(fake.original.Config?.Labels).toBeDefined(); + fake.original.Config!.Labels!["openshell.ai/sandbox-id"] = replacementSandboxId; await expect( adapter.cleanupIncompleteCreate({ diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index b463a820051..3a2df714de6 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -2554,6 +2554,7 @@ export function createDockerManagedBootstrapAdapter( if (normalizedOriginal.hash !== snapshot.specHash) { throw new Error("Managed bootstrap Docker replacement snapshot is not exact."); } + assertNoRootProcessInjectionEnvironment(parsed.inspect.Config?.Env); if (parsed.inspect.HostConfig?.ReadonlyRootfs === true) { throw new Error( "Managed bootstrap cannot stage its root-owned request in a read-only root filesystem.", From e42b09e169e58054c29c4651661ff146f77070ef Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 06:19:08 -0700 Subject: [PATCH 093/117] fix(onboard): freeze canonical Docker launch specs Signed-off-by: Aaron Erickson --- .../managed-bootstrap/docker-spec.test.ts | 25 +++++++++++++++++++ .../onboard/managed-bootstrap/docker-spec.ts | 11 ++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/docker-spec.test.ts b/src/lib/onboard/managed-bootstrap/docker-spec.test.ts index 94124f0127b..3a8da33b9af 100644 --- a/src/lib/onboard/managed-bootstrap/docker-spec.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-spec.test.ts @@ -54,6 +54,31 @@ describe("managed bootstrap Docker launch spec", () => { ); }); + it("detaches and deeply freezes canonical launch state at the hashed boundary", () => { + const inspect = createDockerGpuInspectFixture(); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(inspect); + const { canonicalJson, hash } = normalized; + const config = normalized.spec.inspect.Config as Record; + const hostConfig = normalized.spec.inspect.HostConfig as Record; + const network = normalized.spec.inspect.NetworkSettings!.Networks!["openshell-docker"]!; + + expect(() => Object.assign(config, { StopTimeout: 999 })).toThrow(TypeError); + expect(() => Object.assign(hostConfig, { Runtime: "mutated" })).toThrow(TypeError); + expect(() => network.Aliases!.push("mutated")).toThrow(TypeError); + + Object.assign(inspect.Config!, { StopTimeout: 45 }); + Object.assign(inspect.HostConfig!, { Runtime: "mutated" }); + inspect.NetworkSettings!.Networks!["openshell-docker"]!.Aliases!.push("mutated"); + + expect(normalized.spec.inspect.Config).not.toHaveProperty("StopTimeout"); + expect(normalized.spec.inspect.HostConfig).not.toHaveProperty("Runtime"); + expect(network.Aliases).toEqual(["openshell-alpha"]); + expect(normalized.canonicalJson).toBe(`${JSON.stringify(normalized.spec)}\n`); + expect(normalized.canonicalJson).toBe(canonicalJson); + expect(normalized.hash).toBe(hash); + expect(normalizeDockerManagedBootstrapLaunchSpec(inspect).hash).not.toBe(hash); + }); + it.each([ { name: "anonymous Config.Volumes whose data source cannot be proven", diff --git a/src/lib/onboard/managed-bootstrap/docker-spec.ts b/src/lib/onboard/managed-bootstrap/docker-spec.ts index c3cab4e3296..19eaef7f288 100644 --- a/src/lib/onboard/managed-bootstrap/docker-spec.ts +++ b/src/lib/onboard/managed-bootstrap/docker-spec.ts @@ -221,6 +221,12 @@ function canonicalize(value: unknown): unknown { ); } +function deepFreeze(value: T): T { + if (typeof value !== "object" || value === null || Object.isFrozen(value)) return value; + for (const nested of Object.values(value)) deepFreeze(nested); + return Object.freeze(value); +} + export function parseExactDockerContainerInspect(output: string): DockerContainerInspect { let parsed: unknown; try { @@ -282,11 +288,12 @@ export function normalizeDockerManagedBootstrapLaunchSpec(inspect: DockerContain ...("Platform" in raw && typeof raw.Platform === "string" ? { Platform: raw.Platform } : {}), }, }; - const canonicalJson = `${JSON.stringify(canonicalize(spec))}\n`; + const canonicalSpec = deepFreeze(canonicalize(spec) as DockerManagedBootstrapLaunchSpec); + const canonicalJson = `${JSON.stringify(canonicalSpec)}\n`; return Object.freeze({ canonicalJson, hash: createHash("sha256").update(canonicalJson, "utf8").digest("hex"), - spec: Object.freeze(spec), + spec: canonicalSpec, }); } From 1275cf362e09ec3b3d3634a040e81e9ec9b630c2 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 05:24:20 -0700 Subject: [PATCH 094/117] feat(onboard): add Docker bootstrap transaction primitives Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/README.md | 5 + .../managed-bootstrap/docker-journal.test.ts | 175 +++++++ .../managed-bootstrap/docker-journal.ts | 448 ++++++++++++++++++ .../managed-bootstrap/docker-spec.test.ts | 84 ++++ .../onboard/managed-bootstrap/docker-spec.ts | 316 ++++++++++++ test/runtime-provider-source-shape.test.ts | 2 + 6 files changed, 1030 insertions(+) create mode 100644 src/lib/onboard/managed-bootstrap/docker-journal.test.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-journal.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-spec.test.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-spec.ts diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 32213dd1249..adde636fa56 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -66,6 +66,11 @@ including its supervisor environment, to immutable prepared authority before activation. The native boundary introduces no driver-specific environment policy. +The first Docker-specific groundwork defines a private, monotonic cutover +journal and a canonical launch-spec normalizer. Each surface is independently +validated and remains dormant: no registered runtime provider imports either +module, and neither changes sandbox creation or lifecycle behavior. + ## Architectural disposition The coordinator deliberately lands as a dormant trust-boundary slice before a diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts new file mode 100644 index 00000000000..7ed0d8bba6a --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -0,0 +1,175 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapJournal, + parseDockerManagedBootstrapJournal, + serializeDockerManagedBootstrapJournal, +} from "./docker-journal"; + +const roots: string[] = []; +const IDENTITY = "1".repeat(64); +const journal = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + phase: "staged", + bootstrapIdentity: IDENTITY, + sandbox: { + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + driverId: "docker", + }, + profileFingerprint: "2".repeat(64), + imageReference: `registry.example/image@sha256:${"3".repeat(64)}`, + runtimeImageContentId: `sha256:${"4".repeat(64)}`, + originalRuntimeId: "5".repeat(64), + replacementRuntimeId: "6".repeat(64), + originalName: "openshell-alpha", + replacementStagingName: "openshell-alpha-staged", + backupName: "openshell-alpha-backup", + originalSpecHash: "7".repeat(64), + replacementSpecHash: "8".repeat(64), +} satisfies DockerManagedBootstrapJournal); + +function readPinnedPrivateFile(target: string): { readonly mode: number; readonly text: string } { + const descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + const text = fs.readFileSync(descriptor, "utf8"); + const after = fs.fstatSync(descriptor, { bigint: true }); + expect(after.dev).toBe(before.dev); + expect(after.ino).toBe(before.ino); + expect(after.size).toBe(before.size); + expect(after.mtimeNs).toBe(before.mtimeNs); + expect(after.ctimeNs).toBe(before.ctimeNs); + return { mode: Number(before.mode & 0o777n), text }; + } finally { + fs.closeSync(descriptor); + } +} + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +describe("Docker managed bootstrap journal", () => { + it("publishes private canonical state through only monotonic phases", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + const directory = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); + const file = path.join(directory, `${IDENTITY}.json`); + expect(fs.statSync(directory).mode & 0o777).toBe(0o700); + const persisted = readPinnedPrivateFile(file); + expect(persisted.mode).toBe(0o600); + expect(parseDockerManagedBootstrapJournal(persisted.text)).toEqual(journal); + expect(() => store.create(journal)).toThrow("already exists"); + expect(() => store.transition(IDENTITY, "staged", "shared-state-committed")).toThrow( + "unsupported", + ); + + expect(store.transition(IDENTITY, "staged", "cutover").phase).toBe("cutover"); + expect(store.transition(IDENTITY, "cutover", "shared-state-committed").phase).toBe( + "shared-state-committed", + ); + store.remove(IDENTITY, ["shared-state-committed"]); + expect(store.load(IDENTITY)).toBeNull(); + }); + + it("recovers one durable cutover decision before journal replacement", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + store.transition(IDENTITY, "staged", "cutover"); + const file = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, `${IDENTITY}.json`); + fs.writeFileSync(`${file}.decision`, "rollback-authorized\n", { mode: 0o600 }); + + expect(store.load(IDENTITY)?.phase).toBe("rollback-authorized"); + expect(parseDockerManagedBootstrapJournal(readPinnedPrivateFile(file).text).phase).toBe( + "rollback-authorized", + ); + fs.unlinkSync(`${file}.decision`); + expect(store.load(IDENTITY)?.phase).toBe("rollback-authorized"); + expect(() => store.transition(IDENTITY, "cutover", "shared-state-committed")).toThrow( + "expected phase cutover", + ); + store.remove(IDENTITY, ["rollback-authorized"]); + }); + + it("reconciles an exclusive decision collision by typed durable authority", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + store.transition(IDENTITY, "staged", "cutover"); + const target = path.join( + root, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, + `${IDENTITY}.json.decision`, + ); + const link = vi.spyOn(fs, "linkSync").mockImplementationOnce(() => { + fs.writeFileSync(target, "rollback-authorized\n", { flag: "wx", mode: 0o600 }); + throw Object.assign(new Error("exclusive decision collision"), { code: "EEXIST" }); + }); + try { + expect(store.transition(IDENTITY, "cutover", "rollback-authorized").phase).toBe( + "rollback-authorized", + ); + } finally { + link.mockRestore(); + } + }); + + it("preserves a primary journal write failure when temporary cleanup also fails", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + const rename = vi.spyOn(fs, "renameSync").mockImplementationOnce(() => { + throw new Error("primary journal rename failure"); + }); + const unlink = vi.spyOn(fs, "unlinkSync").mockImplementationOnce(() => { + throw new Error("temporary cleanup failure"); + }); + try { + expect(() => store.transition(IDENTITY, "staged", "cutover")).toThrow( + "primary journal rename failure", + ); + } finally { + rename.mockRestore(); + unlink.mockRestore(); + } + }); + + it.skipIf(process.platform === "win32")("refuses a symlink in place of journal authority", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + const file = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, `${IDENTITY}.json`); + const moved = `${file}.moved`; + fs.renameSync(file, moved); + fs.symlinkSync(moved, file); + + expect(() => store.load(IDENTITY)).toThrow("journal file ownership boundary is invalid"); + }); + + it("rejects non-canonical authority", () => { + expect(() => + parseDockerManagedBootstrapJournal(`${JSON.stringify({ ...journal, phase: "unknown" })}\n`), + ).toThrow("phase is unsupported"); + expect( + serializeDockerManagedBootstrapJournal(Object.freeze({ ...journal, phase: "staged" })), + ).toBe(`${JSON.stringify(journal)}\n`); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts new file mode 100644 index 00000000000..c6043409d7d --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -0,0 +1,448 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import type { ManagedBootstrapSandboxIdentity } from "./adapter"; + +export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 1 as const; +export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY = "managed-bootstrap"; + +const SHA256_RE = /^[a-f0-9]{64}$/u; +const MAX_JOURNAL_BYTES = 32 * 1024; +const JOURNAL_DIRECTORY_MODE = 0o700; +const JOURNAL_FILE_MODE = 0o600; +const DECISION_PHASES = new Set([ + "rollback-authorized", + "shared-state-committed", +]); + +export type DockerManagedBootstrapJournalPhase = + | "staged" + | "cutover" + | "rollback-authorized" + | "shared-state-committed"; + +export interface DockerManagedBootstrapJournal { + readonly schemaVersion: typeof DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION; + readonly phase: DockerManagedBootstrapJournalPhase; + readonly bootstrapIdentity: string; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly profileFingerprint: string; + readonly imageReference: string; + readonly runtimeImageContentId: string; + readonly originalRuntimeId: string; + readonly replacementRuntimeId: string; + readonly originalName: string; + readonly replacementStagingName: string; + readonly backupName: string; + readonly originalSpecHash: string; + readonly replacementSpecHash: string; +} + +export interface DockerManagedBootstrapJournalStore { + create(journal: DockerManagedBootstrapJournal): void; + load(bootstrapIdentity: string): DockerManagedBootstrapJournal | null; + transition( + bootstrapIdentity: string, + expected: DockerManagedBootstrapJournalPhase, + next: DockerManagedBootstrapJournalPhase, + ): DockerManagedBootstrapJournal; + remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]): void; +} + +/** + * Alternate stores may use this only when the durable mutation completed and + * the caller lost its acknowledgement. Ordinary I/O and fsync failures must + * retain their original error type and are never reconciled as success. + */ +export class DockerManagedBootstrapJournalAcknowledgementLostError extends Error { + constructor(message: string) { + super(message); + this.name = "DockerManagedBootstrapJournalAcknowledgementLostError"; + } +} + +class DockerManagedBootstrapJournalExistsError extends Error { + constructor() { + super( + "Managed bootstrap Docker journal is invalid: journal already exists for this bootstrap identity", + ); + this.name = "DockerManagedBootstrapJournalExistsError"; + } +} + +const ALLOWED_TRANSITIONS = new Set([ + "staged->cutover", + "cutover->rollback-authorized", + "cutover->shared-state-committed", +]); + +function fail(message: string): never { + throw new Error(`Managed bootstrap Docker journal is invalid: ${message}`); +} + +function exactString(value: unknown, label: string, maxBytes = 4096): string { + if ( + typeof value !== "string" || + value.length === 0 || + value !== value.trim() || + value.includes("\0") || + Buffer.byteLength(value, "utf8") > maxBytes + ) { + fail(`${label} must be one bounded exact string`); + } + return value; +} + +function exactSha256(value: unknown, label: string): string { + if (typeof value !== "string" || !SHA256_RE.test(value)) { + fail(`${label} must be lowercase SHA-256`); + } + return value; +} + +function exactPhase(value: unknown): DockerManagedBootstrapJournalPhase { + if ( + !["staged", "cutover", "rollback-authorized", "shared-state-committed"].includes(String(value)) + ) { + fail("phase is unsupported"); + } + return value as DockerManagedBootstrapJournalPhase; +} + +function exactSandbox(value: unknown): ManagedBootstrapSandboxIdentity { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("sandbox identity must be an object"); + } + const sandbox = value as Record; + if (Object.keys(sandbox).sort().join(",") !== "driverId,sandboxId,sandboxName") { + fail("sandbox identity schema is invalid"); + } + return Object.freeze({ + sandboxName: exactString(sandbox.sandboxName, "sandbox name"), + sandboxId: exactString(sandbox.sandboxId, "sandbox ID"), + driverId: exactString(sandbox.driverId, "driver ID"), + }); +} + +export function normalizeDockerManagedBootstrapJournal( + value: unknown, +): DockerManagedBootstrapJournal { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("journal must be an object"); + } + const journal = value as Record; + const expectedKeys = [ + "backupName", + "bootstrapIdentity", + "imageReference", + "originalName", + "originalRuntimeId", + "originalSpecHash", + "phase", + "profileFingerprint", + "replacementRuntimeId", + "replacementSpecHash", + "replacementStagingName", + "runtimeImageContentId", + "sandbox", + "schemaVersion", + ]; + if ( + Object.keys(journal).sort().join(",") !== expectedKeys.sort().join(",") || + journal.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION + ) { + fail("journal schema is invalid"); + } + const normalized = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + phase: exactPhase(journal.phase), + bootstrapIdentity: exactSha256(journal.bootstrapIdentity, "bootstrap identity"), + sandbox: exactSandbox(journal.sandbox), + profileFingerprint: exactSha256(journal.profileFingerprint, "profile fingerprint"), + imageReference: exactString(journal.imageReference, "image reference"), + runtimeImageContentId: exactString(journal.runtimeImageContentId, "runtime image content ID"), + originalRuntimeId: exactSha256(journal.originalRuntimeId, "original runtime ID"), + replacementRuntimeId: exactSha256(journal.replacementRuntimeId, "replacement runtime ID"), + originalName: exactString(journal.originalName, "original name", 253), + replacementStagingName: exactString( + journal.replacementStagingName, + "replacement staging name", + 253, + ), + backupName: exactString(journal.backupName, "backup name", 253), + originalSpecHash: exactSha256(journal.originalSpecHash, "original spec hash"), + replacementSpecHash: exactSha256(journal.replacementSpecHash, "replacement spec hash"), + } satisfies DockerManagedBootstrapJournal); + if (normalized.originalRuntimeId === normalized.replacementRuntimeId) { + fail("original and replacement runtime IDs must differ"); + } + if ( + new Set([normalized.originalName, normalized.replacementStagingName, normalized.backupName]) + .size !== 3 + ) { + fail("original, staging, and backup names must be distinct"); + } + return normalized; +} + +export function serializeDockerManagedBootstrapJournal( + journal: DockerManagedBootstrapJournal, +): string { + const normalized = normalizeDockerManagedBootstrapJournal(journal); + const serialized = `${JSON.stringify(normalized)}\n`; + if (Buffer.byteLength(serialized, "utf8") > MAX_JOURNAL_BYTES) { + fail("serialized journal exceeds its bounded transport"); + } + return serialized; +} + +export function parseDockerManagedBootstrapJournal(text: string): DockerManagedBootstrapJournal { + if ( + text.length === 0 || + text.includes("\0") || + Buffer.byteLength(text, "utf8") > MAX_JOURNAL_BYTES + ) { + fail("serialized journal is empty or too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("serialized journal is not valid JSON"); + } + const journal = normalizeDockerManagedBootstrapJournal(parsed); + if (serializeDockerManagedBootstrapJournal(journal) !== text) { + fail("serialized journal is not canonical"); + } + return journal; +} + +function assertDirectory(directory: string): void { + fs.mkdirSync(directory, { recursive: true, mode: JOURNAL_DIRECTORY_MODE }); + const stat = fs.lstatSync(directory); + if (!stat.isDirectory() || stat.isSymbolicLink() || (stat.mode & 0o077) !== 0) { + fail("journal directory must be a private real directory"); + } +} + +function journalPath(directory: string, bootstrapIdentity: string): string { + exactSha256(bootstrapIdentity, "bootstrap identity"); + return path.join(directory, `${bootstrapIdentity}.json`); +} + +function decisionPath(target: string): string { + return `${target}.decision`; +} + +function sameStableMetadata(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.mode === right.mode && + left.nlink === right.nlink && + left.uid === right.uid && + left.gid === right.gid && + left.size === right.size && + left.mtimeNs === right.mtimeNs && + left.ctimeNs === right.ctimeNs + ); +} + +function readPrivateFile(target: string, label: string): string | null { + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") { + fail(`cannot safely open ${label} because O_NOFOLLOW is unavailable`); + } + const nonblock = typeof fs.constants.O_NONBLOCK === "number" ? fs.constants.O_NONBLOCK : 0; + let descriptor: number; + try { + descriptor = fs.openSync(target, fs.constants.O_RDONLY | noFollow | nonblock); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + if ((error as NodeJS.ErrnoException).code === "ELOOP") { + fail(`${label} file ownership boundary is invalid`); + } + throw error; + } + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + (before.mode & 0o077n) !== 0n || + before.size <= 0n || + before.size > BigInt(MAX_JOURNAL_BYTES) + ) { + fail(`${label} file ownership boundary is invalid`); + } + const contents = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < contents.length) { + const count = fs.readSync(descriptor, contents, offset, contents.length - offset, offset); + if (count === 0) break; + offset += count; + } + const overflow = Buffer.alloc(1); + const overflowCount = fs.readSync(descriptor, overflow, 0, 1, offset); + const after = fs.fstatSync(descriptor, { bigint: true }); + if (offset !== contents.length || overflowCount !== 0 || !sameStableMetadata(before, after)) { + fail(`${label} file changed during its stable read`); + } + return contents.toString("utf8"); + } finally { + fs.closeSync(descriptor); + } +} + +function fsyncDirectory(directory: string): void { + const descriptor = fs.openSync(directory, "r"); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +function atomicWrite( + directory: string, + target: string, + contents: string, + exclusive: boolean, +): void { + const temporary = path.join( + directory, + `.${path.basename(target)}.${process.pid}.${Date.now().toString(16)}.tmp`, + ); + let descriptor: number | null = null; + let primaryFailure: { readonly error: unknown } | null = null; + try { + descriptor = fs.openSync(temporary, "wx", JOURNAL_FILE_MODE); + fs.writeFileSync(descriptor, contents, "utf8"); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = null; + if (exclusive) { + try { + fs.linkSync(temporary, target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + throw new DockerManagedBootstrapJournalExistsError(); + } + throw error; + } + fs.unlinkSync(temporary); + } else { + fs.renameSync(temporary, target); + } + fs.chmodSync(target, JOURNAL_FILE_MODE); + fsyncDirectory(directory); + } catch (error) { + primaryFailure = { error }; + } + let cleanupFailure: { readonly error: unknown } | null = null; + if (descriptor !== null) { + try { + fs.closeSync(descriptor); + } catch (error) { + cleanupFailure = { error }; + } + } + try { + fs.unlinkSync(temporary); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT" && cleanupFailure === null) { + cleanupFailure = { error }; + } + } + if (primaryFailure !== null) throw primaryFailure.error; + if (cleanupFailure !== null) throw cleanupFailure.error; +} + +export function createFileDockerManagedBootstrapJournalStore( + stateRoot: string, +): DockerManagedBootstrapJournalStore { + const directory = path.join(stateRoot, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); + const load = (bootstrapIdentity: string): DockerManagedBootstrapJournal | null => { + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const contents = readPrivateFile(target, "journal"); + if (contents === null) return null; + const journal = parseDockerManagedBootstrapJournal(contents); + const decision = readPrivateFile(decisionPath(target), "decision"); + if (decision === null) return journal; + const phase = decision.endsWith("\n") ? decision.slice(0, -1) : ""; + if ( + !DECISION_PHASES.has(phase as DockerManagedBootstrapJournalPhase) || + (journal.phase !== "cutover" && journal.phase !== phase) + ) { + fail("decision does not match its cutover journal"); + } + const decided = normalizeDockerManagedBootstrapJournal({ ...journal, phase }); + if (journal.phase === "cutover") { + atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(decided), false); + } + return decided; + }; + return Object.freeze({ + create(journal: DockerManagedBootstrapJournal) { + const normalized = normalizeDockerManagedBootstrapJournal(journal); + assertDirectory(directory); + const target = journalPath(directory, normalized.bootstrapIdentity); + if (readPrivateFile(decisionPath(target), "decision") !== null) { + fail("stale decision exists for this bootstrap identity"); + } + atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(normalized), true); + }, + load, + transition( + bootstrapIdentity: string, + expected: DockerManagedBootstrapJournalPhase, + next: DockerManagedBootstrapJournalPhase, + ) { + if (!ALLOWED_TRANSITIONS.has(`${expected}->${next}`)) { + fail(`transition ${expected} to ${next} is unsupported`); + } + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const current = load(bootstrapIdentity); + if (current?.phase === next) return current; + if (!current || current.phase !== expected) { + fail(`expected phase ${expected} before transition to ${next}`); + } + const updated = normalizeDockerManagedBootstrapJournal({ ...current, phase: next }); + if (expected === "cutover") { + const decision = decisionPath(target); + try { + atomicWrite(directory, decision, `${next}\n`, true); + } catch (error) { + if ( + !(error instanceof DockerManagedBootstrapJournalExistsError) || + readPrivateFile(decision, "decision") !== `${next}\n` + ) { + throw error; + } + } + } + atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(updated), false); + return updated; + }, + remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]) { + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const current = load(bootstrapIdentity); + if (!current || !expected.includes(current.phase)) { + fail(`journal removal is not authorized from phase ${current?.phase ?? "absent"}`); + } + const decision = decisionPath(target); + if (readPrivateFile(decision, "decision") !== null) { + fs.unlinkSync(decision); + fsyncDirectory(directory); + } + fs.unlinkSync(target); + fsyncDirectory(directory); + }, + }); +} diff --git a/src/lib/onboard/managed-bootstrap/docker-spec.test.ts b/src/lib/onboard/managed-bootstrap/docker-spec.test.ts new file mode 100644 index 00000000000..94124f0127b --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-spec.test.ts @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { createDockerGpuInspectFixture } from "../__test-helpers__/docker-gpu-patch-fixtures"; +import { + normalizeDockerManagedBootstrapLaunchSpec, + parseDockerManagedBootstrapLaunchSpec, +} from "./docker-spec"; + +describe("managed bootstrap Docker launch spec", () => { + it("hashes reproducible launch state while excluding runtime ID, phase, IP, and gateway", () => { + const first = createDockerGpuInspectFixture(); + const second = structuredClone(first); + second.Id = "another-runtime-id"; + Object.assign(second, { State: { Running: false, Dead: true } }); + second.NetworkSettings!.Networks!["openshell-docker"]!.IPAddress = "172.18.0.99"; + second.NetworkSettings!.Networks!["openshell-docker"]!.Gateway = "172.18.0.254"; + + const expected = normalizeDockerManagedBootstrapLaunchSpec(first); + const observed = normalizeDockerManagedBootstrapLaunchSpec(second); + + expect(observed.hash).toBe(expected.hash); + expect(observed.canonicalJson).toBe(expected.canonicalJson); + expect(parseDockerManagedBootstrapLaunchSpec(expected.canonicalJson)).toEqual(expected.spec); + }); + + it("changes the hash when a reproducible launch field changes", () => { + const first = createDockerGpuInspectFixture(); + const second = structuredClone(first); + Object.assign(second.Config!, { StopTimeout: 45 }); + + expect(normalizeDockerManagedBootstrapLaunchSpec(second).hash).not.toBe( + normalizeDockerManagedBootstrapLaunchSpec(first).hash, + ); + }); + + it("orders durable launch keys by code unit across host locale settings", () => { + const inspect = createDockerGpuInspectFixture(); + inspect.Config!.Labels = { + "com.nvidia.foo": "lower", + "com.nvidia.Foo": "upper", + "com.nvidia-foo": "punctuation", + }; + + const canonical = normalizeDockerManagedBootstrapLaunchSpec(inspect).canonicalJson; + + expect(canonical.indexOf('"com.nvidia-foo"')).toBeLessThan( + canonical.indexOf('"com.nvidia.Foo"'), + ); + expect(canonical.indexOf('"com.nvidia.Foo"')).toBeLessThan( + canonical.indexOf('"com.nvidia.foo"'), + ); + }); + + it.each([ + { + name: "anonymous Config.Volumes whose data source cannot be proven", + mutate: (inspect: ReturnType) => { + Object.assign(inspect.Config!, { Volumes: { "/var/lib/state": {} } }); + }, + error: /config fields it cannot reproduce exactly: Volumes\./u, + }, + { + name: "multiple attached networks", + mutate: (inspect: ReturnType) => { + inspect.NetworkSettings!.Networks!.secondary = { Aliases: ["alpha-secondary"] }; + }, + error: /multiple attached networks/u, + }, + { + name: "an unknown HostConfig field", + mutate: (inspect: ReturnType) => { + (inspect.HostConfig as Record).FutureRuntimeField = true; + }, + error: /unsupported fields: FutureRuntimeField/u, + }, + ])("fails closed for $name", ({ mutate, error }) => { + const inspect = createDockerGpuInspectFixture(); + mutate(inspect); + expect(() => normalizeDockerManagedBootstrapLaunchSpec(inspect)).toThrow(error); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker-spec.ts b/src/lib/onboard/managed-bootstrap/docker-spec.ts new file mode 100644 index 00000000000..c3cab4e3296 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-spec.ts @@ -0,0 +1,316 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import type { DockerContainerInspect } from "../docker-gpu-patch-types"; + +const CONFIG_KEYS = new Set([ + "ArgsEscaped", + "AttachStderr", + "AttachStdin", + "AttachStdout", + "Cmd", + "Domainname", + "Entrypoint", + "Env", + "ExposedPorts", + "Healthcheck", + "Hostname", + "Image", + "Labels", + "MacAddress", + "NetworkDisabled", + "OnBuild", + "OpenStdin", + "Shell", + "StdinOnce", + "StopSignal", + "StopTimeout", + "Tty", + "User", + "Volumes", + "WorkingDir", +]); + +const HOST_CONFIG_KEYS = new Set([ + "AutoRemove", + "Binds", + "BlkioDeviceReadBps", + "BlkioDeviceReadIOps", + "BlkioDeviceWriteBps", + "BlkioDeviceWriteIOps", + "BlkioWeight", + "BlkioWeightDevice", + "CapAdd", + "CapDrop", + "Cgroup", + "CgroupParent", + "CgroupnsMode", + "ConsoleSize", + "ContainerIDFile", + "CpuCount", + "CpuPercent", + "CpuPeriod", + "CpuQuota", + "CpuRealtimePeriod", + "CpuRealtimeRuntime", + "CpuShares", + "CpusetCpus", + "CpusetMems", + "DeviceCgroupRules", + "DeviceRequests", + "Devices", + "Dns", + "DnsOptions", + "DnsSearch", + "ExtraHosts", + "GroupAdd", + "IOMaximumBandwidth", + "IOMaximumIOps", + "Init", + "IpcMode", + "Isolation", + "Links", + "LogConfig", + "MaskedPaths", + "Memory", + "MemoryReservation", + "MemorySwap", + "MemorySwappiness", + "Mounts", + "NanoCpus", + "NetworkMode", + "OomKillDisable", + "OomScoreAdj", + "PidMode", + "PidsLimit", + "PortBindings", + "Privileged", + "PublishAllPorts", + "ReadonlyPaths", + "ReadonlyRootfs", + "RestartPolicy", + "Runtime", + "SecurityOpt", + "ShmSize", + "StorageOpt", + "Sysctls", + "Tmpfs", + "UTSMode", + "Ulimits", + "UsernsMode", + "VolumeDriver", + "VolumesFrom", +]); + +const UNSUPPORTED_CONFIG_KEYS = new Set([ + "ArgsEscaped", + "AttachStderr", + "AttachStdin", + "AttachStdout", + "MacAddress", + "OnBuild", + "Shell", + "Volumes", +]); + +const UNSUPPORTED_HOST_CONFIG_KEYS = new Set([ + "BlkioDeviceReadBps", + "BlkioDeviceReadIOps", + "BlkioDeviceWriteBps", + "BlkioDeviceWriteIOps", + "BlkioWeight", + "BlkioWeightDevice", + "Cgroup", + "ConsoleSize", + "ContainerIDFile", + "CpuCount", + "CpuPercent", + "CpuRealtimePeriod", + "CpuRealtimeRuntime", + "IOMaximumBandwidth", + "IOMaximumIOps", + "Isolation", + "Links", + "MaskedPaths", + "MemorySwappiness", + "ReadonlyPaths", + "StorageOpt", + "VolumeDriver", + "VolumesFrom", +]); + +export interface DockerManagedBootstrapLaunchSpec { + readonly schemaVersion: 1; + readonly inspect: Pick< + DockerContainerInspect, + "Name" | "Config" | "HostConfig" | "NetworkSettings" + > & { readonly Platform?: string }; +} + +function isEmptyDefault(value: unknown): boolean { + if (value === undefined || value === null || value === false || value === "" || value === 0) { + return true; + } + if (Array.isArray(value)) return value.length === 0; + if (typeof value === "object") return Object.keys(value as object).length === 0; + return false; +} + +function exactObject(value: unknown, label: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`Managed bootstrap Docker ${label} must be an object.`); + } + return value as Record; +} + +function assertKnownKeys( + record: Record, + allowed: ReadonlySet, + label: string, +): void { + const unknown = Object.keys(record).filter((key) => !allowed.has(key)); + if (unknown.length > 0) { + throw new Error( + `Managed bootstrap Docker ${label} contains unsupported fields: ${unknown.sort().join(", ")}.`, + ); + } +} + +function assertUnsupportedDefaults(host: Record): void { + const active = [...UNSUPPORTED_HOST_CONFIG_KEYS].filter((key) => !isEmptyDefault(host[key])); + if (active.length > 0) { + throw new Error( + `Managed bootstrap refuses Docker launch fields it cannot reproduce exactly: ${active + .sort() + .join(", ")}.`, + ); + } +} + +function byCodeUnit(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function normalizedNetworkSettings( + value: DockerContainerInspect["NetworkSettings"], +): DockerContainerInspect["NetworkSettings"] { + const networks = value?.Networks ?? {}; + return { + Networks: Object.fromEntries( + Object.entries(networks) + .sort(([left], [right]) => byCodeUnit(left, right)) + .map(([name, network]) => [ + name, + { + Aliases: [...(network.Aliases ?? [])].sort(), + }, + ]), + ), + }; +} + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalize); + if (typeof value !== "object" || value === null) return value; + return Object.fromEntries( + Object.entries(value as Record) + .sort(([left], [right]) => byCodeUnit(left, right)) + .map(([key, nested]) => [key, canonicalize(nested)]), + ); +} + +export function parseExactDockerContainerInspect(output: string): DockerContainerInspect { + let parsed: unknown; + try { + parsed = JSON.parse(output); + } catch { + throw new Error("Managed bootstrap Docker inspect output is malformed."); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("Managed bootstrap Docker inspect must return exactly one workload."); + } + return exactObject(parsed[0], "inspect") as DockerContainerInspect; +} + +export function normalizeDockerManagedBootstrapLaunchSpec(inspect: DockerContainerInspect): { + readonly canonicalJson: string; + readonly hash: string; + readonly spec: DockerManagedBootstrapLaunchSpec; +} { + const raw = inspect as DockerContainerInspect & Record; + const config = exactObject(raw.Config, "Config"); + const hostConfig = exactObject(raw.HostConfig, "HostConfig"); + assertKnownKeys(config, CONFIG_KEYS, "Config"); + assertKnownKeys(hostConfig, HOST_CONFIG_KEYS, "HostConfig"); + const unsupportedConfig = [...UNSUPPORTED_CONFIG_KEYS].filter( + (key) => !isEmptyDefault(config[key]), + ); + if (unsupportedConfig.length > 0) { + throw new Error( + `Managed bootstrap refuses Docker config fields it cannot reproduce exactly: ${unsupportedConfig + .sort() + .join(", ")}.`, + ); + } + assertUnsupportedDefaults(hostConfig); + + if (config.NetworkDisabled === true) { + throw new Error("Managed bootstrap does not support Config.NetworkDisabled."); + } + if (config.StdinOnce === true) { + throw new Error("Managed bootstrap does not support Config.StdinOnce."); + } + if (hostConfig.AutoRemove === true) { + throw new Error("Managed bootstrap cannot preserve an auto-remove held workload."); + } + if (hostConfig.PublishAllPorts === true) { + throw new Error("Managed bootstrap requires explicit Docker port bindings."); + } + if (Object.keys(inspect.NetworkSettings?.Networks ?? {}).length > 1) { + throw new Error("Managed bootstrap refuses a Docker workload with multiple attached networks."); + } + + const spec: DockerManagedBootstrapLaunchSpec = { + schemaVersion: 1, + inspect: { + Name: inspect.Name, + Config: config as DockerContainerInspect["Config"], + HostConfig: hostConfig as DockerContainerInspect["HostConfig"], + NetworkSettings: normalizedNetworkSettings(inspect.NetworkSettings), + ...("Platform" in raw && typeof raw.Platform === "string" ? { Platform: raw.Platform } : {}), + }, + }; + const canonicalJson = `${JSON.stringify(canonicalize(spec))}\n`; + return Object.freeze({ + canonicalJson, + hash: createHash("sha256").update(canonicalJson, "utf8").digest("hex"), + spec: Object.freeze(spec), + }); +} + +export function parseDockerManagedBootstrapLaunchSpec( + canonicalJson: string, +): DockerManagedBootstrapLaunchSpec { + let parsed: unknown; + try { + parsed = JSON.parse(canonicalJson); + } catch { + throw new Error("Managed bootstrap Docker launch snapshot is malformed."); + } + const record = exactObject(parsed, "launch snapshot"); + if ( + Object.keys(record).sort().join(",") !== ["inspect", "schemaVersion"].join(",") || + record.schemaVersion !== 1 + ) { + throw new Error("Managed bootstrap Docker launch snapshot schema is invalid."); + } + const normalized = normalizeDockerManagedBootstrapLaunchSpec( + exactObject(record.inspect, "launch snapshot inspect") as DockerContainerInspect, + ); + if (normalized.canonicalJson !== canonicalJson) { + throw new Error("Managed bootstrap Docker launch snapshot is not canonical."); + } + return normalized.spec; +} diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index fe6bd97d130..94d235cad52 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -121,6 +121,8 @@ describe("runtime provider central source boundary", () => { it("inventories every dormant managed-bootstrap protocol source", () => { expect(bootstrapProtocolPaths).toEqual([ "src/lib/onboard/managed-bootstrap/adapter.ts", + "src/lib/onboard/managed-bootstrap/docker-journal.ts", + "src/lib/onboard/managed-bootstrap/docker-spec.ts", "src/lib/onboard/managed-bootstrap/envelope.ts", "src/lib/onboard/managed-bootstrap/index.ts", ]); From 149b3a244fe6707619010cddbad2976e880d89b2 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 06:19:08 -0700 Subject: [PATCH 095/117] fix(onboard): freeze canonical Docker launch specs Signed-off-by: Aaron Erickson --- .../managed-bootstrap/docker-spec.test.ts | 25 +++++++++++++++++++ .../onboard/managed-bootstrap/docker-spec.ts | 11 ++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/docker-spec.test.ts b/src/lib/onboard/managed-bootstrap/docker-spec.test.ts index 94124f0127b..3a8da33b9af 100644 --- a/src/lib/onboard/managed-bootstrap/docker-spec.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-spec.test.ts @@ -54,6 +54,31 @@ describe("managed bootstrap Docker launch spec", () => { ); }); + it("detaches and deeply freezes canonical launch state at the hashed boundary", () => { + const inspect = createDockerGpuInspectFixture(); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(inspect); + const { canonicalJson, hash } = normalized; + const config = normalized.spec.inspect.Config as Record; + const hostConfig = normalized.spec.inspect.HostConfig as Record; + const network = normalized.spec.inspect.NetworkSettings!.Networks!["openshell-docker"]!; + + expect(() => Object.assign(config, { StopTimeout: 999 })).toThrow(TypeError); + expect(() => Object.assign(hostConfig, { Runtime: "mutated" })).toThrow(TypeError); + expect(() => network.Aliases!.push("mutated")).toThrow(TypeError); + + Object.assign(inspect.Config!, { StopTimeout: 45 }); + Object.assign(inspect.HostConfig!, { Runtime: "mutated" }); + inspect.NetworkSettings!.Networks!["openshell-docker"]!.Aliases!.push("mutated"); + + expect(normalized.spec.inspect.Config).not.toHaveProperty("StopTimeout"); + expect(normalized.spec.inspect.HostConfig).not.toHaveProperty("Runtime"); + expect(network.Aliases).toEqual(["openshell-alpha"]); + expect(normalized.canonicalJson).toBe(`${JSON.stringify(normalized.spec)}\n`); + expect(normalized.canonicalJson).toBe(canonicalJson); + expect(normalized.hash).toBe(hash); + expect(normalizeDockerManagedBootstrapLaunchSpec(inspect).hash).not.toBe(hash); + }); + it.each([ { name: "anonymous Config.Volumes whose data source cannot be proven", diff --git a/src/lib/onboard/managed-bootstrap/docker-spec.ts b/src/lib/onboard/managed-bootstrap/docker-spec.ts index c3cab4e3296..19eaef7f288 100644 --- a/src/lib/onboard/managed-bootstrap/docker-spec.ts +++ b/src/lib/onboard/managed-bootstrap/docker-spec.ts @@ -221,6 +221,12 @@ function canonicalize(value: unknown): unknown { ); } +function deepFreeze(value: T): T { + if (typeof value !== "object" || value === null || Object.isFrozen(value)) return value; + for (const nested of Object.values(value)) deepFreeze(nested); + return Object.freeze(value); +} + export function parseExactDockerContainerInspect(output: string): DockerContainerInspect { let parsed: unknown; try { @@ -282,11 +288,12 @@ export function normalizeDockerManagedBootstrapLaunchSpec(inspect: DockerContain ...("Platform" in raw && typeof raw.Platform === "string" ? { Platform: raw.Platform } : {}), }, }; - const canonicalJson = `${JSON.stringify(canonicalize(spec))}\n`; + const canonicalSpec = deepFreeze(canonicalize(spec) as DockerManagedBootstrapLaunchSpec); + const canonicalJson = `${JSON.stringify(canonicalSpec)}\n`; return Object.freeze({ canonicalJson, hash: createHash("sha256").update(canonicalJson, "utf8").digest("hex"), - spec: Object.freeze(spec), + spec: canonicalSpec, }); } From 500656ea0bf7b7bcaba1fa9fbdc39fa402d77d21 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 05:34:10 -0700 Subject: [PATCH 096/117] feat(onboard): add transactional Docker bootstrap adapter Signed-off-by: Aaron Erickson --- .../snapshot-auto-create-failure.test.ts | 18 +- .../sandbox/snapshot-restore-test-fixture.ts | 5 + src/lib/actions/sandbox/snapshot.test.ts | 10 +- .../openshell/sandbox-identity.test.ts | 21 + .../adapters/openshell/sandbox-identity.ts | 16 + src/lib/onboard/docker-gpu-patch-clone.ts | 23 +- src/lib/onboard/docker-gpu-patch-types.ts | 17 + src/lib/onboard/managed-bootstrap/README.md | 48 +- src/lib/onboard/managed-bootstrap/adapter.ts | 4 +- .../managed-bootstrap/docker-shared-state.ts | 629 ++++ .../managed-bootstrap/docker-test-fixture.ts | 479 +++ .../onboard/managed-bootstrap/docker.test.ts | 320 ++ src/lib/onboard/managed-bootstrap/docker.ts | 2967 +++++++++++++++++ .../openshell-docker-sandbox-containers.ts | 1 + test/runtime-provider-source-shape.test.ts | 3 + tsconfig.src.json | 7 +- 16 files changed, 4538 insertions(+), 30 deletions(-) create mode 100644 src/lib/adapters/openshell/sandbox-identity.test.ts create mode 100644 src/lib/adapters/openshell/sandbox-identity.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-shared-state.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-test-fixture.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker.test.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker.ts diff --git a/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts b/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts index 65612e0ae3d..9dc200a96e1 100644 --- a/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts +++ b/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts @@ -28,13 +28,22 @@ const streamSandboxCreateMock = vi.fn(async () forcedReady: false, })); -vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => "") })); +vi.mock("../../adapters/docker", () => ({ + dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(), + dockerRunDetached: vi.fn(), +})); vi.mock("../../adapters/openshell/runtime", () => ({ captureOpenshell: captureOpenshellMock, getOpenshellBinary: vi.fn(() => "openshell"), runOpenshell: vi.fn(() => ({ status: 0, output: "" })), })); -vi.mock("../../credentials/store", () => ({ prompt: vi.fn() })); +vi.mock("../../credentials/store", () => ({ + deleteCredential: vi.fn(), + getCredential: vi.fn(() => null), + prompt: vi.fn(), + saveCredential: vi.fn(), +})); vi.mock("../../domain/sandbox/destroy", () => ({ getSandboxDeleteOutcome: vi.fn(() => ({ alreadyGone: false, gatewayUnreachable: false })), })); @@ -50,6 +59,11 @@ vi.mock("../../inference/nim", () => ({ stopNimContainerByName: vi.fn(), })); vi.mock("../../messaging/channels", () => ({ + BUILT_IN_CHANNEL_MANIFESTS: [], + getMessagingConfigEnvAliases: vi.fn(() => ({})), + getMessagingCredentialEnvKeysByChannel: vi.fn(() => ({})), + getMessagingProviderSuffixesByChannel: vi.fn(() => ({})), + listBuiltInMessagingChannelManifests: vi.fn(() => []), listMessagingProviderSuffixes: vi.fn(() => []), listMessagingCredentialMetadata: vi.fn(() => []), })); diff --git a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts index 66819eb0951..0d99c458141 100644 --- a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts +++ b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts @@ -213,7 +213,9 @@ export { lifecycleMock, shieldsMock }; vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(), dockerInspect: dockerInspectMock, + dockerRunDetached: vi.fn(), })); vi.mock("../../agent/defs", () => ({ @@ -227,7 +229,10 @@ vi.mock("../../adapters/openshell/runtime", () => ({ })); vi.mock("../../credentials/store", () => ({ + deleteCredential: vi.fn(), + getCredential: vi.fn(() => null), prompt: vi.fn(), + saveCredential: vi.fn(), })); vi.mock("../../domain/sandbox/destroy", () => ({ diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 977a2f49822..19cf6428de0 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -144,19 +144,21 @@ const latestBackupFixture = { vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(), dockerInspect: dockerInspectMock, + dockerRunDetached: vi.fn(), })); - vi.mock("../../adapters/openshell/runtime", () => ({ captureOpenshell: captureOpenshellMock, getOpenshellBinary: vi.fn(() => "openshell"), runOpenshell: runOpenshellMock, })); - vi.mock("../../credentials/store", () => ({ + deleteCredential: vi.fn(), + getCredential: vi.fn(() => null), prompt: vi.fn(), + saveCredential: vi.fn(), })); - vi.mock("../../domain/sandbox/destroy", () => ({ getSandboxDeleteOutcome: vi.fn(() => ({ alreadyGone: false, gatewayUnreachable: false })), })); @@ -165,7 +167,6 @@ vi.mock("../../inference/nim", () => ({ stopNimContainer: vi.fn(), stopNimContainerByName: vi.fn(), })); - vi.mock("../../policy", async (importOriginal) => ({ ...(await importOriginal()), applyPreset: applyPresetMock, @@ -176,7 +177,6 @@ vi.mock("../../policy", async (importOriginal) => ({ removePreset: removePresetMock, resolveAgentBaselinePolicy: resolveTestAgentBaselinePolicy, })); - vi.mock("../../runner", () => ({ ROOT: "/repo", run: vi.fn(() => ({ status: 0 })), diff --git a/src/lib/adapters/openshell/sandbox-identity.test.ts b/src/lib/adapters/openshell/sandbox-identity.test.ts new file mode 100644 index 00000000000..422bf71ccd8 --- /dev/null +++ b/src/lib/adapters/openshell/sandbox-identity.test.ts @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { parseOpenShellSandboxId } from "./sandbox-identity"; + +describe("OpenShell sandbox identity parsing", () => { + it("accepts one exact durable ID with optional terminal color", () => { + expect(parseOpenShellSandboxId("Name: alpha\nID: sandbox-alpha\n")).toBe("sandbox-alpha"); + expect(parseOpenShellSandboxId("\u001b[32mId: sandbox.alpha_2\u001b[0m\n")).toBe( + "sandbox.alpha_2", + ); + }); + + it("rejects ambiguous or non-canonical IDs", () => { + expect(parseOpenShellSandboxId("ID: first\nID: second\n")).toBeNull(); + expect(parseOpenShellSandboxId("ID: sandbox/alpha\n")).toBeNull(); + expect(parseOpenShellSandboxId("id: sandbox-alpha\n")).toBeNull(); + }); +}); diff --git a/src/lib/adapters/openshell/sandbox-identity.ts b/src/lib/adapters/openshell/sandbox-identity.ts new file mode 100644 index 00000000000..1820a8f8f7d --- /dev/null +++ b/src/lib/adapters/openshell/sandbox-identity.ts @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const ANSI_RE = /\x1b\[[0-9;]*m/gu; +const SANDBOX_ID_RE = /^[A-Za-z0-9._-]+$/u; + +export function parseOpenShellSandboxId(output: string): string | null { + const matches = [ + ...String(output) + .replace(ANSI_RE, "") + .matchAll(/^\s*(?:Id|ID):\s*(\S+)\s*$/gm), + ].map((match) => match[1] ?? ""); + return matches.length === 1 && SANDBOX_ID_RE.test(matches[0] as string) + ? (matches[0] as string) + : null; +} diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts index c92767adad0..828ce0c540b 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -329,7 +329,15 @@ export function buildDockerGpuCloneRunArgs( const image = String(options.image || config.Image || "").trim(); if (!image) throw new Error("Docker inspect output did not include Config.Image."); - const args: string[] = ["--name", dockerContainerName(inspect), ...mode.args]; + const containerName = String(options.containerName ?? dockerContainerName(inspect)).trim(); + if ( + containerName.length === 0 || + containerName.length > 253 || + !/^[A-Za-z0-9][A-Za-z0-9_.-]*$/u.test(containerName) + ) { + throw new Error("Docker clone container name is invalid."); + } + const args: string[] = ["--name", containerName, ...mode.args]; const gpuAugment = mode.kind !== "startup-command"; // Startup-command recreation must retain OpenShell's native CDI attachment. @@ -435,8 +443,17 @@ export function buildDockerGpuCloneRunArgs( if (host.Init) args.push("--init"); const entrypoint = stringArray(config.Entrypoint); - if (entrypoint.length > 0) args.push("--entrypoint", entrypoint[0]); - const commandArgs = sandboxCommand ? [] : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; + const replacementEntrypoint = String(options.containerEntrypoint ?? "").trim(); + if (replacementEntrypoint) { + args.push("--entrypoint", replacementEntrypoint); + } else if (entrypoint.length > 0) { + args.push("--entrypoint", entrypoint[0]); + } + const commandArgs = options.containerCommand + ? [...options.containerCommand] + : sandboxCommand + ? [] + : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; args.push(image, ...commandArgs); return args; } diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 32be0afe46f..d72046bd320 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -112,6 +112,14 @@ export type DockerGpuCloneRunOptions = { sandboxFallbackDns?: string | null; openshellSandboxCommand?: readonly string[] | null; requiredUlimits?: readonly DockerUlimit[] | null; + /** + * Exact replacement process boundary used only by dormant managed bootstrap. + * Ordinary recreation leaves both fields unset. + */ + containerEntrypoint?: string | null; + containerCommand?: readonly string[] | null; + /** Stopped staging name used before exact-name cutover. */ + containerName?: string | null; /** * Extra supplementary group IDs to add to the recreated container via * `--group-add`. On Jetson these are the host group(s) owning the Tegra GPU @@ -190,6 +198,14 @@ export type DockerContainerInspect = { Hostname?: string; Tty?: boolean; OpenStdin?: boolean; + StopTimeout?: number | null; + Volumes?: Record | null; + } | null; + State?: { + Running?: boolean; + Paused?: boolean; + Restarting?: boolean; + Dead?: boolean; } | null; HostConfig?: { Binds?: string[] | null; @@ -244,6 +260,7 @@ export type DockerContainerInspect = { DeviceIDs?: string[] | null; }> | null; ShmSize?: number; + ReadonlyRootfs?: boolean; ReadonlyPaths?: string[] | null; MaskedPaths?: string[] | null; } | null; diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index adde636fa56..717fcaa048b 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -3,9 +3,9 @@ # Managed bootstrap protocol -This directory defines a dormant, driver-neutral transaction contract. It does -not register a runtime provider or change sandbox creation, onboarding, -snapshot, clone, or restore behavior. +This directory defines a dormant, driver-neutral transaction contract and its +first driver adapter. It does not register a runtime provider or change sandbox +creation, onboarding, snapshot, clone, or restore behavior. The protocol binds one random bootstrap identity to: @@ -71,6 +71,21 @@ journal and a canonical launch-spec normalizer. Each surface is independently validated and remains dormant: no registered runtime provider imports either module, and neither changes sandbox creation or lifecycle behavior. +The Docker adapter creates and validates a stopped replacement under an +identity-derived staging name while the original remains running. It stages the +0400 envelope and returns exact cleanup authority without quiescing, renaming, +or otherwise mutating the original. Only after the coordinator durably records +that complete prepared authority may activation journal both full runtime IDs, +all three names, both launch-spec hashes, image identity, profile fingerprint, +and sandbox ID and then enter the destructive cutover. Rollback publishes +`rollback-authorized` before exact replacement deletion; commit publishes +`shared-state-committed` before exact backup deletion. Cleanup is bound to full +runtime IDs. Mutable OpenShell names are read only to detect ownership reuse, +and unsafe name-only deletion returns a typed retention error. The dormant +adapter assumes the protocol's single coordinator; multi-process +lease/arbitration remains an explicit production-activation gate. Activation +must also inject the selected gateway's canonical state root. + ## Architectural disposition The coordinator deliberately lands as a dormant trust-boundary slice before a @@ -91,17 +106,16 @@ activation slice must add a registered-provider contract test for the same transaction before removing those dormancy assertions. The native entrypoint source is intentionally not compiled into production -artifacts, and neither source is packaged or selected yet. No production -TypeScript module imports this protocol. The current image definitions do not -package `nemoclaw-managed-startup-hold` or -`managed-startup-image-runtime.cjs`. A later -provider integration must compile and verify the freestanding entrypoint -natively for amd64 and arm64 in every agent image. It must add those -prerequisites together with their image-runtime bootstrap modes, implement -driver-specific prepare, durable-record, activate, exact cleanup, and rollback, -and only then wire the coordinator into create. The same contract is exercised -for OpenClaw, Hermes, and Deep Agents Code without a provider-specific central -switch. The remaining integration and qualification work is tracked in -[epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) and its linked -implementation stack. Until that complete boundary lands, every registered -runtime provider keeps its bootstrap surface unsupported. +artifacts, and neither image-owned source is packaged or selected yet. No +production TypeScript module imports this protocol or the Docker adapter. The +current image definitions do not package `nemoclaw-managed-startup-hold`, +`managed-startup-image-runtime.cjs`, or the shared-state bootstrap modes consumed +by the adapter. A later provider integration must compile and verify the +freestanding entrypoint natively for amd64 and arm64 in every agent image. It +must add those prerequisites together with their image-runtime bootstrap modes +and wire the coordinator and Docker adapter into create as one boundary. The +same contract is exercised for OpenClaw, Hermes, and Deep Agents Code without a +provider-specific central switch. The remaining integration and qualification +work is tracked in [epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) +and its linked implementation stack. Until that complete boundary lands, every +registered runtime provider keeps its bootstrap surface unsupported. diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index c29fa7a01d7..b0edaa76627 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -931,7 +931,7 @@ function normalizePreparedReplacement( }); } -function createPreparedAuthority( +export function createManagedBootstrapPreparedAuthority( transaction: ManagedBootstrapPreparedTransaction, ): ManagedBootstrapPreparedAuthority { const { handle, snapshot, prepared } = transaction; @@ -1358,7 +1358,7 @@ export async function activateManagedBootstrapSequence( let durablePreparation: ManagedBootstrapDurablePreparationReceipt | null = null; let replacement: ManagedBootstrapReplacementHandle | null = null; try { - const authority = createPreparedAuthority(input.transaction); + const authority = createManagedBootstrapPreparedAuthority(input.transaction); durablePreparation = normalizeDurablePreparationReceipt( await input.authorityStore.recordPreparedAuthority(authority), authority, diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts new file mode 100644 index 00000000000..5953fc2b356 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts @@ -0,0 +1,629 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { + dockerRm as defaultDockerRm, + dockerStop as defaultDockerStop, +} from "../../adapters/docker/container"; +import { dockerRun as defaultDockerRun } from "../../adapters/docker/run"; +import { hasZeroDockerExitStatus } from "../docker-command-result"; +import { + DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + DOCKER_GPU_PATCH_TIMEOUT_MS, +} from "../docker-gpu-patch-constants"; +import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "../docker-gpu-patch-types"; +import { MANAGED_STARTUP_RUNTIME_EXECUTABLE } from "../managed-startup/image-runtime"; +import { MANAGED_STARTUP_AGENTS, type ManagedStartupAgent } from "../managed-startup/profile"; +import { + MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, +} from "../managed-startup/shared-state-transaction"; +import { isImmutableDockerImageId } from "../openshell-docker-sandbox-containers"; +import { cleanupTempDir, secureTempFile } from "../temp-files"; + +const RECEIPT_TEMP_PREFIX = "nemoclaw-managed-startup-receipt"; +const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY = + "/var/lib/nemoclaw/managed-startup-shared-state-commit-v1"; +const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; +const DURABLE_IDENTITY_RE = /^[a-f0-9]{64}$/u; +const NEUTRALIZED_PROCESS_INJECTION_ENV = [ + "--env", + "NODE_OPTIONS=", + "--env", + "NODE_PATH=", + "--env", + "BASH_ENV=", + "--env", + "ENV=", + "--env", + "LD_PRELOAD=", + "--env", + "LD_AUDIT=", + "--env", + "LD_LIBRARY_PATH=", + "--env", + "SHELLOPTS=", + "--env", + "PS4=", +] as const; + +export interface DockerManagedBootstrapSharedStateTransaction { + readonly agent: ManagedStartupAgent; + readonly bootstrapIdentity: string; + readonly containerId: string; + readonly image: string; + readonly profileFingerprint: string; +} + +export interface DockerManagedStartupSharedStateOutcome { + /** + * True only when the new supervisor is still eligible for successful + * container cutover. A commit failure forces shared-state rollback first. + */ + readonly supervisorReady: boolean; + /** Original commit failure after a successful shared-state rollback. */ + readonly failure: Error | null; +} + +export class DockerManagedStartupSharedStateCommitIndeterminateError extends Error { + constructor(detail: string, options?: ErrorOptions) { + super( + `Managed-startup shared-state commit may have completed, but immutable status is unavailable: ${detail}`, + options, + ); + this.name = "DockerManagedStartupSharedStateCommitIndeterminateError"; + } +} + +export function probeDockerManagedStartupSharedState( + input: { + readonly transaction: DockerManagedBootstrapSharedStateTransaction; + readonly profileFingerprint: string; + }, + deps: DockerGpuPatchDeps = {}, +): "committed" | "none" | "pending" { + const transaction = input.transaction; + assertValidManagedStartupTransaction(transaction); + if (input.profileFingerprint !== transaction.profileFingerprint) { + throw new Error("Managed bootstrap shared-state status fingerprint does not match."); + } + const committedReceiptPath = copyManagedStartupReceiptAt( + transaction, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, + deps, + true, + ); + if (committedReceiptPath) { + let verified = false; + try { + verifyCopiedManagedStartupReceipt( + transaction, + input.profileFingerprint, + committedReceiptPath, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, + "committed", + deps, + ); + verified = true; + return "committed"; + } finally { + if (verified) cleanupReceiptBestEffort(committedReceiptPath); + } + } + const receiptPath = copyManagedStartupReceipt(transaction, deps, true); + if (!receiptPath) return "none"; + let verified = false; + try { + verifyCopiedManagedStartupReceipt( + transaction, + input.profileFingerprint, + receiptPath, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + "pending", + deps, + ); + verified = true; + return "pending"; + } finally { + if (verified) cleanupReceiptBestEffort(receiptPath); + } +} + +function verifyCopiedManagedStartupReceipt( + transaction: DockerManagedBootstrapSharedStateTransaction, + profileFingerprint: string, + receiptPath: string, + receiptDirectory: string, + expectedStatus: "committed" | "pending", + deps: DockerGpuPatchDeps, +): void { + if (!transaction.bootstrapIdentity || !/^[a-f0-9]{64}$/u.test(profileFingerprint)) { + throw new Error("Managed bootstrap copied-receipt identity is incomplete."); + } + const dockerRun = deps.dockerRun ?? defaultDockerRun; + const result = dockerRun( + [ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + "--mount", + transactionReceiptMount(receiptPath, receiptDirectory), + "--entrypoint", + "/usr/local/bin/node", + transaction.image, + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + "--shared-state-transaction-status", + "--agent", + transaction.agent, + "--profile-fingerprint", + profileFingerprint, + "--bootstrap-identity", + transaction.bootstrapIdentity, + ], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(result)) { + throw new Error( + `Immutable managed-startup helper could not verify shared-state status: ${commandDetail(result)}. ` + + `Protected receipt retained at ${receiptPath}`, + ); + } + if (String(result.stdout ?? "").trim() !== expectedStatus) { + throw new Error( + `Immutable managed-startup helper returned an invalid copied transaction status. Protected receipt retained at ${receiptPath}`, + ); + } +} + +function commandDetail(result: { + readonly stderr?: string | Buffer | null; + readonly stdout?: string | Buffer | null; + readonly error?: Error | null; +}): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}` + .trim() + .slice(-800); +} + +function cleanupReceiptBestEffort(receiptPath: string): void { + try { + cleanupTempDir(receiptPath, RECEIPT_TEMP_PREFIX); + } catch (error) { + console.warn( + ` ⚠ Managed-startup shared state is finalized, but its protected host receipt could not be removed (${receiptPath}): ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +function assertValidManagedStartupTransaction( + transaction: DockerManagedBootstrapSharedStateTransaction, +): asserts transaction is DockerManagedBootstrapSharedStateTransaction & { + readonly bootstrapIdentity: string; + readonly profileFingerprint: string; +} { + if (!(MANAGED_STARTUP_AGENTS as readonly string[]).includes(transaction.agent)) { + throw new Error("Managed bootstrap shared-state transaction agent is invalid."); + } + if (!FULL_CONTAINER_ID_RE.test(transaction.containerId)) { + throw new Error("Managed bootstrap shared-state transaction container identity is invalid."); + } + if (!isImmutableDockerImageId(transaction.image)) { + throw new Error("Managed bootstrap shared-state transaction image identity is not immutable."); + } + if (!transaction.bootstrapIdentity || !DURABLE_IDENTITY_RE.test(transaction.bootstrapIdentity)) { + throw new Error("Managed bootstrap shared-state transaction identity is missing or invalid."); + } + if ( + !transaction.profileFingerprint || + !DURABLE_IDENTITY_RE.test(transaction.profileFingerprint) + ) { + throw new Error( + "Managed bootstrap shared-state transaction profile fingerprint is missing or invalid.", + ); + } +} + +function transactionCommand( + action: "clear-shared-state-commit-receipt" | "commit" | "rollback", + transaction: DockerManagedBootstrapSharedStateTransaction, +): string[] { + assertValidManagedStartupTransaction(transaction); + return [ + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + action === "clear-shared-state-commit-receipt" + ? "--clear-shared-state-commit-receipt" + : `--${action}-shared-state-transaction`, + "--agent", + transaction.agent, + "--bootstrap-identity", + transaction.bootstrapIdentity, + ]; +} + +export function clearDockerManagedStartupSharedStateCommitReceipt( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps = {}, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + assertValidManagedStartupTransaction(transaction); + const command = transactionCommand("clear-shared-state-commit-receipt", transaction); + const cleared = dockerRun( + [ + "exec", + "--user", + "0:0", + "--workdir", + "/", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + transaction.containerId, + "/usr/bin/env", + "-i", + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + ...command, + ], + DOCKER_MUTATION_OPTIONS, + ); + // Accept a lost Docker acknowledgement only when both exact image-owned + // receipt paths are independently proven absent by the immutable helper. + let status: "committed" | "none" | "pending"; + try { + status = probeDockerManagedStartupSharedState( + { + transaction, + profileFingerprint: transaction.profileFingerprint, + }, + deps, + ); + } catch (error) { + throw new DockerManagedStartupSharedStateCommitIndeterminateError( + error instanceof Error ? error.message : String(error), + { cause: error }, + ); + } + if (status === "none") return; + if (!hasZeroDockerExitStatus(cleared)) { + throw new Error( + `Managed-startup durable commit receipt cleanup failed and exact absence was not proven (status=${status}): ${commandDetail(cleared)}`, + ); + } + throw new Error( + `Managed-startup durable commit receipt cleanup returned success, but exact absence was not proven (status=${status}).`, + ); +} + +function commitManagedStartupSharedState( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + assertValidManagedStartupTransaction(transaction); + const command = transactionCommand("commit", transaction); + const commit = dockerRun( + [ + "exec", + "--user", + "0:0", + "--workdir", + "/", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + transaction.containerId, + "/usr/bin/env", + "-i", + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + ...command, + ], + DOCKER_MUTATION_OPTIONS, + ); + // The commit helper atomically renames the rollback receipt into a compact + // identity-bound commit receipt before Docker returns. Always probe it + // afterward so a lost daemon acknowledgement is accepted only when durable + // commit state is independently proven. + let status: "committed" | "none" | "pending"; + try { + status = probeDockerManagedStartupSharedState( + { + transaction, + profileFingerprint: transaction.profileFingerprint, + }, + deps, + ); + } catch (error) { + throw new DockerManagedStartupSharedStateCommitIndeterminateError( + error instanceof Error ? error.message : String(error), + { cause: error }, + ); + } + if (status === "committed") return; + if (!hasZeroDockerExitStatus(commit)) { + throw new Error( + `Managed-startup shared-state commit helper failed and durable commit was not proven (status=${status}): ${commandDetail(commit)}`, + ); + } + throw new Error( + `Managed-startup shared-state commit helper returned success, but durable commit was not proven (status=${status}).`, + ); +} + +const DOCKER_MUTATION_OPTIONS = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, +} as const; + +function quiesceManagedStartupContainer( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerStop = deps.dockerStop ?? defaultDockerStop; + const stopped = dockerStop(transaction.containerId, { + ...DOCKER_MUTATION_OPTIONS, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopped)) { + throw new Error( + `Could not quiesce the failed managed-startup container before shared-state rollback: ${commandDetail(stopped)}`, + ); + } +} + +function isExactMissingReceiptCopy( + transaction: DockerManagedBootstrapSharedStateTransaction, + sourcePath: string, + result: { + readonly stderr?: string | Buffer | null; + readonly stdout?: string | Buffer | null; + readonly error?: Error | null; + }, +): boolean { + const detail = commandDetail(result); + const escapedPath = sourcePath.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const escapedContainer = transaction.containerId.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + return [ + new RegExp( + `^(?:Error response from daemon: )?Could not find the file ${escapedPath} in container ${escapedContainer}$`, + "u", + ), + new RegExp(`^(?:lstat|stat) ${escapedPath}: no such file or directory$`, "u"), + ].some((pattern) => pattern.test(detail)); +} + +function transactionReceiptMount(receiptPath: string, receiptDirectory: string): string { + return `type=bind,src=${receiptPath},dst=${receiptDirectory},readonly`; +} + +function copyManagedStartupReceiptAt( + transaction: DockerManagedBootstrapSharedStateTransaction, + sourcePath: string, + deps: DockerGpuPatchDeps, + allowAbsent = false, +): string | null { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + const tempSeed = secureTempFile(RECEIPT_TEMP_PREFIX); + const receiptPath = path.join(path.dirname(tempSeed), path.basename(sourcePath)); + try { + const copy = dockerRun( + ["cp", "-a", `${transaction.containerId}:${sourcePath}`, receiptPath], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(copy)) { + if (allowAbsent && isExactMissingReceiptCopy(transaction, sourcePath, copy)) { + cleanupReceiptBestEffort(receiptPath); + return null; + } + throw new Error( + `Could not copy the managed-startup rollback receipt from the failed container: ${commandDetail(copy)}`, + ); + } + if (receiptPath.includes(",") || /[\r\n\0]/u.test(receiptPath)) { + throw new Error("Managed-startup rollback receipt path is unsafe for a Docker bind mount"); + } + return receiptPath; + } catch (error) { + cleanupReceiptBestEffort(receiptPath); + throw error; + } +} + +function copyManagedStartupReceipt( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, + allowAbsent = false, +): string | null { + return copyManagedStartupReceiptAt( + transaction, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + deps, + allowAbsent, + ); +} + +function rollbackManagedStartupSharedState( + transaction: DockerManagedBootstrapSharedStateTransaction, + receiptPath: string, + deps: DockerGpuPatchDeps, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + let restored = false; + try { + const helper = dockerRun( + [ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--cap-add", + "CHOWN", + "--cap-add", + "DAC_OVERRIDE", + "--cap-add", + "FOWNER", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + "--volumes-from", + transaction.containerId, + "--mount", + `type=bind,src=${receiptPath},dst=${MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY},readonly`, + "--entrypoint", + "/usr/local/bin/node", + transaction.image, + ...transactionCommand("rollback", transaction), + "--read-only-receipt", + ], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(helper)) { + throw new Error( + `Immutable managed-startup helper could not restore and verify shared state: ${commandDetail(helper)}. ` + + `Protected receipt retained at ${receiptPath}`, + ); + } + restored = true; + } finally { + if (restored) { + cleanupReceiptBestEffort(receiptPath); + } + } +} + +function removeFailedUnbackedContainer( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerRm = deps.dockerRm ?? defaultDockerRm; + const removed = dockerRm(transaction.containerId, DOCKER_MUTATION_OPTIONS); + if (!hasZeroDockerExitStatus(removed)) { + throw new Error( + `Could not remove the failed managed-startup container after shared-state rollback: ${commandDetail(removed)}`, + ); + } +} + +/** + * Finalize the shared-state half of managed container cutover before generic + * backup removal or rollback. A shared-state rollback failure deliberately + * throws so callers cannot remove the new container or restart the old one + * while `/sandbox` remains partially applied. + */ +export function finalizeDockerManagedStartupSharedState( + input: { + readonly transaction: DockerManagedBootstrapSharedStateTransaction | null; + readonly patchResult?: DockerGpuPatchResult | null; + /** + * The managed-bootstrap journal owns exact replacement removal. Retaining + * it lets the caller publish rollback authorization after shared-state + * restoration and before the first runtime deletion. + */ + readonly retainContainerAfterRollback?: boolean; + readonly supervisorReady: boolean; + }, + deps: DockerGpuPatchDeps = {}, +): DockerManagedStartupSharedStateOutcome { + const transaction = input.transaction; + if (!transaction) { + return { supervisorReady: input.supervisorReady, failure: null }; + } + assertValidManagedStartupTransaction(transaction); + if (input.supervisorReady) { + // Preserve and validate an explicit writable-layer receipt before logical + // commit. The helper receives the copy read-only and does not delete it; + // this keeps rollback possible when Docker loses the helper acknowledgement. + // --volumes-from exposes shared mounts only; it cannot expose this + // container-local transaction directory to an immutable helper. + let receiptPath: string; + try { + const copiedReceipt = copyManagedStartupReceipt(transaction, deps); + if (!copiedReceipt) { + throw new Error("Managed-startup pending receipt disappeared before commit."); + } + receiptPath = copiedReceipt; + } catch (error) { + try { + quiesceManagedStartupContainer(transaction, deps); + } catch (stopError) { + throw new Error( + `Managed-startup receipt preservation failed and the new workload could not be quiesced: ${ + error instanceof Error ? error.message : String(error) + }; ${stopError instanceof Error ? stopError.message : String(stopError)}`, + ); + } + throw error; + } + let commitFailure: Error | null = null; + try { + verifyCopiedManagedStartupReceipt( + transaction, + transaction.profileFingerprint, + receiptPath, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + "pending", + deps, + ); + commitManagedStartupSharedState(transaction, deps); + cleanupReceiptBestEffort(receiptPath); + return { supervisorReady: true, failure: null }; + } catch (error) { + if (error instanceof DockerManagedStartupSharedStateCommitIndeterminateError) { + throw error; + } + commitFailure = error instanceof Error ? error : new Error(String(error)); + } + const failure = new Error( + `OpenShell supervisor reconnected, but managed shared-state logical commit validation failed: ${commitFailure.message}`, + ); + quiesceManagedStartupContainer(transaction, deps); + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!input.patchResult && !input.retainContainerAfterRollback) { + removeFailedUnbackedContainer(transaction, deps); + } + return { supervisorReady: false, failure }; + } + + quiesceManagedStartupContainer(transaction, deps); + const receiptPath = copyManagedStartupReceipt(transaction, deps, true); + if (!receiptPath) { + if (!input.patchResult && !input.retainContainerAfterRollback) { + removeFailedUnbackedContainer(transaction, deps); + } + return { supervisorReady: false, failure: null }; + } + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!input.patchResult && !input.retainContainerAfterRollback) { + removeFailedUnbackedContainer(transaction, deps); + } + return { supervisorReady: false, failure: null }; +} diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts new file mode 100644 index 00000000000..1c2a6871b6a --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -0,0 +1,479 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { expect, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import type { DockerContainerInspect } from "../docker-gpu-patch-types"; +import { encodeManagedStartupProfile, type ManagedStartupAgent } from "../managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import { + createManagedBootstrapPreparedAuthority, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapCompletionReceipt, + type ManagedBootstrapDurablePreparationReceipt, + type ManagedBootstrapHeldWorkloadHandle, + type ManagedBootstrapObservedSnapshot, + type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapReplacementHandle, +} from "./adapter"; +import type { DockerManagedBootstrapDeps } from "./docker"; +import { + type DockerManagedBootstrapJournal, + DockerManagedBootstrapJournalAcknowledgementLostError, + type DockerManagedBootstrapJournalPhase, + type DockerManagedBootstrapJournalStore, +} from "./docker-journal"; +import { normalizeDockerManagedBootstrapLaunchSpec } from "./docker-spec"; +import { parseManagedBootstrapEnvelope } from "./envelope"; + +export const IDENTITY = "1".repeat(64); +export const OLD_ID = "2".repeat(64); +export const NEW_ID = "3".repeat(64); +const CONFIG_ID = `sha256:${"4".repeat(64)}`; +const MANIFEST = `sha256:${"5".repeat(64)}` as const; +const REPOSITORY = "registry.example/nemoclaw/hermes"; +const IMAGE = `${REPOSITORY}@${MANIFEST}`; +const SUPERVISOR = ["/opt/openshell/bin/openshell-sandbox", "supervise"] as const; +export const SUPPORTED_AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; + +type FixtureCommandResult = { + readonly status: number; + readonly stdout?: string; + readonly stderr?: string; +}; + +export type DockerFixtureAcknowledgement = + | "container:create" + | "container:remove" + | "container:rename" + | "container:start" + | "container:stop" + | "journal:create" + | "journal:cutover" + | "journal:remove" + | "journal:rollback-authorized" + | "journal:staged" + | "journal:shared-state-committed"; + +export type DockerFixtureOptions = { + readonly agent?: ManagedStartupAgent; + readonly dockerStartResults?: Readonly>; + readonly journalTransitionFailures?: Partial< + Readonly> + >; + readonly lostAcknowledgements?: readonly DockerFixtureAcknowledgement[]; + readonly ownerId?: string; + readonly sharedState?: "committed" | "none" | "pending"; +}; + +function agentInputs(agent: ManagedStartupAgent = "hermes") { + const request = createManagedStartupRootApplyRequest({ + agent, + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agent, false, false)), + }); + const heldArgv = [ + "env", + "A=1", + "/usr/local/bin/nemoclaw-managed-startup-hold", + "--agent", + agent, + "--profile-fingerprint", + request.profileFingerprint, + "--bootstrap-identity", + IDENTITY, + ] as const; + return { + request, + heldArgv, + metadata: { "nemoclaw.ai/managed-profile": request.profileFingerprint }, + }; +} + +export const { heldArgv } = agentInputs(); +export const sandbox = { + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + driverId: "docker", +}; + +function shellArgv(argv: readonly string[]): string { + return argv.join(" "); +} + +function originalInspect(inputs = agentInputs()): DockerContainerInspect { + return { + Id: OLD_ID, + Image: CONFIG_ID, + Name: "/openshell-alpha", + Config: { + Image: IMAGE, + Env: ["A=1", `OPENSHELL_SANDBOX_COMMAND=${shellArgv(inputs.heldArgv)}`], + Labels: { + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-name": "alpha", + "openshell.ai/sandbox-id": "sandbox-alpha", + ...inputs.metadata, + }, + Entrypoint: [SUPERVISOR[0]], + Cmd: SUPERVISOR.slice(1), + User: "root", + WorkingDir: "/sandbox", + Hostname: "alpha", + }, + State: { Running: true, Paused: false, Restarting: false, Dead: false }, + HostConfig: { + Binds: ["/host/workspace:/sandbox:rw"], + NetworkMode: "openshell", + RestartPolicy: { Name: "unless-stopped" }, + CapDrop: ["NET_RAW"], + SecurityOpt: ["no-new-privileges"], + Ulimits: [{ Name: "nofile", Soft: 65_536, Hard: 65_536 }], + }, + NetworkSettings: { Networks: { openshell: { Aliases: ["openshell-alpha"] } } }, + }; +} + +export function authority(agent: ManagedStartupAgent = "hermes") { + const inputs = agentInputs(agent); + const inspect = originalInspect(inputs); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(inspect); + const plan = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandboxName: "alpha", + driverId: "docker", + image: { repository: REPOSITORY, manifestDigest: MANIFEST }, + profile: { agent, fingerprint: inputs.request.profileFingerprint }, + agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + intendedWorkloadArgv: ["env", "A=1", "nemoclaw-start"], + expectedSupervisorArgv: SUPERVISOR, + metadata: inputs.metadata, + }; + const handle: ManagedBootstrapHeldWorkloadHandle = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: inputs.heldArgv, + intendedWorkloadArgv: plan.intendedWorkloadArgv, + plan, + createReceipt: { sandbox, ready: true, readyAt: "2026-07-31T12:00:00.000Z" }, + }; + const snapshot: ManagedBootstrapObservedSnapshot = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + runtimeId: OLD_ID, + bootstrapIdentity: IDENTITY, + image: plan.image, + runtimeImageContentId: CONFIG_ID, + specHash: normalized.hash, + specCanonicalJson: normalized.canonicalJson, + agentIdentity: plan.agentIdentity, + supervisorArgv: SUPERVISOR, + heldWorkloadArgv: inputs.heldArgv, + metadata: inputs.metadata, + }; + return { handle, plan, request: inputs.request, snapshot }; +} + +function failFixture(message: string): never { + throw new Error(message); +} + +function readProtectedEnvelope(source: string): ReturnType { + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") throw new Error("test requires O_NOFOLLOW"); + const descriptor = fs.openSync(source, fs.constants.O_RDONLY | noFollow); + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + expect(Number(before.mode & 0o777n)).toBe(0o400); + const parsed = parseManagedBootstrapEnvelope(fs.readFileSync(descriptor, "utf8")); + const after = fs.fstatSync(descriptor, { bigint: true }); + expect(after.dev).toBe(before.dev); + expect(after.ino).toBe(before.ino); + expect(after.size).toBe(before.size); + expect(after.mtimeNs).toBe(before.mtimeNs); + expect(after.ctimeNs).toBe(before.ctimeNs); + return parsed; + } finally { + fs.closeSync(descriptor); + } +} + +export function fixture(options: DockerFixtureOptions = {}) { + let original = originalInspect(agentInputs(options.agent)); + let replacement: DockerContainerInspect | null = null; + let journal: DockerManagedBootstrapJournal | null = null; + let sharedState: "committed" | "none" | "pending" = options.sharedState ?? "none"; + const events: string[] = []; + const lostAcknowledgements = new Set(options.lostAcknowledgements ?? []); + const losesAcknowledgement = (operation: DockerFixtureAcknowledgement) => + lostAcknowledgements.has(operation); + const ok = (stdout = ""): FixtureCommandResult => ({ status: 0, stdout, stderr: "" }); + const copyJournal = () => (journal ? structuredClone(journal) : null); + const store: DockerManagedBootstrapJournalStore = { + create(value) { + journal = structuredClone(value); + events.push("journal:staged"); + if (losesAcknowledgement("journal:create")) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal create acknowledgement", + ); + } + }, + load: () => copyJournal(), + transition(_identity, expected, next) { + const current = + journal !== null && journal.phase === expected + ? journal + : failFixture("stale journal transition"); + journal = { ...current, phase: next }; + events.push(`journal:${next}`); + const injectedFailure = options.journalTransitionFailures?.[next]; + if (injectedFailure) throw injectedFailure; + if (losesAcknowledgement(`journal:${next}`)) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal transition acknowledgement", + ); + } + return structuredClone(journal); + }, + remove(_identity, expected) { + const current = journal; + void (current !== null && expected.includes(current.phase) + ? current + : failFixture("stale journal remove")); + journal = null; + events.push("journal:removed"); + if (losesAcknowledgement("journal:remove")) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal remove acknowledgement", + ); + } + }, + }; + const inspect = (reference: string): DockerContainerInspect => { + const candidates = [original, replacement].filter( + (value): value is DockerContainerInspect => value !== null, + ); + const found = candidates.find( + (value) => + value.Id === reference || String(value.Name ?? "").replace(/^\/+/u, "") === reference, + ); + return found ? structuredClone(found) : failFixture(`No such container: ${reference}`); + }; + const dockerCapture: NonNullable = vi.fn((args) => { + switch (args[0]) { + case "image": + return JSON.stringify([{ Id: CONFIG_ID, RepoDigests: [IMAGE] }]); + default: + return JSON.stringify([inspect(String(args[3] ?? ""))]); + } + }); + const dockerRun: NonNullable = vi.fn( + (args: readonly string[]) => { + switch (args[0]) { + case "create": { + events.push("create:replacement"); + const name = String(args[args.indexOf("--name") + 1] ?? ""); + const entrypoint = String(args[args.indexOf("--entrypoint") + 1] ?? ""); + const imageIndex = args.indexOf(IMAGE); + const env = args.flatMap((value, index) => + value === "--env" ? [String(args[index + 1] ?? "")] : [], + ); + replacement = { + ...structuredClone(original), + Id: NEW_ID, + Name: `/${name}`, + Config: { + ...structuredClone(original.Config), + Image: IMAGE, + Env: env, + Entrypoint: [entrypoint], + Cmd: args.slice(imageIndex + 1), + }, + State: { Running: false, Paused: false, Restarting: false, Dead: false }, + }; + return losesAcknowledgement("container:create") + ? { status: 1, stdout: "", stderr: "lost create acknowledgement" } + : ok(NEW_ID); + } + case "ps": + return ok(original ? OLD_ID : ""); + case "inspect": { + const id = String(args[3] ?? ""); + try { + inspect(id); + return ok(`[{"Id":"${id}"}]`); + } catch { + return { status: 1, stderr: `Error response from daemon: No such container: ${id}` }; + } + } + case "cp": { + const sourceIndex = args[1] === "-a" ? 2 : 1; + const source = String(args[sourceIndex] ?? ""); + const destination = String(args[sourceIndex + 1] ?? ""); + const copyIntoContainer = () => { + events.push("stage:envelope"); + expect(readProtectedEnvelope(source).bootstrapIdentity).toBe(IDENTITY); + return ok(); + }; + const copyFromContainer = () => { + const receipt = source.split(":")[1]; + const expected = receipt?.includes("shared-state-commit") ? "committed" : "pending"; + return sharedState === expected + ? (() => { + fs.mkdirSync(destination, { recursive: true }); + return ok(); + })() + : { + status: 1, + stderr: `Error response from daemon: Could not find the file ${receipt} in container ${NEW_ID}`, + }; + }; + return source.includes(":") ? copyFromContainer() : copyIntoContainer(); + } + case "run": + switch (true) { + case args.includes("--shared-state-transaction-status"): + return ok(`${sharedState}\n`); + case args.includes("--rollback-shared-state-transaction"): + sharedState = "none"; + events.push("shared:rollback"); + return ok(); + } + break; + case "exec": + switch (true) { + case args.includes("--commit-shared-state-transaction"): + sharedState = "committed"; + events.push("shared:commit"); + return ok(); + case args.includes("--clear-shared-state-commit-receipt"): + sharedState = "none"; + events.push("shared:clear"); + return ok(); + } + break; + } + throw new Error(`unexpected Docker command: ${args.join(" ")}`); + }, + ); + const deps: DockerManagedBootstrapDeps = { + journalStore: store, + dockerCapture, + dockerRun, + dockerStop: vi.fn((id) => { + events.push(`stop:${id}`); + const target = id === OLD_ID ? original : replacement; + [target] + .filter((value): value is DockerContainerInspect => value?.State !== undefined) + .forEach((value) => { + value.State = { ...value.State, Running: false }; + }); + return losesAcknowledgement("container:stop") + ? { status: 1, stderr: "lost stop acknowledgement" } + : ok(); + }), + dockerRename: vi.fn((id, name) => { + events.push(`rename:${id}:${name}`); + const target = id === OLD_ID ? original : replacement; + [target] + .filter((value): value is DockerContainerInspect => value !== null) + .forEach((value) => { + value.Name = `/${name}`; + }); + return losesAcknowledgement("container:rename") + ? { status: 1, stderr: "lost rename acknowledgement" } + : ok(); + }), + dockerStart: vi.fn((id) => { + events.push(`start:${id}`); + const result = options.dockerStartResults?.[id] ?? ok(); + const target = id === OLD_ID ? original : replacement; + [target] + .filter( + (value): value is DockerContainerInspect => + value?.State !== undefined && result.status === 0, + ) + .forEach((value) => { + value.State = { ...value.State, Running: true }; + }); + return losesAcknowledgement("container:start") + ? { status: 1, stderr: "lost start acknowledgement" } + : result; + }), + dockerRm: vi.fn((id) => { + events.push(`rm:${id}`); + switch (id) { + case OLD_ID: + original = null as unknown as DockerContainerInspect; + break; + case NEW_ID: + replacement = null; + break; + } + return losesAcknowledgement("container:remove") + ? { status: 1, stderr: "lost rm acknowledgement" } + : ok(); + }), + runCaptureOpenshell: vi.fn(() => `Name: alpha\nID: ${options.ownerId ?? "sandbox-alpha"}\n`), + runOpenshell: vi.fn(() => ok()), + now: () => new Date("2026-07-31T12:30:00.000Z"), + }; + return { + deps, + events, + get journal() { + return journal; + }, + get original() { + return original; + }, + get replacement() { + return replacement; + }, + get sharedState() { + return sharedState; + }, + }; +} + +export function completion( + replacement: ManagedBootstrapReplacementHandle, +): ManagedBootstrapCompletionReceipt { + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + runtimeId: replacement.replacementRuntimeId, + image: replacement.image, + runtimeImageContentId: replacement.runtimeImageContentId, + originalSpecHash: replacement.originalSpecHash, + replacementSpecHash: replacement.replacementSpecHash, + profileFingerprint: replacement.profileFingerprint, + bootstrapIdentity: replacement.bootstrapIdentity, + transactionPending: true, + completedAt: "2026-07-31T12:15:00.000Z", + }; +} + +export function durablePreparation( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, +): ManagedBootstrapDurablePreparationReceipt { + const preparedAuthority = createManagedBootstrapPreparedAuthority({ + handle, + snapshot, + prepared, + }); + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + authorityFingerprint: preparedAuthority.authorityFingerprint, + recordId: `test-authority-${handle.plan.profile.agent}`, + recordedAt: "2026-07-31T12:10:00.000Z", + }; +} diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts new file mode 100644 index 00000000000..0f848d86c18 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -0,0 +1,320 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { ManagedBootstrapOwnerCleanupRequiredError } from "./adapter"; +import { createDockerManagedBootstrapAdapter } from "./docker"; +import { + authority, + completion, + durablePreparation, + fixture, + heldArgv, + IDENTITY, + NEW_ID, + OLD_ID, + SUPPORTED_AGENTS, +} from "./docker-test-fixture"; + +describe("Docker managed bootstrap adapter", () => { + it("publishes durable commit authority before deleting the rollback backup after lost acknowledgements", async () => { + const fake = fixture({ + lostAcknowledgements: [ + "container:create", + "container:remove", + "container:rename", + "container:start", + "container:stop", + "journal:create", + "journal:cutover", + "journal:remove", + "journal:shared-state-committed", + ], + sharedState: "pending", + }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + expect(fake.journal).toBeNull(); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + fake.events.push("authority:recorded"); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + const order = fake.events; + expect(order.indexOf("journal:staged")).toBeGreaterThan(order.indexOf("authority:recorded")); + expect(order.indexOf("journal:cutover")).toBeLessThan(order.indexOf(`stop:${OLD_ID}`)); + expect(fake.journal).toMatchObject({ + phase: "cutover", + originalRuntimeId: OLD_ID, + replacementRuntimeId: NEW_ID, + }); + + await expect( + adapter.finalizeBootstrap({ + outcome: "commit", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: completion(replacement), + }), + ).resolves.toMatchObject({ outcome: "committed" }); + expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( + fake.events.indexOf(`rm:${OLD_ID}`), + ); + expect(fake.journal).toBeNull(); + expect(fake.sharedState).toBe("none"); + expect(fake.replacement?.Id).toBe(NEW_ID); + }); + + it("publishes durable rollback authority before deleting the replacement after restart", async () => { + const fake = fixture({ + dockerStartResults: { + [NEW_ID]: { status: 1, stderr: "injected start failure" }, + }, + }); + const first = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await first.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + await expect( + first.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }), + ).rejects.toThrow("could not prove its exact replacement running"); + expect(fake.journal?.phase).toBe("cutover"); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect( + restarted.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement: null, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( + fake.events.indexOf(`rm:${NEW_ID}`), + ); + expect(fake.journal).toBeNull(); + expect(fake.replacement).toBeNull(); + expect(fake.original.Name).toBe("/openshell-alpha"); + expect(fake.original.State?.Running).toBe(false); + }); + + it("recovers the pre-stop cutover crash state after adapter restart", async () => { + const fake = fixture({ + journalTransitionFailures: { + cutover: new Error("injected crash after durable cutover fence"), + }, + }); + const { handle, request: rootRequest, snapshot } = authority(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + await expect( + adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }), + ).rejects.toThrow("crash after durable cutover fence"); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + await expect( + createDockerManagedBootstrapAdapter(fake.deps).finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement: null, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( + fake.events.indexOf(`rm:${NEW_ID}`), + ); + expect(fake.journal).toBeNull(); + }); + + it("fences rollback when image-owned shared state is already committed", async () => { + const fake = fixture({ sharedState: "committed" }); + const { handle, request: rootRequest, snapshot } = authority(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + const eventCount = fake.events.length; + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: null, + }), + ).rejects.toMatchObject({ name: "ManagedBootstrapDurableCommitCleanupPendingError" }); + expect(fake.journal?.phase).toBe("shared-state-committed"); + expect(fake.events.slice(eventCount)).toEqual(["journal:shared-state-committed"]); + }); + + it("rejects cutover before the exact durable authority receipt", async () => { + const fake = fixture(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const invalid = { + ...durablePreparation(handle, snapshot, prepared), + authorityFingerprint: "f".repeat(64), + }; + await expect( + adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: invalid, + }), + ).rejects.toThrow("exact durable prepared-authority receipt"); + expect(fake.journal).toBeNull(); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: null, + replacement: null, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.replacement).toBeNull(); + }); + + it.each( + SUPPORTED_AGENTS, + )("prepares, activates, and exactly rolls back the %s agent without a central switch", async (agent) => { + const fake = fixture({ agent, sharedState: "pending" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(agent); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.journal).toBeNull(); + expect(fake.replacement).toBeNull(); + expect( + vi.mocked(fake.deps.dockerRun!).mock.calls.some(([args]) => { + const agentIndex = args.indexOf("--agent"); + return args.includes("--shared-state-transaction-status") && args[agentIndex + 1] === agent; + }), + ).toBe(true); + }); + + it("quiesces and retains an exact incomplete create when its mutable name is reused", async () => { + const fake = fixture({ ownerId: "sandbox-alpha-recreated" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, plan } = authority(); + await expect( + adapter.cleanupIncompleteCreate({ + plan, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: heldArgv, + createReceipt: handle.createReceipt, + }), + ).rejects.toMatchObject({ + name: "ManagedBootstrapOwnerCleanupRequiredError", + sandboxId: "sandbox-alpha", + runtimeId: OLD_ID, + }); + expect(fake.original.State?.Running).toBe(false); + expect(fake.events).not.toContain(`rm:${OLD_ID}`); + expect(vi.mocked(fake.deps.runOpenshell!)).not.toHaveBeenCalled(); + }); + + it("retains a same-name workload that differs from the validated create receipt", async () => { + const replacementSandboxId = "sandbox-alpha-recreated"; + const fake = fixture({ ownerId: replacementSandboxId }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, plan } = authority(); + if (!fake.original.Config?.Labels) throw new Error("fixture labels are required"); + fake.original.Config.Labels["openshell.ai/sandbox-id"] = replacementSandboxId; + + await expect( + adapter.cleanupIncompleteCreate({ + plan, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: heldArgv, + createReceipt: handle.createReceipt, + }), + ).rejects.toThrow(/does not match the exact validated create receipt/u); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + expect(fake.events).not.toContain(`rm:${OLD_ID}`); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts new file mode 100644 index 00000000000..b463a820051 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -0,0 +1,2967 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { + dockerRename as defaultDockerRename, + dockerRm as defaultDockerRm, + dockerStart as defaultDockerStart, + dockerStop as defaultDockerStop, +} from "../../adapters/docker/container"; +import { + dockerCapture as defaultDockerCapture, + dockerRun as defaultDockerRun, +} from "../../adapters/docker/run"; +import { parseOpenShellSandboxId } from "../../adapters/openshell/sandbox-identity"; +import { hasZeroDockerExitStatus } from "../docker-command-result"; +import { buildDockerGpuCloneRunArgs, dockerContainerName } from "../docker-gpu-patch-clone"; +import { + DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + DOCKER_GPU_PATCH_TIMEOUT_MS, +} from "../docker-gpu-patch-constants"; +import type { + DockerContainerInspect, + DockerGpuPatchDeps, + DockerGpuPatchMode, + DockerGpuPatchModeKind, + DockerUlimit, +} from "../docker-gpu-patch-types"; +import { waitForOpenShellSupervisorReconnect } from "../docker-gpu-supervisor-reconnect"; +import { openshellSandboxCommandEnvValue } from "../docker-startup-command-env"; +import { + isImmutableDockerImageId, + OPENSHELL_MANAGED_BY_LABEL, + OPENSHELL_MANAGED_BY_VALUE, + OPENSHELL_SANDBOX_ID_LABEL, + OPENSHELL_SANDBOX_NAME_LABEL, + queryOpenShellDockerSandboxContainers, +} from "../openshell-docker-sandbox-containers"; +import { cleanupTempDir, secureTempFile } from "../temp-files"; +import { + assertManagedBootstrapIdentity, + assertManagedBootstrapSafeProcessEnvironmentKey, + attachManagedBootstrapRollbackError, + createManagedBootstrapIdentity, + createManagedBootstrapPreparedAuthority, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapAdapter, + ManagedBootstrapCommitStateIndeterminateError, + type ManagedBootstrapCompletionReceipt, + type ManagedBootstrapDiscoveredWorkload, + type ManagedBootstrapDiscoveryInput, + ManagedBootstrapDurableCommitCleanupPendingError, + type ManagedBootstrapDurablePreparationReceipt, + type ManagedBootstrapFinalizationReceipt, + type ManagedBootstrapHeldWorkloadHandle, + type ManagedBootstrapIncompleteCreateCleanupInput, + type ManagedBootstrapObservedSnapshot, + ManagedBootstrapOwnerCleanupRequiredError, + type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapReplacementHandle, + type ManagedBootstrapReplacementOptions, + type ManagedBootstrapSandboxIdentity, + renderManagedBootstrapHeldCommand, +} from "./adapter"; +import { + createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapJournal, + DockerManagedBootstrapJournalAcknowledgementLostError, + type DockerManagedBootstrapJournalStore, + parseDockerManagedBootstrapJournal, + serializeDockerManagedBootstrapJournal, +} from "./docker-journal"; +import { + clearDockerManagedStartupSharedStateCommitReceipt, + DockerManagedStartupSharedStateCommitIndeterminateError, + finalizeDockerManagedStartupSharedState, + probeDockerManagedStartupSharedState, +} from "./docker-shared-state"; +import { + normalizeDockerManagedBootstrapLaunchSpec, + parseDockerManagedBootstrapLaunchSpec, + parseExactDockerContainerInspect, +} from "./docker-spec"; +import { + MANAGED_BOOTSTRAP_COMPLETION_FILE, + MANAGED_BOOTSTRAP_REQUEST_FILE, + parseManagedBootstrapImageCompletion, + serializeManagedBootstrapEnvelope, +} from "./envelope"; + +const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; +const FULL_SHA256_RE = /^sha256:[a-f0-9]{64}$/u; +const MAX_ARGV_BYTES = 128 * 1024; +const MAX_CONTAINER_NAME_LENGTH = 253; +const REQUEST_TEMP_PREFIX = "nemoclaw-managed-bootstrap-request"; +const COMPLETION_TEMP_PREFIX = "nemoclaw-managed-bootstrap-completion"; +const COMPLETION_MAX_BYTES = 4096; +const DOCKER_DRIVER_ID = "docker"; + +export const MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE = "/usr/local/bin/nemoclaw-managed-bootstrap"; + +type DockerCommandResult = { + readonly status?: number | null; + readonly stdout?: string | Buffer | null; + readonly stderr?: string | Buffer | null; + readonly error?: Error | null; +}; + +export type DockerManagedBootstrapDeps = Pick< + DockerGpuPatchDeps, + | "dockerCapture" + | "dockerRename" + | "dockerRm" + | "dockerRun" + | "dockerStart" + | "dockerStop" + | "runCaptureOpenshell" + | "runOpenshell" + | "sleep" + | "now" +> & { + readonly createBootstrapIdentity?: () => string; + readonly journalStore?: DockerManagedBootstrapJournalStore; + /** Canonical gateway-scoped state root; required when no store is injected. */ + readonly stateRoot?: string; +}; + +type ResolvedDeps = Required< + Pick< + DockerManagedBootstrapDeps, + | "dockerCapture" + | "dockerRename" + | "dockerRm" + | "dockerRun" + | "dockerStart" + | "dockerStop" + | "journalStore" + | "now" + | "createBootstrapIdentity" + > +> & + DockerManagedBootstrapDeps; + +type DockerBootstrapTransaction = DockerManagedBootstrapJournal; + +interface DockerBootstrapRollbackTombstone { + readonly profileFingerprint: string; + readonly imageReference: string; + readonly receipt: ManagedBootstrapFinalizationReceipt; +} + +export interface DockerManagedBootstrapAdapter extends ManagedBootstrapAdapter {} + +function resolveDeps(deps: DockerManagedBootstrapDeps): ResolvedDeps { + const journalStore = + deps.journalStore ?? + (deps.stateRoot ? createFileDockerManagedBootstrapJournalStore(deps.stateRoot) : null); + if (!journalStore) { + throw new Error( + "Managed bootstrap Docker requires its canonical state root or an injected journal store.", + ); + } + return { + dockerCapture: defaultDockerCapture, + dockerRename: defaultDockerRename, + dockerRm: defaultDockerRm, + dockerRun: defaultDockerRun, + dockerStart: defaultDockerStart, + dockerStop: defaultDockerStop, + journalStore, + now: () => new Date(), + createBootstrapIdentity: createManagedBootstrapIdentity, + ...deps, + }; +} + +function commandDetail(result: DockerCommandResult): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}` + .trim() + .slice(-1200); +} + +function isExactMissingDockerContainer(containerId: string, result: DockerCommandResult): boolean { + const escapedContainerId = containerId.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const patterns = [ + new RegExp( + `^(?:Error response from daemon: )?No such (?:container|object): ${escapedContainerId}$`, + "u", + ), + new RegExp(`^Error: No such (?:container|object): ${escapedContainerId}$`, "u"), + ]; + return [result.stderr, result.stdout, result.error?.message] + .map((value) => String(value ?? "").trim()) + .filter(Boolean) + .some((detail) => patterns.some((pattern) => pattern.test(detail))); +} + +function probeExactDockerContainerAbsence( + containerId: string, + deps: ResolvedDeps, +): "absent" | "present" | "unknown" { + let result: DockerCommandResult; + try { + result = deps.dockerRun(["inspect", "--type", "container", containerId], { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + } catch { + return "unknown"; + } + if (hasZeroDockerExitStatus(result)) return "present"; + return isExactMissingDockerContainer(containerId, result) ? "absent" : "unknown"; +} + +function assertZero(result: DockerCommandResult, message: string): void { + if (!hasZeroDockerExitStatus(result)) { + throw new Error(`${message}: ${commandDetail(result) || "Docker command failed"}`); + } +} + +function exactStringArray(value: unknown, label: string): string[] { + if (value === null || value === undefined) return []; + const values = typeof value === "string" ? [value] : value; + if ( + !Array.isArray(values) || + values.some( + (item) => + typeof item !== "string" || + item.length === 0 || + item.includes("\0") || + Buffer.byteLength(item, "utf8") > 64 * 1024, + ) + ) { + throw new Error(`Managed bootstrap Docker ${label} is not an exact bounded argv.`); + } + const result = [...values]; + if (Buffer.byteLength(JSON.stringify(result), "utf8") > MAX_ARGV_BYTES) { + throw new Error(`Managed bootstrap Docker ${label} exceeds its bounded argv transport.`); + } + return result; +} + +function exactSupervisorArgv(inspect: DockerContainerInspect): readonly string[] { + const argv = [ + ...exactStringArray(inspect.Config?.Entrypoint, "entrypoint"), + ...exactStringArray(inspect.Config?.Cmd, "command"), + ]; + if (argv.length === 0 || !argv[0]?.startsWith("/")) { + throw new Error( + "Managed bootstrap requires one bounded absolute supervisor argv from Docker inspect.", + ); + } + return Object.freeze(argv); +} + +function exactArrayEqual(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + +function envValue(env: readonly string[] | null | undefined, key: string): string | null { + const prefix = `${key}=`; + const matches = (env ?? []).filter((value) => value.startsWith(prefix)); + return matches.length === 1 ? (matches[0]?.slice(prefix.length) ?? null) : null; +} + +function assertNoRootProcessInjectionEnvironment(env: readonly string[] | null | undefined): void { + for (const entry of env ?? []) { + const separator = entry.indexOf("="); + const key = separator < 0 ? entry : entry.slice(0, separator); + try { + assertManagedBootstrapSafeProcessEnvironmentKey(key); + } catch { + throw new Error(`Managed bootstrap refuses root-process injection environment '${key}'.`); + } + } +} + +function assertRootSupervisor(inspect: DockerContainerInspect): void { + const user = String(inspect.Config?.User ?? "") + .trim() + .toLowerCase(); + if (!["", "0", "0:0", "root", "root:root"].includes(user)) { + throw new Error("Managed bootstrap Docker workload must retain a root supervisor user."); + } +} + +function isStableRunning(inspect: DockerContainerInspect): boolean { + return inspect.State?.Running !== true || + inspect.State.Paused === true || + inspect.State.Restarting === true || + inspect.State.Dead === true + ? false + : true; +} + +function assertStableRunning(inspect: DockerContainerInspect, label: string): void { + if (!isStableRunning(inspect)) { + throw new Error(`Managed bootstrap Docker ${label} is not stably running.`); + } +} + +function isExplicitlyStopped(inspect: DockerContainerInspect): boolean { + return ( + inspect.State?.Running === false && + inspect.State.Paused === false && + inspect.State.Restarting === false && + inspect.State.Dead === false + ); +} + +function assertExplicitlyStopped(inspect: DockerContainerInspect, label: string): void { + if (!isExplicitlyStopped(inspect)) { + throw new Error(`Managed bootstrap Docker ${label} is not explicitly stopped.`); + } +} + +function expectedImageReference(repository: string, manifestDigest: string): string { + if ( + repository.length === 0 || + repository !== repository.trim() || + repository.includes("@") || + repository.includes("\0") || + !FULL_SHA256_RE.test(manifestDigest) + ) { + throw new Error("Managed bootstrap image repository/manifest identity is invalid."); + } + return `${repository}@${manifestDigest}`; +} + +function assertImage( + inspect: DockerContainerInspect, + image: ManagedBootstrapHeldWorkloadHandle["plan"]["image"], + deps: ResolvedDeps, +): string { + const runtimeContentId = String(inspect.Image ?? "").toLowerCase(); + if (!FULL_SHA256_RE.test(runtimeContentId)) { + throw new Error("Managed bootstrap Docker image does not have an immutable local content ID."); + } + const expectedReference = expectedImageReference(image.repository, image.manifestDigest); + const configuredImage = String(inspect.Config?.Image ?? "").trim(); + if (configuredImage !== expectedReference) { + throw new Error( + "Managed bootstrap Docker configured image is not the exact repository@manifestDigest.", + ); + } + const imageOutput = deps.dockerCapture(["image", "inspect", expectedReference], { + ignoreError: false, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + let parsed: unknown; + try { + parsed = JSON.parse(imageOutput); + } catch { + throw new Error("Managed bootstrap Docker image evidence is malformed."); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("Managed bootstrap Docker image evidence is not exact."); + } + const evidence = parsed[0] as { + readonly Id?: unknown; + readonly RepoDigests?: unknown; + }; + const evidenceId = String(evidence.Id ?? "").toLowerCase(); + const repoDigests = Array.isArray(evidence.RepoDigests) + ? evidence.RepoDigests.filter((value): value is string => typeof value === "string") + : []; + if (evidenceId !== runtimeContentId || !repoDigests.includes(expectedReference)) { + throw new Error( + "Managed bootstrap Docker image manifest evidence does not match its local content ID.", + ); + } + return runtimeContentId; +} + +function assertMetadata( + inspect: DockerContainerInspect, + sandbox: ManagedBootstrapHeldWorkloadHandle["sandbox"], + metadata: Readonly>, +): void { + const labels = inspect.Config?.Labels ?? {}; + if ( + labels[OPENSHELL_MANAGED_BY_LABEL] !== OPENSHELL_MANAGED_BY_VALUE || + labels[OPENSHELL_SANDBOX_NAME_LABEL] !== sandbox.sandboxName || + labels[OPENSHELL_SANDBOX_ID_LABEL] !== sandbox.sandboxId + ) { + throw new Error( + "Managed bootstrap Docker workload does not match the durable OpenShell sandbox identity.", + ); + } + for (const [key, value] of Object.entries(metadata)) { + if (labels[key] !== value) { + throw new Error(`Managed bootstrap Docker metadata label '${key}' changed.`); + } + } +} + +function assertHeldCommand( + inspect: DockerContainerInspect, + heldWorkloadArgv: readonly string[], + bootstrapIdentity: string, +): void { + assertManagedBootstrapIdentity(bootstrapIdentity); + const expected = openshellSandboxCommandEnvValue(heldWorkloadArgv); + const observed = envValue(inspect.Config?.Env, "OPENSHELL_SANDBOX_COMMAND"); + if (!expected || observed !== expected) { + throw new Error( + "Managed bootstrap Docker workload does not contain the exact identity-bound hold.", + ); + } + const identityIndexes = heldWorkloadArgv + .map((value, index) => (value === bootstrapIdentity ? index : -1)) + .filter((index) => index >= 0); + if (identityIndexes.length !== 1) { + throw new Error("Managed bootstrap hold does not contain exactly one bootstrap identity."); + } +} + +function assertBootstrapIdentityInObservedHold( + inspect: DockerContainerInspect, + bootstrapIdentity: string, +): void { + assertManagedBootstrapIdentity(bootstrapIdentity); + const observed = envValue(inspect.Config?.Env, "OPENSHELL_SANDBOX_COMMAND"); + if (!observed) { + throw new Error("Managed bootstrap Docker workload is missing its held command."); + } + const occurrences = observed.split(bootstrapIdentity).length - 1; + if (occurrences !== 1) { + throw new Error( + "Managed bootstrap Docker held command does not contain one exact bootstrap identity.", + ); + } +} + +function inspectExact(containerId: string, deps: ResolvedDeps): DockerContainerInspect { + if (!FULL_CONTAINER_ID_RE.test(containerId)) { + throw new Error("Managed bootstrap requires one full lowercase Docker container ID."); + } + const output = deps.dockerCapture(["inspect", "--type", "container", containerId], { + ignoreError: false, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + const inspect = parseExactDockerContainerInspect(output); + if (String(inspect.Id ?? "").toLowerCase() !== containerId) { + throw new Error("Managed bootstrap Docker workload identity changed during inspection."); + } + return inspect; +} + +function inspectDockerContainerReference( + reference: string, + deps: ResolvedDeps, +): DockerContainerInspect { + if ( + reference.length === 0 || + reference !== reference.trim() || + reference.includes("\0") || + Buffer.byteLength(reference, "utf8") > MAX_CONTAINER_NAME_LENGTH + ) { + throw new Error("Managed bootstrap Docker lookup reference is invalid."); + } + const output = deps.dockerCapture(["inspect", "--type", "container", reference], { + ignoreError: false, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + const inspect = parseExactDockerContainerInspect(output); + const runtimeId = String(inspect.Id ?? "").toLowerCase(); + if (!FULL_CONTAINER_ID_RE.test(runtimeId)) { + throw new Error("Managed bootstrap Docker lookup did not resolve one full runtime ID."); + } + return inspect; +} + +function tryInspectExact(containerId: string, deps: ResolvedDeps): DockerContainerInspect | null { + try { + return inspectExact(containerId, deps); + } catch { + return null; + } +} + +function backupName(originalName: string, bootstrapIdentity: string): string { + const suffix = `-nemoclaw-bootstrap-${bootstrapIdentity.slice(0, 20)}`; + return `${originalName.slice(0, Math.max(1, MAX_CONTAINER_NAME_LENGTH - suffix.length))}${suffix}`; +} + +function replacementStagingName(originalName: string, bootstrapIdentity: string): string { + const suffix = `-nemoclaw-staged-${bootstrapIdentity.slice(0, 20)}`; + return `${originalName.slice(0, Math.max(1, MAX_CONTAINER_NAME_LENGTH - suffix.length))}${suffix}`; +} + +function writeProtectedEnvelope( + bootstrapIdentity: string, + request: Parameters[0]["rootApplyRequest"], +): string { + const file = secureTempFile(REQUEST_TEMP_PREFIX, ".json"); + try { + fs.writeFileSync( + file, + serializeManagedBootstrapEnvelope({ bootstrapIdentity, rootApplyRequest: request }), + { encoding: "utf8", flag: "wx", mode: 0o400 }, + ); + fs.chmodSync(file, 0o400); + const stat = fs.lstatSync(file); + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.nlink !== 1 || + (stat.mode & 0o777) !== 0o400 + ) { + throw new Error("Managed bootstrap request source is not one protected 0400 file."); + } + return file; + } catch (error) { + cleanupTempDir(file, REQUEST_TEMP_PREFIX); + throw error; + } +} + +function readProtectedImageCompletion( + replacementRuntimeId: string, + deps: ResolvedDeps, +): ReturnType { + const file = secureTempFile(COMPLETION_TEMP_PREFIX, ".json"); + let descriptor: number | undefined; + try { + const copied = deps.dockerRun( + ["cp", `${replacementRuntimeId}:${MANAGED_BOOTSTRAP_COMPLETION_FILE}`, file], + { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }, + ); + assertZero(copied, "Managed bootstrap could not retrieve its image completion receipt"); + descriptor = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + const before = fs.fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.isSymbolicLink() || + before.nlink !== 1n || + Number(before.mode & 0o777n) !== 0o444 || + before.size < 1n || + before.size > BigInt(COMPLETION_MAX_BYTES) + ) { + throw new Error("Managed bootstrap image completion is not one protected bounded 0444 file."); + } + const bytes = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < bytes.length) { + const count = fs.readSync(descriptor, bytes, offset, bytes.length - offset, offset); + if (count === 0) break; + offset += count; + } + const after = fs.fstatSync(descriptor, { bigint: true }); + if ( + offset !== bytes.length || + after.dev !== before.dev || + after.ino !== before.ino || + after.size !== before.size || + after.mtimeNs !== before.mtimeNs || + after.ctimeNs !== before.ctimeNs || + after.mode !== before.mode || + after.nlink !== before.nlink + ) { + throw new Error("Managed bootstrap image completion changed during stable read."); + } + return parseManagedBootstrapImageCompletion(bytes.toString("utf8")); + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + cleanupTempDir(file, COMPLETION_TEMP_PREFIX); + } +} + +function parseRequiredUlimits(value: unknown): DockerUlimit[] { + if (value === undefined) return []; + if ( + !Array.isArray(value) || + value.some((entry) => typeof entry !== "string" || entry.includes("\0")) + ) { + throw new Error("Managed bootstrap Docker requiredUlimits must be string entries."); + } + return value.map((entry) => { + const match = /^([a-z][a-z0-9_]*)=(\d+):(\d+)$/u.exec(entry); + if (!match) { + throw new Error(`Managed bootstrap Docker ulimit '${entry}' is invalid.`); + } + const soft = Number(match[2]); + const hard = Number(match[3]); + if (!Number.isSafeInteger(soft) || !Number.isSafeInteger(hard) || hard < soft) { + throw new Error(`Managed bootstrap Docker ulimit '${entry}' is invalid.`); + } + return { name: match[1] as string, soft, hard }; + }); +} + +function replacementPlan(options: ManagedBootstrapReplacementOptions): { + readonly mode: DockerGpuPatchMode; + readonly requiredUlimits: readonly DockerUlimit[]; + readonly extraGroupGids: readonly string[]; +} { + const allowed = new Set([ + "gpuModeArgs", + "gpuModeDevice", + "gpuModeKind", + "gpuModeLabel", + "extraGroupGids", + "requiredUlimits", + ]); + const unknown = Object.keys(options.values).filter((key) => !allowed.has(key)); + if (unknown.length > 0) { + throw new Error( + `Managed bootstrap Docker replacement options are unsupported: ${unknown.sort().join(", ")}.`, + ); + } + const kind = String(options.values.gpuModeKind ?? "startup-command") as DockerGpuPatchModeKind; + if (!["gpus", "nvidia-runtime", "cdi", "startup-command"].includes(kind)) { + throw new Error(`Managed bootstrap Docker GPU mode '${kind}' is invalid.`); + } + const args = exactStringArray(options.values.gpuModeArgs ?? [], "GPU mode arguments"); + return { + mode: { + kind, + label: String(options.values.gpuModeLabel ?? "managed bootstrap"), + device: String(options.values.gpuModeDevice ?? ""), + args, + }, + extraGroupGids: exactStringArray(options.values.extraGroupGids ?? [], "extra group GIDs").map( + (value) => { + if (!/^\d+$/u.test(value)) { + throw new Error(`Managed bootstrap Docker supplementary group '${value}' is invalid.`); + } + return value; + }, + ), + requiredUlimits: parseRequiredUlimits(options.values.requiredUlimits), + }; +} + +function replacementCommand( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, +): readonly string[] { + return Object.freeze([ + "--agent", + handle.plan.profile.agent, + "--profile-fingerprint", + handle.plan.profile.fingerprint, + "--bootstrap-identity", + handle.bootstrapIdentity, + "--agent-uid", + String(snapshot.agentIdentity.uid), + "--agent-gid", + String(snapshot.agentIdentity.gid), + "--agent-workdir", + snapshot.agentIdentity.workdir, + "--request-file", + MANAGED_BOOTSTRAP_REQUEST_FILE, + "--", + ...snapshot.supervisorArgv, + ]); +} + +function assertReplacementBoundary( + inspect: DockerContainerInspect, + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, +): void { + const entrypoint = exactStringArray(inspect.Config?.Entrypoint, "replacement entrypoint"); + const command = exactStringArray(inspect.Config?.Cmd, "replacement command"); + if ( + !exactArrayEqual(entrypoint, [MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE]) || + !exactArrayEqual(command, replacementCommand(handle, snapshot)) + ) { + throw new Error("Managed bootstrap Docker replacement process boundary changed."); + } + const intended = openshellSandboxCommandEnvValue(handle.intendedWorkloadArgv); + if (envValue(inspect.Config?.Env, "OPENSHELL_SANDBOX_COMMAND") !== intended) { + throw new Error( + "Managed bootstrap Docker replacement did not restore the intended sandbox command.", + ); + } +} + +const REPLACED_GPU_ENV_KEYS = new Set([ + "NVIDIA_DISABLE_REQUIRE", + "NVIDIA_DRIVER_CAPABILITIES", + "NVIDIA_REQUIRE_CUDA", + "NVIDIA_VISIBLE_DEVICES", +]); + +function canonicalObject(text: string): Record { + const value = JSON.parse(text) as unknown; + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("Managed bootstrap normalized Docker spec is not an object."); + } + return value as Record; +} + +function objectField(record: Record, key: string): Record { + const value = record[key]; + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`Managed bootstrap normalized Docker spec is missing ${key}.`); + } + return value as Record; +} + +function exactJson(value: unknown): string { + return JSON.stringify(value ?? null); +} + +function stringSet(value: unknown, label: string): string[] { + const values = exactStringArray(value ?? [], label); + if (new Set(values).size !== values.length) { + throw new Error(`Managed bootstrap Docker ${label} contains duplicate entries.`); + } + return values.sort(); +} + +function assertExactStringSet(observed: unknown, expected: readonly string[], label: string): void { + if (!exactArrayEqual(stringSet(observed, label), [...expected].sort())) { + throw new Error(`Managed bootstrap Docker ${label} changed outside declared deltas.`); + } +} + +function modeEnvironment(mode: DockerGpuPatchMode): string[] { + const values: string[] = []; + for (let index = 0; index < mode.args.length; index += 1) { + if (mode.args[index] === "--env") { + const value = mode.args[index + 1]; + if (!value || !value.includes("=")) { + throw new Error("Managed bootstrap Docker GPU mode has an invalid environment delta."); + } + values.push(value); + index += 1; + } + } + return values; +} + +function assertExactEnvironmentDelta( + original: Record, + replacement: Record, + mode: DockerGpuPatchMode, + intendedSandboxCommand: string, +): void { + const gpuAugment = mode.kind !== "startup-command"; + const originalEnv = exactStringArray(original.Env ?? [], "original environment"); + const expected = [ + ...modeEnvironment(mode), + ...originalEnv + .filter((entry) => !gpuAugment || !REPLACED_GPU_ENV_KEYS.has(entry.split("=", 1)[0] ?? "")) + .map((entry) => + entry.startsWith("OPENSHELL_SANDBOX_COMMAND=") + ? `OPENSHELL_SANDBOX_COMMAND=${intendedSandboxCommand}` + : entry, + ), + ]; + const observed = exactStringArray(replacement.Env ?? [], "replacement environment"); + if (!exactArrayEqual(observed, expected)) { + throw new Error( + "Managed bootstrap Docker replacement environment changed outside declared deltas.", + ); + } +} + +function canonicalUlimits(value: unknown, label: string): string { + if (!Array.isArray(value)) { + if (value === undefined || value === null) return "[]"; + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + const normalized = value.map((entry) => { + if (typeof entry !== "object" || entry === null || Array.isArray(entry)) { + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + const record = entry as Record; + const name = String(record.Name ?? ""); + const soft = record.Soft; + const hard = record.Hard; + if (!name || !Number.isSafeInteger(soft) || !Number.isSafeInteger(hard)) { + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + return { Hard: hard, Name: name, Soft: soft }; + }); + if (new Set(normalized.map((entry) => entry.Name)).size !== normalized.length) { + throw new Error(`Managed bootstrap Docker ${label} contains duplicate entries.`); + } + return JSON.stringify(normalized.sort((left, right) => left.Name.localeCompare(right.Name))); +} + +function expectedUlimits(original: unknown, required: readonly DockerUlimit[]): string { + const existing = JSON.parse(canonicalUlimits(original, "original ulimits")) as Array<{ + Hard: number; + Name: string; + Soft: number; + }>; + const merged = new Map(existing.map((entry) => [entry.Name, entry])); + for (const requiredEntry of required) { + merged.set(requiredEntry.name, { + Name: requiredEntry.name, + Soft: requiredEntry.soft, + Hard: requiredEntry.hard, + }); + } + return JSON.stringify( + [...merged.values()].sort((left, right) => left.Name.localeCompare(right.Name)), + ); +} + +function assertExactDeviceRequests( + original: unknown, + observed: unknown, + mode: DockerGpuPatchMode, +): void { + if (mode.kind === "startup-command") { + if (exactJson(observed) !== exactJson(original)) { + throw new Error("Managed bootstrap Docker device requests were not preserved exactly."); + } + return; + } + if (Array.isArray(original) && original.length > 0) { + throw new Error( + "Managed bootstrap Docker GPU augmentation cannot replace an existing device request.", + ); + } + const requests = Array.isArray(observed) ? observed : []; + if (mode.kind === "nvidia-runtime") { + if (requests.length !== 0) { + throw new Error( + "Managed bootstrap Docker NVIDIA runtime added an undeclared device request.", + ); + } + return; + } + if (requests.length !== 1 || typeof requests[0] !== "object" || requests[0] === null) { + throw new Error("Managed bootstrap Docker GPU mode did not add one exact device request."); + } + const request = requests[0] as Record; + if (mode.kind === "gpus") { + const all = mode.device === "all"; + const expectedIds = all ? [] : [mode.device]; + const ids = Array.isArray(request.DeviceIDs) ? request.DeviceIDs : []; + if ( + String(request.Driver ?? "") !== "" || + Number(request.Count) !== (all ? -1 : 0) || + !exactArrayEqual(ids.map(String), expectedIds) || + exactJson(request.Capabilities) !== JSON.stringify([["gpu"]]) || + exactJson(request.Options ?? {}) !== "{}" + ) { + throw new Error("Managed bootstrap Docker --gpus request changed outside its exact delta."); + } + return; + } + const ids = Array.isArray(request.DeviceIDs) ? request.DeviceIDs.map(String) : []; + if ( + request.Driver !== "cdi" || + ![-1, 0].includes(Number(request.Count ?? 0)) || + !exactArrayEqual(ids, [mode.device]) || + (request.Capabilities != null && + (!Array.isArray(request.Capabilities) || request.Capabilities.length > 0)) || + exactJson(request.Options ?? {}) !== "{}" + ) { + throw new Error("Managed bootstrap Docker CDI request changed outside its exact delta."); + } +} + +function scrubVerifiedReplacementDeltas(canonicalJson: string): string { + const root = canonicalObject(canonicalJson); + const inspect = objectField(root, "inspect"); + const config = objectField(inspect, "Config"); + const host = objectField(inspect, "HostConfig"); + config.Image = ""; + config.Entrypoint = [""]; + config.Cmd = [""]; + config.Env = ""; + for (const key of [ + "CapAdd", + "DeviceRequests", + "Devices", + "GroupAdd", + "Runtime", + "SecurityOpt", + "Ulimits", + ]) { + host[key] = ``; + } + return JSON.stringify(root); +} + +function assertReplacementMatchesIntent( + originalCanonicalJson: string, + replacement: DockerContainerInspect, + authoritativeName: string, + plan: { + readonly mode: DockerGpuPatchMode; + readonly requiredUlimits: readonly DockerUlimit[]; + readonly extraGroupGids: readonly string[]; + }, + intendedSandboxCommand: string, +): string { + const original = canonicalObject(originalCanonicalJson); + const originalInspect = objectField(original, "inspect"); + const originalConfig = objectField(originalInspect, "Config"); + const originalHost = objectField(originalInspect, "HostConfig"); + const replacementSpec = normalizeDockerManagedBootstrapLaunchSpec({ + ...replacement, + Name: `/${authoritativeName}`, + }); + const observed = canonicalObject(replacementSpec.canonicalJson); + const observedInspect = objectField(observed, "inspect"); + const observedConfig = objectField(observedInspect, "Config"); + const observedHost = objectField(observedInspect, "HostConfig"); + const gpuAugment = plan.mode.kind !== "startup-command"; + assertExactEnvironmentDelta(originalConfig, observedConfig, plan.mode, intendedSandboxCommand); + assertExactStringSet( + observedHost.CapAdd, + [ + ...stringSet(originalHost.CapAdd, "original capability additions"), + ...(gpuAugment ? ["SYS_PTRACE"] : []), + ].filter((value, index, values) => values.indexOf(value) === index), + "capability additions", + ); + const originalSecurity = stringSet(originalHost.SecurityOpt, "original security options"); + assertExactStringSet( + observedHost.SecurityOpt, + [ + ...originalSecurity, + ...(gpuAugment && !originalSecurity.some((value) => value.startsWith("apparmor")) + ? ["apparmor=unconfined"] + : []), + ], + "security options", + ); + if (exactJson(observedHost.Devices) !== exactJson(originalHost.Devices)) { + throw new Error("Managed bootstrap Docker non-GPU devices were not preserved exactly."); + } + assertExactDeviceRequests(originalHost.DeviceRequests, observedHost.DeviceRequests, plan.mode); + const expectedRuntime = plan.mode.kind === "nvidia-runtime" ? "nvidia" : originalHost.Runtime; + if (exactJson(observedHost.Runtime) !== exactJson(expectedRuntime)) { + throw new Error("Managed bootstrap Docker runtime changed outside its selected GPU delta."); + } + assertExactStringSet( + observedHost.GroupAdd, + [ + ...stringSet(originalHost.GroupAdd, "original supplementary groups"), + ...plan.extraGroupGids, + ].filter((value, index, values) => values.indexOf(value) === index), + "supplementary groups", + ); + if ( + canonicalUlimits(observedHost.Ulimits, "replacement ulimits") !== + expectedUlimits(originalHost.Ulimits, plan.requiredUlimits) + ) { + throw new Error("Managed bootstrap Docker ulimits changed outside declared requirements."); + } + const expectedPreserved = scrubVerifiedReplacementDeltas(originalCanonicalJson); + const observedPreserved = scrubVerifiedReplacementDeltas(replacementSpec.canonicalJson); + if (observedPreserved !== expectedPreserved) { + throw new Error( + "Managed bootstrap Docker replacement normalized spec changed outside declared deltas.", + ); + } + return replacementSpec.hash; +} + +function inspectTransactionRuntime( + transaction: DockerBootstrapTransaction, + runtimeId: string, + deps: ResolvedDeps, +): DockerContainerInspect | null { + const presence = probeExactDockerContainerAbsence(runtimeId, deps); + if (presence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId, + detail: "exact Docker runtime presence could not be proven before mutation", + }); + } + if (presence === "absent") return null; + try { + return inspectExact(runtimeId, deps); + } catch (error) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId, + detail: `exact Docker runtime inspection became unavailable: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + } +} + +function assertTransactionOriginal( + transaction: DockerBootstrapTransaction, + inspect: DockerContainerInspect, +): void { + const name = dockerContainerName(inspect); + if (name !== transaction.originalName && name !== transaction.backupName) { + throw new Error("Managed bootstrap original container has an unexpected transaction name."); + } + const normalized = normalizeDockerManagedBootstrapLaunchSpec({ + ...inspect, + Name: `/${transaction.originalName}`, + }); + if (normalized.hash !== transaction.originalSpecHash) { + throw new Error( + "Managed bootstrap refused mutation because the exact original launch spec changed.", + ); + } +} + +function assertTransactionReplacement( + transaction: DockerBootstrapTransaction, + inspect: DockerContainerInspect, +): void { + const name = dockerContainerName(inspect); + if (name !== transaction.replacementStagingName && name !== transaction.originalName) { + throw new Error("Managed bootstrap replacement container has an unexpected transaction name."); + } + const normalized = normalizeDockerManagedBootstrapLaunchSpec({ + ...inspect, + Name: `/${transaction.originalName}`, + }); + if (normalized.hash !== transaction.replacementSpecHash) { + throw new Error( + "Managed bootstrap refused mutation because the exact replacement launch spec changed.", + ); + } +} + +function assertCompletedCutoverRuntimeState( + transaction: DockerBootstrapTransaction, + deps: ResolvedDeps, +): void { + const original = inspectTransactionRuntime(transaction, transaction.originalRuntimeId, deps); + const replacement = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (!original || !replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: original ? transaction.replacementRuntimeId : transaction.originalRuntimeId, + detail: "completed cutover requires both exact transaction runtimes", + }); + } + assertTransactionOriginal(transaction, original); + assertTransactionReplacement(transaction, replacement); + assertExplicitlyStopped(original, "rollback backup"); + assertStableRunning(replacement, "replacement"); + if ( + dockerContainerName(original) !== transaction.backupName || + dockerContainerName(replacement) !== transaction.originalName + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.replacementRuntimeId, + detail: "completed cutover runtime names do not match durable authority", + }); + } +} + +function removeExactReplacement( + transaction: DockerBootstrapTransaction, + replacement: DockerContainerInspect, + deps: ResolvedDeps, +): void { + assertTransactionReplacement(transaction, replacement); + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + if (replacement.State?.Running === true) { + const stopped = deps.dockerStop(transaction.replacementRuntimeId, { + ...options, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopped)) { + const afterStop = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (!afterStop || afterStop.State?.Running === true) { + throw new Error( + `Managed bootstrap could not quiesce its exact replacement: ${ + commandDetail(stopped) || "Docker stop failed" + }`, + ); + } + assertTransactionReplacement(transaction, afterStop); + } + } + const removed = deps.dockerRm(transaction.replacementRuntimeId, options); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(transaction.replacementRuntimeId, deps) !== "absent" + ) { + throw new Error( + `Managed bootstrap could not remove its exact replacement: ${ + commandDetail(removed) || "Docker removal failed" + }`, + ); + } +} + +function restoreOriginal(transaction: DockerBootstrapTransaction, deps: ResolvedDeps): void { + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + const originalBeforeReplacementRemoval = inspectTransactionRuntime( + transaction, + transaction.originalRuntimeId, + deps, + ); + if (!originalBeforeReplacementRemoval) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "the exact rollback original is absent", + }); + } + assertTransactionOriginal(transaction, originalBeforeReplacementRemoval); + const replacement = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (replacement) { + removeExactReplacement(transaction, replacement, deps); + } + const original = inspectExact(transaction.originalRuntimeId, deps); + assertTransactionOriginal(transaction, original); + const currentName = dockerContainerName(original); + if (currentName !== transaction.originalName) { + if (currentName !== transaction.backupName) { + throw new Error("Managed bootstrap original container has an unexpected rollback name."); + } + const renamed = deps.dockerRename( + transaction.originalRuntimeId, + transaction.originalName, + options, + ); + if (!hasZeroDockerExitStatus(renamed)) { + const afterRename = inspectTransactionRuntime( + transaction, + transaction.originalRuntimeId, + deps, + ); + if (!afterRename || dockerContainerName(afterRename) !== transaction.originalName) { + throw new Error( + `Managed bootstrap could not restore the original container name: ${ + commandDetail(renamed) || "Docker rename failed" + }`, + ); + } + assertTransactionOriginal(transaction, afterRename); + } + } + const restoredBeforeStart = inspectExact(transaction.originalRuntimeId, deps); + if (restoredBeforeStart.State?.Running !== true) { + const started = deps.dockerStart(transaction.originalRuntimeId, options); + if (!hasZeroDockerExitStatus(started)) { + const afterStart = inspectTransactionRuntime( + transaction, + transaction.originalRuntimeId, + deps, + ); + if (!afterStart || afterStart.State?.Running !== true) { + throw new Error( + `Managed bootstrap could not restart the original container: ${ + commandDetail(started) || "Docker start failed" + }`, + ); + } + assertTransactionOriginal(transaction, afterStart); + } + } + const restored = inspectExact(transaction.originalRuntimeId, deps); + assertStableRunning(restored, "restored workload"); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(restored); + if (normalized.hash !== transaction.originalSpecHash) { + throw new Error("Managed bootstrap rollback did not restore the exact launch spec."); + } +} + +function removeOwnedWorkload( + sandbox: ManagedBootstrapSandboxIdentity, + deps: ResolvedDeps, + expectedRuntimeId?: string, +): never { + const expectedIdentity = + expectedRuntimeId === undefined + ? `sandbox ${sandbox.sandboxId} with no previously resolved runtime ID` + : `sandbox ${sandbox.sandboxId} expected runtime ${expectedRuntimeId}`; + let containers: DockerCommandResult; + try { + containers = deps.dockerRun( + [ + "ps", + "-a", + "--no-trunc", + "--filter", + `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, + "--filter", + `label=${OPENSHELL_SANDBOX_ID_LABEL}=${sandbox.sandboxId}`, + "--format", + "{{.ID}}", + ], + { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }, + ); + } catch (error) { + throw new Error( + `Managed bootstrap owner cleanup could not enumerate the exact held runtime for ${expectedIdentity}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + if (Number(containers.status ?? 1) !== 0) { + throw new Error( + `Managed bootstrap owner cleanup could not verify the exact held runtime for ${expectedIdentity}: ${ + commandDetail(containers) || "Docker enumeration failed" + }`, + ); + } + const runtimeIds = String(containers.stdout ?? "") + .trim() + .split(/\r?\n/u) + .map((value) => value.trim().toLowerCase()) + .filter(Boolean); + if ( + runtimeIds.length !== 1 || + !FULL_CONTAINER_ID_RE.test(runtimeIds[0] ?? "") || + (expectedRuntimeId !== undefined && runtimeIds[0] !== expectedRuntimeId) + ) { + throw new Error( + `Managed bootstrap owner cleanup could not bind retention for ${expectedIdentity}; resolved runtime IDs: ${ + runtimeIds.length === 0 ? "none" : runtimeIds.join(", ") + }.`, + ); + } + const runtimeId = runtimeIds[0] as string; + let inspect: DockerContainerInspect; + try { + inspect = inspectExact(runtimeId, deps); + } catch (error) { + throw new Error( + `Managed bootstrap could not inspect retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + const labels = inspect.Config?.Labels ?? {}; + if ( + labels[OPENSHELL_MANAGED_BY_LABEL] !== OPENSHELL_MANAGED_BY_VALUE || + labels[OPENSHELL_SANDBOX_NAME_LABEL] !== sandbox.sandboxName || + labels[OPENSHELL_SANDBOX_ID_LABEL] !== sandbox.sandboxId + ) { + throw new Error( + `Managed bootstrap owner cleanup refused retention after exact runtime ${runtimeId} ownership changed for sandbox ${sandbox.sandboxId}.`, + ); + } + let stopped: DockerCommandResult; + try { + stopped = deps.dockerStop(runtimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + } catch (error) { + throw new Error( + `Managed bootstrap could not quiesce retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + assertZero( + stopped, + `Managed bootstrap could not quiesce retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}`, + ); + let retained: DockerContainerInspect; + try { + retained = inspectExact(runtimeId, deps); + } catch (error) { + throw new Error( + `Managed bootstrap could not re-inspect quiesced sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + if ( + retained.State?.Running !== false || + retained.State.Paused !== false || + retained.State.Restarting !== false + ) { + throw new Error( + `Managed bootstrap retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId} did not prove an explicitly quiescent state.`, + ); + } + if (!deps.runCaptureOpenshell) { + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + }); + } + let getBeforeDelete: string; + try { + getBeforeDelete = deps.runCaptureOpenshell(["sandbox", "get", sandbox.sandboxName], { + ignoreError: false, + }); + } catch (error) { + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + detail: `OpenShell owner lookup also failed: ${ + error instanceof Error ? error.message : String(error) + }.`, + }); + } + const sandboxIdBeforeDelete = parseOpenShellSandboxId(getBeforeDelete); + if (sandboxIdBeforeDelete !== sandbox.sandboxId) { + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + detail: `The same mutable name now resolves to durable sandbox ID ${ + sandboxIdBeforeDelete ?? "unknown" + } instead of ${sandbox.sandboxId}.`, + }); + } + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + }); +} + +function resolveIncompleteCreateSandbox( + input: ManagedBootstrapIncompleteCreateCleanupInput, + deps: ResolvedDeps, +): { + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly runtimeId: string; +} { + if ( + input.plan.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + input.plan.driverId !== DOCKER_DRIVER_ID + ) { + throw new Error("Managed bootstrap Docker incomplete-create cleanup received another driver."); + } + assertManagedBootstrapIdentity(input.bootstrapIdentity); + const query = queryOpenShellDockerSandboxContainers(input.plan.sandboxName, deps); + if (!query.ok) { + throw new Error(`Managed bootstrap Docker incomplete-create discovery failed: ${query.error}`); + } + if (query.ids.length !== 1) { + throw new Error( + `Managed bootstrap incomplete-create cleanup requires exactly one labeled Docker workload; found ${String( + query.ids.length, + )}.`, + ); + } + const runtimeId = String(query.ids[0] ?? "").toLowerCase(); + const inspect = inspectExact(runtimeId, deps); + const sandboxId = String(inspect.Config?.Labels?.[OPENSHELL_SANDBOX_ID_LABEL] ?? ""); + if (parseOpenShellSandboxId(`ID: ${sandboxId}\n`) !== sandboxId) { + throw new Error( + "Managed bootstrap Docker incomplete-create workload has no exact durable sandbox ID.", + ); + } + const sandbox = Object.freeze({ + sandboxName: input.plan.sandboxName, + sandboxId, + driverId: input.plan.driverId, + }); + if ( + input.createReceipt.ready !== true || + input.createReceipt.sandbox.sandboxName !== sandbox.sandboxName || + input.createReceipt.sandbox.sandboxId !== sandbox.sandboxId || + input.createReceipt.sandbox.driverId !== sandbox.driverId + ) { + throw new Error( + "Managed bootstrap Docker incomplete-create workload does not match the exact validated create receipt.", + ); + } + assertImage(inspect, input.plan.image, deps); + assertMetadata(inspect, sandbox, input.plan.metadata); + assertHeldCommand(inspect, input.heldWorkloadArgv, input.bootstrapIdentity); + return { sandbox, runtimeId }; +} + +function managedSharedStateTransaction( + handle: ManagedBootstrapHeldWorkloadHandle, + containerId: string, + image: string, +) { + return { + agent: handle.plan.profile.agent, + bootstrapIdentity: handle.bootstrapIdentity, + containerId, + image, + profileFingerprint: handle.plan.profile.fingerprint, + } as const; +} + +function sameDockerBootstrapJournal( + left: DockerBootstrapTransaction, + right: DockerBootstrapTransaction, +): boolean { + return ( + serializeDockerManagedBootstrapJournal(left) === serializeDockerManagedBootstrapJournal(right) + ); +} + +function createDockerBootstrapJournalDurably( + journal: DockerBootstrapTransaction, + deps: ResolvedDeps, +): DockerBootstrapTransaction { + try { + deps.journalStore.create(journal); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + if (!recovered || !sameDockerBootstrapJournal(recovered, journal)) throw error; + return recovered; + } + const persisted = deps.journalStore.load(journal.bootstrapIdentity); + if (!persisted || !sameDockerBootstrapJournal(persisted, journal)) { + throw new Error("Managed bootstrap Docker staged journal was not durably re-readable."); + } + return persisted; +} + +function transitionDockerBootstrapJournalDurably( + journal: DockerBootstrapTransaction, + next: "cutover" | "rollback-authorized" | "shared-state-committed", + deps: ResolvedDeps, +): DockerBootstrapTransaction { + try { + deps.journalStore.transition(journal.bootstrapIdentity, journal.phase, next); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + const expected = Object.freeze({ ...journal, phase: next }); + if (!recovered || !sameDockerBootstrapJournal(recovered, expected)) throw error; + return recovered; + } + const persisted = deps.journalStore.load(journal.bootstrapIdentity); + const expected = Object.freeze({ ...journal, phase: next }); + if (!persisted || !sameDockerBootstrapJournal(persisted, expected)) { + throw new Error(`Managed bootstrap Docker journal transition to ${next} was not durable.`); + } + return persisted; +} + +function removeDockerBootstrapJournalDurably( + journal: DockerBootstrapTransaction, + deps: ResolvedDeps, +): void { + try { + deps.journalStore.remove(journal.bootstrapIdentity, [journal.phase]); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + if (recovered !== null) throw error; + return; + } + if (deps.journalStore.load(journal.bootstrapIdentity) !== null) { + throw new Error("Managed bootstrap Docker journal removal was not durable."); + } +} + +function assertDockerBootstrapTransactionAuthority( + transaction: DockerBootstrapTransaction, + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared?: ManagedBootstrapPreparedReplacementHandle | null, + replacement?: ManagedBootstrapReplacementHandle | null, +): void { + const originalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, + ); + const expectedSandbox = handle.sandbox; + if ( + transaction.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION || + transaction.bootstrapIdentity !== handle.bootstrapIdentity || + transaction.sandbox.sandboxName !== expectedSandbox.sandboxName || + transaction.sandbox.sandboxId !== expectedSandbox.sandboxId || + transaction.sandbox.driverId !== expectedSandbox.driverId || + transaction.profileFingerprint !== handle.plan.profile.fingerprint || + transaction.imageReference !== + expectedImageReference(snapshot.image.repository, snapshot.image.manifestDigest) || + transaction.runtimeImageContentId !== snapshot.runtimeImageContentId || + transaction.originalRuntimeId !== snapshot.runtimeId || + transaction.originalName !== originalName || + transaction.replacementStagingName !== + replacementStagingName(originalName, handle.bootstrapIdentity) || + transaction.backupName !== backupName(originalName, handle.bootstrapIdentity) || + transaction.originalSpecHash !== snapshot.specHash || + (prepared !== undefined && + prepared !== null && + (transaction.originalRuntimeId !== prepared.originalRuntimeId || + transaction.replacementRuntimeId !== prepared.preparedRuntimeId || + transaction.replacementSpecHash !== prepared.expectedActivatedSpecHash)) || + (replacement !== undefined && + replacement !== null && + (transaction.originalRuntimeId !== replacement.originalRuntimeId || + transaction.replacementRuntimeId !== replacement.replacementRuntimeId || + transaction.replacementSpecHash !== replacement.replacementSpecHash)) + ) { + throw new Error( + "Managed bootstrap receipts do not match the durable Docker transaction authority.", + ); + } +} + +function transactionFromPreparedAuthority( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, +): DockerBootstrapTransaction { + const transaction = parseDockerManagedBootstrapJournal(prepared.rollbackAuthority); + if (transaction.phase !== "staged") { + throw new Error("Managed bootstrap Docker prepared authority must describe a staged runtime."); + } + assertDockerBootstrapTransactionAuthority(transaction, handle, snapshot, prepared); + return transaction; +} + +function assertDurablePreparationAuthority( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, + receipt: ManagedBootstrapDurablePreparationReceipt, +): void { + const authority = createManagedBootstrapPreparedAuthority({ handle, snapshot, prepared }); + const recordedAt = new Date(receipt.recordedAt); + if ( + receipt.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + receipt.sandbox.sandboxName !== authority.sandbox.sandboxName || + receipt.sandbox.sandboxId !== authority.sandbox.sandboxId || + receipt.sandbox.driverId !== authority.sandbox.driverId || + receipt.bootstrapIdentity !== authority.bootstrapIdentity || + receipt.authorityFingerprint !== authority.authorityFingerprint || + typeof receipt.recordId !== "string" || + receipt.recordId.length === 0 || + receipt.recordId.includes("\0") || + typeof receipt.recordedAt !== "string" || + !Number.isFinite(recordedAt.getTime()) || + recordedAt.toISOString() !== receipt.recordedAt + ) { + throw new Error( + "Managed bootstrap Docker activation requires the exact durable prepared-authority receipt.", + ); + } +} + +function reconstructDockerBootstrapTransaction( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + replacement: ManagedBootstrapReplacementHandle, + deps: ResolvedDeps, +): DockerBootstrapTransaction { + if ( + replacement.bootstrapIdentity !== handle.bootstrapIdentity || + replacement.originalRuntimeId !== snapshot.runtimeId || + replacement.originalSpecHash !== snapshot.specHash || + replacement.replacementRuntimeId === replacement.originalRuntimeId + ) { + throw new Error( + "Managed bootstrap finalization receipts do not reconstruct one exact Docker transaction.", + ); + } + const transaction = deps.journalStore.load(handle.bootstrapIdentity); + if (!transaction) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: "the durable Docker cutover journal is absent", + }); + } + assertDockerBootstrapTransactionAuthority(transaction, handle, snapshot, null, replacement); + return transaction; +} + +function rollbackReplacementSharedStateIfPending( + input: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly replacementRuntimeId: string; + readonly runtimeImageContentId: string; + }, + deps: ResolvedDeps, +): void { + if (!tryInspectExact(input.replacementRuntimeId, deps)) { + throw new Error( + "Managed bootstrap replacement disappeared before shared-state rollback could be proven; the preserved original remains stopped.", + ); + } + const transaction = managedSharedStateTransaction( + input.handle, + input.replacementRuntimeId, + input.runtimeImageContentId, + ); + finalizeDockerManagedStartupSharedState( + { transaction, supervisorReady: false, retainContainerAfterRollback: true }, + deps, + ); +} + +function cleanupUnjournaledPreparedContainer( + input: { + readonly snapshot: ManagedBootstrapObservedSnapshot; + readonly preparedRuntimeId: string; + readonly stagingName: string; + }, + deps: ResolvedDeps, +): void { + if (!FULL_CONTAINER_ID_RE.test(input.preparedRuntimeId)) return; + const original = inspectExact(input.snapshot.runtimeId, deps); + const originalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(input.snapshot.specCanonicalJson).inspect, + ); + if ( + !isStableRunning(original) || + dockerContainerName(original) !== originalName || + normalizeDockerManagedBootstrapLaunchSpec(original).hash !== input.snapshot.specHash + ) { + throw new Error( + "Managed bootstrap cannot clean an unjournaled replacement after original drift.", + ); + } + const prepared = tryInspectExact(input.preparedRuntimeId, deps); + if (!prepared) return; + if ( + String(prepared.Id ?? "").toLowerCase() !== input.preparedRuntimeId || + dockerContainerName(prepared) !== input.stagingName || + !isExplicitlyStopped(prepared) + ) { + throw new Error( + "Managed bootstrap refused cleanup because the unjournaled prepared runtime changed.", + ); + } + const removed = deps.dockerRm(input.preparedRuntimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(input.preparedRuntimeId, deps) !== "absent" + ) { + throw new Error( + `Managed bootstrap could not remove its unjournaled prepared runtime: ${ + commandDetail(removed) || "Docker removal failed" + }`, + ); + } +} + +function resolvePreparedRollbackAuthority(input: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot; + readonly prepared: ManagedBootstrapPreparedReplacementHandle | null; + readonly durablePreparation: ManagedBootstrapDurablePreparationReceipt | null; +}): DockerBootstrapTransaction | null { + if (input.durablePreparation && !input.prepared) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: input.handle.bootstrapIdentity, + runtimeId: input.snapshot.runtimeId, + detail: "durable prepared authority is present without its exact prepared handle", + }); + } + if (!input.prepared) return null; + const authority = transactionFromPreparedAuthority(input.handle, input.snapshot, input.prepared); + if (input.durablePreparation) { + assertDurablePreparationAuthority( + input.handle, + input.snapshot, + input.prepared, + input.durablePreparation, + ); + } + return authority; +} + +export function createDockerManagedBootstrapAdapter( + dependencies: DockerManagedBootstrapDeps = {}, +): DockerManagedBootstrapAdapter { + const deps = resolveDeps(dependencies); + const committedTransactions = new Set(); + const rollbackTombstones = new Map(); + const completedRollback = ( + handle: ManagedBootstrapHeldWorkloadHandle, + alreadyRolledBack: boolean, + ): ManagedBootstrapFinalizationReceipt => { + const receipt = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: true, + alreadyRolledBack, + finalizedAt: deps.now().toISOString(), + } satisfies ManagedBootstrapFinalizationReceipt); + rollbackTombstones.set(handle.bootstrapIdentity, { + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + handle.plan.image.repository, + handle.plan.image.manifestDigest, + ), + receipt, + }); + return receipt; + }; + const priorRollback = ( + handle: ManagedBootstrapHeldWorkloadHandle, + ): ManagedBootstrapFinalizationReceipt | null => { + const tombstone = rollbackTombstones.get(handle.bootstrapIdentity); + if (!tombstone) return null; + const receipt = tombstone.receipt; + if ( + receipt.sandbox.sandboxName !== handle.sandbox.sandboxName || + receipt.sandbox.sandboxId !== handle.sandbox.sandboxId || + receipt.sandbox.driverId !== handle.sandbox.driverId || + tombstone.profileFingerprint !== handle.plan.profile.fingerprint || + tombstone.imageReference !== + expectedImageReference(handle.plan.image.repository, handle.plan.image.manifestDigest) + ) { + throw new Error("Managed bootstrap rollback tombstone does not match its durable identity."); + } + return Object.freeze({ + ...receipt, + alreadyRolledBack: true, + }); + }; + const rollbackBootstrapNow = ({ + handle, + snapshot, + prepared, + durablePreparation, + replacement, + sharedStateAlreadyRolledBack = false, + }: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot | null; + readonly prepared: ManagedBootstrapPreparedReplacementHandle | null; + readonly durablePreparation: ManagedBootstrapDurablePreparationReceipt | null; + readonly replacement: ManagedBootstrapReplacementHandle | null; + readonly sharedStateAlreadyRolledBack?: boolean; + }): ManagedBootstrapFinalizationReceipt => { + const finalized = priorRollback(handle); + if (finalized) return finalized; + const journal = deps.journalStore.load(handle.bootstrapIdentity); + if ( + committedTransactions.has(handle.bootstrapIdentity) || + journal?.phase === "shared-state-committed" + ) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: handle.bootstrapIdentity, + cleanupRuntimeId: journal?.originalRuntimeId ?? snapshot?.runtimeId ?? "unknown", + detail: + "rollback is no longer legal after the durable Docker commit fence; retry commit finalization", + }); + } + if (!snapshot) { + if (journal || prepared || durablePreparation || replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal?.originalRuntimeId ?? prepared?.originalRuntimeId ?? "unknown", + detail: "Docker replacement authority exists without its observed snapshot", + }); + } + removeOwnedWorkload(handle.sandbox, deps); + return completedRollback(handle, false); + } + + const preparedAuthority = resolvePreparedRollbackAuthority({ + handle, + snapshot, + prepared, + durablePreparation, + }); + + if (!journal) { + const originalPresence = probeExactDockerContainerAbsence(snapshot.runtimeId, deps); + if (originalPresence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: snapshot.runtimeId, + detail: "the original runtime presence is unknown and no durable journal is available", + }); + } + if (originalPresence === "absent") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: handle.bootstrapIdentity, + cleanupRuntimeId: snapshot.runtimeId, + detail: + "rollback is forbidden because the exact original is absent after journal retirement", + }); + } + const original = inspectExact(snapshot.runtimeId, deps); + const expectedOriginalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, + ); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(original); + if ( + dockerContainerName(original) !== expectedOriginalName || + original.State?.Running !== true || + normalized.hash !== snapshot.specHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: snapshot.runtimeId, + detail: "the journal is absent and the exact original is not a proven restored workload", + }); + } + if (preparedAuthority) { + const observedPrepared = inspectTransactionRuntime( + preparedAuthority, + preparedAuthority.replacementRuntimeId, + deps, + ); + if (observedPrepared) { + assertExplicitlyStopped(observedPrepared, "prepared replacement"); + if ( + dockerContainerName(observedPrepared) !== preparedAuthority.replacementStagingName || + normalizeDockerManagedBootstrapLaunchSpec(observedPrepared).canonicalJson !== + prepared?.preparedSpecCanonicalJson + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: preparedAuthority.replacementRuntimeId, + detail: "the unjournaled prepared runtime changed before exact cleanup", + }); + } + removeExactReplacement(preparedAuthority, observedPrepared, deps); + } + } else if (replacement) { + const replacementPresence = probeExactDockerContainerAbsence( + replacement.replacementRuntimeId, + deps, + ); + if (replacementPresence !== "absent") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: + replacementPresence === "present" + ? "the replacement still exists without durable journal authority" + : "replacement absence is unknown without durable journal authority", + }); + } + } + removeOwnedWorkload(handle.sandbox, deps, snapshot.runtimeId); + return completedRollback(handle, true); + } + + if (!preparedAuthority || !durablePreparation) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker cutover lacks its coordinator-recorded prepared authority", + }); + } + const stagedJournal = Object.freeze({ ...journal, phase: "staged" as const }); + if (!sameDockerBootstrapJournal(stagedJournal, preparedAuthority)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker cutover changed its prepared rollback authority", + }); + } + assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); + const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); + if (!original) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "the exact rollback original is absent", + }); + } + assertTransactionOriginal(journal, original); + const observedReplacement = inspectTransactionRuntime( + journal, + journal.replacementRuntimeId, + deps, + ); + + if (journal.phase === "staged") { + assertStableRunning(original, "staged original"); + if (observedReplacement) { + assertExplicitlyStopped(observedReplacement, "staged replacement"); + } + if ( + dockerContainerName(original) !== journal.originalName || + (observedReplacement !== null && + dockerContainerName(observedReplacement) !== journal.replacementStagingName) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "staged transaction runtime state does not match its pre-cutover fence", + }); + } + if (observedReplacement) { + removeExactReplacement(journal, observedReplacement, deps); + } + removeDockerBootstrapJournalDurably(journal, deps); + removeOwnedWorkload(handle.sandbox, deps, journal.originalRuntimeId); + return completedRollback(handle, false); + } + + if (journal.phase !== "cutover" && journal.phase !== "rollback-authorized") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: journal.bootstrapIdentity, + cleanupRuntimeId: journal.originalRuntimeId, + detail: "rollback is forbidden by the durable Docker commit phase", + }); + } + + const originalNameNow = dockerContainerName(original); + const replacementNameNow = observedReplacement + ? dockerContainerName(observedReplacement) + : null; + const originalAtTargetRecoverable = + originalNameNow === journal.originalName && + (isStableRunning(original) || isExplicitlyStopped(original)); + const originalAtBackupRecoverable = + originalNameNow === journal.backupName && isExplicitlyStopped(original); + const replacementAtStagingRecoverable = + replacementNameNow === journal.replacementStagingName && + observedReplacement !== null && + isExplicitlyStopped(observedReplacement); + const replacementAtTargetRecoverable = + replacementNameNow === journal.originalName && + observedReplacement !== null && + (isStableRunning(observedReplacement) || isExplicitlyStopped(observedReplacement)); + const validCutoverState = + (originalAtTargetRecoverable && replacementAtStagingRecoverable) || + (originalAtBackupRecoverable && replacementAtStagingRecoverable) || + (originalAtBackupRecoverable && replacementAtTargetRecoverable); + let activeJournal = journal; + + if (journal.phase === "cutover") { + if ( + (!sharedStateAlreadyRolledBack && (!observedReplacement || !validCutoverState)) || + (sharedStateAlreadyRolledBack && + (observedReplacement !== null || + !( + (originalNameNow === journal.backupName && isExplicitlyStopped(original)) || + originalAtTargetRecoverable + ))) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: + observedReplacement === null && !sharedStateAlreadyRolledBack + ? "the exact replacement disappeared before rollback authorization was durable" + : "cutover runtime names or states do not match a recoverable phase", + }); + } + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable transaction authority changed before rollback authorization", + }); + } + + let sharedStatus: "committed" | "none" | "pending" = "none"; + const sharedTransaction = managedSharedStateTransaction( + handle, + journal.replacementRuntimeId, + journal.runtimeImageContentId, + ); + if (!sharedStateAlreadyRolledBack) { + sharedStatus = probeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + profileFingerprint: journal.profileFingerprint, + }, + deps, + ); + if (sharedStatus === "committed") { + const committedJournal = transitionDockerBootstrapJournalDurably( + journal, + "shared-state-committed", + deps, + ); + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: committedJournal.bootstrapIdentity, + cleanupRuntimeId: committedJournal.originalRuntimeId, + detail: "image-owned shared state is durably committed; rollback is no longer legal", + }); + } + } + activeJournal = transitionDockerBootstrapJournalDurably(journal, "rollback-authorized", deps); + if (!sharedStateAlreadyRolledBack && sharedStatus === "pending") { + finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: false, + retainContainerAfterRollback: true, + }, + deps, + ); + } + } else { + if (!originalAtTargetRecoverable && !originalAtBackupRecoverable) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "rollback-authorized original runtime state is not recoverable", + }); + } + if ( + observedReplacement && + originalNameNow === journal.originalName && + replacementNameNow === journal.originalName + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "both transaction runtimes claim the authoritative workload name", + }); + } + if (observedReplacement) { + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "rollback authorization changed before replacement cleanup", + }); + } + const sharedTransaction = managedSharedStateTransaction( + handle, + journal.replacementRuntimeId, + journal.runtimeImageContentId, + ); + const sharedStatus = probeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + profileFingerprint: journal.profileFingerprint, + }, + deps, + ); + if (sharedStatus === "committed") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: + "shared state became committed after rollback authorization; no mutation was attempted", + }); + } + if (sharedStatus === "pending") { + finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: false, + retainContainerAfterRollback: true, + }, + deps, + ); + } + } + } + + const beforeRestore = deps.journalStore.load(activeJournal.bootstrapIdentity); + if (!beforeRestore || !sameDockerBootstrapJournal(beforeRestore, activeJournal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "durable transaction authority changed before original restoration", + }); + } + restoreOriginal(activeJournal, deps); + const restored = inspectExact(activeJournal.originalRuntimeId, deps); + assertStableRunning(restored, "restored workload"); + if ( + dockerContainerName(restored) !== activeJournal.originalName || + normalizeDockerManagedBootstrapLaunchSpec(restored).hash !== activeJournal.originalSpecHash + ) { + throw new Error("Managed bootstrap Docker rollback did not restore its exact original."); + } + removeDockerBootstrapJournalDurably(activeJournal, deps); + removeOwnedWorkload(handle.sandbox, deps, activeJournal.originalRuntimeId); + return completedRollback(handle, false); + }; + const commitBootstrapNow = ( + receipt: ManagedBootstrapCompletionReceipt, + transaction: DockerBootstrapTransaction, + input: { + readonly sharedStateStatus: "committed" | "none"; + readonly sharedStateTransaction: ReturnType; + }, + ): void => { + if (committedTransactions.has(receipt.bootstrapIdentity)) return; + if ( + transaction.phase !== "shared-state-committed" || + transaction.replacementRuntimeId !== receipt.runtimeId || + transaction.originalSpecHash !== receipt.originalSpecHash || + transaction.replacementSpecHash !== receipt.replacementSpecHash + ) { + throw new Error("Managed bootstrap Docker commit receipt does not match its commit fence."); + } + const current = deps.journalStore.load(transaction.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, transaction)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "durable commit authority changed before exact cleanup", + }); + } + + const replacement = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (!replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.replacementRuntimeId, + detail: "the exact committed replacement is absent", + }); + } + assertTransactionReplacement(transaction, replacement); + if ( + dockerContainerName(replacement) !== transaction.originalName || + replacement.State?.Running !== true + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.replacementRuntimeId, + detail: "the exact replacement is not running under the authoritative workload name", + }); + } + + const original = inspectTransactionRuntime(transaction, transaction.originalRuntimeId, deps); + if (original) { + assertTransactionOriginal(transaction, original); + if (dockerContainerName(original) !== transaction.backupName) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "the exact rollback backup is not quiescent under its durable backup name", + }); + } + assertExplicitlyStopped(original, "commit rollback backup"); + const beforeRemove = deps.journalStore.load(transaction.bootstrapIdentity); + if (!beforeRemove || !sameDockerBootstrapJournal(beforeRemove, transaction)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "durable commit authority changed before exact rollback-backup removal", + }); + } + const removed = deps.dockerRm(transaction.originalRuntimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(transaction.originalRuntimeId, deps) !== "absent" + ) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: receipt.bootstrapIdentity, + cleanupRuntimeId: transaction.originalRuntimeId, + detail: `${commandDetail(removed) || "Docker removal failed"}; exact backup absence was not proven`, + }); + } + } + + if (input.sharedStateStatus === "committed") { + try { + clearDockerManagedStartupSharedStateCommitReceipt(input.sharedStateTransaction, deps); + } catch (error) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: receipt.bootstrapIdentity, + cleanupRuntimeId: transaction.replacementRuntimeId, + detail: `exact rollback backup is absent, but its image-owned commit receipt could not be retired: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + } + } + removeDockerBootstrapJournalDurably(transaction, deps); + committedTransactions.add(receipt.bootstrapIdentity); + }; + const finalizeBootstrap = async ( + input: Parameters[0], + ): Promise => { + if (input.outcome === "rollback") { + return rollbackBootstrapNow(input); + } + const { completion, durablePreparation, handle, prepared, replacement, snapshot } = input; + if (!completion || !snapshot || !prepared || !durablePreparation || !replacement) { + throw new Error("Managed bootstrap commit requires one complete cutover receipt."); + } + const preparedAuthority = transactionFromPreparedAuthority(handle, snapshot, prepared); + assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); + const sharedTransaction = managedSharedStateTransaction( + handle, + replacement.replacementRuntimeId, + replacement.runtimeImageContentId, + ); + let sharedStatus = probeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + profileFingerprint: completion.profileFingerprint, + }, + deps, + ); + let journal = deps.journalStore.load(handle.bootstrapIdentity); + + if (!journal) { + if (committedTransactions.has(completion.bootstrapIdentity)) { + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + } + const originalPresence = probeExactDockerContainerAbsence(snapshot.runtimeId, deps); + if (originalPresence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: snapshot.runtimeId, + detail: "the retired-journal commit cannot prove exact backup absence", + }); + } + if (originalPresence !== "absent" || sharedStatus !== "none") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: + originalPresence === "absent" ? replacement.replacementRuntimeId : snapshot.runtimeId, + detail: + "the durable journal is absent before both exact backup and shared commit receipt retirement were proven", + }); + } + const committedReplacement = inspectExact(replacement.replacementRuntimeId, deps); + assertStableRunning(committedReplacement, "committed replacement"); + const originalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, + ); + if ( + dockerContainerName(committedReplacement) !== originalName || + normalizeDockerManagedBootstrapLaunchSpec(committedReplacement).hash !== + replacement.replacementSpecHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: "the retired-journal replacement does not match the exact completion receipt", + }); + } + committedTransactions.add(completion.bootstrapIdentity); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + } + + if ( + !sameDockerBootstrapJournal( + Object.freeze({ ...journal, phase: "staged" as const }), + preparedAuthority, + ) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker commit changed its prepared rollback authority", + }); + } + assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); + if (journal.phase === "staged" || journal.phase === "rollback-authorized") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: `commit is forbidden from durable journal phase ${journal.phase}`, + }); + } + if (!completion.transactionPending && sharedStatus !== "none") { + throw new Error( + "Managed bootstrap image completion disagrees with shared-state transaction status.", + ); + } + + if (journal.phase === "cutover") { + if (completion.transactionPending && sharedStatus === "none") { + throw new Error( + "Managed bootstrap image completion lost its shared-state receipt before the durable commit fence.", + ); + } + if (sharedStatus === "pending") { + let outcome; + try { + outcome = finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: true, + retainContainerAfterRollback: true, + }, + deps, + ); + } catch (error) { + if (error instanceof DockerManagedStartupSharedStateCommitIndeterminateError) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: error.message, + }); + } + throw error; + } + if (!outcome.supervisorReady) { + const failure = + outcome.failure ?? new Error("Managed bootstrap shared-state commit did not complete."); + try { + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: + "durable authority changed after shared-state rollback and before restoration", + }); + } + journal = transitionDockerBootstrapJournalDurably(journal, "rollback-authorized", deps); + await rollbackBootstrapNow({ + handle, + snapshot, + prepared, + durablePreparation, + replacement, + sharedStateAlreadyRolledBack: true, + }); + } catch (rollbackError) { + attachManagedBootstrapRollbackError(failure, rollbackError); + } + throw failure; + } + sharedStatus = "committed"; + } + journal = transitionDockerBootstrapJournalDurably(journal, "shared-state-committed", deps); + } else if (completion.transactionPending && sharedStatus === "pending") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "shared state is pending after the durable Docker commit fence", + }); + } + + commitBootstrapNow(completion, journal, { + sharedStateStatus: sharedStatus === "committed" ? "committed" : "none", + sharedStateTransaction: sharedTransaction, + }); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + }; + return { + async createHeldWorkload(input) { + if ( + input.plan.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + input.plan.driverId !== DOCKER_DRIVER_ID || + input.request.agent !== input.plan.profile.agent || + input.request.profileFingerprint !== input.plan.profile.fingerprint + ) { + throw new Error("Managed bootstrap Docker create plan does not match its root request."); + } + const bootstrapIdentity = input.bootstrapIdentity ?? deps.createBootstrapIdentity(); + assertManagedBootstrapIdentity(bootstrapIdentity); + const heldWorkloadArgv = renderManagedBootstrapHeldCommand( + input.request, + bootstrapIdentity, + input.plan.intendedWorkloadArgv, + ); + const createReceipt = await input.launch({ heldWorkloadArgv, bootstrapIdentity }); + if ( + createReceipt.ready !== true || + createReceipt.sandbox.sandboxName !== input.plan.sandboxName || + createReceipt.sandbox.driverId !== input.plan.driverId || + !createReceipt.sandbox.sandboxId + ) { + throw new Error( + "Managed bootstrap Docker create did not return one Ready durable sandbox identity.", + ); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: Object.freeze({ ...createReceipt.sandbox }), + bootstrapIdentity, + heldWorkloadArgv, + intendedWorkloadArgv: Object.freeze([...input.plan.intendedWorkloadArgv]), + plan: input.plan, + createReceipt, + }); + }, + + async cleanupIncompleteCreate(input) { + const { sandbox, runtimeId } = resolveIncompleteCreateSandbox(input, deps); + removeOwnedWorkload(sandbox, deps, runtimeId); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + bootstrapIdentity: input.bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: true, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + }, + + async discoverHeldWorkload( + input: ManagedBootstrapDiscoveryInput, + ): Promise { + if (input.sandbox.driverId !== DOCKER_DRIVER_ID) { + throw new Error("Managed bootstrap Docker adapter received another runtime driver."); + } + const query = queryOpenShellDockerSandboxContainers(input.sandbox.sandboxName, deps); + if (!query.ok) { + throw new Error(`Managed bootstrap Docker discovery failed: ${query.error}`); + } + if (query.ids.length !== 1) { + throw new Error( + `Managed bootstrap requires exactly one labeled Docker workload after Ready; found ${String( + query.ids.length, + )}.`, + ); + } + const runtimeId = String(query.ids[0] ?? "").toLowerCase(); + const inspect = inspectExact(runtimeId, deps); + assertStableRunning(inspect, "held workload"); + assertRootSupervisor(inspect); + assertImage(inspect, input.expectedImage, deps); + assertMetadata(inspect, input.sandbox, input.metadata); + assertBootstrapIdentityInObservedHold(inspect, input.bootstrapIdentity); + return Object.freeze({ + sandbox: input.sandbox, + runtimeId, + bootstrapIdentity: input.bootstrapIdentity, + }); + }, + + async inspectHeldWorkload({ handle, discovered }) { + if ( + discovered.bootstrapIdentity !== handle.bootstrapIdentity || + discovered.sandbox.sandboxId !== handle.sandbox.sandboxId || + discovered.sandbox.driverId !== handle.sandbox.driverId + ) { + throw new Error("Managed bootstrap Docker identity changed before inspection."); + } + const first = inspectExact(discovered.runtimeId, deps); + assertStableRunning(first, "held workload"); + assertRootSupervisor(first); + assertNoRootProcessInjectionEnvironment(first.Config?.Env); + const runtimeImageContentId = assertImage(first, handle.plan.image, deps); + assertMetadata(first, handle.sandbox, handle.plan.metadata); + assertHeldCommand(first, handle.heldWorkloadArgv, handle.bootstrapIdentity); + const firstNormalized = normalizeDockerManagedBootstrapLaunchSpec(first); + const inspect = inspectExact(discovered.runtimeId, deps); + assertStableRunning(inspect, "held workload"); + assertRootSupervisor(inspect); + assertNoRootProcessInjectionEnvironment(inspect.Config?.Env); + if (assertImage(inspect, handle.plan.image, deps) !== runtimeImageContentId) { + throw new Error("Managed bootstrap Docker image content changed during stable capture."); + } + assertMetadata(inspect, handle.sandbox, handle.plan.metadata); + assertHeldCommand(inspect, handle.heldWorkloadArgv, handle.bootstrapIdentity); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(inspect); + if ( + normalized.hash !== firstNormalized.hash || + normalized.canonicalJson !== firstNormalized.canonicalJson + ) { + throw new Error("Managed bootstrap Docker launch spec changed during stable capture."); + } + const supervisorArgv = exactSupervisorArgv(inspect); + if (!exactArrayEqual(supervisorArgv, handle.plan.expectedSupervisorArgv)) { + throw new Error("Managed bootstrap Docker supervisor argv changed before replacement."); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: discovered.runtimeId, + bootstrapIdentity: handle.bootstrapIdentity, + image: handle.plan.image, + runtimeImageContentId, + specHash: normalized.hash, + specCanonicalJson: normalized.canonicalJson, + agentIdentity: Object.freeze({ ...handle.plan.agentIdentity }), + supervisorArgv, + heldWorkloadArgv: handle.heldWorkloadArgv, + metadata: handle.plan.metadata, + }); + }, + + async prepareBootstrapReplacement({ handle, snapshot, request, replacementOptions }) { + if ( + snapshot.bootstrapIdentity !== handle.bootstrapIdentity || + !FULL_CONTAINER_ID_RE.test(snapshot.runtimeId) || + request.agent !== handle.plan.profile.agent || + request.profileFingerprint !== handle.plan.profile.fingerprint + ) { + throw new Error("Managed bootstrap Docker replacement identities do not match."); + } + const parsed = parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson); + const normalizedOriginal = normalizeDockerManagedBootstrapLaunchSpec(parsed.inspect); + if (normalizedOriginal.hash !== snapshot.specHash) { + throw new Error("Managed bootstrap Docker replacement snapshot is not exact."); + } + if (parsed.inspect.HostConfig?.ReadonlyRootfs === true) { + throw new Error( + "Managed bootstrap cannot stage its root-owned request in a read-only root filesystem.", + ); + } + const plan = replacementPlan(replacementOptions); + const originalName = dockerContainerName(parsed.inspect); + const backupContainerName = backupName(originalName, handle.bootstrapIdentity); + const stagingName = replacementStagingName(originalName, handle.bootstrapIdentity); + const existingJournal = deps.journalStore.load(handle.bootstrapIdentity); + if (existingJournal) { + assertDockerBootstrapTransactionAuthority(existingJournal, handle, snapshot); + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: existingJournal.bootstrapIdentity, + runtimeId: existingJournal.replacementRuntimeId, + detail: `preparation requires rollback or commit from durable phase ${existingJournal.phase}`, + }); + } + const trampolineCommand = replacementCommand(handle, snapshot); + const cloneArgs = buildDockerGpuCloneRunArgs(parsed.inspect, plan.mode, { + image: expectedImageReference(snapshot.image.repository, snapshot.image.manifestDigest), + openshellSandboxCommand: handle.intendedWorkloadArgv, + requiredUlimits: plan.requiredUlimits, + extraGroupGids: plan.extraGroupGids, + containerEntrypoint: MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, + containerCommand: trampolineCommand, + containerName: stagingName, + }); + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + + let requestFile = ""; + let replacementRuntimeId = ""; + let stagedAuthority: DockerBootstrapTransaction | null = null; + try { + const created = deps.dockerRun(["create", ...cloneArgs], options); + const returnedRuntimeId = String(created.stdout ?? "") + .trim() + .toLowerCase(); + let createdInspect: DockerContainerInspect; + if (FULL_CONTAINER_ID_RE.test(returnedRuntimeId)) { + replacementRuntimeId = returnedRuntimeId; + createdInspect = inspectExact(replacementRuntimeId, deps); + } else { + try { + createdInspect = inspectDockerContainerReference(stagingName, deps); + } catch (lookupError) { + throw new Error( + "Managed bootstrap could not prove a stopped Docker replacement after create: " + + (commandDetail(created) || + (lookupError instanceof Error ? lookupError.message : String(lookupError))), + ); + } + replacementRuntimeId = String(createdInspect.Id ?? "").toLowerCase(); + } + if ( + !FULL_CONTAINER_ID_RE.test(replacementRuntimeId) || + dockerContainerName(createdInspect) !== stagingName + ) { + throw new Error( + "Managed bootstrap Docker create did not resolve one stopped identity-bound staging container.", + ); + } + assertExplicitlyStopped(createdInspect, "created replacement"); + const createdImageContentId = assertImage(createdInspect, snapshot.image, deps); + if (createdImageContentId !== snapshot.runtimeImageContentId) { + throw new Error( + "Managed bootstrap Docker replacement resolved a different image content ID.", + ); + } + assertMetadata(createdInspect, handle.sandbox, snapshot.metadata); + assertRootSupervisor(createdInspect); + assertReplacementBoundary(createdInspect, handle, snapshot); + const expectedActivatedSpecHash = assertReplacementMatchesIntent( + snapshot.specCanonicalJson, + createdInspect, + originalName, + plan, + openshellSandboxCommandEnvValue(handle.intendedWorkloadArgv) as string, + ); + const preparedSpec = normalizeDockerManagedBootstrapLaunchSpec(createdInspect); + const expectedActivatedSpec = normalizeDockerManagedBootstrapLaunchSpec({ + ...createdInspect, + Name: `/${originalName}`, + }); + if (expectedActivatedSpec.hash !== expectedActivatedSpecHash) { + throw new Error("Managed bootstrap Docker expected activation spec is inconsistent."); + } + stagedAuthority = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + phase: "staged", + bootstrapIdentity: handle.bootstrapIdentity, + sandbox: Object.freeze({ ...handle.sandbox }), + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + snapshot.image.repository, + snapshot.image.manifestDigest, + ), + runtimeImageContentId: snapshot.runtimeImageContentId, + originalRuntimeId: snapshot.runtimeId, + replacementRuntimeId, + originalName, + replacementStagingName: stagingName, + backupName: backupContainerName, + originalSpecHash: snapshot.specHash, + replacementSpecHash: expectedActivatedSpecHash, + }); + + requestFile = writeProtectedEnvelope(handle.bootstrapIdentity, request); + const copied = deps.dockerRun( + ["cp", requestFile, replacementRuntimeId + ":" + MANAGED_BOOTSTRAP_REQUEST_FILE], + options, + ); + assertZero( + copied, + "Managed bootstrap could not stage its protected root-owned 0400 envelope", + ); + + const originalBeforeJournal = inspectExact(snapshot.runtimeId, deps); + assertStableRunning(originalBeforeJournal, "pre-journal original"); + if ( + dockerContainerName(originalBeforeJournal) !== originalName || + normalizeDockerManagedBootstrapLaunchSpec(originalBeforeJournal).hash !== + snapshot.specHash + ) { + throw new Error( + "Managed bootstrap Docker original changed while the replacement was staged.", + ); + } + const replacementBeforeJournal = inspectExact(replacementRuntimeId, deps); + assertTransactionReplacement(stagedAuthority, replacementBeforeJournal); + const observedPreparedSpec = + normalizeDockerManagedBootstrapLaunchSpec(replacementBeforeJournal); + const observedActivatedSpec = normalizeDockerManagedBootstrapLaunchSpec({ + ...replacementBeforeJournal, + Name: `/${originalName}`, + }); + if ( + dockerContainerName(replacementBeforeJournal) !== stagingName || + observedPreparedSpec.canonicalJson !== preparedSpec.canonicalJson || + observedActivatedSpec.canonicalJson !== expectedActivatedSpec.canonicalJson + ) { + throw new Error("Managed bootstrap Docker replacement changed before durable staging."); + } + assertExplicitlyStopped(replacementBeforeJournal, "pre-journal replacement"); + + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + originalRuntimeId: snapshot.runtimeId, + preparedRuntimeId: replacementRuntimeId, + image: snapshot.image, + runtimeImageContentId: snapshot.runtimeImageContentId, + originalSpecHash: snapshot.specHash, + preparedSpecHash: preparedSpec.hash, + preparedSpecCanonicalJson: preparedSpec.canonicalJson, + expectedActivatedSpecHash, + expectedActivatedSpecCanonicalJson: expectedActivatedSpec.canonicalJson, + profileFingerprint: handle.plan.profile.fingerprint, + rollbackAuthority: serializeDockerManagedBootstrapJournal(stagedAuthority), + }); + } catch (error) { + let rollbackError: unknown = null; + try { + const durable = deps.journalStore.load(handle.bootstrapIdentity); + if (!durable) { + cleanupUnjournaledPreparedContainer( + { snapshot, preparedRuntimeId: replacementRuntimeId, stagingName }, + deps, + ); + } + } catch (cleanupError) { + rollbackError = cleanupError; + } + const failure = error instanceof Error ? error : new Error(String(error)); + if (rollbackError) attachManagedBootstrapRollbackError(failure, rollbackError); + throw failure; + } finally { + if (requestFile) cleanupTempDir(requestFile, REQUEST_TEMP_PREFIX); + } + }, + async activateBootstrapReplacement({ handle, snapshot, prepared, durablePreparation }) { + const authority = transactionFromPreparedAuthority(handle, snapshot, prepared); + assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); + const existingJournal = deps.journalStore.load(handle.bootstrapIdentity); + if (existingJournal) { + assertDockerBootstrapTransactionAuthority(existingJournal, handle, snapshot, prepared); + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: existingJournal.bootstrapIdentity, + runtimeId: existingJournal.replacementRuntimeId, + detail: `activation requires rollback or commit from durable phase ${existingJournal.phase}`, + }); + } + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + try { + const originalBeforeJournal = inspectExact(snapshot.runtimeId, deps); + const preparedBeforeJournal = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionOriginal(authority, originalBeforeJournal); + assertTransactionReplacement(authority, preparedBeforeJournal); + assertStableRunning(originalBeforeJournal, "pre-activation original"); + assertExplicitlyStopped(preparedBeforeJournal, "pre-activation replacement"); + if ( + dockerContainerName(originalBeforeJournal) !== authority.originalName || + dockerContainerName(preparedBeforeJournal) !== authority.replacementStagingName || + normalizeDockerManagedBootstrapLaunchSpec(preparedBeforeJournal).canonicalJson !== + prepared.preparedSpecCanonicalJson + ) { + throw new Error( + "Managed bootstrap Docker prepared runtimes changed before durable activation.", + ); + } + + let journal = createDockerBootstrapJournalDurably(authority, deps); + const originalAtFence = inspectExact(snapshot.runtimeId, deps); + const replacementAtFence = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionOriginal(journal, originalAtFence); + assertTransactionReplacement(journal, replacementAtFence); + if ( + dockerContainerName(originalAtFence) !== journal.originalName || + originalAtFence.State?.Running !== true || + dockerContainerName(replacementAtFence) !== journal.replacementStagingName || + normalizeDockerManagedBootstrapLaunchSpec(replacementAtFence).canonicalJson !== + prepared.preparedSpecCanonicalJson + ) { + throw new Error( + "Managed bootstrap Docker staged runtimes changed before the cutover fence.", + ); + } + assertExplicitlyStopped(replacementAtFence, "staged replacement"); + journal = transitionDockerBootstrapJournalDurably(journal, "cutover", deps); + + const stopped = deps.dockerStop(snapshot.runtimeId, { + ...options, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + const afterStop = inspectExact(snapshot.runtimeId, deps); + assertTransactionOriginal(journal, afterStop); + if (dockerContainerName(afterStop) !== journal.originalName) { + throw new Error( + "Managed bootstrap could not prove its exact original stopped after Docker stop: " + + (commandDetail(stopped) || "state did not reach stopped"), + ); + } + assertExplicitlyStopped(afterStop, "stopped original"); + + const renamedOriginal = deps.dockerRename(snapshot.runtimeId, journal.backupName, options); + const afterOriginalRename = inspectExact(snapshot.runtimeId, deps); + assertTransactionOriginal(journal, afterOriginalRename); + if (dockerContainerName(afterOriginalRename) !== journal.backupName) { + throw new Error( + "Managed bootstrap could not prove its exact original backup rename: " + + (commandDetail(renamedOriginal) || "name did not reach backup"), + ); + } + assertExplicitlyStopped(afterOriginalRename, "renamed rollback backup"); + + const renamedReplacement = deps.dockerRename( + prepared.preparedRuntimeId, + journal.originalName, + options, + ); + const afterReplacementRename = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionReplacement(journal, afterReplacementRename); + if (dockerContainerName(afterReplacementRename) !== journal.originalName) { + throw new Error( + "Managed bootstrap could not prove its exact replacement cutover rename: " + + (commandDetail(renamedReplacement) || "name did not reach target"), + ); + } + + const started = deps.dockerStart(prepared.preparedRuntimeId, options); + const running = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionReplacement(journal, running); + const runningSpec = normalizeDockerManagedBootstrapLaunchSpec(running); + if ( + dockerContainerName(running) !== journal.originalName || + running.State?.Running !== true || + running.State.Paused === true || + running.State.Restarting === true || + running.State.Dead === true || + runningSpec.canonicalJson !== prepared.expectedActivatedSpecCanonicalJson + ) { + throw new Error( + "Managed bootstrap could not prove its exact replacement running after Docker start: " + + (commandDetail(started) || "state did not reach running"), + ); + } + assertReplacementBoundary(running, handle, snapshot); + const preservedOriginal = inspectExact(snapshot.runtimeId, deps); + assertTransactionOriginal(journal, preservedOriginal); + if (dockerContainerName(preservedOriginal) !== journal.backupName) { + throw new Error("Managed bootstrap Docker rollback backup changed during cutover."); + } + assertExplicitlyStopped(preservedOriginal, "preserved rollback backup"); + + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + originalRuntimeId: snapshot.runtimeId, + replacementRuntimeId: prepared.preparedRuntimeId, + image: snapshot.image, + runtimeImageContentId: snapshot.runtimeImageContentId, + originalSpecHash: snapshot.specHash, + replacementSpecHash: prepared.expectedActivatedSpecHash, + replacementSpecCanonicalJson: prepared.expectedActivatedSpecCanonicalJson, + profileFingerprint: handle.plan.profile.fingerprint, + }); + } catch (error) { + let rollbackError: unknown = null; + try { + if (!deps.journalStore.load(handle.bootstrapIdentity)) { + cleanupUnjournaledPreparedContainer( + { + snapshot, + preparedRuntimeId: prepared.preparedRuntimeId, + stagingName: authority.replacementStagingName, + }, + deps, + ); + } + } catch (cleanupError) { + rollbackError = cleanupError; + } + const failure = error instanceof Error ? error : new Error(String(error)); + if (rollbackError) attachManagedBootstrapRollbackError(failure, rollbackError); + throw failure; + } + }, + async awaitBootstrap({ handle, snapshot, replacement, timeoutSecs }) { + if ( + replacement.bootstrapIdentity !== handle.bootstrapIdentity || + replacement.originalRuntimeId !== snapshot.runtimeId || + replacement.profileFingerprint !== handle.plan.profile.fingerprint + ) { + throw new Error("Managed bootstrap Docker completion identities do not match."); + } + const journal = reconstructDockerBootstrapTransaction(handle, snapshot, replacement, deps); + if (journal.phase !== "cutover") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: `bootstrap completion is invalid from durable journal phase ${journal.phase}`, + }); + } + assertCompletedCutoverRuntimeState(journal, deps); + const before = inspectExact(replacement.replacementRuntimeId, deps); + assertStableRunning(before, "replacement"); + const beforeImageContentId = assertImage(before, replacement.image, deps); + if (beforeImageContentId !== replacement.runtimeImageContentId) { + throw new Error("Managed bootstrap Docker replacement image content changed."); + } + assertReplacementBoundary(before, handle, snapshot); + if (!waitForOpenShellSupervisorReconnect(handle.sandbox.sandboxName, timeoutSecs, deps)) { + throw new Error("Managed bootstrap Docker supervisor did not reconnect."); + } + const afterWaitJournal = deps.journalStore.load(journal.bootstrapIdentity); + if (!afterWaitJournal || !sameDockerBootstrapJournal(afterWaitJournal, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable transaction authority changed while awaiting bootstrap", + }); + } + assertCompletedCutoverRuntimeState(afterWaitJournal, deps); + const after = inspectExact(replacement.replacementRuntimeId, deps); + assertStableRunning(after, "completed replacement"); + if (assertImage(after, replacement.image, deps) !== replacement.runtimeImageContentId) { + throw new Error("Managed bootstrap Docker completed image content changed."); + } + assertReplacementBoundary(after, handle, snapshot); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(after); + if (normalized.hash !== replacement.replacementSpecHash) { + throw new Error("Managed bootstrap Docker replacement changed during bootstrap."); + } + const imageCompletion = readProtectedImageCompletion(replacement.replacementRuntimeId, deps); + if ( + imageCompletion.bootstrapIdentity !== replacement.bootstrapIdentity || + imageCompletion.agent !== handle.plan.profile.agent || + imageCompletion.profileFingerprint !== replacement.profileFingerprint + ) { + throw new Error( + "Managed bootstrap Docker image completion identities do not match the transaction.", + ); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: replacement.replacementRuntimeId, + image: replacement.image, + runtimeImageContentId: replacement.runtimeImageContentId, + originalSpecHash: replacement.originalSpecHash, + replacementSpecHash: replacement.replacementSpecHash, + profileFingerprint: replacement.profileFingerprint, + bootstrapIdentity: replacement.bootstrapIdentity, + transactionPending: imageCompletion.transactionPending, + completedAt: deps.now().toISOString(), + }); + }, + + finalizeBootstrap, + }; +} diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.ts b/src/lib/onboard/openshell-docker-sandbox-containers.ts index a8934027fd6..94c067f027c 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.ts @@ -7,6 +7,7 @@ import type { DockerGpuPatchDeps } from "./docker-gpu-patch-types"; export const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; export const OPENSHELL_MANAGED_BY_VALUE = "openshell"; export const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; +export const OPENSHELL_SANDBOX_ID_LABEL = "openshell.ai/sandbox-id"; const DOCKER_SANDBOX_QUERY_TIMEOUT_MS = 30_000; diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 94d235cad52..48e889d78c1 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -122,7 +122,10 @@ describe("runtime provider central source boundary", () => { expect(bootstrapProtocolPaths).toEqual([ "src/lib/onboard/managed-bootstrap/adapter.ts", "src/lib/onboard/managed-bootstrap/docker-journal.ts", + "src/lib/onboard/managed-bootstrap/docker-shared-state.ts", "src/lib/onboard/managed-bootstrap/docker-spec.ts", + "src/lib/onboard/managed-bootstrap/docker-test-fixture.ts", + "src/lib/onboard/managed-bootstrap/docker.ts", "src/lib/onboard/managed-bootstrap/envelope.ts", "src/lib/onboard/managed-bootstrap/index.ts", ]); diff --git a/tsconfig.src.json b/tsconfig.src.json index da1287ae436..13c12fb1310 100644 --- a/tsconfig.src.json +++ b/tsconfig.src.json @@ -16,5 +16,10 @@ "types": ["node"] }, "include": ["src/**/*.ts"], - "exclude": ["node_modules", "nemoclaw", "src/**/*.test.ts"] + "exclude": [ + "node_modules", + "nemoclaw", + "src/**/*.test.ts", + "src/**/*-test-fixture.ts" + ] } From 88adfe1b150e3dc4dc4be6f3157ba4bfcd9e3edb Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 06:13:36 -0700 Subject: [PATCH 097/117] fix(onboard): validate managed bootstrap clone inputs Signed-off-by: Aaron Erickson --- .../onboard/docker-gpu-patch-clone.test.ts | 35 ++++++++++++++++++ src/lib/onboard/managed-bootstrap/adapter.ts | 1 + .../onboard/managed-bootstrap/docker.test.ts | 37 ++++++++++++++++++- src/lib/onboard/managed-bootstrap/docker.ts | 1 + 4 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard/docker-gpu-patch-clone.test.ts b/src/lib/onboard/docker-gpu-patch-clone.test.ts index 8ad39b30162..cbdbfb1e252 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.test.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.test.ts @@ -122,6 +122,41 @@ describe("Docker GPU clone envelope", () => { expect(args).not.toContain("nofile=1024:1024"); }); + it("uses exact managed-bootstrap container, entrypoint, and command overrides", () => { + const args = buildDockerGpuCloneRunArgs( + inspectFixture(), + buildDockerGpuMode("startup-command"), + { + containerName: "openshell-alpha-bootstrap-stage", + containerEntrypoint: "/usr/local/bin/nemoclaw-managed-bootstrap", + containerCommand: ["--request", "/run/nemoclaw/bootstrap-request.json"], + }, + ); + + expect(args.slice(0, 2)).toEqual(["--name", "openshell-alpha-bootstrap-stage"]); + expect(args).toEqual( + expect.arrayContaining(["--entrypoint", "/usr/local/bin/nemoclaw-managed-bootstrap"]), + ); + expect(args.slice(args.indexOf("openshell/sandbox:abc"))).toEqual([ + "openshell/sandbox:abc", + "--request", + "/run/nemoclaw/bootstrap-request.json", + ]); + }); + + it.each([ + "", + "-starts-with-dash", + "contains/slash", + "a".repeat(254), + ])("rejects invalid managed-bootstrap container name %j", (containerName) => { + expect(() => + buildDockerGpuCloneRunArgs(inspectFixture(), buildDockerGpuMode("startup-command"), { + containerName, + }), + ).toThrow("Docker clone container name is invalid."); + }); + it("adds SYS_PTRACE to the GPU clone when the baseline container lacks it", () => { const inspect = inspectFixture(); inspect.HostConfig!.CapAdd = ["SYS_ADMIN", "NET_ADMIN"]; diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index b0edaa76627..4721d059ef1 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -25,6 +25,7 @@ const PROCESS_INJECTION_ENV_KEYS = new Set([ "LD_AUDIT", "LD_LIBRARY_PATH", "LD_PRELOAD", + "NODE_OPTIONS", "PS4", "SHELLOPTS", ]); diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 0f848d86c18..5c534649575 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -5,6 +5,10 @@ import { describe, expect, it, vi } from "vitest"; import { ManagedBootstrapOwnerCleanupRequiredError } from "./adapter"; import { createDockerManagedBootstrapAdapter } from "./docker"; +import { + normalizeDockerManagedBootstrapLaunchSpec, + parseDockerManagedBootstrapLaunchSpec, +} from "./docker-spec"; import { authority, completion, @@ -277,6 +281,35 @@ describe("Docker managed bootstrap adapter", () => { ).toBe(true); }); + it.each([ + "NODE_OPTIONS", + "LD_PRELOAD", + "BASH_ENV", + ])("rejects hostile %s from the launch snapshot before replacement creation", async (key) => { + const fake = fixture(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority(); + const parsed = parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson); + const hostileInspect = structuredClone(parsed.inspect); + hostileInspect.Config!.Env = [...(hostileInspect.Config!.Env ?? []), `${key}=/tmp/hostile`]; + const hostileSpec = normalizeDockerManagedBootstrapLaunchSpec(hostileInspect); + + await expect( + adapter.prepareBootstrapReplacement({ + handle, + snapshot: { + ...snapshot, + specHash: hostileSpec.hash, + specCanonicalJson: hostileSpec.canonicalJson, + }, + request, + replacementOptions: { values: {} }, + }), + ).rejects.toThrow(`Managed bootstrap refuses root-process injection environment '${key}'.`); + expect(fake.events).not.toContain("create:replacement"); + expect(fake.replacement).toBeNull(); + }); + it("quiesces and retains an exact incomplete create when its mutable name is reused", async () => { const fake = fixture({ ownerId: "sandbox-alpha-recreated" }); const adapter = createDockerManagedBootstrapAdapter(fake.deps); @@ -303,8 +336,8 @@ describe("Docker managed bootstrap adapter", () => { const fake = fixture({ ownerId: replacementSandboxId }); const adapter = createDockerManagedBootstrapAdapter(fake.deps); const { handle, plan } = authority(); - if (!fake.original.Config?.Labels) throw new Error("fixture labels are required"); - fake.original.Config.Labels["openshell.ai/sandbox-id"] = replacementSandboxId; + expect(fake.original.Config?.Labels).toBeDefined(); + fake.original.Config!.Labels!["openshell.ai/sandbox-id"] = replacementSandboxId; await expect( adapter.cleanupIncompleteCreate({ diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index b463a820051..3a2df714de6 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -2554,6 +2554,7 @@ export function createDockerManagedBootstrapAdapter( if (normalizedOriginal.hash !== snapshot.specHash) { throw new Error("Managed bootstrap Docker replacement snapshot is not exact."); } + assertNoRootProcessInjectionEnvironment(parsed.inspect.Config?.Env); if (parsed.inspect.HostConfig?.ReadonlyRootfs === true) { throw new Error( "Managed bootstrap cannot stage its root-owned request in a read-only root filesystem.", From 51b75ae18bd0c3faf33aaee74f1fc60bccb3e543 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 06:41:02 -0700 Subject: [PATCH 098/117] feat(onboard): bind managed create to runtime providers Signed-off-by: Aaron Erickson --- ci/source-shape-test-budget.json | 2 +- docs/inference/verify-inference-route.mdx | 5 +- docs/reference/commands.mdx | 5 +- docs/reference/troubleshooting.mdx | 12 +- .../adapters/openshell/sandbox-identity.ts | 16 + src/lib/onboard.ts | 7 +- .../sandbox-gpu-create-flow.ts | 2 + .../docker-gpu-local-inference.test.ts | 100 +++-- src/lib/onboard/docker-gpu-local-inference.ts | 87 ++++- .../docker-gpu-route-consumers.test.ts | 10 +- ...ocker-gpu-sandbox-create-lifecycle.test.ts | 95 ++++- src/lib/onboard/docker-gpu-sandbox-create.ts | 349 +++++++++++++----- ...ker-startup-command-sandbox-create.test.ts | 103 +++++- src/lib/onboard/managed-bootstrap/README.md | 64 ++-- .../managed-bootstrap/docker-runtime.ts | 294 +++++++++++++++ src/lib/onboard/managed-bootstrap/index.ts | 4 + .../managed-bootstrap/runtime-create.ts | 145 ++++++++ src/lib/onboard/runtime-provider/contract.ts | 13 +- src/lib/onboard/runtime-provider/registry.ts | 3 +- .../runtime-provider-contract.test.ts | 76 ++++ src/lib/onboard/sandbox-create-launch.test.ts | 46 +++ src/lib/onboard/sandbox-create-launch.ts | 29 +- .../onboard/sandbox-gpu-create-flow.test.ts | 154 ++++++++ src/lib/onboard/sandbox-gpu-create-flow.ts | 116 ++++-- .../onboard/sandbox-gpu-create-run-attempt.ts | 302 +++++++++++---- test/onboard-prepared-build-context.test.ts | 9 +- test/onboard-sandbox-recreation.test.ts | 5 + test/onboard-terminal-dashboard.test.ts | 9 +- test/runtime-provider-source-shape.test.ts | 33 +- 29 files changed, 1784 insertions(+), 311 deletions(-) create mode 100644 src/lib/onboard/managed-bootstrap/docker-runtime.ts create mode 100644 src/lib/onboard/managed-bootstrap/runtime-create.ts diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 8e473b6d06a..c5f7af36e78 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -528,7 +528,7 @@ }, { "file": "test/runtime-provider-source-shape.test.ts", - "test": "keeps production activation paths disconnected from managed bootstrap", + "test": "keeps production activation paths disconnected from driver bootstrap adapters", "category": "security" }, { diff --git a/docs/inference/verify-inference-route.mdx b/docs/inference/verify-inference-route.mdx index b13874e8184..f2c480eafc4 100644 --- a/docs/inference/verify-inference-route.mdx +++ b/docs/inference/verify-inference-route.mdx @@ -43,7 +43,10 @@ Restore the configured endpoint or proxy, run `$$nemoclaw onboard --resume` to c For local Ollama and vLLM on Docker GPU sandboxes using the compatibility route, onboarding performs an additional check after the sandbox becomes ready. It requests `https://inference.local/v1/models` from inside the sandbox and accepts only a 2xx response. -When this check fails, onboarding reports the endpoint and local-provider recovery steps before the first agent prompt. +If this check fails after compatibility recreation, onboarding prints failure diagnostics and attempts to restore the pre-patch container before it exits. +If that rollback fails, onboarding reports that the pre-patch container was not restored and prints container-cleanup guidance. +The local-provider failure output includes the endpoint and recovery steps before the first agent prompt. +GPU-proof diagnostics are captured before rollback and can also print cleanup guidance before the final container state is known, so inspect the sandbox and its labeled Docker containers before running a deletion command. NVIDIA NIM and other compatible endpoints receive their provider validation during onboarding but do not receive this local-provider post-ready check. For those routes, continue to the final route check, then use the status command and a short agent request after onboarding. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index b7d6dbb3fcd..eb7ac46f20a 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -813,7 +813,10 @@ On ordinary native Linux, the compatibility path uses an available NVIDIA CDI sp On Docker Desktop WSL, the compatibility path skips CDI and tries Docker `--gpus all` before the NVIDIA runtime. On Jetson/Tegra hosts, the compatibility path uses the NVIDIA runtime and adds eligible host group IDs for the supported GPU device nodes. These include selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices. -If the compatibility attempt fails, onboarding keeps its diagnostics and the failed sandbox in place and prints a manual cleanup command. +After compatibility recreation starts, onboarding keeps the pre-patch container as a rollback backup until the replacement passes the Ready, GPU, and applicable local-inference checks. +If a later check fails, onboarding prints failure diagnostics and attempts to restore the pre-patch container. +If rollback fails, onboarding reports that the pre-patch container was not restored and prints container-cleanup guidance. +GPU-proof diagnostics are captured before rollback and can print that guidance before the final container state is known, so inspect the sandbox and its labeled Docker containers before running a deletion command. Prerequisites: diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 514f5ea50e4..5c72b91f47a 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2542,20 +2542,26 @@ Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses t #### Common compatibility-path recovery -If the compatibility attempt fails on any host, onboarding leaves the failed sandbox and diagnostic bundle in place so you can inspect the OpenShell and Docker state. +After compatibility recreation starts, onboarding keeps the pre-patch container as a rollback backup until the replacement passes the Ready, direct GPU, and applicable local-inference checks. +If a later check fails, onboarding prints failure diagnostics and attempts to restore the pre-patch container before it exits. +When rollback succeeds, the pre-patch sandbox remains available. +When rollback fails, onboarding reports that the pre-patch container was not restored and prints container-cleanup guidance. +GPU-proof diagnostics are captured before rollback and can print that guidance before the final container state is known. +Inspect the sandbox and its labeled Docker containers before running a deletion command. Starting with NemoClaw v0.0.43, the standard installer handles the `/proc//task//comm` permission case during this patch path. If an older release fails direct GPU proof with that path and `Permission denied`, upgrade NemoClaw and rerun onboarding. -The output includes a cleanup command such as: +When inspection confirms that the failed sandbox remains, delete it with a command such as: ```bash openshell sandbox delete ``` -Fix the NVIDIA Container Toolkit or CDI configuration reported in the diagnostics, clean up the failed sandbox, then rerun onboarding. +Fix the NVIDIA Container Toolkit or CDI configuration reported in the diagnostics. +Run the deletion command only after confirming that the pre-patch sandbox was not restored, then rerun onboarding. If you do not need GPU access inside the sandbox, rerun with `--no-sandbox-gpu`. If sandbox creation fails with `CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all`, the OpenShell gateway tried `docker create --device nvidia.com/gpu=all` and Docker could not resolve the CDI spec. diff --git a/src/lib/adapters/openshell/sandbox-identity.ts b/src/lib/adapters/openshell/sandbox-identity.ts index 1820a8f8f7d..dbcc47696c4 100644 --- a/src/lib/adapters/openshell/sandbox-identity.ts +++ b/src/lib/adapters/openshell/sandbox-identity.ts @@ -14,3 +14,19 @@ export function parseOpenShellSandboxId(output: string): string | null { ? (matches[0] as string) : null; } + +export function resolveOpenShellSandboxId( + sandboxName: string, + runCaptureOpenshell: (args: string[], options?: Record) => string, +): string { + const output = runCaptureOpenshell(["sandbox", "get", sandboxName], { + ignoreError: false, + }); + const sandboxId = parseOpenShellSandboxId(output); + if (!sandboxId) { + throw new Error( + `OpenShell sandbox '${sandboxName}' did not return one exact durable sandbox ID.`, + ); + } + return sandboxId; +} diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index f07ded4ddee..b87721a9405 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2709,7 +2709,7 @@ async function createSandboxWithBaseImageResolution( recreateRuntime.advance("creating"); const { createResult, - dockerGpuCreatePatch, + runtimePatch, route: selectedGpuRoute, firstCreateOutput, registryImageRef, @@ -2765,7 +2765,7 @@ async function createSandboxWithBaseImageResolution( } if (effectiveSandboxGpuConfig.sandboxGpuEnabled) { - dockerGpuLocalInference.verifyGpuSandboxLocalInferenceAfterReady( + await dockerGpuLocalInference.verifyGpuSandboxLocalInferenceAndCommitAfterReady( effectiveSandboxGpuConfig, provider, { @@ -2773,11 +2773,10 @@ async function createSandboxWithBaseImageResolution( dockerDriverGateway, selectedRoute: selectedGpuRoute, verifyDirectSandboxGpu, - verifyGpuOrExit: dockerGpuCreatePatch.verifyGpuOrExit, - selectedMode: dockerGpuCreatePatch.selectedMode, runCaptureOpenshell, log: console.log, }, + runtimePatch, ); } diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index 7061252f12c..eebacbacd3d 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -64,8 +64,10 @@ export function createGpuPatchFixture() { maybeApplyDuringCreate: vi.fn(), createFailureMessage: vi.fn(() => null), exitOnPatchError: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), ensureApplied: vi.fn(), waitForSupervisorReconnectIfNeeded: vi.fn(), + commitAfterReady: vi.fn(), selectedMode: vi.fn(() => null), printReadinessFailureIfEnabled: vi.fn(), verifyGpuOrExit: vi.fn(() => VERIFIED_GPU_PROOF), diff --git a/src/lib/onboard/docker-gpu-local-inference.test.ts b/src/lib/onboard/docker-gpu-local-inference.test.ts index 60dfe28c639..3f0f6180ec4 100644 --- a/src/lib/onboard/docker-gpu-local-inference.test.ts +++ b/src/lib/onboard/docker-gpu-local-inference.test.ts @@ -11,6 +11,7 @@ import { shouldUseDockerGpuPatchHostNetwork, verifyDockerGpuSandboxLocalInference, verifyGpuSandboxAfterReady, + verifyGpuSandboxLocalInferenceAndCommitAfterReady, } from "./docker-gpu-local-inference"; const HOST_NETWORK_ENV = { @@ -311,10 +312,10 @@ describe("verifyGpuSandboxAfterReady", () => { }; } - it("runs the GPU proof and the runtime inference gate when the patch is active", () => { + it("runs the GPU proof and the runtime inference gate when the patch is active", async () => { const log = vi.fn(); const verifyDirectSandboxGpu = vi.fn(); - verifyGpuSandboxAfterReady( + await verifyGpuSandboxAfterReady( GPU_CONFIG, "vllm-local", baseOptions({ @@ -327,12 +328,12 @@ describe("verifyGpuSandboxAfterReady", () => { expect(log).toHaveBeenCalledWith(expect.stringContaining("reached local inference")); }); - it("captures the CUDA-usability proof onto the config for status persistence (#4231)", () => { + it("captures the CUDA-usability proof onto the config for status persistence (#4231)", async () => { const proof = { status: "verified" as const, cudaVerified: true, at: "t" }; const config: { sandboxGpuEnabled: boolean; sandboxGpuProof?: typeof proof | null } = { sandboxGpuEnabled: true, }; - verifyGpuSandboxAfterReady( + await verifyGpuSandboxAfterReady( config, "vllm-local", baseOptions({ @@ -343,45 +344,86 @@ describe("verifyGpuSandboxAfterReady", () => { expect(config.sandboxGpuProof).toEqual(proof); }); - it("does not duplicate proof diagnostics when Docker GPU patch verifier handles them", () => { + it("does not duplicate proof diagnostics when Docker GPU patch verifier handles them", async () => { const proofError = new Error("process.exit"); const verifyGpuOrExit = vi.fn(() => { throw proofError; }); const logError = vi.fn(); - expect(() => + await expect( verifyGpuSandboxAfterReady( GPU_CONFIG, "ollama-local", baseOptions({ verifyGpuOrExit, logError }), ), - ).toThrow(proofError); + ).rejects.toBe(proofError); expect(logError).not.toHaveBeenCalled(); }); - it("routes failure diagnostics through the provided error sink and exits", () => { + it("routes failure diagnostics through the provided error sink and throws for rollback", async () => { const logError = vi.fn(); - const exitSpy = vi.spyOn(process, "exit").mockImplementation((() => { - throw new Error("process.exit"); - }) as never); - try { - expect(() => - verifyGpuSandboxAfterReady( - GPU_CONFIG, - "ollama-local", - baseOptions({ - logError, - deps: { execInSandbox: execEmitting("HTTP_000"), sleep: vi.fn() }, - }), - ), - ).toThrow("process.exit"); - expect(exitSpy).toHaveBeenCalledWith(1); - expect(logError).toHaveBeenCalledWith( - expect.stringContaining("Local inference reachability check failed"), - ); - } finally { - exitSpy.mockRestore(); - } + await expect( + verifyGpuSandboxAfterReady( + GPU_CONFIG, + "ollama-local", + baseOptions({ + logError, + deps: { execInSandbox: execEmitting("HTTP_000"), sleep: vi.fn() }, + }), + ), + ).rejects.toThrow("GPU sandbox local inference reachability failed"); + expect(logError).toHaveBeenCalledWith( + expect.stringContaining("Local inference reachability check failed"), + ); + }); +}); + +describe("verifyGpuSandboxLocalInferenceAndCommitAfterReady", () => { + function options() { + return { + ...gpuPatchOptions(), + verifyDirectSandboxGpu: vi.fn(), + runCaptureOpenshell: vi.fn(() => ""), + log: vi.fn(), + }; + } + + it("commits only after local-inference reachability returns HTTP 2xx", async () => { + const runtimePatch = { + commitAfterReady: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + }; + await verifyGpuSandboxLocalInferenceAndCommitAfterReady( + GPU_CONFIG, + "ollama-local", + { + ...options(), + deps: { execInSandbox: execEmitting("HTTP_200"), sleep: vi.fn() }, + }, + runtimePatch, + ); + expect(runtimePatch.commitAfterReady).toHaveBeenCalledOnce(); + expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + }); + + it("rolls back before propagating an inference verification failure", async () => { + const runtimePatch = { + commitAfterReady: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + }; + await expect( + verifyGpuSandboxLocalInferenceAndCommitAfterReady( + GPU_CONFIG, + "ollama-local", + { + ...options(), + deps: { execInSandbox: execEmitting("HTTP_000"), sleep: vi.fn() }, + }, + runtimePatch, + ), + ).rejects.toThrow("GPU sandbox local inference reachability failed"); + expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce(); + expect(runtimePatch.commitAfterReady).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index da58f8c44d8..496ee830448 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -11,6 +11,7 @@ import { import type { DockerGpuPatchMode } from "./docker-gpu-patch-types"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; +import type { ManagedBootstrapRuntimePatch } from "./managed-bootstrap/runtime-create"; import { executeSandboxCommandForVerification } from "./sandbox-verification-exec"; const { @@ -385,9 +386,9 @@ export type GpuSandboxAfterReadyOptions = { verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult; verifyGpuOrExit?: ( verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult, - ) => SandboxGpuProofResult; + ) => Promise; reportGpuProofFailure?: boolean; - selectedMode: () => DockerGpuPatchMode | null; + selectedMode: ManagedBootstrapRuntimePatch["selectedMode"]; runCaptureOpenshell: (args: string[], opts?: Record) => string; env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform; @@ -396,33 +397,47 @@ export type GpuSandboxAfterReadyOptions = { deps?: DockerGpuSandboxInferenceVerifyDeps; }; +function asDockerGpuPatchMode( + selected: ReturnType, +): DockerGpuPatchMode | null { + if (!selected || !["gpus", "nvidia-runtime", "cdi", "startup-command"].includes(selected.kind)) { + return null; + } + return { + kind: selected.kind as DockerGpuPatchMode["kind"], + label: selected.label, + device: selected.device, + args: [...selected.args], + }; +} + /** * Post-readiness GPU sandbox verification orchestrator (kept out of the * ~12k-line onboard.ts entrypoint per the codebase-growth guardrail). Runs the * direct GPU proof, then — only when the Docker GPU patch is active for a local * inference provider — gates on local inference reachability from the sandbox - * runtime (#4509). Exits the process with actionable output if either proof - * fails. + * runtime (#4509). Throws with actionable output if either proof fails so the + * caller can complete rollback before selecting a terminal exit status. */ -export function verifyGpuSandboxAfterReady( +export async function verifyGpuSandboxAfterReady( config: DockerGpuLocalInferenceConfig, provider: string | null | undefined, options: GpuSandboxAfterReadyOptions, -): void { - verifyGpuSandboxAccessAfterReady(config, options); +): Promise { + await verifyGpuSandboxAccessAfterReady(config, options); verifyGpuSandboxLocalInferenceAfterReady(config, provider, options); } -export function verifyGpuSandboxAccessAfterReady( +export async function verifyGpuSandboxAccessAfterReady( config: DockerGpuLocalInferenceConfig, options: GpuSandboxAfterReadyOptions, -): SandboxGpuProofResult { +): Promise { try { // Capture the CUDA-usability proof result and write it back onto the shared // config so onboarding can persist it to the registry and `status` can // report proven usability rather than mere configuration (#4231). const proof = options.verifyGpuOrExit - ? options.verifyGpuOrExit(options.verifyDirectSandboxGpu) + ? await options.verifyGpuOrExit(options.verifyDirectSandboxGpu) : options.verifyDirectSandboxGpu(options.sandboxName); config.sandboxGpuProof = proof; return proof; @@ -431,11 +446,16 @@ export function verifyGpuSandboxAccessAfterReady( // prints the richer Error-phase / patched-container diagnostics before // rethrowing. Avoid a second generic proof-failure block in that path. if (!options.verifyGpuOrExit && options.reportGpuProofFailure !== false) { - printDockerGpuProofFailure(options.sandboxName, error, options.selectedMode(), { - runCaptureOpenshell: options.runCaptureOpenshell, - additionalSummaryLines: adaptDockerGpuRouteForPatch(options.selectedRoute) - .additionalSummaryLines, - }); + printDockerGpuProofFailure( + options.sandboxName, + error, + asDockerGpuPatchMode(options.selectedMode()), + { + runCaptureOpenshell: options.runCaptureOpenshell, + additionalSummaryLines: adaptDockerGpuRouteForPatch(options.selectedRoute) + .additionalSummaryLines, + }, + ); } throw error; } @@ -444,7 +464,7 @@ export function verifyGpuSandboxAccessAfterReady( export function verifyGpuSandboxLocalInferenceAfterReady( config: DockerGpuLocalInferenceConfig, provider: string | null | undefined, - options: GpuSandboxAfterReadyOptions, + options: Omit, ): void { if (options.selectedRoute !== "compatibility") return; const verification = verifyDockerGpuSandboxLocalInference(config, provider, { @@ -469,6 +489,39 @@ export function verifyGpuSandboxLocalInferenceAfterReady( verification, options.logError ?? ((message) => console.error(message)), ); - process.exit(1); + throw new Error( + `GPU sandbox local inference reachability failed for ${verification.endpoint}.`, + ); + } +} + +/** + * Keep the managed create transaction reversible until the sandbox's real + * local-inference reachability check returns HTTP 2xx. Rollback failures are + * attached to the original verification failure so callers retain both pieces + * of evidence. + */ +export async function verifyGpuSandboxLocalInferenceAndCommitAfterReady( + config: DockerGpuLocalInferenceConfig, + provider: string | null | undefined, + options: Omit, + runtimePatch: Pick< + ManagedBootstrapRuntimePatch, + "commitAfterReady" | "rollbackManagedStartupAfterCreateFailure" + >, +): Promise { + try { + verifyGpuSandboxLocalInferenceAfterReady(config, provider, options); + await runtimePatch.commitAfterReady(); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + try { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); + } catch (rollbackError) { + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + throw failure; } } diff --git a/src/lib/onboard/docker-gpu-route-consumers.test.ts b/src/lib/onboard/docker-gpu-route-consumers.test.ts index f2454e5d45d..b11215d340e 100644 --- a/src/lib/onboard/docker-gpu-route-consumers.test.ts +++ b/src/lib/onboard/docker-gpu-route-consumers.test.ts @@ -137,7 +137,7 @@ describe("selected route consumers", () => { expect(reverifyBridgeReachability).not.toHaveBeenCalled(); }); - it("skips compatibility-only inference gates after native wins", () => { + it("skips compatibility-only inference gates after native wins", async () => { const execInSandbox = vi.fn(); expect( verifyDockerGpuSandboxLocalInference(GPU_CONFIG, "ollama-local", { @@ -149,7 +149,7 @@ describe("selected route consumers", () => { ).toEqual({ status: "skipped", reason: "not-docker-gpu-patch" }); const verifyDirectSandboxGpu = vi.fn(); - verifyGpuSandboxAfterReady(GPU_CONFIG, "ollama-local", { + await verifyGpuSandboxAfterReady(GPU_CONFIG, "ollama-local", { sandboxName: "alpha", dockerDriverGateway: true, selectedRoute: "native", @@ -162,11 +162,11 @@ describe("selected route consumers", () => { expect(execInSandbox).not.toHaveBeenCalled(); }); - it("defers native proof diagnostics while automatic fallback owns recovery", () => { + it("defers native proof diagnostics while automatic fallback owns recovery", async () => { const proofError = new Error("native CUDA proof failed"); const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); try { - expect(() => + await expect( verifyGpuSandboxAfterReady(GPU_CONFIG, "ollama-local", { sandboxName: "alpha", dockerDriverGateway: true, @@ -178,7 +178,7 @@ describe("selected route consumers", () => { selectedMode: () => null, runCaptureOpenshell: vi.fn(() => ""), }), - ).toThrow(proofError); + ).rejects.toThrow(proofError); expect(consoleError).not.toHaveBeenCalled(); } finally { consoleError.mockRestore(); diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index fb94eba175e..6ddea611aa9 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -42,7 +42,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { vi.restoreAllMocks(); }); - it("defers backup removal until waitForSupervisorReconnectIfNeeded sees supervisorReady=true", () => { + it("retains the backup after reconnect and removes it only after post-Ready commit", async () => { const deps = makeDeps(); const result = deferredCreateResult(); const recreatePatch = vi.fn(() => result); @@ -84,13 +84,57 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.waitForSupervisorReconnectIfNeeded(); expect(waitForSupervisor).toHaveBeenCalledTimes(1); + expect(finalizeBackup).not.toHaveBeenCalled(); + + await patch.commitAfterReady(); expect(finalizeBackup).toHaveBeenCalledTimes(1); expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: true }, deps); expect(capturePreRollbackDiagnostics).not.toHaveBeenCalled(); expect(onPatchFailureExit).not.toHaveBeenCalled(); }); - it("refuses compatibility success when the backup container cannot be removed", () => { + it("reports a failed post-Ready rollback instead of treating it as restored", async () => { + const deps = makeDeps(); + const result = deferredCreateResult(); + const finalizeBackup = vi.fn(() => ({ + backupRemoved: false, + rolledBack: false, + })); + const onPatchFailureExit = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => ["existing-container"]), + recreatePatch: vi.fn(() => result), + waitForSupervisor: vi.fn(() => true), + finalizeBackup, + onPatchFailureExit, + }, + }); + + patch.maybeApplyDuringCreate(); + patch.waitForSupervisorReconnectIfNeeded(); + await patch.rollbackManagedStartupAfterCreateFailure(); + + expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: false }, deps); + expect(onPatchFailureExit).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + message: expect.stringContaining("pre-patch container was not restored"), + }), + expect.objectContaining({ + context: expect.objectContaining({ + backupContainerName: result.backupContainerName, + rolledBack: false, + }), + }), + ); + }); + + it("refuses compatibility success when the backup container cannot be removed", async () => { const deps = makeDeps(); const result = deferredCreateResult(); const onPatchFailureExit = vi.fn(); @@ -113,11 +157,14 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); patch.waitForSupervisorReconnectIfNeeded(); + expect(onPatchFailureExit).not.toHaveBeenCalled(); + + await patch.commitAfterReady(); expect(onPatchFailureExit).toHaveBeenCalledOnce(); expect(onPatchFailureExit.mock.calls[0]?.[1]).toEqual( expect.objectContaining({ - message: expect.stringContaining("backup container"), + message: expect.stringContaining("rollback backup"), }), ); expect(onPatchFailureExit.mock.calls[0]?.[2]).toEqual( @@ -245,7 +292,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(onPatchFailureExit).not.toHaveBeenCalled(); }); - it("records patchError when recreate throws and exitOnPatchError reports it via printDockerGpuPatchFailureAndExit", () => { + it("records patchError when recreate throws and exitOnPatchError reports it via printDockerGpuPatchFailureAndExit", async () => { const deps = makeDeps(); const recreatePatch = vi.fn(() => { throw new Error("docker rename failed"); @@ -271,7 +318,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); expect(patch.createFailureMessage()).toMatch(/Docker GPU patch failed/); - patch.exitOnPatchError(); + await patch.exitOnPatchError(); expect(onPatchFailureExit).toHaveBeenCalledTimes(1); // Supervisor wait must be skipped because needsSupervisorWait stayed false. patch.waitForSupervisorReconnectIfNeeded(); @@ -279,7 +326,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(finalizeBackup).not.toHaveBeenCalled(); }); - it("hard-stops a structured failed GPU proof on the compatibility route", () => { + it("hard-stops a structured failed GPU proof on the compatibility route", async () => { const deps = makeDeps(); const patch = createDockerGpuSandboxCreatePatch({ route: "compatibility", @@ -291,7 +338,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { }, }); - expect(() => + await expect( patch.verifyGpuOrExit(() => ({ status: "failed", cudaVerified: false, @@ -299,6 +346,38 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { detail: "No devices were found", at: "2026-07-07T00:00:00.000Z", })), - ).toThrow("Sandbox GPU proof returned failed status: nvidia-smi when available"); + ).rejects.toThrow("Sandbox GPU proof returned failed status: nvidia-smi when available"); + }); + + it("reports a failed rollback after GPU-proof diagnostics", async () => { + const deps = makeDeps(); + const result = deferredCreateResult(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => ["existing-container"]), + recreatePatch: vi.fn(() => result), + waitForSupervisor: vi.fn(() => true), + finalizeBackup: vi.fn(() => ({ + backupRemoved: false, + rolledBack: false, + })), + }, + }); + + patch.maybeApplyDuringCreate(); + patch.waitForSupervisorReconnectIfNeeded(); + + await expect( + patch.verifyGpuOrExit(() => { + throw new Error("nvidia-smi failed"); + }), + ).rejects.toThrow("nvidia-smi failed"); + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("pre-patch container was not restored"), + ); }); }); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index b683cbb1c77..52c15d0b5be 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -42,7 +42,7 @@ export { type DockerGpuSandboxCreateDeps = Pick< DockerGpuPatchDeps, - "runOpenshell" | "runCaptureOpenshell" | "sleep" | "dockerCapture" + "runOpenshell" | "runCaptureOpenshell" | "sleep" | "dockerCapture" | "dockerRun" | "dockerStop" >; type WaitSupervisorFn = typeof waitForOpenShellSupervisorReconnect; @@ -61,6 +61,11 @@ type PatchFailureExitFn = ( type DockerGpuSandboxCreatePatchOptions = { route: SelectedDockerGpuRoute; persistStartupCommand?: boolean; + /** + * A managed bootstrap owns the one permitted recreation after Ready. Keep + * route diagnostics/proof active without running the legacy recreator. + */ + externalRecreation?: boolean; sandboxName: string; gpuDevice?: string | null; openshellSandboxCommand?: readonly string[] | null; @@ -91,12 +96,27 @@ type DockerGpuSandboxCreatePatchOptions = { }; }; +export interface DockerManagedBootstrapDeferredCutover { + readonly selectedMode: DockerGpuPatchMode; + readonly failureContext: DockerGpuPatchFailureContext; + rollback(): Promise; + commit(): Promise; +} + export type DockerGpuSandboxCreatePatch = { maybeApplyDuringCreate: () => void; createFailureMessage: () => string | null; - exitOnPatchError: () => void; - ensureApplied: () => void; + exitOnPatchError: () => Promise; + attachManagedBootstrapCutover: (cutover: DockerManagedBootstrapDeferredCutover) => void; + rollbackManagedStartupAfterCreateFailure: () => Promise; + ensureApplied: () => Promise; waitForSupervisorReconnectIfNeeded: () => void; + /** + * Commit an attached managed cutover or remove a legacy recreation backup. + * Call only after authoritative Ready and the required GPU and applicable + * local-inference checks pass. + */ + commitAfterReady: () => Promise; selectedMode: () => DockerGpuPatchMode | null; /** * Print the Docker GPU readiness-failure block (including the Error-phase @@ -106,14 +126,14 @@ export type DockerGpuSandboxCreatePatch = { printReadinessFailureIfEnabled: () => void; /** * Run the GPU proof while distinguishing "sandbox in terminal phase" from - * "proof failed inside a live sandbox". Calls `process.exit(1)` for the - * former and rethrows after printing diagnostics for the latter so the - * onboarding flow surfaces the right failure cause (#4316). Returns the - * CUDA-usability proof result on success so callers can persist it (#4231). + * "proof failed inside a live sandbox". Awaits rollback and throws after + * printing diagnostics so the onboarding flow can select the terminal exit + * status without racing the rollback (#4316). Returns the CUDA-usability + * proof result on success so callers can persist it (#4231). */ verifyGpuOrExit: ( verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult, - ) => SandboxGpuProofResult; + ) => Promise; }; export function createDockerGpuSandboxCreatePatch( @@ -121,8 +141,12 @@ export function createDockerGpuSandboxCreatePatch( ): DockerGpuSandboxCreatePatch { const routeAdapter = adaptDockerGpuRouteForPatch(options.route); let result: DockerGpuPatchResult | null = null; + let managedBootstrapCutover: DockerManagedBootstrapDeferredCutover | null = null; let patchError: unknown = null; let needsSupervisorWait = false; + let cutoverFinalized = false; + let cutoverFinalization: Promise | null = null; + let cutoverFinalizationOutcome: "commit" | "rollback" | null = null; const findContainerIds = options.overrides?.findContainerIds ?? findOpenShellDockerSandboxContainerIds; @@ -145,7 +169,10 @@ export function createDockerGpuSandboxCreatePatch( backend: options.backend, dockerDesktopWsl: options.dockerDesktopWsl ?? isDockerDesktopWslRuntime(), }; - const patchEnabled = routeAdapter.enabled || options.persistStartupCommand === true; + const recreationEnabled = + options.externalRecreation !== true && + (routeAdapter.enabled || options.persistStartupCommand === true); + const patchEnabled = recreationEnabled; const patchTarget = routeAdapter.enabled ? "NVIDIA GPU access" : "restart-safe startup"; const recreateSelectedPatch = createDockerSandboxRecreator({ gpuEnabled: routeAdapter.enabled, @@ -156,21 +183,92 @@ export function createDockerGpuSandboxCreatePatch( recreateStartup: recreateStartupPatch, }); + const applyPatch = (deps: DockerGpuPatchDeps): void => { + if (!recreationEnabled) return; + result = recreateSelectedPatch(false, deps); + needsSupervisorWait = true; + console.log(` ✓ Docker container mode selected: ${result.mode.label}`); + }; + + const rollbackAfterFailure = async (): Promise => { + if (cutoverFinalized || (!managedBootstrapCutover && !result)) return null; + if (cutoverFinalization) { + try { + if (cutoverFinalizationOutcome !== "rollback") { + throw new Error("Managed startup rollback raced an in-progress commit finalization."); + } + await cutoverFinalization; + return null; + } catch (error) { + return error instanceof Error ? error : new Error(String(error)); + } + } + const finalization = (async () => { + await managedBootstrapCutover?.rollback(); + const finalizeOutcome = result + ? finalizeBackup({ result, supervisorReady: false }, options.deps) + : null; + cutoverFinalized = true; + needsSupervisorWait = false; + if (finalizeOutcome && !finalizeOutcome.rolledBack) { + throw new Error( + "Docker container rollback failed; the pre-patch container was not restored.", + ); + } + })(); + cutoverFinalization = finalization; + cutoverFinalizationOutcome = "rollback"; + try { + await finalization; + return null; + } catch (error) { + return error instanceof Error ? error : new Error(String(error)); + } finally { + if (!cutoverFinalized) { + cutoverFinalization = null; + cutoverFinalizationOutcome = null; + } + } + }; + + const reportPatchErrorAndExit = async (): Promise => { + if (!patchError) return; + const rollbackError = await rollbackAfterFailure(); + if (rollbackError) { + patchError = new Error( + `${patchError instanceof Error ? patchError.message : String(patchError)}; managed startup rollback failed: ${rollbackError.message}`, + ); + } + onPatchFailureExit(options.sandboxName, patchError, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }); + }; + const selectedMode = (): DockerGpuPatchMode | null => + managedBootstrapCutover?.selectedMode ?? result?.mode ?? null; + const failureContext = (): DockerGpuPatchFailureContext => + managedBootstrapCutover?.failureContext ?? buildFailureContext(options.sandboxName, result); + return { maybeApplyDuringCreate() { if (!patchEnabled || result || patchError) return; const containerIds = findContainerIds(options.sandboxName); if (containerIds.length === 0) return; + if (containerIds.length !== 1) { + patchError = new Error( + `Docker recreation observed ${String(containerIds.length)} matching containers; refusing an ambiguous replacement.`, + ); + return; + } console.log( ` OpenShell Docker container detected; recreating it with ${patchTarget} before readiness wait...`, ); try { - result = recreateSelectedPatch(false, { + applyPatch({ runCaptureOpenshell: options.deps.runCaptureOpenshell, sleep: options.deps.sleep, }); - needsSupervisorWait = true; - console.log(` ✓ Docker container mode selected: ${result.mode.label}`); } catch (error) { patchError = error; } @@ -183,33 +281,44 @@ export function createDockerGpuSandboxCreatePatch( : "Docker startup-command patch failed while OpenShell sandbox create was still waiting."; }, - exitOnPatchError() { - if (!patchError) return; - onPatchFailureExit(options.sandboxName, patchError, { + async exitOnPatchError() { + await reportPatchErrorAndExit(); + }, + + attachManagedBootstrapCutover(cutover) { + if (managedBootstrapCutover || result || cutoverFinalized) { + throw new Error("Managed bootstrap cutover may be attached exactly once."); + } + managedBootstrapCutover = cutover; + }, + + async rollbackManagedStartupAfterCreateFailure() { + const rollbackError = await rollbackAfterFailure(); + if (!rollbackError) return; + onPatchFailureExit(options.sandboxName, rollbackError, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: { + ...failureContext(), + rolledBack: false, + }, }); }, - ensureApplied() { + async ensureApplied() { if (!patchEnabled || result) return; console.log(` Recreating OpenShell Docker sandbox container with ${patchTarget}...`); try { - result = recreateSelectedPatch(false, options.deps); - needsSupervisorWait = true; - console.log(` ✓ Docker container mode selected: ${result.mode.label}`); + applyPatch(options.deps); } catch (error) { - onPatchFailureExit(options.sandboxName, error, { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - }); + patchError = error; + await reportPatchErrorAndExit(); } }, waitForSupervisorReconnectIfNeeded() { - if (!needsSupervisorWait) return; + if (!needsSupervisorWait || cutoverFinalized) return; const supervisorReconnectTimeoutSecs = getDockerGpuSupervisorReconnectTimeoutSecs( options.timeoutSecs, ); @@ -221,14 +330,17 @@ export function createDockerGpuSandboxCreatePatch( supervisorReconnectTimeoutSecs, { runOpenshell: options.deps.runOpenshell, - // Pass `runCaptureOpenshell` so the supervisor-reconnect wait can - // short-circuit on a terminal sandbox phase instead of burning - // the full reconnect timeout window when the patched container - // crashed on startup (#4316). runCaptureOpenshell: options.deps.runCaptureOpenshell, sleep: options.deps.sleep, }, ); + if (supervisorReady) { + // Reconnect completes the legacy recreation check. Keep its rollback + // backup until the caller accepts authoritative Ready and the required + // GPU checks. + needsSupervisorWait = false; + return; + } if (!supervisorReady && result) { try { captureFailedClone(options.sandboxName, result, options.deps); @@ -239,40 +351,13 @@ export function createDockerGpuSandboxCreatePatch( } } const finalizeOutcome = result - ? finalizeBackup({ result, supervisorReady }, options.deps) + ? finalizeBackup({ result, supervisorReady: false }, options.deps) : null; - if (supervisorReady) { - if (finalizeOutcome && !finalizeOutcome.backupRemoved) { - onPatchFailureExit( - options.sandboxName, - new Error( - "OpenShell supervisor reconnected, but the recreated backup container could not be removed.", - ), - { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - context: { - sandboxName: options.sandboxName, - oldContainerId: result?.oldContainerId, - newContainerId: result?.newContainerId, - backupContainerName: result?.backupContainerName, - selectedMode: result?.mode ?? null, - rolledBack: false, - }, - }, - ); - } - return; - } - const failureMessage = (() => { - if (!finalizeOutcome) { - return "OpenShell supervisor did not reconnect to the recreated container."; - } - return finalizeOutcome.rolledBack - ? "OpenShell supervisor did not reconnect to the recreated container; pre-patch sandbox restored." - : "OpenShell supervisor did not reconnect to the recreated container and rollback failed; pre-patch sandbox was NOT restored."; - })(); + cutoverFinalized = true; + needsSupervisorWait = false; + const failureMessage = finalizeOutcome?.rolledBack + ? "OpenShell supervisor did not reconnect to the recreated container; pre-patch sandbox restored." + : "OpenShell supervisor did not reconnect to the recreated container and rollback failed; pre-patch sandbox was NOT restored."; onPatchFailureExit(options.sandboxName, new Error(failureMessage), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, @@ -288,21 +373,108 @@ export function createDockerGpuSandboxCreatePatch( }); }, + async commitAfterReady() { + if (cutoverFinalized || (!managedBootstrapCutover && !result)) return; + if (needsSupervisorWait) { + const error = new Error( + "Managed startup cannot commit before the recreated OpenShell supervisor reconnects.", + ); + const rollbackError = await rollbackAfterFailure(); + onPatchFailureExit( + options.sandboxName, + rollbackError + ? new Error(`${error.message} Rollback failed: ${rollbackError.message}`) + : error, + { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }, + ); + return; + } + if (cutoverFinalization) { + if (cutoverFinalizationOutcome !== "commit") { + throw new Error("Managed startup commit raced an in-progress rollback finalization."); + } + await cutoverFinalization; + return; + } + const finalization = (async () => { + if (managedBootstrapCutover) { + try { + await managedBootstrapCutover.commit(); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + let rollbackError: Error | null = null; + try { + await managedBootstrapCutover.rollback(); + cutoverFinalized = true; + needsSupervisorWait = false; + } catch (rollbackFailure) { + rollbackError = + rollbackFailure instanceof Error + ? rollbackFailure + : new Error(String(rollbackFailure)); + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + onPatchFailureExit(options.sandboxName, failure, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: { + ...failureContext(), + rolledBack: rollbackError === null, + }, + }); + return; + } + } + const finalizeOutcome = result + ? finalizeBackup({ result, supervisorReady: true }, options.deps) + : null; + cutoverFinalized = true; + if (!finalizeOutcome || finalizeOutcome.backupRemoved) return; + onPatchFailureExit( + options.sandboxName, + new Error("Managed startup passed Ready, but its rollback backup could not be removed."), + { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: failureContext(), + }, + ); + })(); + cutoverFinalization = finalization; + cutoverFinalizationOutcome = "commit"; + try { + await finalization; + } finally { + if (!cutoverFinalized) { + cutoverFinalization = null; + cutoverFinalizationOutcome = null; + } + } + }, + selectedMode() { - return result?.mode ?? null; + return selectedMode(); }, printReadinessFailureIfEnabled() { if (!routeAdapter.enabled) return; - printDockerGpuReadinessFailure(options.sandboxName, result?.mode ?? null, { + printDockerGpuReadinessFailure(options.sandboxName, selectedMode(), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, - context: buildFailureContext(options.sandboxName, result), + context: failureContext(), additionalSummaryLines: routeAdapter.additionalSummaryLines, }); }, - verifyGpuOrExit(verifyDirectSandboxGpu) { + async verifyGpuOrExit(verifyDirectSandboxGpu) { // Before issuing GPU proof commands through `openshell sandbox exec`, // confirm the sandbox is still in a live phase. A sandbox that // transitioned to Error after the readiness wait succeeded (e.g. the @@ -312,7 +484,7 @@ export function createDockerGpuSandboxCreatePatch( // container/Error-phase classification instead of running the proof // (#4316). const sandboxName = options.sandboxName; - const failureContext = buildFailureContext(sandboxName, result); + const currentFailureContext = failureContext(); if (routeAdapter.enabled && options.deps.runCaptureOpenshell) { const list = options.deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true, @@ -321,20 +493,23 @@ export function createDockerGpuSandboxCreatePatch( if (phase) { console.error(""); console.error(` Skipping GPU proof: sandbox '${sandboxName}' is in ${phase} phase.`); - printDockerGpuProofFailure( - sandboxName, - new Error( - `Sandbox '${sandboxName}' entered ${phase} phase after readiness; GPU proof skipped.`, - ), - result?.mode ?? null, - { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - context: failureContext, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - }, + const failure = new Error( + `Sandbox '${sandboxName}' entered ${phase} phase after readiness; GPU proof skipped.`, ); - process.exit(1); + printDockerGpuProofFailure(sandboxName, failure, selectedMode(), { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + context: currentFailureContext, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }); + const rollbackError = await rollbackAfterFailure(); + if (rollbackError) { + console.error(` ${rollbackError.message}`); + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + throw failure; } } try { @@ -346,13 +521,21 @@ export function createDockerGpuSandboxCreatePatch( } return proof; } catch (error) { - printDockerGpuProofFailure(sandboxName, error, result?.mode ?? null, { + const failure = error instanceof Error ? error : new Error(String(error)); + printDockerGpuProofFailure(sandboxName, failure, selectedMode(), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, - context: routeAdapter.enabled ? failureContext : null, + context: routeAdapter.enabled ? currentFailureContext : null, additionalSummaryLines: routeAdapter.additionalSummaryLines, }); - throw error; + const rollbackError = await rollbackAfterFailure(); + if (rollbackError) { + console.error(` ${rollbackError.message}`); + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + throw failure; } }, }; diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts index 17cd02f6432..ec0ad3cb306 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts @@ -68,7 +68,7 @@ describe("Docker startup-command sandbox creation", () => { vi.restoreAllMocks(); }); - it("uses the startup-command recreation path with DCode's exact resource limits", () => { + it("uses the startup-command recreation path with DCode's exact resource limits", async () => { const dockerCaptureOutput: Record = { ps: "old-container-id\n", inspect: JSON.stringify([inspectFixture()]), @@ -102,7 +102,7 @@ describe("Docker startup-command sandbox creation", () => { }, }); - patch.ensureApplied(); + await patch.ensureApplied(); expect(recreatePatch).not.toHaveBeenCalled(); expect(dockerRunDetached.mock.calls[0]?.[0]).toEqual( @@ -156,7 +156,102 @@ describe("Docker startup-command sandbox creation", () => { expect(context.rolledBack).toBe(true); }); - it("reports startup-command creation failures through the composed patch boundary", () => { + it("defers a driver-owned managed cutover until the authoritative caller commits", async () => { + const deps = makeDeps(); + let releaseCommit = () => {}; + const commit = vi.fn( + () => + new Promise((resolve) => { + releaseCommit = resolve; + }), + ); + const rollback = vi.fn(async () => {}); + const patch = createDockerGpuSandboxCreatePatch({ + route: "native", + externalRecreation: true, + sandboxName: "alpha", + timeoutSecs: 60, + deps, + }); + patch.attachManagedBootstrapCutover({ + selectedMode: { + kind: "startup-command", + label: "managed bootstrap", + device: "", + args: [], + }, + failureContext: { sandboxName: "alpha" }, + commit, + rollback, + }); + patch.maybeApplyDuringCreate(); + await patch.ensureApplied(); + patch.waitForSupervisorReconnectIfNeeded(); + expect(commit).not.toHaveBeenCalled(); + const firstCommit = patch.commitAfterReady(); + const duplicateCommit = patch.commitAfterReady(); + expect(commit).toHaveBeenCalledOnce(); + expect(rollback).not.toHaveBeenCalled(); + releaseCommit(); + await Promise.all([firstCommit, duplicateCommit]); + }); + + it("rolls back a driver-owned cutover before reporting commit failure", async () => { + const deps = makeDeps(); + const events: string[] = []; + const commit = vi.fn(async () => { + events.push("commit"); + throw new Error("receipt validation failed"); + }); + const rollback = vi.fn(async () => { + events.push("rollback"); + }); + const onPatchFailureExit = vi.fn(() => { + events.push("exit"); + }); + const patch = createDockerGpuSandboxCreatePatch({ + route: "native", + externalRecreation: true, + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { onPatchFailureExit }, + }); + patch.attachManagedBootstrapCutover({ + selectedMode: { + kind: "startup-command", + label: "managed bootstrap", + device: "", + args: [], + }, + failureContext: { + sandboxName: "alpha", + oldContainerId: "held-container", + newContainerId: "replacement-container", + }, + commit, + rollback, + }); + + await patch.commitAfterReady(); + + expect(events).toEqual(["commit", "rollback", "exit"]); + expect(onPatchFailureExit).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ message: "receipt validation failed" }), + expect.objectContaining({ + context: expect.objectContaining({ + oldContainerId: "held-container", + newContainerId: "replacement-container", + rolledBack: true, + }), + }), + ); + await patch.rollbackManagedStartupAfterCreateFailure(); + expect(rollback).toHaveBeenCalledOnce(); + }); + + it("reports startup-command creation failures through the composed patch boundary", async () => { const deps = makeDeps(); const onPatchFailureExit = vi.fn(); const patch = createDockerGpuSandboxCreatePatch({ @@ -177,7 +272,7 @@ describe("Docker startup-command sandbox creation", () => { patch.maybeApplyDuringCreate(); expect(patch.createFailureMessage()).toMatch(/startup-command patch failed/); - patch.exitOnPatchError(); + await patch.exitOnPatchError(); expect(onPatchFailureExit).toHaveBeenCalledWith( "alpha", expect.objectContaining({ message: "startup recreate failed" }), diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 717fcaa048b..63a790993f5 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -3,9 +3,11 @@ # Managed bootstrap protocol -This directory defines a dormant, driver-neutral transaction contract and its -first driver adapter. It does not register a runtime provider or change sandbox -creation, onboarding, snapshot, clone, or restore behavior. +This directory defines a dormant, driver-neutral transaction contract, its +first driver adapter, and an injectable sandbox-create lifecycle. Production +runtime bundles still report bootstrap as unsupported, so the candidate +lifecycle remains inert. The shared finalization path keeps existing Docker +recreation reversible through later Ready, GPU, and local-inference checks. The protocol binds one random bootstrap identity to: @@ -66,10 +68,12 @@ including its supervisor environment, to immutable prepared authority before activation. The native boundary introduces no driver-specific environment policy. -The first Docker-specific groundwork defines a private, monotonic cutover -journal and a canonical launch-spec normalizer. Each surface is independently -validated and remains dormant: no registered runtime provider imports either -module, and neither changes sandbox creation or lifecycle behavior. +The Docker-specific layers define a private, monotonic cutover journal, a +canonical launch-spec normalizer, and an injectable provider create lifecycle. +The candidate provider surface composes these layers without registering them +in a production runtime bundle. It remains inert, while the shared finalization +surface extends rollback ownership for the existing Docker compatibility and +startup recreation paths. The Docker adapter creates and validates a stopped replacement under an identity-derived staging name while the original remains running. It stages the @@ -88,34 +92,28 @@ must also inject the selected gateway's canonical state root. ## Architectural disposition -The coordinator deliberately lands as a dormant trust-boundary slice before a -provider or image activates it. This keeps the driver-neutral transaction -authority review separate from the first driver implementation instead of -making that implementation the de facto central contract. The coordinator -module remains cohesive because its receipt shapes, normalization, state -transitions, and rollback proofs form one authority boundary; provider-specific -logic must live outside it rather than growing this file. +The runtime-provider bundle is the only bootstrap registration boundary. The +candidate Docker surface owns create routing, replacement construction, +native-to-compatibility fallback evidence, and deferred commit or rollback. +Central onboarding accepts that provider-neutral surface without a Docker or +Podman selection branch. Tests register an MXC-style surface through the same +bundle and render held launches for OpenClaw, Hermes, and LangChain Deep Agents +Code. -This is executable, bounded groundwork rather than an untested placeholder. -`adapter.test.ts` drives prepare, durable record, activation, finalization, and -failure rollback for OpenClaw, Hermes, and LangChain Deep Agents Code through an -MXC-named fake driver. `runtime-provider-source-shape.test.ts` separately -inventories the protocol, provider, and image-packaging surfaces and proves that -production activation cannot import or package the protocol yet. The later -activation slice must add a registered-provider contract test for the same -transaction before removing those dormancy assertions. +The coordinator remains the driver-neutral transaction authority: its receipt +shapes, normalization, state transitions, and rollback proofs form one cohesive +boundary, while provider-specific routing and runtime operations stay outside +it. The candidate composition is executable, bounded groundwork rather than an +untested placeholder, but no registered provider selects it yet. The native entrypoint source is intentionally not compiled into production -artifacts, and neither image-owned source is packaged or selected yet. No -production TypeScript module imports this protocol or the Docker adapter. The +artifacts, and neither image-owned source is packaged or selected yet. The current image definitions do not package `nemoclaw-managed-startup-hold`, `managed-startup-image-runtime.cjs`, or the shared-state bootstrap modes consumed -by the adapter. A later provider integration must compile and verify the -freestanding entrypoint natively for amd64 and arm64 in every agent image. It -must add those prerequisites together with their image-runtime bootstrap modes -and wire the coordinator and Docker adapter into create as one boundary. The -same contract is exercised for OpenClaw, Hermes, and Deep Agents Code without a -provider-specific central switch. The remaining integration and qualification -work is tracked in [epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) -and its linked implementation stack. Until that complete boundary lands, every -registered runtime provider keeps its bootstrap surface unsupported. +by the adapter. Later persistence and qualification slices must compile and +verify the freestanding entrypoint for amd64 and arm64 in every agent image, add +the image-runtime prerequisites, and provide the canonical durable authority +store. The remaining integration and qualification work is tracked in +[epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744). Until that complete +boundary passes protected E2E, every production runtime provider keeps +bootstrap unsupported. diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.ts new file mode 100644 index 00000000000..21932a2207f --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.ts @@ -0,0 +1,294 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { RuntimeProviderBootstrapSurface } from "../runtime-provider/contract"; +import { detectTegraDeviceGroupGids } from "../docker-gpu-jetson-groups"; +import { buildDockerGpuMode, selectDockerGpuPatchMode } from "../docker-gpu-patch-mode"; +import type { DockerGpuPatchMode } from "../docker-gpu-patch-types"; +import { renderCompatibilityFallbackCreateArgs } from "../docker-gpu-route"; +import { + createDockerGpuSandboxCreatePatch, + isDockerDesktopWslRuntime, +} from "../docker-gpu-sandbox-create"; +import { + isImmutableDockerImageId, + queryOpenShellDockerSandboxContainers, + queryOpenShellDockerSandboxRuntimeSnapshot, +} from "../openshell-docker-sandbox-containers"; +import * as sandboxGpuCreateAttempt from "../sandbox-gpu-create-attempt"; +import { + activateManagedBootstrapSequence, + finalizeManagedBootstrapSequence, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + prepareManagedBootstrapSequence, +} from "./adapter"; +import { createDockerManagedBootstrapAdapter } from "./docker"; +import type { + ManagedBootstrapRuntimeCompatibilityLaunchInput, + ManagedBootstrapRuntimeCreateLaunchResult, + ManagedBootstrapRuntimeCreateLifecycle, + ManagedBootstrapRuntimeCreateLifecycleInput, + ManagedBootstrapRuntimeOnboardRoutingInput, +} from "./runtime-create"; + +type SupportedBootstrapSurface = Extract< + RuntimeProviderBootstrapSurface, + { readonly supported: true } +>; + +function dockerReplacementOptions( + mode: DockerGpuPatchMode, + input: ManagedBootstrapRuntimeCreateLifecycleInput, +) { + const backend = input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic"; + return { + values: { + gpuModeArgs: [...mode.args], + gpuModeDevice: mode.device, + gpuModeKind: mode.kind, + gpuModeLabel: mode.label, + requiredUlimits: input.requiredLimits.map( + (limit) => `${limit.name}=${limit.soft}:${limit.hard}`, + ), + extraGroupGids: + backend === "jetson" && input.route === "compatibility" ? detectTegraDeviceGroupGids() : [], + }, + }; +} + +function selectedDockerMode( + input: ManagedBootstrapRuntimeCreateLifecycleInput, + dockerDesktopWsl: boolean | undefined, +): DockerGpuPatchMode { + const backend = input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic"; + if (input.route !== "compatibility" || !input.sandboxGpuConfig.sandboxGpuEnabled) { + return buildDockerGpuMode("startup-command"); + } + const selection = selectDockerGpuPatchMode( + { + image: `${input.image.repository}@${input.image.manifestDigest}`, + device: input.sandboxGpuConfig.sandboxGpuDevice, + backend, + dockerDesktopWsl, + }, + input.dependencies, + ); + if (selection.mode) return selection.mode; + throw new Error( + backend === "jetson" + ? "Docker did not accept the Jetson NVIDIA runtime GPU mode for managed bootstrap." + : "Docker did not accept a compatibility GPU mode for managed bootstrap.", + ); +} + +function createDockerLifecycle( + providerId: string, + input: ManagedBootstrapRuntimeCreateLifecycleInput, +): ManagedBootstrapRuntimeCreateLifecycle { + if (input.providerId !== providerId) { + throw new Error( + `Managed bootstrap provider '${providerId}' cannot run authority for '${input.providerId}'.`, + ); + } + const dockerDesktopWsl = + input.route === "compatibility" ? isDockerDesktopWslRuntime() : undefined; + const mode = selectedDockerMode(input, dockerDesktopWsl); + const backend = input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic"; + const persistStartupCommand = + input.persistStartupCommand && (input.route !== "native" || input.requiredLimits.length > 0); + const patch = createDockerGpuSandboxCreatePatch({ + route: input.route, + persistStartupCommand, + externalRecreation: true, + sandboxName: input.sandboxName, + gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, + openshellSandboxCommand: input.heldWorkloadArgv, + requiredUlimits: input.requiredLimits, + timeoutSecs: input.timeoutSecs, + backend, + dockerDesktopWsl, + deps: input.dependencies, + ...(input.onPatchFailure + ? { + overrides: { + onPatchFailureExit: (_sandboxName: string, error: unknown) => + input.onPatchFailure?.(error), + }, + } + : {}), + }); + const adapter = input.adapterOverride ?? createDockerManagedBootstrapAdapter(input.dependencies); + const createPlan = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandboxName: input.sandboxName, + driverId: providerId, + image: input.image, + profile: { + agent: input.request.agent, + fingerprint: input.request.profileFingerprint, + }, + agentIdentity: input.agentIdentity, + intendedWorkloadArgv: input.intendedWorkloadArgv, + expectedSupervisorArgv: input.expectedSupervisorArgv, + metadata: {}, + } as const; + const replacementOptions = dockerReplacementOptions(mode, input); + + return { + launchArgv: input.launchArgv, + patch, + async prepareNetwork() { + if (input.route !== "compatibility") return; + const { enforceDockerGpuPatchPreserveNetwork } = await import( + "../docker-gpu-local-inference" + ); + await enforceDockerGpuPatchPreserveNetwork( + input.network.inferenceProvider, + input.sandboxGpuConfig, + { + dockerDriverGateway: input.network.dockerDriverGateway, + selectedRoute: input.route, + gatewayPort: input.network.gatewayPort, + log: console.log, + }, + ); + }, + async runCreate( + launch: (input: { + readonly heldWorkloadArgv: readonly string[]; + readonly bootstrapIdentity: string; + }) => Promise>, + ): Promise { + const launchState: { value?: ManagedBootstrapRuntimeCreateLaunchResult } = {}; + const prepared = await prepareManagedBootstrapSequence(adapter, { + create: { + bootstrapIdentity: input.bootstrapIdentity, + plan: createPlan, + request: input.request, + launch: async (launchInput) => { + const launched = await launch(launchInput); + launchState.value = launched; + return launched.receipt; + }, + }, + request: input.request, + replacementOptions, + }); + const activated = await activateManagedBootstrapSequence(adapter, { + transaction: prepared, + authorityStore: input.authorityStore, + timeoutSecs: input.timeoutSecs, + }); + const launched = launchState.value; + if (!launched) { + await finalizeManagedBootstrapSequence(adapter, { + outcome: "rollback", + transaction: activated, + }); + throw new Error("Managed bootstrap did not return its OpenShell create receipt."); + } + let finalized = false; + patch.attachManagedBootstrapCutover({ + selectedMode: mode, + failureContext: { + sandboxName: input.sandboxName, + oldContainerId: activated.snapshot.runtimeId, + newContainerId: activated.replacement.replacementRuntimeId, + backupContainerName: null, + selectedMode: mode, + }, + async rollback() { + if (finalized) return; + await finalizeManagedBootstrapSequence(adapter, { + outcome: "rollback", + transaction: activated, + }); + finalized = true; + }, + async commit() { + if (finalized) return; + await finalizeManagedBootstrapSequence(adapter, { + outcome: "commit", + transaction: activated, + }); + finalized = true; + }, + }); + return launched.value; + }, + }; +} + +function createDockerOnboardRouting(input: ManagedBootstrapRuntimeOnboardRoutingInput) { + const baseline = input.nativeFallbackEnabled + ? queryOpenShellDockerSandboxContainers(input.sandboxName) + : null; + const inspectNativeRuntime = () => { + const snapshot = queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName); + return snapshot.ok + ? { + imageId: snapshot.imageId, + bookkeepingImageRef: snapshot.bookkeepingImageRef, + stateError: snapshot.stateError, + nativeGpuAttachmentState: snapshot.nativeGpuAttachmentState, + } + : null; + }; + return { + nativeFallbackHasCleanBaseline: baseline?.ok === true && baseline.ids.length === 0, + inspectNativeRuntime, + isNativeCreateRoutingFailure: (output: string, sawProgress: boolean): boolean => + sandboxGpuCreateAttempt.isNativeGpuCreateRoutingFailure(output, { sawProgress }), + isTrustedNativeRuntimeError: (error: string): boolean => + sandboxGpuCreateAttempt.isTrustedNativeGpuRuntimeError(error), + isNativeReadinessRoutingFailure: (failure: { + readonly failurePhase: string | null; + readonly runtimeError: string; + }): boolean => sandboxGpuCreateAttempt.isNativeGpuReadinessRoutingFailure(failure), + prepareCompatibilityLaunch: ( + compatibility: ManagedBootstrapRuntimeCompatibilityLaunchInput, + ) => { + const runtime = compatibility.runtimeSnapshot; + const imageId = + runtime?.imageId ?? + (compatibility.prebuildImageId && isImmutableDockerImageId(compatibility.prebuildImageId) + ? compatibility.prebuildImageId.toLowerCase() + : null); + let registryImageRef = compatibility.currentRegistryImageRef; + if ( + !registryImageRef && + runtime?.bookkeepingImageRef && + !isImmutableDockerImageId(runtime.bookkeepingImageRef) + ) { + registryImageRef = runtime.bookkeepingImageRef; + } + const createArgs = renderCompatibilityFallbackCreateArgs(compatibility.createArgs, { + imageRef: imageId, + allowUnbuiltSource: compatibility.allowUnbuiltSource, + compatibilityPolicyPath: compatibility.compatibilityPolicyPath, + }); + return { + createArgv: input.openshellArgv([ + "sandbox", + "create", + ...createArgs, + "--", + ...compatibility.startupCommand, + ]), + registryImageRef, + }; + }, + }; +} + +/** Candidate Docker surface. Production activation remains a later qualification slice. */ +export function createDockerManagedBootstrapSurface( + providerId = "docker", +): SupportedBootstrapSurface { + return { + providerId, + supported: true, + createLifecycle: (input) => createDockerLifecycle(providerId, input), + createOnboardRouting: createDockerOnboardRouting, + }; +} diff --git a/src/lib/onboard/managed-bootstrap/index.ts b/src/lib/onboard/managed-bootstrap/index.ts index 17099572608..c55768afe02 100644 --- a/src/lib/onboard/managed-bootstrap/index.ts +++ b/src/lib/onboard/managed-bootstrap/index.ts @@ -20,3 +20,7 @@ export { serializeManagedBootstrapEnvelope, serializeManagedBootstrapImageCompletion, } from "./envelope"; +export type { + ManagedBootstrapRuntimeCreateLifecycle, + ManagedBootstrapRuntimePatch, +} from "./runtime-create"; diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.ts b/src/lib/onboard/managed-bootstrap/runtime-create.ts new file mode 100644 index 00000000000..9154b0de44b --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/runtime-create.ts @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxGpuProofResult } from "../../state/registry"; +import type { ManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import type { SandboxGpuConfig } from "../sandbox-gpu-mode"; +import type { + ManagedBootstrapAdapter, + ManagedBootstrapAgentIdentity, + ManagedBootstrapAuthorityStore, + ManagedBootstrapCreateReceipt, + ManagedBootstrapImageIdentity, +} from "./adapter"; + +export interface ManagedBootstrapRuntimeCommandResult { + readonly status?: number | null; + readonly stdout?: string | Buffer | null; + readonly stderr?: string | Buffer | null; + readonly error?: Error | null; +} + +export interface ManagedBootstrapRuntimeDependencies { + readonly runCaptureOpenshell?: (args: string[], options?: Record) => string; + readonly runOpenshell?: ( + args: string[], + options?: Record, + ) => ManagedBootstrapRuntimeCommandResult; + readonly sleep?: (seconds: number) => void; +} + +export type ManagedBootstrapRuntimeRoute = "none" | "native" | "compatibility"; + +export interface ManagedBootstrapRuntimeLimit { + readonly name: string; + readonly soft: number; + readonly hard: number; +} + +/** Provider-neutral lifecycle surface consumed by sandbox-create coordinators. */ +export interface ManagedBootstrapRuntimePatch { + maybeApplyDuringCreate(): void | Promise; + createFailureMessage(): string | null; + exitOnPatchError(): void | Promise; + rollbackManagedStartupAfterCreateFailure(): void | Promise; + ensureApplied(): void | Promise; + waitForSupervisorReconnectIfNeeded(): void | Promise; + commitAfterReady(): void | Promise; + selectedMode(): { + readonly kind: string; + readonly label: string; + readonly device: string; + readonly args: readonly string[]; + } | null; + printReadinessFailureIfEnabled(): void; + verifyGpuOrExit( + verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult, + ): Promise; +} + +export interface ManagedBootstrapRuntimeCreateLifecycleInput { + readonly providerId: string; + readonly bootstrapIdentity: string; + readonly request: ManagedStartupRootApplyRequest; + readonly image: ManagedBootstrapImageIdentity; + readonly agentIdentity: ManagedBootstrapAgentIdentity; + readonly intendedWorkloadArgv: readonly string[]; + readonly expectedSupervisorArgv: readonly string[]; + readonly launchArgv: readonly string[]; + readonly heldWorkloadArgv: readonly string[]; + readonly authorityStore: ManagedBootstrapAuthorityStore; + readonly adapterOverride?: ManagedBootstrapAdapter; + readonly route: ManagedBootstrapRuntimeRoute; + readonly persistStartupCommand: boolean; + readonly sandboxName: string; + readonly sandboxGpuConfig: SandboxGpuConfig; + readonly requiredLimits: readonly ManagedBootstrapRuntimeLimit[]; + readonly timeoutSecs: number; + readonly onPatchFailure?: (error: unknown) => never; + readonly network: { + readonly inferenceProvider: string; + readonly dockerDriverGateway: boolean; + readonly gatewayPort: number; + }; + readonly dependencies: ManagedBootstrapRuntimeDependencies; +} + +export interface ManagedBootstrapRuntimeCreateLaunchResult { + readonly value: T; + readonly receipt: ManagedBootstrapCreateReceipt; +} + +export interface ManagedBootstrapRuntimeCreateLifecycle { + readonly launchArgv: readonly string[]; + readonly patch: ManagedBootstrapRuntimePatch; + prepareNetwork(): Promise; + runCreate( + launch: (input: { + readonly heldWorkloadArgv: readonly string[]; + readonly bootstrapIdentity: string; + }) => Promise>, + ): Promise; +} + +export interface ManagedBootstrapRuntimeSnapshot { + readonly imageId: string | null; + readonly bookkeepingImageRef: string | null; + readonly stateError: string; + readonly nativeGpuAttachmentState: "present" | "absent" | "unknown"; +} + +export interface ManagedBootstrapRuntimeCompatibilityLaunchInput { + readonly createArgs: readonly string[]; + readonly currentRegistryImageRef: string | null; + readonly prebuildImageId: string | null; + readonly allowUnbuiltSource: boolean; + readonly compatibilityPolicyPath: string; + readonly startupCommand: readonly string[]; + readonly runtimeSnapshot: ManagedBootstrapRuntimeSnapshot | null; +} + +export interface ManagedBootstrapRuntimeCompatibilityLaunch { + readonly createArgv: readonly string[]; + readonly registryImageRef: string | null; +} + +/** Provider-owned native-to-compatibility evidence and launch preparation. */ +export interface ManagedBootstrapRuntimeOnboardRouting { + readonly nativeFallbackHasCleanBaseline: boolean; + inspectNativeRuntime(): ManagedBootstrapRuntimeSnapshot | null; + isNativeCreateRoutingFailure(output: string, sawProgress: boolean): boolean; + isTrustedNativeRuntimeError(error: string): boolean; + isNativeReadinessRoutingFailure(input: { + readonly failurePhase: string | null; + readonly runtimeError: string; + }): boolean; + prepareCompatibilityLaunch( + input: ManagedBootstrapRuntimeCompatibilityLaunchInput, + ): ManagedBootstrapRuntimeCompatibilityLaunch; +} + +export interface ManagedBootstrapRuntimeOnboardRoutingInput { + readonly sandboxName: string; + readonly openshellArgv: (args: string[]) => string[]; + readonly nativeFallbackEnabled: boolean; +} diff --git a/src/lib/onboard/runtime-provider/contract.ts b/src/lib/onboard/runtime-provider/contract.ts index 8888d95f83a..8938b6244ff 100644 --- a/src/lib/onboard/runtime-provider/contract.ts +++ b/src/lib/onboard/runtime-provider/contract.ts @@ -3,6 +3,12 @@ import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; import type { ManagedImageSelectionPolicy } from "../workload/source"; +import type { + ManagedBootstrapRuntimeCreateLifecycle, + ManagedBootstrapRuntimeCreateLifecycleInput, + ManagedBootstrapRuntimeOnboardRouting, + ManagedBootstrapRuntimeOnboardRoutingInput, +} from "../managed-bootstrap/runtime-create"; export const RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION = 1 as const; export const RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION = 1 as const; @@ -261,7 +267,12 @@ export type RuntimeProviderMutationAuthoritySurface = export type RuntimeProviderBootstrapSurface = | RuntimeProviderSupportedSurface<{ - prepare(sandbox: SandboxEntry): unknown; + createLifecycle( + input: ManagedBootstrapRuntimeCreateLifecycleInput, + ): ManagedBootstrapRuntimeCreateLifecycle; + createOnboardRouting( + input: ManagedBootstrapRuntimeOnboardRoutingInput, + ): ManagedBootstrapRuntimeOnboardRouting; }> | RuntimeProviderUnsupportedSurface; diff --git a/src/lib/onboard/runtime-provider/registry.ts b/src/lib/onboard/runtime-provider/registry.ts index 332792d2dc2..1d0a31ae6aa 100644 --- a/src/lib/onboard/runtime-provider/registry.ts +++ b/src/lib/onboard/runtime-provider/registry.ts @@ -337,7 +337,8 @@ function validateMutationAuthoritySurface( function validateBootstrapSurface(surface: Record): void { if (surface.supported === true) { - requireFunction(surface, "prepare", "bootstrap"); + requireFunction(surface, "createLifecycle", "bootstrap"); + requireFunction(surface, "createOnboardRouting", "bootstrap"); } } diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index bf2a47fe4b4..3ac4da9e4fc 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -21,6 +21,7 @@ import { loadAgent } from "../../agent/defs"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; import { cloneSandboxWorkloadReceipt } from "../../state/registry/workload"; import { MANAGED_IMAGE_REPOSITORIES } from "../managed-image/contract"; +import { createDockerManagedBootstrapSurface } from "../managed-bootstrap/docker-runtime"; import { encodeManagedStartupProfile, type ManagedStartupProfile, @@ -145,6 +146,28 @@ describe("RuntimeProviderBundle registry contract", () => { } }); + it("validates the dormant Docker bootstrap candidate through the same bundle registry", () => { + const docker = createDockerRuntimeProviderBundle(); + const providers = createRuntimeProviderBundleRegistry([ + [ + "docker", + { + ...docker, + bootstrap: createDockerManagedBootstrapSurface(), + }, + ], + ]); + + expect(providers.docker?.bootstrap).toMatchObject({ + providerId: "docker", + supported: true, + }); + expect(CURRENT_RUNTIME_PROVIDER_BUNDLES.docker?.bootstrap).toMatchObject({ + providerId: "docker", + supported: false, + }); + }); + it("deeply clones and freezes every registered nested value", () => { const source = mxcBundle(); const registry = createRuntimeProviderBundleRegistry([["mxc", source]]); @@ -170,6 +193,59 @@ describe("RuntimeProviderBundle registry contract", () => { }).toThrow(TypeError); }); + it("registers an MXC-style managed-bootstrap provider through the bundle surface", () => { + const bundle = mxcBundle(); + const createLifecycle = vi.fn(() => ({ + launchArgv: ["mxc", "create"], + patch: { + maybeApplyDuringCreate: vi.fn(), + createFailureMessage: vi.fn(() => null), + exitOnPatchError: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + ensureApplied: vi.fn(), + waitForSupervisorReconnectIfNeeded: vi.fn(), + commitAfterReady: vi.fn(), + selectedMode: vi.fn(() => null), + printReadinessFailureIfEnabled: vi.fn(), + verifyGpuOrExit: vi.fn(async (verify) => verify("alpha")), + }, + prepareNetwork: vi.fn(async () => undefined), + runCreate: vi.fn(), + })); + const createOnboardRouting = vi.fn(() => ({ + nativeFallbackHasCleanBaseline: false, + inspectNativeRuntime: vi.fn(() => null), + isNativeCreateRoutingFailure: vi.fn(() => false), + isTrustedNativeRuntimeError: vi.fn(() => false), + isNativeReadinessRoutingFailure: vi.fn(() => false), + prepareCompatibilityLaunch: vi.fn(() => ({ createArgv: [], registryImageRef: null })), + })); + const providers = createRuntimeProviderBundleRegistry([ + [ + "mxc", + replaceSurface(bundle, "bootstrap", { + providerId: "mxc", + supported: true, + createLifecycle, + createOnboardRouting, + }), + ], + ]); + const registered = providers.mxc!; + expectSupportedSurface(registered.bootstrap); + + const routing = registered.bootstrap.createOnboardRouting({ + sandboxName: "alpha", + openshellArgv: (args) => args, + nativeFallbackEnabled: false, + }); + + expect(registered.identity.id).toBe("mxc"); + expect(routing.nativeFallbackHasCleanBaseline).toBe(false); + expect(createOnboardRouting).toHaveBeenCalledOnce(); + expect(createLifecycle).not.toHaveBeenCalled(); + }); + it("rejects an omitted managed platform without changing legacy receipt acceptance", () => { const { platform: _omittedPlatform, ...managedWithoutPlatform } = MANAGED_RECEIPT; const persistedManaged = cloneSandboxWorkloadReceipt(managedWithoutPlatform); diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index 3a650fc9c11..8e822e0ce65 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -8,9 +8,12 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; import { loadAgent } from "../agent/defs"; import { SANDBOX_BUILD_CONTEXT_PREFIX } from "../sandbox/build-context"; import { createOpenshellCliHelpers } from "./openshell-cli"; +import { encodeManagedStartupProfile } from "./managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; import { buildSandboxRuntimeEnvArgs, prepareSandboxCreateLaunch, @@ -104,6 +107,49 @@ describe("buildSandboxRuntimeEnvArgs", () => { }); describe("prepareSandboxCreateLaunch", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("renders one identity-bound held launch for %s without exposing the startup profile", (agentName) => { + const request = createManagedStartupRootApplyRequest({ + agent: agentName, + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agentName)), + }); + const result = prepareSandboxCreateLaunch({ + agent: loadAgent(agentName), + chatUiUrl: "", + createArgs: ["--name", `${agentName}-sandbox`], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: disabledHermesDashboardState, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + openshellArgv: (args) => ["openshell", ...args], + buildEnv: () => ({}), + managedStartupRootApplyRequest: request, + }); + + expect(result.intendedSandboxStartupCommand).toEqual([ + "env", + ...result.envArgs, + "nemoclaw-start", + ]); + expect(result.managedBootstrapIdentity).toMatch(/^[a-f0-9]{64}$/u); + expect(result.sandboxStartupCommand).toEqual([ + ...result.intendedSandboxStartupCommand.slice(0, -1), + "/usr/local/bin/nemoclaw-managed-startup-hold", + "--agent", + agentName, + "--profile-fingerprint", + request.profileFingerprint, + "--bootstrap-identity", + result.managedBootstrapIdentity, + ]); + expect(result.createArgv.join("\n")).not.toContain(request.encodedProfile); + }); + it("builds the sandbox create command and runtime env envelope", () => { const openshellShellCommand = vi.fn((args: string[]) => `openshell ${args.join(" ")}`); const result = prepareSandboxCreateLaunch({ diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index 1f3aa3cd01b..09e703f6d4b 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -9,6 +9,11 @@ import { appendExtraPlaceholderKeysEnvArg } from "./extra-placeholder-keys"; import type { HermesDashboardOnboardState } from "./hermes-dashboard"; import { appendHermesDashboardEnvArgs } from "./hermes-dashboard"; import { appendHostProxyEnvArgs } from "./host-proxy-env"; +import { + createManagedBootstrapIdentity, + renderManagedBootstrapHeldCommand, +} from "./managed-bootstrap/adapter"; +import type { ManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; import { appendOpenClawRuntimeEnvArgs } from "./openclaw-runtime-env"; import { prebuildSandboxImageIfEligible, @@ -57,6 +62,8 @@ export interface SandboxCreateLaunchInput { openshellShellCommand: OpenshellShellCommand; openshellArgv?: OpenshellArgv; buildEnv?(): Record; + /** Dormant until a complete runtime bundle and durable authority store are selected. */ + managedStartupRootApplyRequest?: ManagedStartupRootApplyRequest | null; } export interface SandboxCreateLaunch { @@ -66,6 +73,9 @@ export interface SandboxCreateLaunch { envArgs: string[]; sandboxEnv: Record; sandboxStartupCommand: string[]; + intendedSandboxStartupCommand: string[]; + managedBootstrapIdentity: string | null; + managedStartupRootApplyRequest: ManagedStartupRootApplyRequest | null; } export interface SandboxCreateLaunchWithPrebuildInput extends SandboxCreateLaunchInput { @@ -203,7 +213,21 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San // from openshell because bash returns the status of the last pipeline // command (awk, always 0) unless pipefail is set. Removing the pipe // lets the real exit code flow through to run(). - const sandboxStartupCommand = ["env", ...envArgs, "nemoclaw-start"]; + const intendedSandboxStartupCommand = ["env", ...envArgs, "nemoclaw-start"]; + const managedStartupRootApplyRequest = input.managedStartupRootApplyRequest ?? null; + const managedBootstrapIdentity = managedStartupRootApplyRequest + ? createManagedBootstrapIdentity() + : null; + const sandboxStartupCommand = + managedStartupRootApplyRequest && managedBootstrapIdentity + ? [ + ...renderManagedBootstrapHeldCommand( + managedStartupRootApplyRequest, + managedBootstrapIdentity, + intendedSandboxStartupCommand, + ), + ] + : intendedSandboxStartupCommand; const openshellArgs = ["sandbox", "create", ...input.createArgs, "--", ...sandboxStartupCommand]; const createCommand = renderSandboxCreateCommand( input.createArgs, @@ -221,6 +245,9 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San envArgs, sandboxEnv, sandboxStartupCommand, + intendedSandboxStartupCommand, + managedBootstrapIdentity, + managedStartupRootApplyRequest, }; } diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index c7671ff4e60..720306078f4 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -3,6 +3,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { createInMemoryRuntimeProviderBundle } from "../../../test/helpers/runtime-provider-bundle"; + const mocks = vi.hoisted(() => ({ streamSandboxCreate: vi.fn(), waitForCreatedSandboxReadyWithTrace: vi.fn(), @@ -59,11 +62,23 @@ import { setupGpuFlowMocks, VERIFIED_GPU_PROOF as VERIFIED_PROOF, } from "./__test-helpers__/sandbox-gpu-create-flow"; +import type { + ManagedBootstrapRuntimeCreateLifecycleInput, + ManagedBootstrapRuntimePatch, +} from "./managed-bootstrap/runtime-create"; +import { encodeManagedStartupProfile } from "./managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; +import { prepareSandboxCreateLaunch } from "./sandbox-create-launch"; import { runSandboxGpuCreateFlow, type SandboxGpuCreateFlowDeps, type SandboxGpuCreateFlowInput, } from "./sandbox-gpu-create-flow"; +import type { + RuntimeProviderBootstrapSurface, + RuntimeProviderBundle, +} from "./runtime-provider/contract"; +import { createRuntimeProviderBundleRegistry } from "./runtime-provider/registry"; const FAILED_PROOF: SandboxGpuProofResult = { status: "failed", @@ -150,6 +165,145 @@ function createSourceInput(): SandboxGpuCreateFlowInput { beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); +describe("runSandboxGpuCreateFlow provider-owned managed create", () => { + it("runs an MXC-style bundle without a Docker branch in central orchestration", async () => { + const input = createInput(); + input.sandboxGpuConfig = { + mode: "0", + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }; + input.gpuRoutePlan = "none"; + input.initialGpuRoute = "none"; + const request = createManagedStartupRootApplyRequest({ + agent: "openclaw", + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")), + }); + const launch = prepareSandboxCreateLaunch({ + agent: null, + sandboxName: "alpha", + chatUiUrl: "", + createArgs: ["--name", "alpha"], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: { config: null, enabled: false }, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + openshellArgv: (args) => ["openshell", ...args], + buildEnv: () => ({}), + managedStartupRootApplyRequest: request, + }); + input.createArgv = launch.createArgv; + input.sandboxEnv = launch.sandboxEnv; + input.sandboxStartupCommand = launch.sandboxStartupCommand; + const patch = createPatch() as unknown as ManagedBootstrapRuntimePatch; + const createLifecycle = vi.fn( + (lifecycleInput: ManagedBootstrapRuntimeCreateLifecycleInput) => ({ + launchArgv: ["mxc-launch", ...lifecycleInput.launchArgv.slice(1)], + patch, + prepareNetwork: vi.fn(async () => undefined), + runCreate: async ( + start: (held: { + readonly heldWorkloadArgv: readonly string[]; + readonly bootstrapIdentity: string; + }) => Promise<{ readonly value: T }>, + ): Promise => + ( + await start({ + heldWorkloadArgv: lifecycleInput.heldWorkloadArgv, + bootstrapIdentity: lifecycleInput.bootstrapIdentity, + }) + ).value, + }), + ); + const source = createInMemoryRuntimeProviderBundle({ + providerId: "mxc", + workloadProfile: { + support: null, + hostArchitectures: [], + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: true, + }, + }); + const registered = createRuntimeProviderBundleRegistry([ + [ + "mxc", + { + ...source, + bootstrap: { + providerId: "mxc", + supported: true, + createLifecycle, + createOnboardRouting: vi.fn(() => ({ + nativeFallbackHasCleanBaseline: false, + inspectNativeRuntime: vi.fn(() => null), + isNativeCreateRoutingFailure: vi.fn(() => false), + isTrustedNativeRuntimeError: vi.fn(() => false), + isNativeReadinessRoutingFailure: vi.fn(() => false), + prepareCompatibilityLaunch: vi.fn(() => ({ + createArgv: [], + registryImageRef: null, + })), + })), + }, + }, + ], + ]); + const runtimeProvider = registered.mxc as RuntimeProviderBundle & { + readonly bootstrap: Extract; + }; + input.managedBootstrap = { + bootstrapIdentity: launch.managedBootstrapIdentity!, + runtimeProvider, + authorityStore: { + async recordPreparedAuthority(authority) { + return { + schemaVersion: 1, + sandbox: authority.sandbox, + bootstrapIdentity: authority.bootstrapIdentity, + authorityFingerprint: authority.authorityFingerprint, + recordId: "mxc-record-alpha", + recordedAt: "2026-07-31T00:00:00.000Z", + }; + }, + }, + request, + image: { + repository: "registry.example/nemoclaw-openclaw", + manifestDigest: `sha256:${"d".repeat(64)}`, + }, + agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + intendedWorkloadArgv: launch.intendedSandboxStartupCommand, + expectedSupervisorArgv: ["/mxc/supervisor"], + }; + const deps = createDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementation((args) => + args[1] === "get" ? "ID: mxc-alpha\n" : "alpha Ready", + ); + + const result = await runSandboxGpuCreateFlow(input, deps); + + expect(result).toMatchObject({ route: "none", runtimePatch: patch }); + expect(createLifecycle).toHaveBeenCalledWith( + expect.objectContaining({ providerId: "mxc", route: "none" }), + ); + expect(mocks.streamSandboxCreate).toHaveBeenCalledWith( + "mxc-launch", + input.createArgv.slice(1), + input.sandboxEnv, + expect.anything(), + ); + expect(mocks.createDockerGpuSandboxCreatePatch).not.toHaveBeenCalled(); + expect(mocks.queryOpenShellDockerSandboxContainers).not.toHaveBeenCalled(); + expect(mocks.queryOpenShellDockerSandboxRuntimeSnapshot).not.toHaveBeenCalled(); + expect(mocks.enforceDockerGpuPatchPreserveNetwork).not.toHaveBeenCalled(); + }); +}); + describe("runSandboxGpuCreateFlow proof authorization", () => { it("does not retry compatibility when the native proof throws an exec/policy error (#6110)", async () => { const deps = createDeps(); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 98574891e0b..878363389e9 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -10,12 +10,23 @@ import type { DockerGpuPatchDeps, DockerUlimit } from "./docker-gpu-patch-types" import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { renderCompatibilityFallbackCreateArgs } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; -import type { DockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +import type { + ManagedBootstrapAdapter, + ManagedBootstrapAgentIdentity, + ManagedBootstrapAuthorityStore, + ManagedBootstrapImageIdentity, +} from "./managed-bootstrap/adapter"; +import type { ManagedBootstrapRuntimePatch } from "./managed-bootstrap/runtime-create"; +import type { ManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; import { isImmutableDockerImageId } from "./openshell-docker-sandbox-containers"; import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; import { createSandboxGpuCreateAttemptRunner } from "./sandbox-gpu-create-run-attempt"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; import type { SandboxPrebuildResult } from "./sandbox-prebuild"; +import type { + RuntimeProviderBootstrapSurface, + RuntimeProviderBundle, +} from "./runtime-provider/contract"; import { addTraceEvent } from "./tracing"; export { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; @@ -41,6 +52,18 @@ export interface SandboxGpuCreateFlowInput { restoreBackupPath: string | null; terminalAgent: boolean; persistStartupCommand?: boolean; + managedBootstrap?: { + readonly bootstrapIdentity: string; + readonly runtimeProvider: RuntimeProviderBundle & { + readonly bootstrap: Extract; + }; + readonly authorityStore: ManagedBootstrapAuthorityStore; + readonly request: ManagedStartupRootApplyRequest; + readonly image: ManagedBootstrapImageIdentity; + readonly agentIdentity: ManagedBootstrapAgentIdentity; + readonly intendedWorkloadArgv: readonly string[]; + readonly expectedSupervisorArgv: readonly string[]; + } | null; requiredUlimits?: readonly DockerUlimit[] | null; } @@ -50,11 +73,13 @@ export interface SandboxGpuCreateFlowDeps { sleep: Sleep; openshellArgv(args: string[]): string[]; verifyDirectSandboxGpu(sandboxName: string): SandboxGpuProofResult; + /** Production callers omit this factory and use the runtime provider's adapter. */ + createManagedBootstrapAdapter?: () => ManagedBootstrapAdapter; } export interface SandboxGpuCreateFlowResult { createResult: StreamSandboxCreateResult; - dockerGpuCreatePatch: DockerGpuSandboxCreatePatch; + runtimePatch: ManagedBootstrapRuntimePatch; route: SelectedDockerGpuRoute; firstCreateOutput: string; /** Mutable tag/reference retained only for registry and image-GC bookkeeping. */ @@ -105,46 +130,65 @@ export async function runSandboxGpuCreateFlow( throw new Error("Compatibility retry policy was not materialized."); } const nativeRuntimeSnapshot = attemptRunner.state.nativeRuntimeSnapshot; - const prebuildImageId = input.prebuild.imageId; - const imageId = - nativeRuntimeSnapshot?.imageId ?? - (prebuildImageId && isImmutableDockerImageId(prebuildImageId) - ? prebuildImageId.toLowerCase() - : null); - if ( - !registryImageRef && - nativeRuntimeSnapshot?.bookkeepingImageRef && - !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) - ) { - registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; + if (attemptRunner.managedRouting) { + const prepared = attemptRunner.managedRouting.prepareCompatibilityLaunch({ + createArgs: input.prebuild.createArgs, + currentRegistryImageRef: registryImageRef, + prebuildImageId: input.prebuild.imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + startupCommand: input.sandboxStartupCommand, + runtimeSnapshot: nativeRuntimeSnapshot, + }); + attemptRunner.state.compatibilityArgv = [...prepared.createArgv]; + registryImageRef = prepared.registryImageRef; + } else { + const prebuildImageId = input.prebuild.imageId; + const imageId = + nativeRuntimeSnapshot?.imageId ?? + (prebuildImageId && isImmutableDockerImageId(prebuildImageId) + ? prebuildImageId.toLowerCase() + : null); + if ( + !registryImageRef && + nativeRuntimeSnapshot?.bookkeepingImageRef && + !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) + ) { + registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; + } + const compatibilityArgs = renderCompatibilityFallbackCreateArgs( + input.prebuild.createArgs, + { + imageRef: imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + }, + ); + attemptRunner.state.compatibilityArgv = deps.openshellArgv([ + "sandbox", + "create", + ...compatibilityArgs, + "--", + ...input.sandboxStartupCommand, + ]); } - const compatibilityArgs = renderCompatibilityFallbackCreateArgs(input.prebuild.createArgs, { - imageRef: imageId, - allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, - compatibilityPolicyPath: input.compatibilityPolicyPath, - }); - attemptRunner.state.compatibilityArgv = deps.openshellArgv([ - "sandbox", - "create", - ...compatibilityArgs, - "--", - ...input.sandboxStartupCommand, - ]); if (attemptRunner.state.compatibilityArgv.length === 0) { throw new Error("Compatibility sandbox create executable is missing."); } }, activateCompatibilityAttempt: async () => { - await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( - input.provider, - input.sandboxGpuConfig, - { - dockerDriverGateway: input.dockerDriverGateway, - selectedRoute: "compatibility", - gatewayPort: input.gatewayPort, - log: console.log, - }, - ); + if (!input.managedBootstrap) { + await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( + input.provider, + input.sandboxGpuConfig, + { + dockerDriverGateway: input.dockerDriverGateway, + selectedRoute: "compatibility", + gatewayPort: input.gatewayPort, + log: console.log, + }, + ); + } input.sandboxGpuConfig.sandboxGpuProof = null; }, traceEvent: addTraceEvent, diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 726758cc0ac..8d33c1a40d4 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { printSandboxCreateRecoveryHints } from "../build-context"; +import { resolveOpenShellSandboxId } from "../adapters/openshell/sandbox-identity"; import { getSandboxDeleteOutcome } from "../domain/sandbox/destroy"; import { streamSandboxCreate } from "../sandbox/create-stream"; import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-ready-gate"; @@ -13,8 +14,8 @@ import { reportSandboxCreateFailure } from "./created-sandbox-failure"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +import type { ManagedBootstrapRuntimeSnapshot } from "./managed-bootstrap/runtime-create"; import { - type OpenShellDockerSandboxRuntimeSnapshotQuery, queryOpenShellDockerSandboxContainers, queryOpenShellDockerSandboxRuntimeSnapshot, } from "./openshell-docker-sandbox-containers"; @@ -28,7 +29,7 @@ import * as sandboxGpuPreflight from "./sandbox-gpu-preflight"; import * as sandboxReadinessTracing from "./sandbox-readiness-tracing"; import { addTraceEvent } from "./tracing"; -type NativeRuntimeSnapshot = Extract; +type NativeRuntimeSnapshot = ManagedBootstrapRuntimeSnapshot; export type SandboxGpuCreateAttemptState = { firstCreateOutput: string; @@ -41,6 +42,12 @@ export type SandboxGpuCreateAttemptState = { // Ready row. Require one confirmation poll before advancing to the GPU proof. const COMPATIBILITY_STABLE_READY_POLLS = 2; +class ManagedBootstrapCreateStreamFailure extends Error { + constructor(readonly result: Awaited>) { + super("Managed bootstrap held workload did not complete its create stream."); + } +} + export function createSandboxGpuCreateAttemptRunner( input: SandboxGpuCreateFlowInput, deps: SandboxGpuCreateFlowDeps, @@ -51,13 +58,26 @@ export function createSandboxGpuCreateAttemptRunner( allowUnbuiltCompatibilitySource: false, nativeRuntimeSnapshot: null, }; + const managedRouting = input.managedBootstrap?.runtimeProvider.bootstrap.createOnboardRouting({ + sandboxName: input.sandboxName, + openshellArgv: deps.openshellArgv, + nativeFallbackEnabled: + input.initialGpuRoute === "native" && input.gpuRoutePlan === "native-with-fallback", + }); const nativeFallbackBaseline = - input.initialGpuRoute === "native" && input.gpuRoutePlan === "native-with-fallback" + !managedRouting && + input.initialGpuRoute === "native" && + input.gpuRoutePlan === "native-with-fallback" ? queryOpenShellDockerSandboxContainers(input.sandboxName) : null; const nativeFallbackHasCleanBaseline = - nativeFallbackBaseline?.ok === true && nativeFallbackBaseline.ids.length === 0; - const inspectNativeRuntime = () => queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName); + managedRouting?.nativeFallbackHasCleanBaseline ?? + (nativeFallbackBaseline?.ok === true && nativeFallbackBaseline.ids.length === 0); + const inspectNativeRuntime = (): NativeRuntimeSnapshot | null => { + if (managedRouting) return managedRouting.inspectNativeRuntime(); + const snapshot = queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName); + return snapshot.ok ? snapshot : null; + }; const runAttempt = async (route: SelectedDockerGpuRoute) => { const compatibility = route === "compatibility"; @@ -70,73 +90,199 @@ export function createSandboxGpuCreateAttemptRunner( ); } const hasRequiredUlimits = (input.requiredUlimits?.length ?? 0) > 0; - const dockerGpuCreatePatch = createDockerGpuSandboxCreatePatch({ - route, - // The startup clone preserves native CDI devices, so DCode can apply its - // exact required limits without replacing the native GPU envelope. - // Other native routes are not swapped solely to persist a command. - persistStartupCommand: - input.persistStartupCommand === true && (route !== "native" || hasRequiredUlimits), - sandboxName: input.sandboxName, - gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, - openshellSandboxCommand: input.sandboxStartupCommand, - requiredUlimits: input.requiredUlimits, - timeoutSecs: input.sandboxReadyTimeoutSecs, - backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", - deps, - }); + const managedBootstrap = input.managedBootstrap ?? null; const attemptArgv = state.compatibilityArgv ?? input.createArgv; - const [createExecutable, ...createExecutableArgs] = attemptArgv; + const managedLifecycle = managedBootstrap + ? managedBootstrap.runtimeProvider.bootstrap.createLifecycle({ + providerId: managedBootstrap.runtimeProvider.identity.id, + bootstrapIdentity: managedBootstrap.bootstrapIdentity, + request: managedBootstrap.request, + image: managedBootstrap.image, + agentIdentity: managedBootstrap.agentIdentity, + intendedWorkloadArgv: managedBootstrap.intendedWorkloadArgv, + expectedSupervisorArgv: managedBootstrap.expectedSupervisorArgv, + launchArgv: attemptArgv, + heldWorkloadArgv: input.sandboxStartupCommand, + authorityStore: managedBootstrap.authorityStore, + ...(deps.createManagedBootstrapAdapter + ? { adapterOverride: deps.createManagedBootstrapAdapter() } + : {}), + route, + persistStartupCommand: input.persistStartupCommand === true, + sandboxName: input.sandboxName, + sandboxGpuConfig: input.sandboxGpuConfig, + requiredLimits: input.requiredUlimits ?? [], + timeoutSecs: input.sandboxReadyTimeoutSecs, + network: { + inferenceProvider: input.provider, + dockerDriverGateway: input.dockerDriverGateway, + gatewayPort: input.gatewayPort, + }, + dependencies: { + runCaptureOpenshell: deps.runCaptureOpenshell, + runOpenshell: deps.runOpenshell, + sleep: deps.sleep, + }, + }) + : null; + const runtimePatch = + managedLifecycle?.patch ?? + createDockerGpuSandboxCreatePatch({ + route, + // The startup clone preserves native CDI devices, so DCode can apply its + // exact required limits without replacing the native GPU envelope. + // Other native routes are not swapped solely to persist a command. + persistStartupCommand: + input.persistStartupCommand === true && (route !== "native" || hasRequiredUlimits), + externalRecreation: false, + sandboxName: input.sandboxName, + gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, + openshellSandboxCommand: input.sandboxStartupCommand, + requiredUlimits: input.requiredUlimits, + timeoutSecs: input.sandboxReadyTimeoutSecs, + backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", + deps, + }); + await managedLifecycle?.prepareNetwork(); + const [createExecutable, ...createExecutableArgs] = managedLifecycle?.launchArgv ?? attemptArgv; if (!createExecutable) throw new Error("Sandbox create executable is missing."); - const createResult = await streamSandboxCreate( - createExecutable, - createExecutableArgs, - input.sandboxEnv, - { + const streamCreate = () => + streamSandboxCreate(createExecutable, createExecutableArgs, input.sandboxEnv, { readyCheck: () => { const list = deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); return isSandboxReady(list, input.sandboxName); }, - onPoll: () => dockerGpuCreatePatch.maybeApplyDuringCreate(), + onPoll: () => runtimePatch.maybeApplyDuringCreate(), readyCheckOutputPatterns: getReadyCheckOutputPatternsForAgent( input.terminalAgent, input.sandboxEnv, ), - failureCheck: dockerGpuCreatePatch.createFailureMessage, + failureCheck: runtimePatch.createFailureMessage, traceEvent: addTraceEvent, initialPhase: compatibility && (input.prebuild.imageRef || state.compatibilityArgv) ? "create" : undefined, - }, - ); + }); + let createResult: Awaited>; + let managedIncompleteCreateRecovered = false; + if (managedBootstrap && managedLifecycle) { + try { + createResult = await managedLifecycle.runCreate( + async ({ heldWorkloadArgv, bootstrapIdentity }) => { + if ( + bootstrapIdentity !== managedBootstrap.bootstrapIdentity || + heldWorkloadArgv.length !== input.sandboxStartupCommand.length || + heldWorkloadArgv.some((value, index) => value !== input.sandboxStartupCommand[index]) + ) { + throw new Error( + "Managed bootstrap launch does not match the rendered identity-bound hold.", + ); + } + const result = await streamCreate(); + const createFailure = + result.status === 0 ? null : classifySandboxCreateFailure(result.output); + if (result.status !== 0 && createFailure?.kind !== "sandbox_create_incomplete") { + throw new ManagedBootstrapCreateStreamFailure(result); + } + if (createFailure?.kind === "sandbox_create_incomplete") { + const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ + sandboxName: input.sandboxName, + timeoutSecs: input.sandboxReadyTimeoutSecs, + runCaptureOpenshell: deps.runCaptureOpenshell, + isSandboxReady, + getSandboxFailurePhase, + stableReadyPolls: 1, + sleep: deps.sleep, + }); + if (!readiness.ready) { + throw new Error( + `Managed bootstrap incomplete create did not reach authoritative Ready state (${readiness.reason}).`, + ); + } + } else { + const list = deps.runCaptureOpenshell(["sandbox", "list"], { + ignoreError: true, + }); + if (!isSandboxReady(list, input.sandboxName)) { + throw new Error( + "Managed bootstrap create completed without an authoritative Ready sandbox.", + ); + } + } + let sandboxId: string; + try { + sandboxId = resolveOpenShellSandboxId(input.sandboxName, deps.runCaptureOpenshell); + } catch (error) { + throw new Error( + createFailure?.kind === "sandbox_create_incomplete" + ? "Managed bootstrap incomplete create did not return one exact durable sandbox identity after Ready." + : "Managed bootstrap create did not return one exact durable sandbox identity after Ready.", + { cause: error }, + ); + } + managedIncompleteCreateRecovered = createFailure?.kind === "sandbox_create_incomplete"; + return { + value: result, + receipt: { + sandbox: { + sandboxName: input.sandboxName, + sandboxId, + driverId: managedBootstrap.runtimeProvider.identity.id, + }, + ready: true, + readyAt: new Date().toISOString(), + }, + }; + }, + ); + } catch (error) { + if (!(error instanceof ManagedBootstrapCreateStreamFailure)) throw error; + createResult = error.result; + } + } else { + createResult = await streamCreate(); + } if (!state.firstCreateOutput) state.firstCreateOutput = createResult.output; - dockerGpuCreatePatch.exitOnPatchError(); + await runtimePatch.exitOnPatchError(); if (createResult.status !== 0) { const failure = classifySandboxCreateFailure(createResult.output); if (failure.kind === "sandbox_create_incomplete") { console.warn(""); - console.warn( - ` Create stream exited with code ${createResult.status} after sandbox was created.`, - ); - console.warn(" Checking whether the sandbox reaches Ready state..."); + if (managedIncompleteCreateRecovered) { + console.warn( + ` Create stream exited with code ${createResult.status}; the exact durable sandbox reached Ready, and onboarding is continuing with final checks.`, + ); + } else { + console.warn( + ` Create stream exited with code ${createResult.status} after sandbox was created.`, + ); + console.warn(" Checking whether the sandbox reaches Ready state..."); + } } else if ( route === "native" && input.gpuRoutePlan === "native-with-fallback" && nativeFallbackHasCleanBaseline && (() => { if ( - sandboxGpuCreateAttempt.isNativeGpuCreateRoutingFailure(createResult.output, { - sawProgress: createResult.sawProgress, - }) + managedRouting + ? managedRouting.isNativeCreateRoutingFailure( + createResult.output, + createResult.sawProgress, + ) + : sandboxGpuCreateAttempt.isNativeGpuCreateRoutingFailure(createResult.output, { + sawProgress: createResult.sawProgress, + }) ) { state.allowUnbuiltCompatibilitySource = input.prebuild.imageRef === null; return true; } const snapshot = inspectNativeRuntime(); if ( - snapshot.ok && - sandboxGpuCreateAttempt.isTrustedNativeGpuRuntimeError(snapshot.stateError) + snapshot && + (managedRouting + ? managedRouting.isTrustedNativeRuntimeError(snapshot.stateError) + : sandboxGpuCreateAttempt.isTrustedNativeGpuRuntimeError(snapshot.stateError)) ) { state.nativeRuntimeSnapshot = snapshot; return true; @@ -144,6 +290,7 @@ export function createSandboxGpuCreateAttemptRunner( return false; })() ) { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); return { ok: false, route, @@ -152,6 +299,7 @@ export function createSandboxGpuCreateAttemptRunner( fallbackEligible: true, } as const; } else { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); reportSandboxCreateFailure( { sandboxName: input.sandboxName, @@ -171,8 +319,8 @@ export function createSandboxGpuCreateAttemptRunner( ); } } - dockerGpuCreatePatch.ensureApplied(); - dockerGpuCreatePatch.waitForSupervisorReconnectIfNeeded(); + await runtimePatch.ensureApplied(); + await runtimePatch.waitForSupervisorReconnectIfNeeded(); console.log(" Waiting for sandbox to become ready..."); const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ sandboxName: input.sandboxName, @@ -197,13 +345,19 @@ export function createSandboxGpuCreateAttemptRunner( const runtimeSnapshot = canClassifyNativeReadiness ? inspectNativeRuntime() : null; if ( canClassifyNativeReadiness && - runtimeSnapshot?.ok && - sandboxGpuCreateAttempt.isNativeGpuReadinessRoutingFailure({ - failurePhase: readiness.failurePhase, - runtimeError: runtimeSnapshot.stateError, - }) + runtimeSnapshot && + (managedRouting + ? managedRouting.isNativeReadinessRoutingFailure({ + failurePhase: readiness.failurePhase, + runtimeError: runtimeSnapshot.stateError, + }) + : sandboxGpuCreateAttempt.isNativeGpuReadinessRoutingFailure({ + failurePhase: readiness.failurePhase, + runtimeError: runtimeSnapshot.stateError, + })) ) { state.nativeRuntimeSnapshot = runtimeSnapshot; + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); return { ok: false, route, @@ -214,10 +368,11 @@ export function createSandboxGpuCreateAttemptRunner( fallbackEligible: true, } as const; } + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); printSandboxCreateFailureDiagnostics(input.sandboxName, { backupPath: input.restoreBackupPath, }); - if (compatibility) dockerGpuCreatePatch.printReadinessFailureIfEnabled(); + if (compatibility) runtimePatch.printReadinessFailureIfEnabled(); else { const deletion = deps.runOpenshell(["sandbox", "delete", input.sandboxName], { ignoreError: true, @@ -240,27 +395,32 @@ export function createSandboxGpuCreateAttemptRunner( route === "native" && input.gpuRoutePlan === "native-with-fallback" && nativeFallbackHasCleanBaseline; - const proof: SandboxGpuProofResult = dockerGpuLocalInference.verifyGpuSandboxAccessAfterReady( - input.sandboxGpuConfig, - { - sandboxName: input.sandboxName, - dockerDriverGateway: input.dockerDriverGateway, - selectedRoute: route, - verifyDirectSandboxGpu: deps.verifyDirectSandboxGpu, - verifyGpuOrExit: deferNativeProofFailure - ? undefined - : dockerGpuCreatePatch.verifyGpuOrExit, - reportGpuProofFailure: !deferNativeProofFailure, - selectedMode: dockerGpuCreatePatch.selectedMode, - runCaptureOpenshell: deps.runCaptureOpenshell, - log: console.log, - }, - ); + let proof: SandboxGpuProofResult; + try { + proof = await dockerGpuLocalInference.verifyGpuSandboxAccessAfterReady( + input.sandboxGpuConfig, + { + sandboxName: input.sandboxName, + dockerDriverGateway: input.dockerDriverGateway, + selectedRoute: route, + verifyDirectSandboxGpu: deps.verifyDirectSandboxGpu, + verifyGpuOrExit: deferNativeProofFailure ? undefined : runtimePatch.verifyGpuOrExit, + reportGpuProofFailure: !deferNativeProofFailure, + selectedMode: runtimePatch.selectedMode, + runCaptureOpenshell: deps.runCaptureOpenshell, + log: console.log, + }, + ); + } catch (error) { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); + throw error; + } if (deferNativeProofFailure && proof.status === "failed") { if (sandboxGpuPreflight.isExplicitNvidiaSmiDriverProofFailure(proof)) { const snapshot = inspectNativeRuntime(); - if (snapshot.ok && snapshot.nativeGpuAttachmentState === "absent") { + if (snapshot?.nativeGpuAttachmentState === "absent") { state.nativeRuntimeSnapshot = snapshot; + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); return { ok: false, route, @@ -272,6 +432,7 @@ export function createSandboxGpuCreateAttemptRunner( } as const; } } + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); console.error(""); console.error(" Native sandbox GPU proof failed."); console.error( @@ -283,15 +444,22 @@ export function createSandboxGpuCreateAttemptRunner( process.exit(1); } if (proof.status === "failed") { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); throw new Error("Sandbox GPU proof returned failed status."); } } + // GPU-enabled cutover stays reversible until the caller also proves the + // configured host-local inference path. Non-GPU workloads have completed + // their final authoritative Ready gate here. + if (!input.sandboxGpuConfig.sandboxGpuEnabled) { + await runtimePatch.commitAfterReady(); + } return { ok: true, route, - value: { createResult, dockerGpuCreatePatch }, + value: { createResult, runtimePatch }, } as const; }; - return { state, runAttempt }; + return { state, managedRouting, runAttempt }; } diff --git a/test/onboard-prepared-build-context.test.ts b/test/onboard-prepared-build-context.test.ts index 0d91d612ae4..5babec1765c 100644 --- a/test/onboard-prepared-build-context.test.ts +++ b/test/onboard-prepared-build-context.test.ts @@ -98,12 +98,15 @@ let stageCalls = 0; dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch = () => ({ maybeApplyDuringCreate: () => {}, createFailureMessage: () => null, - exitOnPatchError: () => {}, - ensureApplied: () => {}, + exitOnPatchError: async () => {}, + attachManagedBootstrapCutover: () => {}, + rollbackManagedStartupAfterCreateFailure: async () => {}, + ensureApplied: async () => {}, waitForSupervisorReconnectIfNeeded: () => {}, + commitAfterReady: async () => {}, selectedMode: () => null, printReadinessFailureIfEnabled: () => {}, - verifyGpuOrExit: (verify) => verify(sandboxName), + verifyGpuOrExit: async (verify) => verify(sandboxName), }); buildContextStage.stageCreateSandboxBuildContext = () => { diff --git a/test/onboard-sandbox-recreation.test.ts b/test/onboard-sandbox-recreation.test.ts index 11b35a58b6e..c0a4eb2f6f3 100644 --- a/test/onboard-sandbox-recreation.test.ts +++ b/test/onboard-sandbox-recreation.test.ts @@ -1203,6 +1203,7 @@ const fs = require("node:fs"); const commands = []; let sandboxListCalls = 0; +let dockerPsCalls = 0; const keepAlive = setInterval(() => {}, 1000); runner.run = (command, opts = {}) => { _deleted = _deleted || _n(command).includes("sandbox delete"); @@ -1210,6 +1211,10 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { + if (_n(command).startsWith("docker ps -a --no-trunc ")) { + dockerPsCalls += 1; + if (dockerPsCalls === 1) return "a".repeat(64); + } if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) { sandboxListCalls += 1; diff --git a/test/onboard-terminal-dashboard.test.ts b/test/onboard-terminal-dashboard.test.ts index 61baa83310a..0dd2686eadd 100644 --- a/test/onboard-terminal-dashboard.test.ts +++ b/test/onboard-terminal-dashboard.test.ts @@ -67,12 +67,15 @@ const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, "") dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch = () => ({ maybeApplyDuringCreate: () => {}, createFailureMessage: () => null, - exitOnPatchError: () => {}, - ensureApplied: () => {}, + exitOnPatchError: async () => {}, + attachManagedBootstrapCutover: () => {}, + rollbackManagedStartupAfterCreateFailure: async () => {}, + ensureApplied: async () => {}, waitForSupervisorReconnectIfNeeded: () => {}, + commitAfterReady: async () => {}, selectedMode: () => null, printReadinessFailureIfEnabled: () => {}, - verifyGpuOrExit: (verify) => verify(sandboxName), + verifyGpuOrExit: async (verify) => verify(sandboxName), }); agentOnboard.createAgentSandbox = () => { diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 48e889d78c1..9045d5faad4 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -28,8 +28,8 @@ let activationPaths: string[] = []; let providerPaths: string[] = []; let dockerfilePaths: string[] = []; let packagingPaths: string[] = []; -const bootstrapLoad = - /(?:from\s*|import\s*|import\s*\(\s*|require\s*\(\s*)["'][^"']*managed-bootstrap/iu; +const driverBootstrapLoad = + /(?:from\s*|import\s*|import\s*\(\s*|require\s*\(\s*)["'][^"']*managed-bootstrap\/(?:docker|docker-journal|docker-runtime)/iu; const packagedBootstrapAsset = /(?:nemoclaw-managed-bootstrap|managed-bootstrap-trampoline|managed-startup-image-runtime\.cjs|nemoclaw-managed-startup-hold)/u; @@ -70,7 +70,7 @@ beforeAll(() => { }); describe("runtime provider central source boundary", () => { - // source-shape-contract: compatibility -- Migrated lifecycle and mutation consumers must stay provider-neutral while production selection excludes unqualified future providers and managed-bootstrap dependencies + // source-shape-contract: compatibility -- Migrated lifecycle and mutation consumers must stay provider-neutral while production selection excludes unqualified future providers and driver-specific bootstrap dependencies it("keeps migrated provider identities and implementations behind the one bundle composition", () => { const driverNeutralActions = { "actions/inference-set.ts": read("src/lib/actions/inference-set.ts"), @@ -114,7 +114,12 @@ describe("runtime provider central source boundary", () => { expect(driverNeutralActions["actions/sandbox/start.ts"]).toMatch( /resolved\.lifecycle\.verifyStarted\(/u, ); - expect(Object.values(providerContract).join("\n")).not.toMatch(/managed-bootstrap/u); + expect(providerContract.contract).toMatch( + /import type[\s\S]*from ["']\.\.\/managed-bootstrap\/runtime-create["']/u, + ); + expect( + [providerContract.current, providerContract.docker, providerContract.registry].join("\n"), + ).not.toMatch(/managed-bootstrap/u); expect(providerContract.current).not.toMatch(/\b(?:podman|mxc)\b/iu); }); @@ -122,12 +127,14 @@ describe("runtime provider central source boundary", () => { expect(bootstrapProtocolPaths).toEqual([ "src/lib/onboard/managed-bootstrap/adapter.ts", "src/lib/onboard/managed-bootstrap/docker-journal.ts", + "src/lib/onboard/managed-bootstrap/docker-runtime.ts", "src/lib/onboard/managed-bootstrap/docker-shared-state.ts", "src/lib/onboard/managed-bootstrap/docker-spec.ts", "src/lib/onboard/managed-bootstrap/docker-test-fixture.ts", "src/lib/onboard/managed-bootstrap/docker.ts", "src/lib/onboard/managed-bootstrap/envelope.ts", "src/lib/onboard/managed-bootstrap/index.ts", + "src/lib/onboard/managed-bootstrap/runtime-create.ts", ]); }); @@ -167,19 +174,25 @@ describe("runtime provider central source boundary", () => { expect(bootstrapProtocolSource).not.toMatch(/\b(?:docker|podman|openshell|mxc)\b/iu); }); - // source-shape-contract: security -- Production onboarding must not activate managed bootstrap until a complete provider image and rollback implementation lands together - it("keeps production activation paths disconnected from managed bootstrap", () => { + // source-shape-contract: security -- Production onboarding may consume the provider-neutral create contract but cannot select a driver-specific bootstrap implementation + it("keeps production activation paths disconnected from driver bootstrap adapters", () => { const onboardEntry = read("src/lib/onboard.ts"); const activationSource = activationPaths.map(read).join("\n"); - expect(onboardEntry).not.toMatch(bootstrapLoad); - expect(activationSource).not.toMatch(bootstrapLoad); + expect(onboardEntry).not.toMatch(driverBootstrapLoad); + expect(activationSource).not.toMatch(driverBootstrapLoad); }); // source-shape-contract: security -- Registered runtime providers must remain bootstrap-unsupported until their complete transaction implementations are qualified it("keeps registered providers bootstrap-unsupported", () => { const dockerProvider = read("src/lib/onboard/runtime-provider/docker.ts"); - const providerSource = providerPaths.map(read).join("\n"); - expect(providerSource).not.toMatch(/managed-bootstrap/iu); + const providerImplementationSource = providerPaths + .filter((path) => path !== "src/lib/onboard/runtime-provider/contract.ts") + .map(read) + .join("\n"); + expect(dockerProvider).not.toMatch( + /(?:from\s+["'][^"']*managed-bootstrap|require\([^)]*managed-bootstrap)/u, + ); + expect(providerImplementationSource).not.toMatch(/managed-bootstrap/iu); expect(dockerProvider.match(/bootstrap:\s*unsupported\(/gu)).toHaveLength(2); expect(dockerProvider.match(/recovery:\s*unsupported\(/gu)).toHaveLength(2); }); From 7bcc8525c7f3570cf0f143bf049a466b2783bac4 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 07:20:58 -0700 Subject: [PATCH 099/117] fix(onboard): address Docker bootstrap review feedback Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/adapter.ts | 1 + .../managed-bootstrap/docker-shared-state.ts | 22 +++-- .../managed-bootstrap/docker-test-fixture.ts | 16 ++-- .../onboard/managed-bootstrap/docker.test.ts | 83 +++++++++++++++++-- src/lib/onboard/managed-bootstrap/docker.ts | 47 ++++------- 5 files changed, 118 insertions(+), 51 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index 4721d059ef1..415fced86cd 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -26,6 +26,7 @@ const PROCESS_INJECTION_ENV_KEYS = new Set([ "LD_LIBRARY_PATH", "LD_PRELOAD", "NODE_OPTIONS", + "NODE_PATH", "PS4", "SHELLOPTS", ]); diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts index 5953fc2b356..86d42507a54 100644 --- a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts @@ -28,6 +28,11 @@ const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY = "/var/lib/nemoclaw/managed-startup-shared-state-commit-v1"; const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; const DURABLE_IDENTITY_RE = /^[a-f0-9]{64}$/u; +const DOCKER_MUTATION_OPTIONS = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, +} as const; const NEUTRALIZED_PROCESS_INJECTION_ENV = [ "--env", "NODE_OPTIONS=", @@ -371,12 +376,6 @@ function commitManagedStartupSharedState( ); } -const DOCKER_MUTATION_OPTIONS = { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, -} as const; - function quiesceManagedStartupContainer( transaction: DockerManagedBootstrapSharedStateTransaction, deps: DockerGpuPatchDeps, @@ -605,7 +604,16 @@ export function finalizeDockerManagedStartupSharedState( const failure = new Error( `OpenShell supervisor reconnected, but managed shared-state logical commit validation failed: ${commitFailure.message}`, ); - quiesceManagedStartupContainer(transaction, deps); + try { + quiesceManagedStartupContainer(transaction, deps); + } catch (stopError) { + throw new Error( + `${failure.message}; the new workload could not be quiesced: ${ + stopError instanceof Error ? stopError.message : String(stopError) + }`, + { cause: stopError }, + ); + } rollbackManagedStartupSharedState(transaction, receiptPath, deps); if (!input.patchResult && !input.retainContainerAfterRollback) { removeFailedUnbackedContainer(transaction, deps); diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index 1c2a6871b6a..8432960be47 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -67,6 +67,7 @@ export type DockerFixtureOptions = { readonly lostAcknowledgements?: readonly DockerFixtureAcknowledgement[]; readonly ownerId?: string; readonly sharedState?: "committed" | "none" | "pending"; + readonly sharedStateCommitResult?: FixtureCommandResult; }; function agentInputs(agent: ManagedStartupAgent = "hermes") { @@ -202,7 +203,7 @@ function readProtectedEnvelope(source: string): ReturnType { }); const order = fake.events; expect(order.indexOf("journal:staged")).toBeGreaterThan(order.indexOf("authority:recorded")); + expect(order).toContain("journal:cutover"); + expect(order).toContain(`stop:${OLD_ID}`); expect(order.indexOf("journal:cutover")).toBeLessThan(order.indexOf(`stop:${OLD_ID}`)); expect(fake.journal).toMatchObject({ phase: "cutover", @@ -75,6 +77,8 @@ describe("Docker managed bootstrap adapter", () => { completion: completion(replacement), }), ).resolves.toMatchObject({ outcome: "committed" }); + expect(fake.events).toContain("journal:shared-state-committed"); + expect(fake.events).toContain(`rm:${OLD_ID}`); expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( fake.events.indexOf(`rm:${OLD_ID}`), ); @@ -83,6 +87,46 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.replacement?.Id).toBe(NEW_ID); }); + it("preserves commit validation failure details when the replacement cannot be quiesced", async () => { + const fake = fixture({ + sharedState: "pending", + sharedStateCommitResult: { status: 1, stderr: "injected commit failure" }, + }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority(); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + vi.mocked(fake.deps.dockerStop!).mockReturnValue({ + status: 1, + stderr: "injected quiesce failure", + }); + + await expect( + adapter.finalizeBootstrap({ + outcome: "commit", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: completion(replacement), + }), + ).rejects.toThrow( + /logical commit validation failed: Managed-startup shared-state commit helper failed.*injected commit failure.*new workload could not be quiesced.*injected quiesce failure/u, + ); + }); + it("publishes durable rollback authority before deleting the replacement after restart", async () => { const fake = fixture({ dockerStartResults: { @@ -120,13 +164,16 @@ describe("Docker managed bootstrap adapter", () => { completion: null, }), ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.events).toContain("journal:rollback-authorized"); + expect(fake.events).toContain(`rm:${NEW_ID}`); expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( fake.events.indexOf(`rm:${NEW_ID}`), ); expect(fake.journal).toBeNull(); expect(fake.replacement).toBeNull(); - expect(fake.original.Name).toBe("/openshell-alpha"); - expect(fake.original.State?.Running).toBe(false); + expect(fake.original).not.toBeNull(); + expect(fake.original?.Name).toBe("/openshell-alpha"); + expect(fake.original?.State?.Running).toBe(false); }); it("recovers the pre-stop cutover crash state after adapter restart", async () => { @@ -164,6 +211,8 @@ describe("Docker managed bootstrap adapter", () => { completion: null, }), ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.events).toContain("journal:rollback-authorized"); + expect(fake.events).toContain(`rm:${NEW_ID}`); expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( fake.events.indexOf(`rm:${NEW_ID}`), ); @@ -281,8 +330,29 @@ describe("Docker managed bootstrap adapter", () => { ).toBe(true); }); + it("rejects an empty intended workload argv with a precise boundary error", async () => { + const fake = fixture(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority(); + const plan = { ...handle.plan, intendedWorkloadArgv: [] }; + const emptyArgvHandle = { ...handle, intendedWorkloadArgv: [], plan }; + await expect( + adapter.prepareBootstrapReplacement({ + handle: emptyArgvHandle, + snapshot, + request, + replacementOptions: { values: {} }, + }), + ).rejects.toThrow( + "Managed bootstrap Docker replacement requires one bounded intended workload argv.", + ); + expect(fake.events).toContain("create:replacement"); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + }); + it.each([ "NODE_OPTIONS", + "NODE_PATH", "LD_PRELOAD", "BASH_ENV", ])("rejects hostile %s from the launch snapshot before replacement creation", async (key) => { @@ -326,7 +396,8 @@ describe("Docker managed bootstrap adapter", () => { sandboxId: "sandbox-alpha", runtimeId: OLD_ID, }); - expect(fake.original.State?.Running).toBe(false); + expect(fake.original).not.toBeNull(); + expect(fake.original?.State?.Running).toBe(false); expect(fake.events).not.toContain(`rm:${OLD_ID}`); expect(vi.mocked(fake.deps.runOpenshell!)).not.toHaveBeenCalled(); }); @@ -336,8 +407,10 @@ describe("Docker managed bootstrap adapter", () => { const fake = fixture({ ownerId: replacementSandboxId }); const adapter = createDockerManagedBootstrapAdapter(fake.deps); const { handle, plan } = authority(); - expect(fake.original.Config?.Labels).toBeDefined(); - fake.original.Config!.Labels!["openshell.ai/sandbox-id"] = replacementSandboxId; + expect(fake.original).not.toBeNull(); + const labels = fake.original?.Config?.Labels; + expect(labels).toBeDefined(); + Object.assign(labels ?? {}, { "openshell.ai/sandbox-id": replacementSandboxId }); await expect( adapter.cleanupIncompleteCreate({ diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index 3a2df714de6..03afb2ba280 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -30,7 +30,6 @@ import type { import { waitForOpenShellSupervisorReconnect } from "../docker-gpu-supervisor-reconnect"; import { openshellSandboxCommandEnvValue } from "../docker-startup-command-env"; import { - isImmutableDockerImageId, OPENSHELL_MANAGED_BY_LABEL, OPENSHELL_MANAGED_BY_VALUE, OPENSHELL_SANDBOX_ID_LABEL, @@ -1193,7 +1192,7 @@ function restoreOriginal(transaction: DockerBootstrapTransaction, deps: Resolved } } -function removeOwnedWorkload( +function retainOwnedWorkloadForOwnerCleanup( sandbox: ManagedBootstrapSandboxIdentity, deps: ResolvedDeps, expectedRuntimeId?: string, @@ -1598,30 +1597,6 @@ function reconstructDockerBootstrapTransaction( return transaction; } -function rollbackReplacementSharedStateIfPending( - input: { - readonly handle: ManagedBootstrapHeldWorkloadHandle; - readonly replacementRuntimeId: string; - readonly runtimeImageContentId: string; - }, - deps: ResolvedDeps, -): void { - if (!tryInspectExact(input.replacementRuntimeId, deps)) { - throw new Error( - "Managed bootstrap replacement disappeared before shared-state rollback could be proven; the preserved original remains stopped.", - ); - } - const transaction = managedSharedStateTransaction( - input.handle, - input.replacementRuntimeId, - input.runtimeImageContentId, - ); - finalizeDockerManagedStartupSharedState( - { transaction, supervisorReady: false, retainContainerAfterRollback: true }, - deps, - ); -} - function cleanupUnjournaledPreparedContainer( input: { readonly snapshot: ManagedBootstrapObservedSnapshot; @@ -1787,7 +1762,7 @@ export function createDockerManagedBootstrapAdapter( detail: "Docker replacement authority exists without its observed snapshot", }); } - removeOwnedWorkload(handle.sandbox, deps); + retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps); return completedRollback(handle, false); } @@ -1868,7 +1843,7 @@ export function createDockerManagedBootstrapAdapter( }); } } - removeOwnedWorkload(handle.sandbox, deps, snapshot.runtimeId); + retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, snapshot.runtimeId); return completedRollback(handle, true); } @@ -1923,7 +1898,7 @@ export function createDockerManagedBootstrapAdapter( removeExactReplacement(journal, observedReplacement, deps); } removeDockerBootstrapJournalDurably(journal, deps); - removeOwnedWorkload(handle.sandbox, deps, journal.originalRuntimeId); + retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, journal.originalRuntimeId); return completedRollback(handle, false); } @@ -2103,7 +2078,7 @@ export function createDockerManagedBootstrapAdapter( throw new Error("Managed bootstrap Docker rollback did not restore its exact original."); } removeDockerBootstrapJournalDurably(activeJournal, deps); - removeOwnedWorkload(handle.sandbox, deps, activeJournal.originalRuntimeId); + retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, activeJournal.originalRuntimeId); return completedRollback(handle, false); }; const commitBootstrapNow = ( @@ -2361,7 +2336,7 @@ export function createDockerManagedBootstrapAdapter( "durable authority changed after shared-state rollback and before restoration", }); } - journal = transitionDockerBootstrapJournalDurably(journal, "rollback-authorized", deps); + transitionDockerBootstrapJournalDurably(journal, "rollback-authorized", deps); await rollbackBootstrapNow({ handle, snapshot, @@ -2443,7 +2418,7 @@ export function createDockerManagedBootstrapAdapter( async cleanupIncompleteCreate(input) { const { sandbox, runtimeId } = resolveIncompleteCreateSandbox(input, deps); - removeOwnedWorkload(sandbox, deps, runtimeId); + retainOwnedWorkloadForOwnerCleanup(sandbox, deps, runtimeId); return Object.freeze({ schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, sandbox, @@ -2630,13 +2605,19 @@ export function createDockerManagedBootstrapAdapter( } assertMetadata(createdInspect, handle.sandbox, snapshot.metadata); assertRootSupervisor(createdInspect); + const intendedSandboxCommand = openshellSandboxCommandEnvValue(handle.intendedWorkloadArgv); + if (!intendedSandboxCommand) { + throw new Error( + "Managed bootstrap Docker replacement requires one bounded intended workload argv.", + ); + } assertReplacementBoundary(createdInspect, handle, snapshot); const expectedActivatedSpecHash = assertReplacementMatchesIntent( snapshot.specCanonicalJson, createdInspect, originalName, plan, - openshellSandboxCommandEnvValue(handle.intendedWorkloadArgv) as string, + intendedSandboxCommand, ); const preparedSpec = normalizeDockerManagedBootstrapLaunchSpec(createdInspect); const expectedActivatedSpec = normalizeDockerManagedBootstrapLaunchSpec({ From c874abc64a256bba1363d23c7cde34456483c970 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 07:28:56 -0700 Subject: [PATCH 100/117] test(onboard): harden Docker bootstrap assertions Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/docker.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 2bf8d527343..cc68f6825ce 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { assert, describe, expect, it, vi } from "vitest"; import { ManagedBootstrapOwnerCleanupRequiredError } from "./adapter"; import { createDockerManagedBootstrapAdapter } from "./docker"; @@ -56,6 +56,8 @@ describe("Docker managed bootstrap adapter", () => { durablePreparation: durable, }); const order = fake.events; + expect(order).toContain("authority:recorded"); + expect(order).toContain("journal:staged"); expect(order.indexOf("journal:staged")).toBeGreaterThan(order.indexOf("authority:recorded")); expect(order).toContain("journal:cutover"); expect(order).toContain(`stop:${OLD_ID}`); @@ -125,6 +127,7 @@ describe("Docker managed bootstrap adapter", () => { ).rejects.toThrow( /logical commit validation failed: Managed-startup shared-state commit helper failed.*injected commit failure.*new workload could not be quiesced.*injected quiesce failure/u, ); + expect(fake.events).not.toContain("shared:rollback"); }); it("publishes durable rollback authority before deleting the replacement after restart", async () => { @@ -409,8 +412,8 @@ describe("Docker managed bootstrap adapter", () => { const { handle, plan } = authority(); expect(fake.original).not.toBeNull(); const labels = fake.original?.Config?.Labels; - expect(labels).toBeDefined(); - Object.assign(labels ?? {}, { "openshell.ai/sandbox-id": replacementSandboxId }); + assert(labels, "fixture labels are required"); + labels["openshell.ai/sandbox-id"] = replacementSandboxId; await expect( adapter.cleanupIncompleteCreate({ From d3f532e517bfcb3a197065621fbf10ac3faa38a3 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 07:32:48 -0700 Subject: [PATCH 101/117] test(onboard): scope bootstrap fixture switch state Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/docker-test-fixture.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index 8432960be47..cb069cb5bca 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -349,11 +349,12 @@ export function fixture(options: DockerFixtureOptions = {}) { break; case "exec": switch (true) { - case args.includes("--commit-shared-state-transaction"): + case args.includes("--commit-shared-state-transaction"): { const result = options.sharedStateCommitResult ?? ok(); sharedState = result.status === 0 ? "committed" : sharedState; events.push("shared:commit"); return result; + } case args.includes("--clear-shared-state-commit-receipt"): sharedState = "none"; events.push("shared:clear"); From 633e0dcc7df2116374c94d687f27632dbb0324bb Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 07:38:17 -0700 Subject: [PATCH 102/117] test(onboard): guard Docker agent argument lookup Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/docker.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index cc68f6825ce..7979330b180 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -328,7 +328,11 @@ describe("Docker managed bootstrap adapter", () => { expect( vi.mocked(fake.deps.dockerRun!).mock.calls.some(([args]) => { const agentIndex = args.indexOf("--agent"); - return args.includes("--shared-state-transaction-status") && args[agentIndex + 1] === agent; + return ( + args.includes("--shared-state-transaction-status") && + agentIndex >= 0 && + args[agentIndex + 1] === agent + ); }), ).toBe(true); }); From 908c2dc2bc81fd7817257c2d9fd81a7da4b3d283 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 08:30:22 -0700 Subject: [PATCH 103/117] feat(onboard): add Docker managed bootstrap lifecycle Signed-off-by: Aaron Erickson --- .../snapshot-auto-create-failure.test.ts | 18 +- .../sandbox/snapshot-restore-test-fixture.ts | 5 + src/lib/actions/sandbox/snapshot.test.ts | 10 +- .../openshell/sandbox-identity.test.ts | 21 + .../adapters/openshell/sandbox-identity.ts | 16 + .../onboard/docker-gpu-patch-clone.test.ts | 35 + src/lib/onboard/docker-gpu-patch-clone.ts | 23 +- src/lib/onboard/docker-gpu-patch-types.ts | 17 + src/lib/onboard/managed-bootstrap/README.md | 48 +- .../onboard/managed-bootstrap/adapter.test.ts | 7 +- src/lib/onboard/managed-bootstrap/adapter.ts | 6 +- .../docker-shared-state.test.ts | 117 + .../managed-bootstrap/docker-shared-state.ts | 630 ++++ .../managed-bootstrap/docker-test-fixture.ts | 484 +++ .../onboard/managed-bootstrap/docker.test.ts | 433 +++ src/lib/onboard/managed-bootstrap/docker.ts | 2949 +++++++++++++++++ .../openshell-docker-sandbox-containers.ts | 1 + test/runtime-provider-source-shape.test.ts | 3 + tsconfig.src.json | 7 +- 19 files changed, 4798 insertions(+), 32 deletions(-) create mode 100644 src/lib/adapters/openshell/sandbox-identity.test.ts create mode 100644 src/lib/adapters/openshell/sandbox-identity.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-shared-state.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker-test-fixture.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker.test.ts create mode 100644 src/lib/onboard/managed-bootstrap/docker.ts diff --git a/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts b/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts index 65612e0ae3d..9dc200a96e1 100644 --- a/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts +++ b/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts @@ -28,13 +28,22 @@ const streamSandboxCreateMock = vi.fn(async () forcedReady: false, })); -vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => "") })); +vi.mock("../../adapters/docker", () => ({ + dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(), + dockerRunDetached: vi.fn(), +})); vi.mock("../../adapters/openshell/runtime", () => ({ captureOpenshell: captureOpenshellMock, getOpenshellBinary: vi.fn(() => "openshell"), runOpenshell: vi.fn(() => ({ status: 0, output: "" })), })); -vi.mock("../../credentials/store", () => ({ prompt: vi.fn() })); +vi.mock("../../credentials/store", () => ({ + deleteCredential: vi.fn(), + getCredential: vi.fn(() => null), + prompt: vi.fn(), + saveCredential: vi.fn(), +})); vi.mock("../../domain/sandbox/destroy", () => ({ getSandboxDeleteOutcome: vi.fn(() => ({ alreadyGone: false, gatewayUnreachable: false })), })); @@ -50,6 +59,11 @@ vi.mock("../../inference/nim", () => ({ stopNimContainerByName: vi.fn(), })); vi.mock("../../messaging/channels", () => ({ + BUILT_IN_CHANNEL_MANIFESTS: [], + getMessagingConfigEnvAliases: vi.fn(() => ({})), + getMessagingCredentialEnvKeysByChannel: vi.fn(() => ({})), + getMessagingProviderSuffixesByChannel: vi.fn(() => ({})), + listBuiltInMessagingChannelManifests: vi.fn(() => []), listMessagingProviderSuffixes: vi.fn(() => []), listMessagingCredentialMetadata: vi.fn(() => []), })); diff --git a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts index 66819eb0951..0d99c458141 100644 --- a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts +++ b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts @@ -213,7 +213,9 @@ export { lifecycleMock, shieldsMock }; vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(), dockerInspect: dockerInspectMock, + dockerRunDetached: vi.fn(), })); vi.mock("../../agent/defs", () => ({ @@ -227,7 +229,10 @@ vi.mock("../../adapters/openshell/runtime", () => ({ })); vi.mock("../../credentials/store", () => ({ + deleteCredential: vi.fn(), + getCredential: vi.fn(() => null), prompt: vi.fn(), + saveCredential: vi.fn(), })); vi.mock("../../domain/sandbox/destroy", () => ({ diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 977a2f49822..19cf6428de0 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -144,19 +144,21 @@ const latestBackupFixture = { vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(), dockerInspect: dockerInspectMock, + dockerRunDetached: vi.fn(), })); - vi.mock("../../adapters/openshell/runtime", () => ({ captureOpenshell: captureOpenshellMock, getOpenshellBinary: vi.fn(() => "openshell"), runOpenshell: runOpenshellMock, })); - vi.mock("../../credentials/store", () => ({ + deleteCredential: vi.fn(), + getCredential: vi.fn(() => null), prompt: vi.fn(), + saveCredential: vi.fn(), })); - vi.mock("../../domain/sandbox/destroy", () => ({ getSandboxDeleteOutcome: vi.fn(() => ({ alreadyGone: false, gatewayUnreachable: false })), })); @@ -165,7 +167,6 @@ vi.mock("../../inference/nim", () => ({ stopNimContainer: vi.fn(), stopNimContainerByName: vi.fn(), })); - vi.mock("../../policy", async (importOriginal) => ({ ...(await importOriginal()), applyPreset: applyPresetMock, @@ -176,7 +177,6 @@ vi.mock("../../policy", async (importOriginal) => ({ removePreset: removePresetMock, resolveAgentBaselinePolicy: resolveTestAgentBaselinePolicy, })); - vi.mock("../../runner", () => ({ ROOT: "/repo", run: vi.fn(() => ({ status: 0 })), diff --git a/src/lib/adapters/openshell/sandbox-identity.test.ts b/src/lib/adapters/openshell/sandbox-identity.test.ts new file mode 100644 index 00000000000..422bf71ccd8 --- /dev/null +++ b/src/lib/adapters/openshell/sandbox-identity.test.ts @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { parseOpenShellSandboxId } from "./sandbox-identity"; + +describe("OpenShell sandbox identity parsing", () => { + it("accepts one exact durable ID with optional terminal color", () => { + expect(parseOpenShellSandboxId("Name: alpha\nID: sandbox-alpha\n")).toBe("sandbox-alpha"); + expect(parseOpenShellSandboxId("\u001b[32mId: sandbox.alpha_2\u001b[0m\n")).toBe( + "sandbox.alpha_2", + ); + }); + + it("rejects ambiguous or non-canonical IDs", () => { + expect(parseOpenShellSandboxId("ID: first\nID: second\n")).toBeNull(); + expect(parseOpenShellSandboxId("ID: sandbox/alpha\n")).toBeNull(); + expect(parseOpenShellSandboxId("id: sandbox-alpha\n")).toBeNull(); + }); +}); diff --git a/src/lib/adapters/openshell/sandbox-identity.ts b/src/lib/adapters/openshell/sandbox-identity.ts new file mode 100644 index 00000000000..1820a8f8f7d --- /dev/null +++ b/src/lib/adapters/openshell/sandbox-identity.ts @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const ANSI_RE = /\x1b\[[0-9;]*m/gu; +const SANDBOX_ID_RE = /^[A-Za-z0-9._-]+$/u; + +export function parseOpenShellSandboxId(output: string): string | null { + const matches = [ + ...String(output) + .replace(ANSI_RE, "") + .matchAll(/^\s*(?:Id|ID):\s*(\S+)\s*$/gm), + ].map((match) => match[1] ?? ""); + return matches.length === 1 && SANDBOX_ID_RE.test(matches[0] as string) + ? (matches[0] as string) + : null; +} diff --git a/src/lib/onboard/docker-gpu-patch-clone.test.ts b/src/lib/onboard/docker-gpu-patch-clone.test.ts index 8ad39b30162..cbdbfb1e252 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.test.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.test.ts @@ -122,6 +122,41 @@ describe("Docker GPU clone envelope", () => { expect(args).not.toContain("nofile=1024:1024"); }); + it("uses exact managed-bootstrap container, entrypoint, and command overrides", () => { + const args = buildDockerGpuCloneRunArgs( + inspectFixture(), + buildDockerGpuMode("startup-command"), + { + containerName: "openshell-alpha-bootstrap-stage", + containerEntrypoint: "/usr/local/bin/nemoclaw-managed-bootstrap", + containerCommand: ["--request", "/run/nemoclaw/bootstrap-request.json"], + }, + ); + + expect(args.slice(0, 2)).toEqual(["--name", "openshell-alpha-bootstrap-stage"]); + expect(args).toEqual( + expect.arrayContaining(["--entrypoint", "/usr/local/bin/nemoclaw-managed-bootstrap"]), + ); + expect(args.slice(args.indexOf("openshell/sandbox:abc"))).toEqual([ + "openshell/sandbox:abc", + "--request", + "/run/nemoclaw/bootstrap-request.json", + ]); + }); + + it.each([ + "", + "-starts-with-dash", + "contains/slash", + "a".repeat(254), + ])("rejects invalid managed-bootstrap container name %j", (containerName) => { + expect(() => + buildDockerGpuCloneRunArgs(inspectFixture(), buildDockerGpuMode("startup-command"), { + containerName, + }), + ).toThrow("Docker clone container name is invalid."); + }); + it("adds SYS_PTRACE to the GPU clone when the baseline container lacks it", () => { const inspect = inspectFixture(); inspect.HostConfig!.CapAdd = ["SYS_ADMIN", "NET_ADMIN"]; diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts index c92767adad0..828ce0c540b 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -329,7 +329,15 @@ export function buildDockerGpuCloneRunArgs( const image = String(options.image || config.Image || "").trim(); if (!image) throw new Error("Docker inspect output did not include Config.Image."); - const args: string[] = ["--name", dockerContainerName(inspect), ...mode.args]; + const containerName = String(options.containerName ?? dockerContainerName(inspect)).trim(); + if ( + containerName.length === 0 || + containerName.length > 253 || + !/^[A-Za-z0-9][A-Za-z0-9_.-]*$/u.test(containerName) + ) { + throw new Error("Docker clone container name is invalid."); + } + const args: string[] = ["--name", containerName, ...mode.args]; const gpuAugment = mode.kind !== "startup-command"; // Startup-command recreation must retain OpenShell's native CDI attachment. @@ -435,8 +443,17 @@ export function buildDockerGpuCloneRunArgs( if (host.Init) args.push("--init"); const entrypoint = stringArray(config.Entrypoint); - if (entrypoint.length > 0) args.push("--entrypoint", entrypoint[0]); - const commandArgs = sandboxCommand ? [] : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; + const replacementEntrypoint = String(options.containerEntrypoint ?? "").trim(); + if (replacementEntrypoint) { + args.push("--entrypoint", replacementEntrypoint); + } else if (entrypoint.length > 0) { + args.push("--entrypoint", entrypoint[0]); + } + const commandArgs = options.containerCommand + ? [...options.containerCommand] + : sandboxCommand + ? [] + : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; args.push(image, ...commandArgs); return args; } diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 32be0afe46f..d72046bd320 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -112,6 +112,14 @@ export type DockerGpuCloneRunOptions = { sandboxFallbackDns?: string | null; openshellSandboxCommand?: readonly string[] | null; requiredUlimits?: readonly DockerUlimit[] | null; + /** + * Exact replacement process boundary used only by dormant managed bootstrap. + * Ordinary recreation leaves both fields unset. + */ + containerEntrypoint?: string | null; + containerCommand?: readonly string[] | null; + /** Stopped staging name used before exact-name cutover. */ + containerName?: string | null; /** * Extra supplementary group IDs to add to the recreated container via * `--group-add`. On Jetson these are the host group(s) owning the Tegra GPU @@ -190,6 +198,14 @@ export type DockerContainerInspect = { Hostname?: string; Tty?: boolean; OpenStdin?: boolean; + StopTimeout?: number | null; + Volumes?: Record | null; + } | null; + State?: { + Running?: boolean; + Paused?: boolean; + Restarting?: boolean; + Dead?: boolean; } | null; HostConfig?: { Binds?: string[] | null; @@ -244,6 +260,7 @@ export type DockerContainerInspect = { DeviceIDs?: string[] | null; }> | null; ShmSize?: number; + ReadonlyRootfs?: boolean; ReadonlyPaths?: string[] | null; MaskedPaths?: string[] | null; } | null; diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index adde636fa56..717fcaa048b 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -3,9 +3,9 @@ # Managed bootstrap protocol -This directory defines a dormant, driver-neutral transaction contract. It does -not register a runtime provider or change sandbox creation, onboarding, -snapshot, clone, or restore behavior. +This directory defines a dormant, driver-neutral transaction contract and its +first driver adapter. It does not register a runtime provider or change sandbox +creation, onboarding, snapshot, clone, or restore behavior. The protocol binds one random bootstrap identity to: @@ -71,6 +71,21 @@ journal and a canonical launch-spec normalizer. Each surface is independently validated and remains dormant: no registered runtime provider imports either module, and neither changes sandbox creation or lifecycle behavior. +The Docker adapter creates and validates a stopped replacement under an +identity-derived staging name while the original remains running. It stages the +0400 envelope and returns exact cleanup authority without quiescing, renaming, +or otherwise mutating the original. Only after the coordinator durably records +that complete prepared authority may activation journal both full runtime IDs, +all three names, both launch-spec hashes, image identity, profile fingerprint, +and sandbox ID and then enter the destructive cutover. Rollback publishes +`rollback-authorized` before exact replacement deletion; commit publishes +`shared-state-committed` before exact backup deletion. Cleanup is bound to full +runtime IDs. Mutable OpenShell names are read only to detect ownership reuse, +and unsafe name-only deletion returns a typed retention error. The dormant +adapter assumes the protocol's single coordinator; multi-process +lease/arbitration remains an explicit production-activation gate. Activation +must also inject the selected gateway's canonical state root. + ## Architectural disposition The coordinator deliberately lands as a dormant trust-boundary slice before a @@ -91,17 +106,16 @@ activation slice must add a registered-provider contract test for the same transaction before removing those dormancy assertions. The native entrypoint source is intentionally not compiled into production -artifacts, and neither source is packaged or selected yet. No production -TypeScript module imports this protocol. The current image definitions do not -package `nemoclaw-managed-startup-hold` or -`managed-startup-image-runtime.cjs`. A later -provider integration must compile and verify the freestanding entrypoint -natively for amd64 and arm64 in every agent image. It must add those -prerequisites together with their image-runtime bootstrap modes, implement -driver-specific prepare, durable-record, activate, exact cleanup, and rollback, -and only then wire the coordinator into create. The same contract is exercised -for OpenClaw, Hermes, and Deep Agents Code without a provider-specific central -switch. The remaining integration and qualification work is tracked in -[epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) and its linked -implementation stack. Until that complete boundary lands, every registered -runtime provider keeps its bootstrap surface unsupported. +artifacts, and neither image-owned source is packaged or selected yet. No +production TypeScript module imports this protocol or the Docker adapter. The +current image definitions do not package `nemoclaw-managed-startup-hold`, +`managed-startup-image-runtime.cjs`, or the shared-state bootstrap modes consumed +by the adapter. A later provider integration must compile and verify the +freestanding entrypoint natively for amd64 and arm64 in every agent image. It +must add those prerequisites together with their image-runtime bootstrap modes +and wire the coordinator and Docker adapter into create as one boundary. The +same contract is exercised for OpenClaw, Hermes, and Deep Agents Code without a +provider-specific central switch. The remaining integration and qualification +work is tracked in [epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) +and its linked implementation stack. Until that complete boundary lands, every +registered runtime provider keeps its bootstrap surface unsupported. diff --git a/src/lib/onboard/managed-bootstrap/adapter.test.ts b/src/lib/onboard/managed-bootstrap/adapter.test.ts index 7941dc8686b..9968171289d 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.test.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.test.ts @@ -909,13 +909,16 @@ describe("managed bootstrap adapter contract", () => { }); it.each([ + "BASHOPTS=extdebug", "BASH_ENV=/sandbox/attacker", "ENV=/sandbox/attacker", - "LD_PRELOAD=/sandbox/attacker.so", "LD_AUDIT=/sandbox/attacker.so", "LD_LIBRARY_PATH=/sandbox/lib", - "SHELLOPTS=xtrace", + "LD_PRELOAD=/sandbox/attacker.so", + "NODE_OPTIONS=--require=/sandbox/attacker.cjs", + "NODE_PATH=/sandbox/attacker-modules", "PS4=$(touch /sandbox/bypass)", + "SHELLOPTS=xtrace", "BASH_FUNC_attacker%%=() { touch /sandbox/bypass; }", ])("rejects a process-control assignment before rendering the held command: %s", (assignment) => { const request = requestFor("hermes"); diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index c29fa7a01d7..415fced86cd 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -25,6 +25,8 @@ const PROCESS_INJECTION_ENV_KEYS = new Set([ "LD_AUDIT", "LD_LIBRARY_PATH", "LD_PRELOAD", + "NODE_OPTIONS", + "NODE_PATH", "PS4", "SHELLOPTS", ]); @@ -931,7 +933,7 @@ function normalizePreparedReplacement( }); } -function createPreparedAuthority( +export function createManagedBootstrapPreparedAuthority( transaction: ManagedBootstrapPreparedTransaction, ): ManagedBootstrapPreparedAuthority { const { handle, snapshot, prepared } = transaction; @@ -1358,7 +1360,7 @@ export async function activateManagedBootstrapSequence( let durablePreparation: ManagedBootstrapDurablePreparationReceipt | null = null; let replacement: ManagedBootstrapReplacementHandle | null = null; try { - const authority = createPreparedAuthority(input.transaction); + const authority = createManagedBootstrapPreparedAuthority(input.transaction); durablePreparation = normalizeDurablePreparationReceipt( await input.authorityStore.recordPreparedAuthority(authority), authority, diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts new file mode 100644 index 00000000000..740eed20f72 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { DockerGpuPatchDeps } from "../docker-gpu-patch-types"; +import { + clearDockerManagedStartupSharedStateCommitReceipt, + type DockerManagedBootstrapSharedStateTransaction, + finalizeDockerManagedStartupSharedState, +} from "./docker-shared-state"; +import { authority, fixture, IDENTITY, NEW_ID } from "./docker-test-fixture"; + +const CLEAN_NODE_COMMAND = [ + "/usr/bin/env", + "-i", + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", +] as const; +const LOADER_ENV_OVERRIDES = [ + "--env", + "LD_AUDIT=", + "--env", + "LD_LIBRARY_PATH=", + "--env", + "LD_PRELOAD=", +] as const; + +function sharedStateTransaction(): DockerManagedBootstrapSharedStateTransaction { + const { handle } = authority("hermes"); + return { + agent: "hermes", + bootstrapIdentity: IDENTITY, + containerId: NEW_ID, + image: `sha256:${"4".repeat(64)}`, + profileFingerprint: handle.plan.profile.fingerprint, + }; +} + +function nodeHelperCalls(deps: DockerGpuPatchDeps): readonly (readonly string[])[] { + return vi + .mocked(deps.dockerRun!) + .mock.calls.map(([args]) => args) + .filter((args) => args[0] === "run" || args[0] === "exec") + .filter((args) => args.includes("/usr/local/bin/node")); +} + +function expectCleanNodeHelper(args: readonly string[]): void { + expect(args).toEqual(expect.arrayContaining([...LOADER_ENV_OVERRIDES])); + expect(args).not.toContain("BASH_FUNC_*"); + const nodeIndex = args.indexOf("/usr/local/bin/node"); + expect(nodeIndex).toBeGreaterThan(0); + if (args[0] === "run") { + const entrypointIndex = args.indexOf("--entrypoint"); + expect(args[entrypointIndex + 1]).toBe(CLEAN_NODE_COMMAND[0]); + expect(args.slice(nodeIndex - CLEAN_NODE_COMMAND.length + 2, nodeIndex + 1)).toEqual( + CLEAN_NODE_COMMAND.slice(1), + ); + return; + } + expect(args.slice(nodeIndex - CLEAN_NODE_COMMAND.length + 1, nodeIndex + 1)).toEqual( + CLEAN_NODE_COMMAND, + ); +} + +describe("Docker managed-bootstrap shared-state helper environment", () => { + it("clears arbitrary image and container environment before every verification and commit helper", () => { + const fake = fixture({ sharedState: "pending" }); + const outcome = finalizeDockerManagedStartupSharedState( + { + transaction: sharedStateTransaction(), + retainContainerAfterRollback: true, + supervisorReady: true, + }, + fake.deps, + ); + + expect(outcome).toEqual({ supervisorReady: true, failure: null }); + const helpers = nodeHelperCalls(fake.deps); + expect(helpers.some((args) => args.includes("--shared-state-transaction-status"))).toBe(true); + expect(helpers.some((args) => args.includes("--commit-shared-state-transaction"))).toBe(true); + expect(helpers).not.toHaveLength(0); + helpers.forEach(expectCleanNodeHelper); + }); + + it("clears arbitrary image environment before the immutable rollback helper", () => { + const fake = fixture({ sharedState: "pending" }); + const outcome = finalizeDockerManagedStartupSharedState( + { + transaction: sharedStateTransaction(), + retainContainerAfterRollback: true, + supervisorReady: false, + }, + fake.deps, + ); + + expect(outcome).toEqual({ supervisorReady: false, failure: null }); + const helpers = nodeHelperCalls(fake.deps); + expect(helpers).toHaveLength(1); + expect(helpers[0]).toContain("--rollback-shared-state-transaction"); + expectCleanNodeHelper(helpers[0]!); + }); + + it("clears arbitrary container environment before the durable receipt-clear helper", () => { + const fake = fixture({ sharedState: "committed" }); + clearDockerManagedStartupSharedStateCommitReceipt(sharedStateTransaction(), fake.deps); + + const helpers = nodeHelperCalls(fake.deps); + expect(helpers).toHaveLength(1); + expect(helpers[0]).toContain("--clear-shared-state-commit-receipt"); + expectCleanNodeHelper(helpers[0]!); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts new file mode 100644 index 00000000000..8d41877a788 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts @@ -0,0 +1,630 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { + dockerRm as defaultDockerRm, + dockerStop as defaultDockerStop, +} from "../../adapters/docker/container"; +import { dockerRun as defaultDockerRun } from "../../adapters/docker/run"; +import { hasZeroDockerExitStatus } from "../docker-command-result"; +import { + DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + DOCKER_GPU_PATCH_TIMEOUT_MS, +} from "../docker-gpu-patch-constants"; +import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "../docker-gpu-patch-types"; +import { MANAGED_STARTUP_RUNTIME_EXECUTABLE } from "../managed-startup/image-runtime"; +import { MANAGED_STARTUP_AGENTS, type ManagedStartupAgent } from "../managed-startup/profile"; +import { + MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, +} from "../managed-startup/shared-state-transaction"; +import { isImmutableDockerImageId } from "../openshell-docker-sandbox-containers"; +import { cleanupTempDir, secureTempFile } from "../temp-files"; + +const RECEIPT_TEMP_PREFIX = "nemoclaw-managed-startup-receipt"; +const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY = + "/var/lib/nemoclaw/managed-startup-shared-state-commit-v1"; +const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; +const DURABLE_IDENTITY_RE = /^[a-f0-9]{64}$/u; +const DOCKER_MUTATION_OPTIONS = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, +} as const; +/** + * The dynamic loader consumes these before `/usr/bin/env -i` can clear the + * inherited image or container environment. Every other variable is removed + * by the shared clean-Node argv below before Node starts. + */ +const NEUTRALIZED_PRE_ENV_LOADER_ENV = [ + "--env", + "LD_AUDIT=", + "--env", + "LD_LIBRARY_PATH=", + "--env", + "LD_PRELOAD=", +] as const; +const CLEAN_NODE_ENTRYPOINT = "/usr/bin/env"; +const CLEAN_NODE_ARGV = [ + "-i", + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", +] as const; + +export interface DockerManagedBootstrapSharedStateTransaction { + readonly agent: ManagedStartupAgent; + readonly bootstrapIdentity: string; + readonly containerId: string; + readonly image: string; + readonly profileFingerprint: string; +} + +export interface DockerManagedStartupSharedStateOutcome { + /** + * True only when the new supervisor is still eligible for successful + * container cutover. A commit failure forces shared-state rollback first. + */ + readonly supervisorReady: boolean; + /** Original commit failure after a successful shared-state rollback. */ + readonly failure: Error | null; +} + +export class DockerManagedStartupSharedStateCommitIndeterminateError extends Error { + constructor(detail: string, options?: ErrorOptions) { + super( + `Managed-startup shared-state commit may have completed, but immutable status is unavailable: ${detail}`, + options, + ); + this.name = "DockerManagedStartupSharedStateCommitIndeterminateError"; + } +} + +export function probeDockerManagedStartupSharedState( + input: { + readonly transaction: DockerManagedBootstrapSharedStateTransaction; + readonly profileFingerprint: string; + }, + deps: DockerGpuPatchDeps = {}, +): "committed" | "none" | "pending" { + const transaction = input.transaction; + assertValidManagedStartupTransaction(transaction); + if (input.profileFingerprint !== transaction.profileFingerprint) { + throw new Error("Managed bootstrap shared-state status fingerprint does not match."); + } + const committedReceiptPath = copyManagedStartupReceiptAt( + transaction, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, + deps, + true, + ); + if (committedReceiptPath) { + let verified = false; + try { + verifyCopiedManagedStartupReceipt( + transaction, + input.profileFingerprint, + committedReceiptPath, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, + "committed", + deps, + ); + verified = true; + return "committed"; + } finally { + if (verified) cleanupReceiptBestEffort(committedReceiptPath); + } + } + const receiptPath = copyManagedStartupReceipt(transaction, deps, true); + if (!receiptPath) return "none"; + let verified = false; + try { + verifyCopiedManagedStartupReceipt( + transaction, + input.profileFingerprint, + receiptPath, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + "pending", + deps, + ); + verified = true; + return "pending"; + } finally { + if (verified) cleanupReceiptBestEffort(receiptPath); + } +} + +function verifyCopiedManagedStartupReceipt( + transaction: DockerManagedBootstrapSharedStateTransaction, + profileFingerprint: string, + receiptPath: string, + receiptDirectory: string, + expectedStatus: "committed" | "pending", + deps: DockerGpuPatchDeps, +): void { + if (!transaction.bootstrapIdentity || !/^[a-f0-9]{64}$/u.test(profileFingerprint)) { + throw new Error("Managed bootstrap copied-receipt identity is incomplete."); + } + const dockerRun = deps.dockerRun ?? defaultDockerRun; + const result = dockerRun( + [ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + ...NEUTRALIZED_PRE_ENV_LOADER_ENV, + "--mount", + transactionReceiptMount(receiptPath, receiptDirectory), + "--entrypoint", + CLEAN_NODE_ENTRYPOINT, + transaction.image, + ...CLEAN_NODE_ARGV, + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + "--shared-state-transaction-status", + "--agent", + transaction.agent, + "--profile-fingerprint", + profileFingerprint, + "--bootstrap-identity", + transaction.bootstrapIdentity, + ], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(result)) { + throw new Error( + `Immutable managed-startup helper could not verify shared-state status: ${commandDetail(result)}. ` + + `Protected receipt retained at ${receiptPath}`, + ); + } + if (String(result.stdout ?? "").trim() !== expectedStatus) { + throw new Error( + `Immutable managed-startup helper returned an invalid copied transaction status. Protected receipt retained at ${receiptPath}`, + ); + } +} + +function commandDetail(result: { + readonly stderr?: string | Buffer | null; + readonly stdout?: string | Buffer | null; + readonly error?: Error | null; +}): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}` + .trim() + .slice(-800); +} + +function cleanupReceiptBestEffort(receiptPath: string): void { + try { + cleanupTempDir(receiptPath, RECEIPT_TEMP_PREFIX); + } catch (error) { + console.warn( + ` ⚠ Managed-startup shared state is finalized, but its protected host receipt could not be removed (${receiptPath}): ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +function assertValidManagedStartupTransaction( + transaction: DockerManagedBootstrapSharedStateTransaction, +): asserts transaction is DockerManagedBootstrapSharedStateTransaction & { + readonly bootstrapIdentity: string; + readonly profileFingerprint: string; +} { + if (!(MANAGED_STARTUP_AGENTS as readonly string[]).includes(transaction.agent)) { + throw new Error("Managed bootstrap shared-state transaction agent is invalid."); + } + if (!FULL_CONTAINER_ID_RE.test(transaction.containerId)) { + throw new Error("Managed bootstrap shared-state transaction container identity is invalid."); + } + if (!isImmutableDockerImageId(transaction.image)) { + throw new Error("Managed bootstrap shared-state transaction image identity is not immutable."); + } + if (!transaction.bootstrapIdentity || !DURABLE_IDENTITY_RE.test(transaction.bootstrapIdentity)) { + throw new Error("Managed bootstrap shared-state transaction identity is missing or invalid."); + } + if ( + !transaction.profileFingerprint || + !DURABLE_IDENTITY_RE.test(transaction.profileFingerprint) + ) { + throw new Error( + "Managed bootstrap shared-state transaction profile fingerprint is missing or invalid.", + ); + } +} + +function transactionCommand( + action: "clear-shared-state-commit-receipt" | "commit" | "rollback", + transaction: DockerManagedBootstrapSharedStateTransaction, +): string[] { + assertValidManagedStartupTransaction(transaction); + return [ + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + action === "clear-shared-state-commit-receipt" + ? "--clear-shared-state-commit-receipt" + : `--${action}-shared-state-transaction`, + "--agent", + transaction.agent, + "--bootstrap-identity", + transaction.bootstrapIdentity, + ]; +} + +export function clearDockerManagedStartupSharedStateCommitReceipt( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps = {}, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + assertValidManagedStartupTransaction(transaction); + const command = transactionCommand("clear-shared-state-commit-receipt", transaction); + const cleared = dockerRun( + [ + "exec", + "--user", + "0:0", + "--workdir", + "/", + ...NEUTRALIZED_PRE_ENV_LOADER_ENV, + transaction.containerId, + CLEAN_NODE_ENTRYPOINT, + ...CLEAN_NODE_ARGV, + ...command, + ], + DOCKER_MUTATION_OPTIONS, + ); + // Accept a lost Docker acknowledgement only when both exact image-owned + // receipt paths are independently proven absent by the immutable helper. + let status: "committed" | "none" | "pending"; + try { + status = probeDockerManagedStartupSharedState( + { + transaction, + profileFingerprint: transaction.profileFingerprint, + }, + deps, + ); + } catch (error) { + throw new DockerManagedStartupSharedStateCommitIndeterminateError( + error instanceof Error ? error.message : String(error), + { cause: error }, + ); + } + if (status === "none") return; + if (!hasZeroDockerExitStatus(cleared)) { + throw new Error( + `Managed-startup durable commit receipt cleanup failed and exact absence was not proven (status=${status}): ${commandDetail(cleared)}`, + ); + } + throw new Error( + `Managed-startup durable commit receipt cleanup returned success, but exact absence was not proven (status=${status}).`, + ); +} + +function commitManagedStartupSharedState( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + assertValidManagedStartupTransaction(transaction); + const command = transactionCommand("commit", transaction); + const commit = dockerRun( + [ + "exec", + "--user", + "0:0", + "--workdir", + "/", + ...NEUTRALIZED_PRE_ENV_LOADER_ENV, + transaction.containerId, + CLEAN_NODE_ENTRYPOINT, + ...CLEAN_NODE_ARGV, + ...command, + ], + DOCKER_MUTATION_OPTIONS, + ); + // The commit helper atomically renames the rollback receipt into a compact + // identity-bound commit receipt before Docker returns. Always probe it + // afterward so a lost daemon acknowledgement is accepted only when durable + // commit state is independently proven. + let status: "committed" | "none" | "pending"; + try { + status = probeDockerManagedStartupSharedState( + { + transaction, + profileFingerprint: transaction.profileFingerprint, + }, + deps, + ); + } catch (error) { + throw new DockerManagedStartupSharedStateCommitIndeterminateError( + error instanceof Error ? error.message : String(error), + { cause: error }, + ); + } + if (status === "committed") return; + if (!hasZeroDockerExitStatus(commit)) { + throw new Error( + `Managed-startup shared-state commit helper failed and durable commit was not proven (status=${status}): ${commandDetail(commit)}`, + ); + } + throw new Error( + `Managed-startup shared-state commit helper returned success, but durable commit was not proven (status=${status}).`, + ); +} + +function quiesceManagedStartupContainer( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerStop = deps.dockerStop ?? defaultDockerStop; + const stopped = dockerStop(transaction.containerId, { + ...DOCKER_MUTATION_OPTIONS, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopped)) { + throw new Error( + `Could not quiesce the failed managed-startup container before shared-state rollback: ${commandDetail(stopped)}`, + ); + } +} + +function isExactMissingReceiptCopy( + transaction: DockerManagedBootstrapSharedStateTransaction, + sourcePath: string, + result: { + readonly stderr?: string | Buffer | null; + readonly stdout?: string | Buffer | null; + readonly error?: Error | null; + }, +): boolean { + const detail = commandDetail(result); + const escapedPath = sourcePath.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const escapedContainer = transaction.containerId.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + return [ + new RegExp( + `^(?:Error response from daemon: )?Could not find the file ${escapedPath} in container ${escapedContainer}$`, + "u", + ), + new RegExp(`^(?:lstat|stat) ${escapedPath}: no such file or directory$`, "u"), + ].some((pattern) => pattern.test(detail)); +} + +function transactionReceiptMount(receiptPath: string, receiptDirectory: string): string { + return `type=bind,src=${receiptPath},dst=${receiptDirectory},readonly`; +} + +function copyManagedStartupReceiptAt( + transaction: DockerManagedBootstrapSharedStateTransaction, + sourcePath: string, + deps: DockerGpuPatchDeps, + allowAbsent = false, +): string | null { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + const tempSeed = secureTempFile(RECEIPT_TEMP_PREFIX); + const receiptPath = path.join(path.dirname(tempSeed), path.basename(sourcePath)); + try { + const copy = dockerRun( + ["cp", "-a", `${transaction.containerId}:${sourcePath}`, receiptPath], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(copy)) { + if (allowAbsent && isExactMissingReceiptCopy(transaction, sourcePath, copy)) { + cleanupReceiptBestEffort(receiptPath); + return null; + } + throw new Error( + `Could not copy the managed-startup rollback receipt from the failed container: ${commandDetail(copy)}`, + ); + } + if (receiptPath.includes(",") || /[\r\n\0]/u.test(receiptPath)) { + throw new Error("Managed-startup rollback receipt path is unsafe for a Docker bind mount"); + } + return receiptPath; + } catch (error) { + cleanupReceiptBestEffort(receiptPath); + throw error; + } +} + +function copyManagedStartupReceipt( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, + allowAbsent = false, +): string | null { + return copyManagedStartupReceiptAt( + transaction, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + deps, + allowAbsent, + ); +} + +function rollbackManagedStartupSharedState( + transaction: DockerManagedBootstrapSharedStateTransaction, + receiptPath: string, + deps: DockerGpuPatchDeps, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + let restored = false; + try { + const helper = dockerRun( + [ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--cap-add", + "CHOWN", + "--cap-add", + "DAC_OVERRIDE", + "--cap-add", + "FOWNER", + ...NEUTRALIZED_PRE_ENV_LOADER_ENV, + "--volumes-from", + transaction.containerId, + "--mount", + `type=bind,src=${receiptPath},dst=${MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY},readonly`, + "--entrypoint", + CLEAN_NODE_ENTRYPOINT, + transaction.image, + ...CLEAN_NODE_ARGV, + ...transactionCommand("rollback", transaction), + "--read-only-receipt", + ], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(helper)) { + throw new Error( + `Immutable managed-startup helper could not restore and verify shared state: ${commandDetail(helper)}. ` + + `Protected receipt retained at ${receiptPath}`, + ); + } + restored = true; + } finally { + if (restored) { + cleanupReceiptBestEffort(receiptPath); + } + } +} + +function removeFailedUnbackedContainer( + transaction: DockerManagedBootstrapSharedStateTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerRm = deps.dockerRm ?? defaultDockerRm; + const removed = dockerRm(transaction.containerId, DOCKER_MUTATION_OPTIONS); + if (!hasZeroDockerExitStatus(removed)) { + throw new Error( + `Could not remove the failed managed-startup container after shared-state rollback: ${commandDetail(removed)}`, + ); + } +} + +/** + * Finalize the shared-state half of managed container cutover before generic + * backup removal or rollback. A shared-state rollback failure deliberately + * throws so callers cannot remove the new container or restart the old one + * while `/sandbox` remains partially applied. + */ +export function finalizeDockerManagedStartupSharedState( + input: { + readonly transaction: DockerManagedBootstrapSharedStateTransaction | null; + readonly patchResult?: DockerGpuPatchResult | null; + /** + * The managed-bootstrap journal owns exact replacement removal. Retaining + * it lets the caller publish rollback authorization after shared-state + * restoration and before the first runtime deletion. + */ + readonly retainContainerAfterRollback?: boolean; + readonly supervisorReady: boolean; + }, + deps: DockerGpuPatchDeps = {}, +): DockerManagedStartupSharedStateOutcome { + const transaction = input.transaction; + if (!transaction) { + return { supervisorReady: input.supervisorReady, failure: null }; + } + assertValidManagedStartupTransaction(transaction); + if (input.supervisorReady) { + // Preserve and validate an explicit writable-layer receipt before logical + // commit. The helper receives the copy read-only and does not delete it; + // this keeps rollback possible when Docker loses the helper acknowledgement. + // --volumes-from exposes shared mounts only; it cannot expose this + // container-local transaction directory to an immutable helper. + let receiptPath: string; + try { + const copiedReceipt = copyManagedStartupReceipt(transaction, deps); + if (!copiedReceipt) { + throw new Error("Managed-startup pending receipt disappeared before commit."); + } + receiptPath = copiedReceipt; + } catch (error) { + try { + quiesceManagedStartupContainer(transaction, deps); + } catch (stopError) { + throw new Error( + `Managed-startup receipt preservation failed and the new workload could not be quiesced: ${ + error instanceof Error ? error.message : String(error) + }; ${stopError instanceof Error ? stopError.message : String(stopError)}`, + ); + } + throw error; + } + let commitFailure: Error | null = null; + try { + verifyCopiedManagedStartupReceipt( + transaction, + transaction.profileFingerprint, + receiptPath, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + "pending", + deps, + ); + commitManagedStartupSharedState(transaction, deps); + cleanupReceiptBestEffort(receiptPath); + return { supervisorReady: true, failure: null }; + } catch (error) { + if (error instanceof DockerManagedStartupSharedStateCommitIndeterminateError) { + throw error; + } + commitFailure = error instanceof Error ? error : new Error(String(error)); + } + const failure = new Error( + `OpenShell supervisor reconnected, but managed shared-state logical commit validation failed: ${commitFailure.message}`, + ); + try { + quiesceManagedStartupContainer(transaction, deps); + } catch (stopError) { + throw new Error( + `${failure.message}; the new workload could not be quiesced: ${ + stopError instanceof Error ? stopError.message : String(stopError) + }`, + { cause: stopError }, + ); + } + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!input.patchResult && !input.retainContainerAfterRollback) { + removeFailedUnbackedContainer(transaction, deps); + } + return { supervisorReady: false, failure }; + } + + quiesceManagedStartupContainer(transaction, deps); + const receiptPath = copyManagedStartupReceipt(transaction, deps, true); + if (!receiptPath) { + if (!input.patchResult && !input.retainContainerAfterRollback) { + removeFailedUnbackedContainer(transaction, deps); + } + return { supervisorReady: false, failure: null }; + } + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!input.patchResult && !input.retainContainerAfterRollback) { + removeFailedUnbackedContainer(transaction, deps); + } + return { supervisorReady: false, failure: null }; +} diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts new file mode 100644 index 00000000000..cb069cb5bca --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -0,0 +1,484 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { expect, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import type { DockerContainerInspect } from "../docker-gpu-patch-types"; +import { encodeManagedStartupProfile, type ManagedStartupAgent } from "../managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import { + createManagedBootstrapPreparedAuthority, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapCompletionReceipt, + type ManagedBootstrapDurablePreparationReceipt, + type ManagedBootstrapHeldWorkloadHandle, + type ManagedBootstrapObservedSnapshot, + type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapReplacementHandle, +} from "./adapter"; +import type { DockerManagedBootstrapDeps } from "./docker"; +import { + type DockerManagedBootstrapJournal, + DockerManagedBootstrapJournalAcknowledgementLostError, + type DockerManagedBootstrapJournalPhase, + type DockerManagedBootstrapJournalStore, +} from "./docker-journal"; +import { normalizeDockerManagedBootstrapLaunchSpec } from "./docker-spec"; +import { parseManagedBootstrapEnvelope } from "./envelope"; + +export const IDENTITY = "1".repeat(64); +export const OLD_ID = "2".repeat(64); +export const NEW_ID = "3".repeat(64); +const CONFIG_ID = `sha256:${"4".repeat(64)}`; +const MANIFEST = `sha256:${"5".repeat(64)}` as const; +const REPOSITORY = "registry.example/nemoclaw/hermes"; +const IMAGE = `${REPOSITORY}@${MANIFEST}`; +const SUPERVISOR = ["/opt/openshell/bin/openshell-sandbox", "supervise"] as const; +export const SUPPORTED_AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; + +type FixtureCommandResult = { + readonly status: number; + readonly stdout?: string; + readonly stderr?: string; +}; + +export type DockerFixtureAcknowledgement = + | "container:create" + | "container:remove" + | "container:rename" + | "container:start" + | "container:stop" + | "journal:create" + | "journal:cutover" + | "journal:remove" + | "journal:rollback-authorized" + | "journal:staged" + | "journal:shared-state-committed"; + +export type DockerFixtureOptions = { + readonly agent?: ManagedStartupAgent; + readonly dockerStartResults?: Readonly>; + readonly journalTransitionFailures?: Partial< + Readonly> + >; + readonly lostAcknowledgements?: readonly DockerFixtureAcknowledgement[]; + readonly ownerId?: string; + readonly sharedState?: "committed" | "none" | "pending"; + readonly sharedStateCommitResult?: FixtureCommandResult; +}; + +function agentInputs(agent: ManagedStartupAgent = "hermes") { + const request = createManagedStartupRootApplyRequest({ + agent, + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agent, false, false)), + }); + const heldArgv = [ + "env", + "A=1", + "/usr/local/bin/nemoclaw-managed-startup-hold", + "--agent", + agent, + "--profile-fingerprint", + request.profileFingerprint, + "--bootstrap-identity", + IDENTITY, + ] as const; + return { + request, + heldArgv, + metadata: { "nemoclaw.ai/managed-profile": request.profileFingerprint }, + }; +} + +export const { heldArgv } = agentInputs(); +export const sandbox = { + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + driverId: "docker", +}; + +function shellArgv(argv: readonly string[]): string { + return argv.join(" "); +} + +function originalInspect(inputs = agentInputs()): DockerContainerInspect { + return { + Id: OLD_ID, + Image: CONFIG_ID, + Name: "/openshell-alpha", + Config: { + Image: IMAGE, + Env: ["A=1", `OPENSHELL_SANDBOX_COMMAND=${shellArgv(inputs.heldArgv)}`], + Labels: { + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-name": "alpha", + "openshell.ai/sandbox-id": "sandbox-alpha", + ...inputs.metadata, + }, + Entrypoint: [SUPERVISOR[0]], + Cmd: SUPERVISOR.slice(1), + User: "root", + WorkingDir: "/sandbox", + Hostname: "alpha", + }, + State: { Running: true, Paused: false, Restarting: false, Dead: false }, + HostConfig: { + Binds: ["/host/workspace:/sandbox:rw"], + NetworkMode: "openshell", + RestartPolicy: { Name: "unless-stopped" }, + CapDrop: ["NET_RAW"], + SecurityOpt: ["no-new-privileges"], + Ulimits: [{ Name: "nofile", Soft: 65_536, Hard: 65_536 }], + }, + NetworkSettings: { Networks: { openshell: { Aliases: ["openshell-alpha"] } } }, + }; +} + +export function authority(agent: ManagedStartupAgent = "hermes") { + const inputs = agentInputs(agent); + const inspect = originalInspect(inputs); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(inspect); + const plan = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandboxName: "alpha", + driverId: "docker", + image: { repository: REPOSITORY, manifestDigest: MANIFEST }, + profile: { agent, fingerprint: inputs.request.profileFingerprint }, + agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + intendedWorkloadArgv: ["env", "A=1", "nemoclaw-start"], + expectedSupervisorArgv: SUPERVISOR, + metadata: inputs.metadata, + }; + const handle: ManagedBootstrapHeldWorkloadHandle = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: inputs.heldArgv, + intendedWorkloadArgv: plan.intendedWorkloadArgv, + plan, + createReceipt: { sandbox, ready: true, readyAt: "2026-07-31T12:00:00.000Z" }, + }; + const snapshot: ManagedBootstrapObservedSnapshot = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + runtimeId: OLD_ID, + bootstrapIdentity: IDENTITY, + image: plan.image, + runtimeImageContentId: CONFIG_ID, + specHash: normalized.hash, + specCanonicalJson: normalized.canonicalJson, + agentIdentity: plan.agentIdentity, + supervisorArgv: SUPERVISOR, + heldWorkloadArgv: inputs.heldArgv, + metadata: inputs.metadata, + }; + return { handle, plan, request: inputs.request, snapshot }; +} + +function failFixture(message: string): never { + throw new Error(message); +} + +function readProtectedEnvelope(source: string): ReturnType { + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") throw new Error("test requires O_NOFOLLOW"); + const descriptor = fs.openSync(source, fs.constants.O_RDONLY | noFollow); + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + expect(Number(before.mode & 0o777n)).toBe(0o400); + const parsed = parseManagedBootstrapEnvelope(fs.readFileSync(descriptor, "utf8")); + const after = fs.fstatSync(descriptor, { bigint: true }); + expect(after.dev).toBe(before.dev); + expect(after.ino).toBe(before.ino); + expect(after.size).toBe(before.size); + expect(after.mtimeNs).toBe(before.mtimeNs); + expect(after.ctimeNs).toBe(before.ctimeNs); + return parsed; + } finally { + fs.closeSync(descriptor); + } +} + +export function fixture(options: DockerFixtureOptions = {}) { + let original: DockerContainerInspect | null = originalInspect(agentInputs(options.agent)); + let replacement: DockerContainerInspect | null = null; + let journal: DockerManagedBootstrapJournal | null = null; + let sharedState: "committed" | "none" | "pending" = options.sharedState ?? "none"; + const events: string[] = []; + const lostAcknowledgements = new Set(options.lostAcknowledgements ?? []); + const losesAcknowledgement = (operation: DockerFixtureAcknowledgement) => + lostAcknowledgements.has(operation); + const ok = (stdout = ""): FixtureCommandResult => ({ status: 0, stdout, stderr: "" }); + const copyJournal = () => (journal ? structuredClone(journal) : null); + const store: DockerManagedBootstrapJournalStore = { + create(value) { + journal = structuredClone(value); + events.push("journal:staged"); + if (losesAcknowledgement("journal:create")) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal create acknowledgement", + ); + } + }, + load: () => copyJournal(), + transition(_identity, expected, next) { + const current = + journal !== null && journal.phase === expected + ? journal + : failFixture("stale journal transition"); + journal = { ...current, phase: next }; + events.push(`journal:${next}`); + const injectedFailure = options.journalTransitionFailures?.[next]; + if (injectedFailure) throw injectedFailure; + if (losesAcknowledgement(`journal:${next}`)) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal transition acknowledgement", + ); + } + return structuredClone(journal); + }, + remove(_identity, expected) { + const current = journal; + void (current !== null && expected.includes(current.phase) + ? current + : failFixture("stale journal remove")); + journal = null; + events.push("journal:removed"); + if (losesAcknowledgement("journal:remove")) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal remove acknowledgement", + ); + } + }, + }; + const inspect = (reference: string): DockerContainerInspect => { + const candidates = [original, replacement].filter( + (value): value is DockerContainerInspect => value !== null, + ); + const found = candidates.find( + (value) => + value.Id === reference || String(value.Name ?? "").replace(/^\/+/u, "") === reference, + ); + return found ? structuredClone(found) : failFixture(`No such container: ${reference}`); + }; + const dockerCapture: NonNullable = vi.fn((args) => { + switch (args[0]) { + case "image": + return JSON.stringify([{ Id: CONFIG_ID, RepoDigests: [IMAGE] }]); + default: + return JSON.stringify([inspect(String(args[3] ?? ""))]); + } + }); + const dockerRun: NonNullable = vi.fn( + (args: readonly string[]) => { + switch (args[0]) { + case "create": { + events.push("create:replacement"); + const source = + original ?? failFixture("original disappeared before replacement creation"); + const name = String(args[args.indexOf("--name") + 1] ?? ""); + const entrypoint = String(args[args.indexOf("--entrypoint") + 1] ?? ""); + const imageIndex = args.indexOf(IMAGE); + const env = args.flatMap((value, index) => + value === "--env" ? [String(args[index + 1] ?? "")] : [], + ); + replacement = { + ...structuredClone(source), + Id: NEW_ID, + Name: `/${name}`, + Config: { + ...structuredClone(source.Config), + Image: IMAGE, + Env: env, + Entrypoint: [entrypoint], + Cmd: args.slice(imageIndex + 1), + }, + State: { Running: false, Paused: false, Restarting: false, Dead: false }, + }; + return losesAcknowledgement("container:create") + ? { status: 1, stdout: "", stderr: "lost create acknowledgement" } + : ok(NEW_ID); + } + case "ps": + return ok(original ? OLD_ID : ""); + case "inspect": { + const id = String(args[3] ?? ""); + try { + inspect(id); + return ok(`[{"Id":"${id}"}]`); + } catch { + return { status: 1, stderr: `Error response from daemon: No such container: ${id}` }; + } + } + case "cp": { + const sourceIndex = args[1] === "-a" ? 2 : 1; + const source = String(args[sourceIndex] ?? ""); + const destination = String(args[sourceIndex + 1] ?? ""); + const copyIntoContainer = () => { + events.push("stage:envelope"); + expect(readProtectedEnvelope(source).bootstrapIdentity).toBe(IDENTITY); + return ok(); + }; + const copyFromContainer = () => { + const receipt = source.split(":")[1]; + const expected = receipt?.includes("shared-state-commit") ? "committed" : "pending"; + return sharedState === expected + ? (() => { + fs.mkdirSync(destination, { recursive: true }); + return ok(); + })() + : { + status: 1, + stderr: `Error response from daemon: Could not find the file ${receipt} in container ${NEW_ID}`, + }; + }; + return source.includes(":") ? copyFromContainer() : copyIntoContainer(); + } + case "run": + switch (true) { + case args.includes("--shared-state-transaction-status"): + return ok(`${sharedState}\n`); + case args.includes("--rollback-shared-state-transaction"): + sharedState = "none"; + events.push("shared:rollback"); + return ok(); + } + break; + case "exec": + switch (true) { + case args.includes("--commit-shared-state-transaction"): { + const result = options.sharedStateCommitResult ?? ok(); + sharedState = result.status === 0 ? "committed" : sharedState; + events.push("shared:commit"); + return result; + } + case args.includes("--clear-shared-state-commit-receipt"): + sharedState = "none"; + events.push("shared:clear"); + return ok(); + } + break; + } + throw new Error(`unexpected Docker command: ${args.join(" ")}`); + }, + ); + const deps: DockerManagedBootstrapDeps = { + journalStore: store, + dockerCapture, + dockerRun, + dockerStop: vi.fn((id) => { + events.push(`stop:${id}`); + const target = id === OLD_ID ? original : replacement; + [target] + .filter((value): value is DockerContainerInspect => value?.State !== undefined) + .forEach((value) => { + value.State = { ...value.State, Running: false }; + }); + return losesAcknowledgement("container:stop") + ? { status: 1, stderr: "lost stop acknowledgement" } + : ok(); + }), + dockerRename: vi.fn((id, name) => { + events.push(`rename:${id}:${name}`); + const target = id === OLD_ID ? original : replacement; + [target] + .filter((value): value is DockerContainerInspect => value !== null) + .forEach((value) => { + value.Name = `/${name}`; + }); + return losesAcknowledgement("container:rename") + ? { status: 1, stderr: "lost rename acknowledgement" } + : ok(); + }), + dockerStart: vi.fn((id) => { + events.push(`start:${id}`); + const result = options.dockerStartResults?.[id] ?? ok(); + const target = id === OLD_ID ? original : replacement; + [target] + .filter( + (value): value is DockerContainerInspect => + value?.State !== undefined && result.status === 0, + ) + .forEach((value) => { + value.State = { ...value.State, Running: true }; + }); + return losesAcknowledgement("container:start") + ? { status: 1, stderr: "lost start acknowledgement" } + : result; + }), + dockerRm: vi.fn((id) => { + events.push(`rm:${id}`); + switch (id) { + case OLD_ID: + original = null; + break; + case NEW_ID: + replacement = null; + break; + } + return losesAcknowledgement("container:remove") + ? { status: 1, stderr: "lost rm acknowledgement" } + : ok(); + }), + runCaptureOpenshell: vi.fn(() => `Name: alpha\nID: ${options.ownerId ?? "sandbox-alpha"}\n`), + runOpenshell: vi.fn(() => ok()), + now: () => new Date("2026-07-31T12:30:00.000Z"), + }; + return { + deps, + events, + get journal() { + return journal; + }, + get original() { + return original; + }, + get replacement() { + return replacement; + }, + get sharedState() { + return sharedState; + }, + }; +} + +export function completion( + replacement: ManagedBootstrapReplacementHandle, +): ManagedBootstrapCompletionReceipt { + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + runtimeId: replacement.replacementRuntimeId, + image: replacement.image, + runtimeImageContentId: replacement.runtimeImageContentId, + originalSpecHash: replacement.originalSpecHash, + replacementSpecHash: replacement.replacementSpecHash, + profileFingerprint: replacement.profileFingerprint, + bootstrapIdentity: replacement.bootstrapIdentity, + transactionPending: true, + completedAt: "2026-07-31T12:15:00.000Z", + }; +} + +export function durablePreparation( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, +): ManagedBootstrapDurablePreparationReceipt { + const preparedAuthority = createManagedBootstrapPreparedAuthority({ + handle, + snapshot, + prepared, + }); + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + authorityFingerprint: preparedAuthority.authorityFingerprint, + recordId: `test-authority-${handle.plan.profile.agent}`, + recordedAt: "2026-07-31T12:10:00.000Z", + }; +} diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts new file mode 100644 index 00000000000..7979330b180 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -0,0 +1,433 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { assert, describe, expect, it, vi } from "vitest"; + +import { ManagedBootstrapOwnerCleanupRequiredError } from "./adapter"; +import { createDockerManagedBootstrapAdapter } from "./docker"; +import { + normalizeDockerManagedBootstrapLaunchSpec, + parseDockerManagedBootstrapLaunchSpec, +} from "./docker-spec"; +import { + authority, + completion, + durablePreparation, + fixture, + heldArgv, + IDENTITY, + NEW_ID, + OLD_ID, + SUPPORTED_AGENTS, +} from "./docker-test-fixture"; + +describe("Docker managed bootstrap adapter", () => { + it("publishes durable commit authority before deleting the rollback backup after lost acknowledgements", async () => { + const fake = fixture({ + lostAcknowledgements: [ + "container:create", + "container:remove", + "container:rename", + "container:start", + "container:stop", + "journal:create", + "journal:cutover", + "journal:remove", + "journal:shared-state-committed", + ], + sharedState: "pending", + }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + expect(fake.journal).toBeNull(); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + fake.events.push("authority:recorded"); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + const order = fake.events; + expect(order).toContain("authority:recorded"); + expect(order).toContain("journal:staged"); + expect(order.indexOf("journal:staged")).toBeGreaterThan(order.indexOf("authority:recorded")); + expect(order).toContain("journal:cutover"); + expect(order).toContain(`stop:${OLD_ID}`); + expect(order.indexOf("journal:cutover")).toBeLessThan(order.indexOf(`stop:${OLD_ID}`)); + expect(fake.journal).toMatchObject({ + phase: "cutover", + originalRuntimeId: OLD_ID, + replacementRuntimeId: NEW_ID, + }); + + await expect( + adapter.finalizeBootstrap({ + outcome: "commit", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: completion(replacement), + }), + ).resolves.toMatchObject({ outcome: "committed" }); + expect(fake.events).toContain("journal:shared-state-committed"); + expect(fake.events).toContain(`rm:${OLD_ID}`); + expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( + fake.events.indexOf(`rm:${OLD_ID}`), + ); + expect(fake.journal).toBeNull(); + expect(fake.sharedState).toBe("none"); + expect(fake.replacement?.Id).toBe(NEW_ID); + }); + + it("preserves commit validation failure details when the replacement cannot be quiesced", async () => { + const fake = fixture({ + sharedState: "pending", + sharedStateCommitResult: { status: 1, stderr: "injected commit failure" }, + }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority(); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + vi.mocked(fake.deps.dockerStop!).mockReturnValue({ + status: 1, + stderr: "injected quiesce failure", + }); + + await expect( + adapter.finalizeBootstrap({ + outcome: "commit", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: completion(replacement), + }), + ).rejects.toThrow( + /logical commit validation failed: Managed-startup shared-state commit helper failed.*injected commit failure.*new workload could not be quiesced.*injected quiesce failure/u, + ); + expect(fake.events).not.toContain("shared:rollback"); + }); + + it("publishes durable rollback authority before deleting the replacement after restart", async () => { + const fake = fixture({ + dockerStartResults: { + [NEW_ID]: { status: 1, stderr: "injected start failure" }, + }, + }); + const first = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await first.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + await expect( + first.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }), + ).rejects.toThrow("could not prove its exact replacement running"); + expect(fake.journal?.phase).toBe("cutover"); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect( + restarted.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement: null, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.events).toContain("journal:rollback-authorized"); + expect(fake.events).toContain(`rm:${NEW_ID}`); + expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( + fake.events.indexOf(`rm:${NEW_ID}`), + ); + expect(fake.journal).toBeNull(); + expect(fake.replacement).toBeNull(); + expect(fake.original).not.toBeNull(); + expect(fake.original?.Name).toBe("/openshell-alpha"); + expect(fake.original?.State?.Running).toBe(false); + }); + + it("recovers the pre-stop cutover crash state after adapter restart", async () => { + const fake = fixture({ + journalTransitionFailures: { + cutover: new Error("injected crash after durable cutover fence"), + }, + }); + const { handle, request: rootRequest, snapshot } = authority(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + await expect( + adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }), + ).rejects.toThrow("crash after durable cutover fence"); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + await expect( + createDockerManagedBootstrapAdapter(fake.deps).finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement: null, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.events).toContain("journal:rollback-authorized"); + expect(fake.events).toContain(`rm:${NEW_ID}`); + expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( + fake.events.indexOf(`rm:${NEW_ID}`), + ); + expect(fake.journal).toBeNull(); + }); + + it("fences rollback when image-owned shared state is already committed", async () => { + const fake = fixture({ sharedState: "committed" }); + const { handle, request: rootRequest, snapshot } = authority(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + const eventCount = fake.events.length; + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: null, + }), + ).rejects.toMatchObject({ name: "ManagedBootstrapDurableCommitCleanupPendingError" }); + expect(fake.journal?.phase).toBe("shared-state-committed"); + expect(fake.events.slice(eventCount)).toEqual(["journal:shared-state-committed"]); + }); + + it("rejects cutover before the exact durable authority receipt", async () => { + const fake = fixture(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const invalid = { + ...durablePreparation(handle, snapshot, prepared), + authorityFingerprint: "f".repeat(64), + }; + await expect( + adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: invalid, + }), + ).rejects.toThrow("exact durable prepared-authority receipt"); + expect(fake.journal).toBeNull(); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: null, + replacement: null, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.replacement).toBeNull(); + }); + + it.each( + SUPPORTED_AGENTS, + )("prepares, activates, and exactly rolls back the %s agent without a central switch", async (agent) => { + const fake = fixture({ agent, sharedState: "pending" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(agent); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request: rootRequest, + replacementOptions: { values: {} }, + }); + const durable = durablePreparation(handle, snapshot, prepared); + const replacement = await adapter.activateBootstrapReplacement({ + handle, + snapshot, + prepared, + durablePreparation: durable, + }); + await expect( + adapter.finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: durable, + replacement, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); + expect(fake.journal).toBeNull(); + expect(fake.replacement).toBeNull(); + expect( + vi.mocked(fake.deps.dockerRun!).mock.calls.some(([args]) => { + const agentIndex = args.indexOf("--agent"); + return ( + args.includes("--shared-state-transaction-status") && + agentIndex >= 0 && + args[agentIndex + 1] === agent + ); + }), + ).toBe(true); + }); + + it("rejects an empty intended workload argv with a precise boundary error", async () => { + const fake = fixture(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority(); + const plan = { ...handle.plan, intendedWorkloadArgv: [] }; + const emptyArgvHandle = { ...handle, intendedWorkloadArgv: [], plan }; + await expect( + adapter.prepareBootstrapReplacement({ + handle: emptyArgvHandle, + snapshot, + request, + replacementOptions: { values: {} }, + }), + ).rejects.toThrow( + "Managed bootstrap Docker replacement requires one bounded intended workload argv.", + ); + expect(fake.events).toContain("create:replacement"); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + }); + + it.each([ + "NODE_OPTIONS", + "NODE_PATH", + "LD_PRELOAD", + "BASH_ENV", + ])("rejects hostile %s from the launch snapshot before replacement creation", async (key) => { + const fake = fixture(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority(); + const parsed = parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson); + const hostileInspect = structuredClone(parsed.inspect); + hostileInspect.Config!.Env = [...(hostileInspect.Config!.Env ?? []), `${key}=/tmp/hostile`]; + const hostileSpec = normalizeDockerManagedBootstrapLaunchSpec(hostileInspect); + + await expect( + adapter.prepareBootstrapReplacement({ + handle, + snapshot: { + ...snapshot, + specHash: hostileSpec.hash, + specCanonicalJson: hostileSpec.canonicalJson, + }, + request, + replacementOptions: { values: {} }, + }), + ).rejects.toThrow(`Managed bootstrap refuses root-process injection environment '${key}'.`); + expect(fake.events).not.toContain("create:replacement"); + expect(fake.replacement).toBeNull(); + }); + + it("quiesces and retains an exact incomplete create when its mutable name is reused", async () => { + const fake = fixture({ ownerId: "sandbox-alpha-recreated" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, plan } = authority(); + await expect( + adapter.cleanupIncompleteCreate({ + plan, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: heldArgv, + createReceipt: handle.createReceipt, + }), + ).rejects.toMatchObject({ + name: "ManagedBootstrapOwnerCleanupRequiredError", + sandboxId: "sandbox-alpha", + runtimeId: OLD_ID, + }); + expect(fake.original).not.toBeNull(); + expect(fake.original?.State?.Running).toBe(false); + expect(fake.events).not.toContain(`rm:${OLD_ID}`); + expect(vi.mocked(fake.deps.runOpenshell!)).not.toHaveBeenCalled(); + }); + + it("retains a same-name workload that differs from the validated create receipt", async () => { + const replacementSandboxId = "sandbox-alpha-recreated"; + const fake = fixture({ ownerId: replacementSandboxId }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, plan } = authority(); + expect(fake.original).not.toBeNull(); + const labels = fake.original?.Config?.Labels; + assert(labels, "fixture labels are required"); + labels["openshell.ai/sandbox-id"] = replacementSandboxId; + + await expect( + adapter.cleanupIncompleteCreate({ + plan, + bootstrapIdentity: IDENTITY, + heldWorkloadArgv: heldArgv, + createReceipt: handle.createReceipt, + }), + ).rejects.toThrow(/does not match the exact validated create receipt/u); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + expect(fake.events).not.toContain(`rm:${OLD_ID}`); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts new file mode 100644 index 00000000000..03afb2ba280 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -0,0 +1,2949 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { + dockerRename as defaultDockerRename, + dockerRm as defaultDockerRm, + dockerStart as defaultDockerStart, + dockerStop as defaultDockerStop, +} from "../../adapters/docker/container"; +import { + dockerCapture as defaultDockerCapture, + dockerRun as defaultDockerRun, +} from "../../adapters/docker/run"; +import { parseOpenShellSandboxId } from "../../adapters/openshell/sandbox-identity"; +import { hasZeroDockerExitStatus } from "../docker-command-result"; +import { buildDockerGpuCloneRunArgs, dockerContainerName } from "../docker-gpu-patch-clone"; +import { + DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + DOCKER_GPU_PATCH_TIMEOUT_MS, +} from "../docker-gpu-patch-constants"; +import type { + DockerContainerInspect, + DockerGpuPatchDeps, + DockerGpuPatchMode, + DockerGpuPatchModeKind, + DockerUlimit, +} from "../docker-gpu-patch-types"; +import { waitForOpenShellSupervisorReconnect } from "../docker-gpu-supervisor-reconnect"; +import { openshellSandboxCommandEnvValue } from "../docker-startup-command-env"; +import { + OPENSHELL_MANAGED_BY_LABEL, + OPENSHELL_MANAGED_BY_VALUE, + OPENSHELL_SANDBOX_ID_LABEL, + OPENSHELL_SANDBOX_NAME_LABEL, + queryOpenShellDockerSandboxContainers, +} from "../openshell-docker-sandbox-containers"; +import { cleanupTempDir, secureTempFile } from "../temp-files"; +import { + assertManagedBootstrapIdentity, + assertManagedBootstrapSafeProcessEnvironmentKey, + attachManagedBootstrapRollbackError, + createManagedBootstrapIdentity, + createManagedBootstrapPreparedAuthority, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapAdapter, + ManagedBootstrapCommitStateIndeterminateError, + type ManagedBootstrapCompletionReceipt, + type ManagedBootstrapDiscoveredWorkload, + type ManagedBootstrapDiscoveryInput, + ManagedBootstrapDurableCommitCleanupPendingError, + type ManagedBootstrapDurablePreparationReceipt, + type ManagedBootstrapFinalizationReceipt, + type ManagedBootstrapHeldWorkloadHandle, + type ManagedBootstrapIncompleteCreateCleanupInput, + type ManagedBootstrapObservedSnapshot, + ManagedBootstrapOwnerCleanupRequiredError, + type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapReplacementHandle, + type ManagedBootstrapReplacementOptions, + type ManagedBootstrapSandboxIdentity, + renderManagedBootstrapHeldCommand, +} from "./adapter"; +import { + createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapJournal, + DockerManagedBootstrapJournalAcknowledgementLostError, + type DockerManagedBootstrapJournalStore, + parseDockerManagedBootstrapJournal, + serializeDockerManagedBootstrapJournal, +} from "./docker-journal"; +import { + clearDockerManagedStartupSharedStateCommitReceipt, + DockerManagedStartupSharedStateCommitIndeterminateError, + finalizeDockerManagedStartupSharedState, + probeDockerManagedStartupSharedState, +} from "./docker-shared-state"; +import { + normalizeDockerManagedBootstrapLaunchSpec, + parseDockerManagedBootstrapLaunchSpec, + parseExactDockerContainerInspect, +} from "./docker-spec"; +import { + MANAGED_BOOTSTRAP_COMPLETION_FILE, + MANAGED_BOOTSTRAP_REQUEST_FILE, + parseManagedBootstrapImageCompletion, + serializeManagedBootstrapEnvelope, +} from "./envelope"; + +const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; +const FULL_SHA256_RE = /^sha256:[a-f0-9]{64}$/u; +const MAX_ARGV_BYTES = 128 * 1024; +const MAX_CONTAINER_NAME_LENGTH = 253; +const REQUEST_TEMP_PREFIX = "nemoclaw-managed-bootstrap-request"; +const COMPLETION_TEMP_PREFIX = "nemoclaw-managed-bootstrap-completion"; +const COMPLETION_MAX_BYTES = 4096; +const DOCKER_DRIVER_ID = "docker"; + +export const MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE = "/usr/local/bin/nemoclaw-managed-bootstrap"; + +type DockerCommandResult = { + readonly status?: number | null; + readonly stdout?: string | Buffer | null; + readonly stderr?: string | Buffer | null; + readonly error?: Error | null; +}; + +export type DockerManagedBootstrapDeps = Pick< + DockerGpuPatchDeps, + | "dockerCapture" + | "dockerRename" + | "dockerRm" + | "dockerRun" + | "dockerStart" + | "dockerStop" + | "runCaptureOpenshell" + | "runOpenshell" + | "sleep" + | "now" +> & { + readonly createBootstrapIdentity?: () => string; + readonly journalStore?: DockerManagedBootstrapJournalStore; + /** Canonical gateway-scoped state root; required when no store is injected. */ + readonly stateRoot?: string; +}; + +type ResolvedDeps = Required< + Pick< + DockerManagedBootstrapDeps, + | "dockerCapture" + | "dockerRename" + | "dockerRm" + | "dockerRun" + | "dockerStart" + | "dockerStop" + | "journalStore" + | "now" + | "createBootstrapIdentity" + > +> & + DockerManagedBootstrapDeps; + +type DockerBootstrapTransaction = DockerManagedBootstrapJournal; + +interface DockerBootstrapRollbackTombstone { + readonly profileFingerprint: string; + readonly imageReference: string; + readonly receipt: ManagedBootstrapFinalizationReceipt; +} + +export interface DockerManagedBootstrapAdapter extends ManagedBootstrapAdapter {} + +function resolveDeps(deps: DockerManagedBootstrapDeps): ResolvedDeps { + const journalStore = + deps.journalStore ?? + (deps.stateRoot ? createFileDockerManagedBootstrapJournalStore(deps.stateRoot) : null); + if (!journalStore) { + throw new Error( + "Managed bootstrap Docker requires its canonical state root or an injected journal store.", + ); + } + return { + dockerCapture: defaultDockerCapture, + dockerRename: defaultDockerRename, + dockerRm: defaultDockerRm, + dockerRun: defaultDockerRun, + dockerStart: defaultDockerStart, + dockerStop: defaultDockerStop, + journalStore, + now: () => new Date(), + createBootstrapIdentity: createManagedBootstrapIdentity, + ...deps, + }; +} + +function commandDetail(result: DockerCommandResult): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}` + .trim() + .slice(-1200); +} + +function isExactMissingDockerContainer(containerId: string, result: DockerCommandResult): boolean { + const escapedContainerId = containerId.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const patterns = [ + new RegExp( + `^(?:Error response from daemon: )?No such (?:container|object): ${escapedContainerId}$`, + "u", + ), + new RegExp(`^Error: No such (?:container|object): ${escapedContainerId}$`, "u"), + ]; + return [result.stderr, result.stdout, result.error?.message] + .map((value) => String(value ?? "").trim()) + .filter(Boolean) + .some((detail) => patterns.some((pattern) => pattern.test(detail))); +} + +function probeExactDockerContainerAbsence( + containerId: string, + deps: ResolvedDeps, +): "absent" | "present" | "unknown" { + let result: DockerCommandResult; + try { + result = deps.dockerRun(["inspect", "--type", "container", containerId], { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + } catch { + return "unknown"; + } + if (hasZeroDockerExitStatus(result)) return "present"; + return isExactMissingDockerContainer(containerId, result) ? "absent" : "unknown"; +} + +function assertZero(result: DockerCommandResult, message: string): void { + if (!hasZeroDockerExitStatus(result)) { + throw new Error(`${message}: ${commandDetail(result) || "Docker command failed"}`); + } +} + +function exactStringArray(value: unknown, label: string): string[] { + if (value === null || value === undefined) return []; + const values = typeof value === "string" ? [value] : value; + if ( + !Array.isArray(values) || + values.some( + (item) => + typeof item !== "string" || + item.length === 0 || + item.includes("\0") || + Buffer.byteLength(item, "utf8") > 64 * 1024, + ) + ) { + throw new Error(`Managed bootstrap Docker ${label} is not an exact bounded argv.`); + } + const result = [...values]; + if (Buffer.byteLength(JSON.stringify(result), "utf8") > MAX_ARGV_BYTES) { + throw new Error(`Managed bootstrap Docker ${label} exceeds its bounded argv transport.`); + } + return result; +} + +function exactSupervisorArgv(inspect: DockerContainerInspect): readonly string[] { + const argv = [ + ...exactStringArray(inspect.Config?.Entrypoint, "entrypoint"), + ...exactStringArray(inspect.Config?.Cmd, "command"), + ]; + if (argv.length === 0 || !argv[0]?.startsWith("/")) { + throw new Error( + "Managed bootstrap requires one bounded absolute supervisor argv from Docker inspect.", + ); + } + return Object.freeze(argv); +} + +function exactArrayEqual(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + +function envValue(env: readonly string[] | null | undefined, key: string): string | null { + const prefix = `${key}=`; + const matches = (env ?? []).filter((value) => value.startsWith(prefix)); + return matches.length === 1 ? (matches[0]?.slice(prefix.length) ?? null) : null; +} + +function assertNoRootProcessInjectionEnvironment(env: readonly string[] | null | undefined): void { + for (const entry of env ?? []) { + const separator = entry.indexOf("="); + const key = separator < 0 ? entry : entry.slice(0, separator); + try { + assertManagedBootstrapSafeProcessEnvironmentKey(key); + } catch { + throw new Error(`Managed bootstrap refuses root-process injection environment '${key}'.`); + } + } +} + +function assertRootSupervisor(inspect: DockerContainerInspect): void { + const user = String(inspect.Config?.User ?? "") + .trim() + .toLowerCase(); + if (!["", "0", "0:0", "root", "root:root"].includes(user)) { + throw new Error("Managed bootstrap Docker workload must retain a root supervisor user."); + } +} + +function isStableRunning(inspect: DockerContainerInspect): boolean { + return inspect.State?.Running !== true || + inspect.State.Paused === true || + inspect.State.Restarting === true || + inspect.State.Dead === true + ? false + : true; +} + +function assertStableRunning(inspect: DockerContainerInspect, label: string): void { + if (!isStableRunning(inspect)) { + throw new Error(`Managed bootstrap Docker ${label} is not stably running.`); + } +} + +function isExplicitlyStopped(inspect: DockerContainerInspect): boolean { + return ( + inspect.State?.Running === false && + inspect.State.Paused === false && + inspect.State.Restarting === false && + inspect.State.Dead === false + ); +} + +function assertExplicitlyStopped(inspect: DockerContainerInspect, label: string): void { + if (!isExplicitlyStopped(inspect)) { + throw new Error(`Managed bootstrap Docker ${label} is not explicitly stopped.`); + } +} + +function expectedImageReference(repository: string, manifestDigest: string): string { + if ( + repository.length === 0 || + repository !== repository.trim() || + repository.includes("@") || + repository.includes("\0") || + !FULL_SHA256_RE.test(manifestDigest) + ) { + throw new Error("Managed bootstrap image repository/manifest identity is invalid."); + } + return `${repository}@${manifestDigest}`; +} + +function assertImage( + inspect: DockerContainerInspect, + image: ManagedBootstrapHeldWorkloadHandle["plan"]["image"], + deps: ResolvedDeps, +): string { + const runtimeContentId = String(inspect.Image ?? "").toLowerCase(); + if (!FULL_SHA256_RE.test(runtimeContentId)) { + throw new Error("Managed bootstrap Docker image does not have an immutable local content ID."); + } + const expectedReference = expectedImageReference(image.repository, image.manifestDigest); + const configuredImage = String(inspect.Config?.Image ?? "").trim(); + if (configuredImage !== expectedReference) { + throw new Error( + "Managed bootstrap Docker configured image is not the exact repository@manifestDigest.", + ); + } + const imageOutput = deps.dockerCapture(["image", "inspect", expectedReference], { + ignoreError: false, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + let parsed: unknown; + try { + parsed = JSON.parse(imageOutput); + } catch { + throw new Error("Managed bootstrap Docker image evidence is malformed."); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("Managed bootstrap Docker image evidence is not exact."); + } + const evidence = parsed[0] as { + readonly Id?: unknown; + readonly RepoDigests?: unknown; + }; + const evidenceId = String(evidence.Id ?? "").toLowerCase(); + const repoDigests = Array.isArray(evidence.RepoDigests) + ? evidence.RepoDigests.filter((value): value is string => typeof value === "string") + : []; + if (evidenceId !== runtimeContentId || !repoDigests.includes(expectedReference)) { + throw new Error( + "Managed bootstrap Docker image manifest evidence does not match its local content ID.", + ); + } + return runtimeContentId; +} + +function assertMetadata( + inspect: DockerContainerInspect, + sandbox: ManagedBootstrapHeldWorkloadHandle["sandbox"], + metadata: Readonly>, +): void { + const labels = inspect.Config?.Labels ?? {}; + if ( + labels[OPENSHELL_MANAGED_BY_LABEL] !== OPENSHELL_MANAGED_BY_VALUE || + labels[OPENSHELL_SANDBOX_NAME_LABEL] !== sandbox.sandboxName || + labels[OPENSHELL_SANDBOX_ID_LABEL] !== sandbox.sandboxId + ) { + throw new Error( + "Managed bootstrap Docker workload does not match the durable OpenShell sandbox identity.", + ); + } + for (const [key, value] of Object.entries(metadata)) { + if (labels[key] !== value) { + throw new Error(`Managed bootstrap Docker metadata label '${key}' changed.`); + } + } +} + +function assertHeldCommand( + inspect: DockerContainerInspect, + heldWorkloadArgv: readonly string[], + bootstrapIdentity: string, +): void { + assertManagedBootstrapIdentity(bootstrapIdentity); + const expected = openshellSandboxCommandEnvValue(heldWorkloadArgv); + const observed = envValue(inspect.Config?.Env, "OPENSHELL_SANDBOX_COMMAND"); + if (!expected || observed !== expected) { + throw new Error( + "Managed bootstrap Docker workload does not contain the exact identity-bound hold.", + ); + } + const identityIndexes = heldWorkloadArgv + .map((value, index) => (value === bootstrapIdentity ? index : -1)) + .filter((index) => index >= 0); + if (identityIndexes.length !== 1) { + throw new Error("Managed bootstrap hold does not contain exactly one bootstrap identity."); + } +} + +function assertBootstrapIdentityInObservedHold( + inspect: DockerContainerInspect, + bootstrapIdentity: string, +): void { + assertManagedBootstrapIdentity(bootstrapIdentity); + const observed = envValue(inspect.Config?.Env, "OPENSHELL_SANDBOX_COMMAND"); + if (!observed) { + throw new Error("Managed bootstrap Docker workload is missing its held command."); + } + const occurrences = observed.split(bootstrapIdentity).length - 1; + if (occurrences !== 1) { + throw new Error( + "Managed bootstrap Docker held command does not contain one exact bootstrap identity.", + ); + } +} + +function inspectExact(containerId: string, deps: ResolvedDeps): DockerContainerInspect { + if (!FULL_CONTAINER_ID_RE.test(containerId)) { + throw new Error("Managed bootstrap requires one full lowercase Docker container ID."); + } + const output = deps.dockerCapture(["inspect", "--type", "container", containerId], { + ignoreError: false, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + const inspect = parseExactDockerContainerInspect(output); + if (String(inspect.Id ?? "").toLowerCase() !== containerId) { + throw new Error("Managed bootstrap Docker workload identity changed during inspection."); + } + return inspect; +} + +function inspectDockerContainerReference( + reference: string, + deps: ResolvedDeps, +): DockerContainerInspect { + if ( + reference.length === 0 || + reference !== reference.trim() || + reference.includes("\0") || + Buffer.byteLength(reference, "utf8") > MAX_CONTAINER_NAME_LENGTH + ) { + throw new Error("Managed bootstrap Docker lookup reference is invalid."); + } + const output = deps.dockerCapture(["inspect", "--type", "container", reference], { + ignoreError: false, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + const inspect = parseExactDockerContainerInspect(output); + const runtimeId = String(inspect.Id ?? "").toLowerCase(); + if (!FULL_CONTAINER_ID_RE.test(runtimeId)) { + throw new Error("Managed bootstrap Docker lookup did not resolve one full runtime ID."); + } + return inspect; +} + +function tryInspectExact(containerId: string, deps: ResolvedDeps): DockerContainerInspect | null { + try { + return inspectExact(containerId, deps); + } catch { + return null; + } +} + +function backupName(originalName: string, bootstrapIdentity: string): string { + const suffix = `-nemoclaw-bootstrap-${bootstrapIdentity.slice(0, 20)}`; + return `${originalName.slice(0, Math.max(1, MAX_CONTAINER_NAME_LENGTH - suffix.length))}${suffix}`; +} + +function replacementStagingName(originalName: string, bootstrapIdentity: string): string { + const suffix = `-nemoclaw-staged-${bootstrapIdentity.slice(0, 20)}`; + return `${originalName.slice(0, Math.max(1, MAX_CONTAINER_NAME_LENGTH - suffix.length))}${suffix}`; +} + +function writeProtectedEnvelope( + bootstrapIdentity: string, + request: Parameters[0]["rootApplyRequest"], +): string { + const file = secureTempFile(REQUEST_TEMP_PREFIX, ".json"); + try { + fs.writeFileSync( + file, + serializeManagedBootstrapEnvelope({ bootstrapIdentity, rootApplyRequest: request }), + { encoding: "utf8", flag: "wx", mode: 0o400 }, + ); + fs.chmodSync(file, 0o400); + const stat = fs.lstatSync(file); + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.nlink !== 1 || + (stat.mode & 0o777) !== 0o400 + ) { + throw new Error("Managed bootstrap request source is not one protected 0400 file."); + } + return file; + } catch (error) { + cleanupTempDir(file, REQUEST_TEMP_PREFIX); + throw error; + } +} + +function readProtectedImageCompletion( + replacementRuntimeId: string, + deps: ResolvedDeps, +): ReturnType { + const file = secureTempFile(COMPLETION_TEMP_PREFIX, ".json"); + let descriptor: number | undefined; + try { + const copied = deps.dockerRun( + ["cp", `${replacementRuntimeId}:${MANAGED_BOOTSTRAP_COMPLETION_FILE}`, file], + { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }, + ); + assertZero(copied, "Managed bootstrap could not retrieve its image completion receipt"); + descriptor = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + const before = fs.fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.isSymbolicLink() || + before.nlink !== 1n || + Number(before.mode & 0o777n) !== 0o444 || + before.size < 1n || + before.size > BigInt(COMPLETION_MAX_BYTES) + ) { + throw new Error("Managed bootstrap image completion is not one protected bounded 0444 file."); + } + const bytes = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < bytes.length) { + const count = fs.readSync(descriptor, bytes, offset, bytes.length - offset, offset); + if (count === 0) break; + offset += count; + } + const after = fs.fstatSync(descriptor, { bigint: true }); + if ( + offset !== bytes.length || + after.dev !== before.dev || + after.ino !== before.ino || + after.size !== before.size || + after.mtimeNs !== before.mtimeNs || + after.ctimeNs !== before.ctimeNs || + after.mode !== before.mode || + after.nlink !== before.nlink + ) { + throw new Error("Managed bootstrap image completion changed during stable read."); + } + return parseManagedBootstrapImageCompletion(bytes.toString("utf8")); + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + cleanupTempDir(file, COMPLETION_TEMP_PREFIX); + } +} + +function parseRequiredUlimits(value: unknown): DockerUlimit[] { + if (value === undefined) return []; + if ( + !Array.isArray(value) || + value.some((entry) => typeof entry !== "string" || entry.includes("\0")) + ) { + throw new Error("Managed bootstrap Docker requiredUlimits must be string entries."); + } + return value.map((entry) => { + const match = /^([a-z][a-z0-9_]*)=(\d+):(\d+)$/u.exec(entry); + if (!match) { + throw new Error(`Managed bootstrap Docker ulimit '${entry}' is invalid.`); + } + const soft = Number(match[2]); + const hard = Number(match[3]); + if (!Number.isSafeInteger(soft) || !Number.isSafeInteger(hard) || hard < soft) { + throw new Error(`Managed bootstrap Docker ulimit '${entry}' is invalid.`); + } + return { name: match[1] as string, soft, hard }; + }); +} + +function replacementPlan(options: ManagedBootstrapReplacementOptions): { + readonly mode: DockerGpuPatchMode; + readonly requiredUlimits: readonly DockerUlimit[]; + readonly extraGroupGids: readonly string[]; +} { + const allowed = new Set([ + "gpuModeArgs", + "gpuModeDevice", + "gpuModeKind", + "gpuModeLabel", + "extraGroupGids", + "requiredUlimits", + ]); + const unknown = Object.keys(options.values).filter((key) => !allowed.has(key)); + if (unknown.length > 0) { + throw new Error( + `Managed bootstrap Docker replacement options are unsupported: ${unknown.sort().join(", ")}.`, + ); + } + const kind = String(options.values.gpuModeKind ?? "startup-command") as DockerGpuPatchModeKind; + if (!["gpus", "nvidia-runtime", "cdi", "startup-command"].includes(kind)) { + throw new Error(`Managed bootstrap Docker GPU mode '${kind}' is invalid.`); + } + const args = exactStringArray(options.values.gpuModeArgs ?? [], "GPU mode arguments"); + return { + mode: { + kind, + label: String(options.values.gpuModeLabel ?? "managed bootstrap"), + device: String(options.values.gpuModeDevice ?? ""), + args, + }, + extraGroupGids: exactStringArray(options.values.extraGroupGids ?? [], "extra group GIDs").map( + (value) => { + if (!/^\d+$/u.test(value)) { + throw new Error(`Managed bootstrap Docker supplementary group '${value}' is invalid.`); + } + return value; + }, + ), + requiredUlimits: parseRequiredUlimits(options.values.requiredUlimits), + }; +} + +function replacementCommand( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, +): readonly string[] { + return Object.freeze([ + "--agent", + handle.plan.profile.agent, + "--profile-fingerprint", + handle.plan.profile.fingerprint, + "--bootstrap-identity", + handle.bootstrapIdentity, + "--agent-uid", + String(snapshot.agentIdentity.uid), + "--agent-gid", + String(snapshot.agentIdentity.gid), + "--agent-workdir", + snapshot.agentIdentity.workdir, + "--request-file", + MANAGED_BOOTSTRAP_REQUEST_FILE, + "--", + ...snapshot.supervisorArgv, + ]); +} + +function assertReplacementBoundary( + inspect: DockerContainerInspect, + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, +): void { + const entrypoint = exactStringArray(inspect.Config?.Entrypoint, "replacement entrypoint"); + const command = exactStringArray(inspect.Config?.Cmd, "replacement command"); + if ( + !exactArrayEqual(entrypoint, [MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE]) || + !exactArrayEqual(command, replacementCommand(handle, snapshot)) + ) { + throw new Error("Managed bootstrap Docker replacement process boundary changed."); + } + const intended = openshellSandboxCommandEnvValue(handle.intendedWorkloadArgv); + if (envValue(inspect.Config?.Env, "OPENSHELL_SANDBOX_COMMAND") !== intended) { + throw new Error( + "Managed bootstrap Docker replacement did not restore the intended sandbox command.", + ); + } +} + +const REPLACED_GPU_ENV_KEYS = new Set([ + "NVIDIA_DISABLE_REQUIRE", + "NVIDIA_DRIVER_CAPABILITIES", + "NVIDIA_REQUIRE_CUDA", + "NVIDIA_VISIBLE_DEVICES", +]); + +function canonicalObject(text: string): Record { + const value = JSON.parse(text) as unknown; + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("Managed bootstrap normalized Docker spec is not an object."); + } + return value as Record; +} + +function objectField(record: Record, key: string): Record { + const value = record[key]; + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`Managed bootstrap normalized Docker spec is missing ${key}.`); + } + return value as Record; +} + +function exactJson(value: unknown): string { + return JSON.stringify(value ?? null); +} + +function stringSet(value: unknown, label: string): string[] { + const values = exactStringArray(value ?? [], label); + if (new Set(values).size !== values.length) { + throw new Error(`Managed bootstrap Docker ${label} contains duplicate entries.`); + } + return values.sort(); +} + +function assertExactStringSet(observed: unknown, expected: readonly string[], label: string): void { + if (!exactArrayEqual(stringSet(observed, label), [...expected].sort())) { + throw new Error(`Managed bootstrap Docker ${label} changed outside declared deltas.`); + } +} + +function modeEnvironment(mode: DockerGpuPatchMode): string[] { + const values: string[] = []; + for (let index = 0; index < mode.args.length; index += 1) { + if (mode.args[index] === "--env") { + const value = mode.args[index + 1]; + if (!value || !value.includes("=")) { + throw new Error("Managed bootstrap Docker GPU mode has an invalid environment delta."); + } + values.push(value); + index += 1; + } + } + return values; +} + +function assertExactEnvironmentDelta( + original: Record, + replacement: Record, + mode: DockerGpuPatchMode, + intendedSandboxCommand: string, +): void { + const gpuAugment = mode.kind !== "startup-command"; + const originalEnv = exactStringArray(original.Env ?? [], "original environment"); + const expected = [ + ...modeEnvironment(mode), + ...originalEnv + .filter((entry) => !gpuAugment || !REPLACED_GPU_ENV_KEYS.has(entry.split("=", 1)[0] ?? "")) + .map((entry) => + entry.startsWith("OPENSHELL_SANDBOX_COMMAND=") + ? `OPENSHELL_SANDBOX_COMMAND=${intendedSandboxCommand}` + : entry, + ), + ]; + const observed = exactStringArray(replacement.Env ?? [], "replacement environment"); + if (!exactArrayEqual(observed, expected)) { + throw new Error( + "Managed bootstrap Docker replacement environment changed outside declared deltas.", + ); + } +} + +function canonicalUlimits(value: unknown, label: string): string { + if (!Array.isArray(value)) { + if (value === undefined || value === null) return "[]"; + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + const normalized = value.map((entry) => { + if (typeof entry !== "object" || entry === null || Array.isArray(entry)) { + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + const record = entry as Record; + const name = String(record.Name ?? ""); + const soft = record.Soft; + const hard = record.Hard; + if (!name || !Number.isSafeInteger(soft) || !Number.isSafeInteger(hard)) { + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + return { Hard: hard, Name: name, Soft: soft }; + }); + if (new Set(normalized.map((entry) => entry.Name)).size !== normalized.length) { + throw new Error(`Managed bootstrap Docker ${label} contains duplicate entries.`); + } + return JSON.stringify(normalized.sort((left, right) => left.Name.localeCompare(right.Name))); +} + +function expectedUlimits(original: unknown, required: readonly DockerUlimit[]): string { + const existing = JSON.parse(canonicalUlimits(original, "original ulimits")) as Array<{ + Hard: number; + Name: string; + Soft: number; + }>; + const merged = new Map(existing.map((entry) => [entry.Name, entry])); + for (const requiredEntry of required) { + merged.set(requiredEntry.name, { + Name: requiredEntry.name, + Soft: requiredEntry.soft, + Hard: requiredEntry.hard, + }); + } + return JSON.stringify( + [...merged.values()].sort((left, right) => left.Name.localeCompare(right.Name)), + ); +} + +function assertExactDeviceRequests( + original: unknown, + observed: unknown, + mode: DockerGpuPatchMode, +): void { + if (mode.kind === "startup-command") { + if (exactJson(observed) !== exactJson(original)) { + throw new Error("Managed bootstrap Docker device requests were not preserved exactly."); + } + return; + } + if (Array.isArray(original) && original.length > 0) { + throw new Error( + "Managed bootstrap Docker GPU augmentation cannot replace an existing device request.", + ); + } + const requests = Array.isArray(observed) ? observed : []; + if (mode.kind === "nvidia-runtime") { + if (requests.length !== 0) { + throw new Error( + "Managed bootstrap Docker NVIDIA runtime added an undeclared device request.", + ); + } + return; + } + if (requests.length !== 1 || typeof requests[0] !== "object" || requests[0] === null) { + throw new Error("Managed bootstrap Docker GPU mode did not add one exact device request."); + } + const request = requests[0] as Record; + if (mode.kind === "gpus") { + const all = mode.device === "all"; + const expectedIds = all ? [] : [mode.device]; + const ids = Array.isArray(request.DeviceIDs) ? request.DeviceIDs : []; + if ( + String(request.Driver ?? "") !== "" || + Number(request.Count) !== (all ? -1 : 0) || + !exactArrayEqual(ids.map(String), expectedIds) || + exactJson(request.Capabilities) !== JSON.stringify([["gpu"]]) || + exactJson(request.Options ?? {}) !== "{}" + ) { + throw new Error("Managed bootstrap Docker --gpus request changed outside its exact delta."); + } + return; + } + const ids = Array.isArray(request.DeviceIDs) ? request.DeviceIDs.map(String) : []; + if ( + request.Driver !== "cdi" || + ![-1, 0].includes(Number(request.Count ?? 0)) || + !exactArrayEqual(ids, [mode.device]) || + (request.Capabilities != null && + (!Array.isArray(request.Capabilities) || request.Capabilities.length > 0)) || + exactJson(request.Options ?? {}) !== "{}" + ) { + throw new Error("Managed bootstrap Docker CDI request changed outside its exact delta."); + } +} + +function scrubVerifiedReplacementDeltas(canonicalJson: string): string { + const root = canonicalObject(canonicalJson); + const inspect = objectField(root, "inspect"); + const config = objectField(inspect, "Config"); + const host = objectField(inspect, "HostConfig"); + config.Image = ""; + config.Entrypoint = [""]; + config.Cmd = [""]; + config.Env = ""; + for (const key of [ + "CapAdd", + "DeviceRequests", + "Devices", + "GroupAdd", + "Runtime", + "SecurityOpt", + "Ulimits", + ]) { + host[key] = ``; + } + return JSON.stringify(root); +} + +function assertReplacementMatchesIntent( + originalCanonicalJson: string, + replacement: DockerContainerInspect, + authoritativeName: string, + plan: { + readonly mode: DockerGpuPatchMode; + readonly requiredUlimits: readonly DockerUlimit[]; + readonly extraGroupGids: readonly string[]; + }, + intendedSandboxCommand: string, +): string { + const original = canonicalObject(originalCanonicalJson); + const originalInspect = objectField(original, "inspect"); + const originalConfig = objectField(originalInspect, "Config"); + const originalHost = objectField(originalInspect, "HostConfig"); + const replacementSpec = normalizeDockerManagedBootstrapLaunchSpec({ + ...replacement, + Name: `/${authoritativeName}`, + }); + const observed = canonicalObject(replacementSpec.canonicalJson); + const observedInspect = objectField(observed, "inspect"); + const observedConfig = objectField(observedInspect, "Config"); + const observedHost = objectField(observedInspect, "HostConfig"); + const gpuAugment = plan.mode.kind !== "startup-command"; + assertExactEnvironmentDelta(originalConfig, observedConfig, plan.mode, intendedSandboxCommand); + assertExactStringSet( + observedHost.CapAdd, + [ + ...stringSet(originalHost.CapAdd, "original capability additions"), + ...(gpuAugment ? ["SYS_PTRACE"] : []), + ].filter((value, index, values) => values.indexOf(value) === index), + "capability additions", + ); + const originalSecurity = stringSet(originalHost.SecurityOpt, "original security options"); + assertExactStringSet( + observedHost.SecurityOpt, + [ + ...originalSecurity, + ...(gpuAugment && !originalSecurity.some((value) => value.startsWith("apparmor")) + ? ["apparmor=unconfined"] + : []), + ], + "security options", + ); + if (exactJson(observedHost.Devices) !== exactJson(originalHost.Devices)) { + throw new Error("Managed bootstrap Docker non-GPU devices were not preserved exactly."); + } + assertExactDeviceRequests(originalHost.DeviceRequests, observedHost.DeviceRequests, plan.mode); + const expectedRuntime = plan.mode.kind === "nvidia-runtime" ? "nvidia" : originalHost.Runtime; + if (exactJson(observedHost.Runtime) !== exactJson(expectedRuntime)) { + throw new Error("Managed bootstrap Docker runtime changed outside its selected GPU delta."); + } + assertExactStringSet( + observedHost.GroupAdd, + [ + ...stringSet(originalHost.GroupAdd, "original supplementary groups"), + ...plan.extraGroupGids, + ].filter((value, index, values) => values.indexOf(value) === index), + "supplementary groups", + ); + if ( + canonicalUlimits(observedHost.Ulimits, "replacement ulimits") !== + expectedUlimits(originalHost.Ulimits, plan.requiredUlimits) + ) { + throw new Error("Managed bootstrap Docker ulimits changed outside declared requirements."); + } + const expectedPreserved = scrubVerifiedReplacementDeltas(originalCanonicalJson); + const observedPreserved = scrubVerifiedReplacementDeltas(replacementSpec.canonicalJson); + if (observedPreserved !== expectedPreserved) { + throw new Error( + "Managed bootstrap Docker replacement normalized spec changed outside declared deltas.", + ); + } + return replacementSpec.hash; +} + +function inspectTransactionRuntime( + transaction: DockerBootstrapTransaction, + runtimeId: string, + deps: ResolvedDeps, +): DockerContainerInspect | null { + const presence = probeExactDockerContainerAbsence(runtimeId, deps); + if (presence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId, + detail: "exact Docker runtime presence could not be proven before mutation", + }); + } + if (presence === "absent") return null; + try { + return inspectExact(runtimeId, deps); + } catch (error) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId, + detail: `exact Docker runtime inspection became unavailable: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + } +} + +function assertTransactionOriginal( + transaction: DockerBootstrapTransaction, + inspect: DockerContainerInspect, +): void { + const name = dockerContainerName(inspect); + if (name !== transaction.originalName && name !== transaction.backupName) { + throw new Error("Managed bootstrap original container has an unexpected transaction name."); + } + const normalized = normalizeDockerManagedBootstrapLaunchSpec({ + ...inspect, + Name: `/${transaction.originalName}`, + }); + if (normalized.hash !== transaction.originalSpecHash) { + throw new Error( + "Managed bootstrap refused mutation because the exact original launch spec changed.", + ); + } +} + +function assertTransactionReplacement( + transaction: DockerBootstrapTransaction, + inspect: DockerContainerInspect, +): void { + const name = dockerContainerName(inspect); + if (name !== transaction.replacementStagingName && name !== transaction.originalName) { + throw new Error("Managed bootstrap replacement container has an unexpected transaction name."); + } + const normalized = normalizeDockerManagedBootstrapLaunchSpec({ + ...inspect, + Name: `/${transaction.originalName}`, + }); + if (normalized.hash !== transaction.replacementSpecHash) { + throw new Error( + "Managed bootstrap refused mutation because the exact replacement launch spec changed.", + ); + } +} + +function assertCompletedCutoverRuntimeState( + transaction: DockerBootstrapTransaction, + deps: ResolvedDeps, +): void { + const original = inspectTransactionRuntime(transaction, transaction.originalRuntimeId, deps); + const replacement = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (!original || !replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: original ? transaction.replacementRuntimeId : transaction.originalRuntimeId, + detail: "completed cutover requires both exact transaction runtimes", + }); + } + assertTransactionOriginal(transaction, original); + assertTransactionReplacement(transaction, replacement); + assertExplicitlyStopped(original, "rollback backup"); + assertStableRunning(replacement, "replacement"); + if ( + dockerContainerName(original) !== transaction.backupName || + dockerContainerName(replacement) !== transaction.originalName + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.replacementRuntimeId, + detail: "completed cutover runtime names do not match durable authority", + }); + } +} + +function removeExactReplacement( + transaction: DockerBootstrapTransaction, + replacement: DockerContainerInspect, + deps: ResolvedDeps, +): void { + assertTransactionReplacement(transaction, replacement); + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + if (replacement.State?.Running === true) { + const stopped = deps.dockerStop(transaction.replacementRuntimeId, { + ...options, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopped)) { + const afterStop = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (!afterStop || afterStop.State?.Running === true) { + throw new Error( + `Managed bootstrap could not quiesce its exact replacement: ${ + commandDetail(stopped) || "Docker stop failed" + }`, + ); + } + assertTransactionReplacement(transaction, afterStop); + } + } + const removed = deps.dockerRm(transaction.replacementRuntimeId, options); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(transaction.replacementRuntimeId, deps) !== "absent" + ) { + throw new Error( + `Managed bootstrap could not remove its exact replacement: ${ + commandDetail(removed) || "Docker removal failed" + }`, + ); + } +} + +function restoreOriginal(transaction: DockerBootstrapTransaction, deps: ResolvedDeps): void { + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + const originalBeforeReplacementRemoval = inspectTransactionRuntime( + transaction, + transaction.originalRuntimeId, + deps, + ); + if (!originalBeforeReplacementRemoval) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "the exact rollback original is absent", + }); + } + assertTransactionOriginal(transaction, originalBeforeReplacementRemoval); + const replacement = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (replacement) { + removeExactReplacement(transaction, replacement, deps); + } + const original = inspectExact(transaction.originalRuntimeId, deps); + assertTransactionOriginal(transaction, original); + const currentName = dockerContainerName(original); + if (currentName !== transaction.originalName) { + if (currentName !== transaction.backupName) { + throw new Error("Managed bootstrap original container has an unexpected rollback name."); + } + const renamed = deps.dockerRename( + transaction.originalRuntimeId, + transaction.originalName, + options, + ); + if (!hasZeroDockerExitStatus(renamed)) { + const afterRename = inspectTransactionRuntime( + transaction, + transaction.originalRuntimeId, + deps, + ); + if (!afterRename || dockerContainerName(afterRename) !== transaction.originalName) { + throw new Error( + `Managed bootstrap could not restore the original container name: ${ + commandDetail(renamed) || "Docker rename failed" + }`, + ); + } + assertTransactionOriginal(transaction, afterRename); + } + } + const restoredBeforeStart = inspectExact(transaction.originalRuntimeId, deps); + if (restoredBeforeStart.State?.Running !== true) { + const started = deps.dockerStart(transaction.originalRuntimeId, options); + if (!hasZeroDockerExitStatus(started)) { + const afterStart = inspectTransactionRuntime( + transaction, + transaction.originalRuntimeId, + deps, + ); + if (!afterStart || afterStart.State?.Running !== true) { + throw new Error( + `Managed bootstrap could not restart the original container: ${ + commandDetail(started) || "Docker start failed" + }`, + ); + } + assertTransactionOriginal(transaction, afterStart); + } + } + const restored = inspectExact(transaction.originalRuntimeId, deps); + assertStableRunning(restored, "restored workload"); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(restored); + if (normalized.hash !== transaction.originalSpecHash) { + throw new Error("Managed bootstrap rollback did not restore the exact launch spec."); + } +} + +function retainOwnedWorkloadForOwnerCleanup( + sandbox: ManagedBootstrapSandboxIdentity, + deps: ResolvedDeps, + expectedRuntimeId?: string, +): never { + const expectedIdentity = + expectedRuntimeId === undefined + ? `sandbox ${sandbox.sandboxId} with no previously resolved runtime ID` + : `sandbox ${sandbox.sandboxId} expected runtime ${expectedRuntimeId}`; + let containers: DockerCommandResult; + try { + containers = deps.dockerRun( + [ + "ps", + "-a", + "--no-trunc", + "--filter", + `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, + "--filter", + `label=${OPENSHELL_SANDBOX_ID_LABEL}=${sandbox.sandboxId}`, + "--format", + "{{.ID}}", + ], + { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }, + ); + } catch (error) { + throw new Error( + `Managed bootstrap owner cleanup could not enumerate the exact held runtime for ${expectedIdentity}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + if (Number(containers.status ?? 1) !== 0) { + throw new Error( + `Managed bootstrap owner cleanup could not verify the exact held runtime for ${expectedIdentity}: ${ + commandDetail(containers) || "Docker enumeration failed" + }`, + ); + } + const runtimeIds = String(containers.stdout ?? "") + .trim() + .split(/\r?\n/u) + .map((value) => value.trim().toLowerCase()) + .filter(Boolean); + if ( + runtimeIds.length !== 1 || + !FULL_CONTAINER_ID_RE.test(runtimeIds[0] ?? "") || + (expectedRuntimeId !== undefined && runtimeIds[0] !== expectedRuntimeId) + ) { + throw new Error( + `Managed bootstrap owner cleanup could not bind retention for ${expectedIdentity}; resolved runtime IDs: ${ + runtimeIds.length === 0 ? "none" : runtimeIds.join(", ") + }.`, + ); + } + const runtimeId = runtimeIds[0] as string; + let inspect: DockerContainerInspect; + try { + inspect = inspectExact(runtimeId, deps); + } catch (error) { + throw new Error( + `Managed bootstrap could not inspect retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + const labels = inspect.Config?.Labels ?? {}; + if ( + labels[OPENSHELL_MANAGED_BY_LABEL] !== OPENSHELL_MANAGED_BY_VALUE || + labels[OPENSHELL_SANDBOX_NAME_LABEL] !== sandbox.sandboxName || + labels[OPENSHELL_SANDBOX_ID_LABEL] !== sandbox.sandboxId + ) { + throw new Error( + `Managed bootstrap owner cleanup refused retention after exact runtime ${runtimeId} ownership changed for sandbox ${sandbox.sandboxId}.`, + ); + } + let stopped: DockerCommandResult; + try { + stopped = deps.dockerStop(runtimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + } catch (error) { + throw new Error( + `Managed bootstrap could not quiesce retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + assertZero( + stopped, + `Managed bootstrap could not quiesce retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}`, + ); + let retained: DockerContainerInspect; + try { + retained = inspectExact(runtimeId, deps); + } catch (error) { + throw new Error( + `Managed bootstrap could not re-inspect quiesced sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + if ( + retained.State?.Running !== false || + retained.State.Paused !== false || + retained.State.Restarting !== false + ) { + throw new Error( + `Managed bootstrap retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId} did not prove an explicitly quiescent state.`, + ); + } + if (!deps.runCaptureOpenshell) { + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + }); + } + let getBeforeDelete: string; + try { + getBeforeDelete = deps.runCaptureOpenshell(["sandbox", "get", sandbox.sandboxName], { + ignoreError: false, + }); + } catch (error) { + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + detail: `OpenShell owner lookup also failed: ${ + error instanceof Error ? error.message : String(error) + }.`, + }); + } + const sandboxIdBeforeDelete = parseOpenShellSandboxId(getBeforeDelete); + if (sandboxIdBeforeDelete !== sandbox.sandboxId) { + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + detail: `The same mutable name now resolves to durable sandbox ID ${ + sandboxIdBeforeDelete ?? "unknown" + } instead of ${sandbox.sandboxId}.`, + }); + } + throw new ManagedBootstrapOwnerCleanupRequiredError({ + sandboxName: sandbox.sandboxName, + sandboxId: sandbox.sandboxId, + runtimeId, + }); +} + +function resolveIncompleteCreateSandbox( + input: ManagedBootstrapIncompleteCreateCleanupInput, + deps: ResolvedDeps, +): { + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly runtimeId: string; +} { + if ( + input.plan.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + input.plan.driverId !== DOCKER_DRIVER_ID + ) { + throw new Error("Managed bootstrap Docker incomplete-create cleanup received another driver."); + } + assertManagedBootstrapIdentity(input.bootstrapIdentity); + const query = queryOpenShellDockerSandboxContainers(input.plan.sandboxName, deps); + if (!query.ok) { + throw new Error(`Managed bootstrap Docker incomplete-create discovery failed: ${query.error}`); + } + if (query.ids.length !== 1) { + throw new Error( + `Managed bootstrap incomplete-create cleanup requires exactly one labeled Docker workload; found ${String( + query.ids.length, + )}.`, + ); + } + const runtimeId = String(query.ids[0] ?? "").toLowerCase(); + const inspect = inspectExact(runtimeId, deps); + const sandboxId = String(inspect.Config?.Labels?.[OPENSHELL_SANDBOX_ID_LABEL] ?? ""); + if (parseOpenShellSandboxId(`ID: ${sandboxId}\n`) !== sandboxId) { + throw new Error( + "Managed bootstrap Docker incomplete-create workload has no exact durable sandbox ID.", + ); + } + const sandbox = Object.freeze({ + sandboxName: input.plan.sandboxName, + sandboxId, + driverId: input.plan.driverId, + }); + if ( + input.createReceipt.ready !== true || + input.createReceipt.sandbox.sandboxName !== sandbox.sandboxName || + input.createReceipt.sandbox.sandboxId !== sandbox.sandboxId || + input.createReceipt.sandbox.driverId !== sandbox.driverId + ) { + throw new Error( + "Managed bootstrap Docker incomplete-create workload does not match the exact validated create receipt.", + ); + } + assertImage(inspect, input.plan.image, deps); + assertMetadata(inspect, sandbox, input.plan.metadata); + assertHeldCommand(inspect, input.heldWorkloadArgv, input.bootstrapIdentity); + return { sandbox, runtimeId }; +} + +function managedSharedStateTransaction( + handle: ManagedBootstrapHeldWorkloadHandle, + containerId: string, + image: string, +) { + return { + agent: handle.plan.profile.agent, + bootstrapIdentity: handle.bootstrapIdentity, + containerId, + image, + profileFingerprint: handle.plan.profile.fingerprint, + } as const; +} + +function sameDockerBootstrapJournal( + left: DockerBootstrapTransaction, + right: DockerBootstrapTransaction, +): boolean { + return ( + serializeDockerManagedBootstrapJournal(left) === serializeDockerManagedBootstrapJournal(right) + ); +} + +function createDockerBootstrapJournalDurably( + journal: DockerBootstrapTransaction, + deps: ResolvedDeps, +): DockerBootstrapTransaction { + try { + deps.journalStore.create(journal); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + if (!recovered || !sameDockerBootstrapJournal(recovered, journal)) throw error; + return recovered; + } + const persisted = deps.journalStore.load(journal.bootstrapIdentity); + if (!persisted || !sameDockerBootstrapJournal(persisted, journal)) { + throw new Error("Managed bootstrap Docker staged journal was not durably re-readable."); + } + return persisted; +} + +function transitionDockerBootstrapJournalDurably( + journal: DockerBootstrapTransaction, + next: "cutover" | "rollback-authorized" | "shared-state-committed", + deps: ResolvedDeps, +): DockerBootstrapTransaction { + try { + deps.journalStore.transition(journal.bootstrapIdentity, journal.phase, next); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + const expected = Object.freeze({ ...journal, phase: next }); + if (!recovered || !sameDockerBootstrapJournal(recovered, expected)) throw error; + return recovered; + } + const persisted = deps.journalStore.load(journal.bootstrapIdentity); + const expected = Object.freeze({ ...journal, phase: next }); + if (!persisted || !sameDockerBootstrapJournal(persisted, expected)) { + throw new Error(`Managed bootstrap Docker journal transition to ${next} was not durable.`); + } + return persisted; +} + +function removeDockerBootstrapJournalDurably( + journal: DockerBootstrapTransaction, + deps: ResolvedDeps, +): void { + try { + deps.journalStore.remove(journal.bootstrapIdentity, [journal.phase]); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + if (recovered !== null) throw error; + return; + } + if (deps.journalStore.load(journal.bootstrapIdentity) !== null) { + throw new Error("Managed bootstrap Docker journal removal was not durable."); + } +} + +function assertDockerBootstrapTransactionAuthority( + transaction: DockerBootstrapTransaction, + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared?: ManagedBootstrapPreparedReplacementHandle | null, + replacement?: ManagedBootstrapReplacementHandle | null, +): void { + const originalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, + ); + const expectedSandbox = handle.sandbox; + if ( + transaction.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION || + transaction.bootstrapIdentity !== handle.bootstrapIdentity || + transaction.sandbox.sandboxName !== expectedSandbox.sandboxName || + transaction.sandbox.sandboxId !== expectedSandbox.sandboxId || + transaction.sandbox.driverId !== expectedSandbox.driverId || + transaction.profileFingerprint !== handle.plan.profile.fingerprint || + transaction.imageReference !== + expectedImageReference(snapshot.image.repository, snapshot.image.manifestDigest) || + transaction.runtimeImageContentId !== snapshot.runtimeImageContentId || + transaction.originalRuntimeId !== snapshot.runtimeId || + transaction.originalName !== originalName || + transaction.replacementStagingName !== + replacementStagingName(originalName, handle.bootstrapIdentity) || + transaction.backupName !== backupName(originalName, handle.bootstrapIdentity) || + transaction.originalSpecHash !== snapshot.specHash || + (prepared !== undefined && + prepared !== null && + (transaction.originalRuntimeId !== prepared.originalRuntimeId || + transaction.replacementRuntimeId !== prepared.preparedRuntimeId || + transaction.replacementSpecHash !== prepared.expectedActivatedSpecHash)) || + (replacement !== undefined && + replacement !== null && + (transaction.originalRuntimeId !== replacement.originalRuntimeId || + transaction.replacementRuntimeId !== replacement.replacementRuntimeId || + transaction.replacementSpecHash !== replacement.replacementSpecHash)) + ) { + throw new Error( + "Managed bootstrap receipts do not match the durable Docker transaction authority.", + ); + } +} + +function transactionFromPreparedAuthority( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, +): DockerBootstrapTransaction { + const transaction = parseDockerManagedBootstrapJournal(prepared.rollbackAuthority); + if (transaction.phase !== "staged") { + throw new Error("Managed bootstrap Docker prepared authority must describe a staged runtime."); + } + assertDockerBootstrapTransactionAuthority(transaction, handle, snapshot, prepared); + return transaction; +} + +function assertDurablePreparationAuthority( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + prepared: ManagedBootstrapPreparedReplacementHandle, + receipt: ManagedBootstrapDurablePreparationReceipt, +): void { + const authority = createManagedBootstrapPreparedAuthority({ handle, snapshot, prepared }); + const recordedAt = new Date(receipt.recordedAt); + if ( + receipt.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + receipt.sandbox.sandboxName !== authority.sandbox.sandboxName || + receipt.sandbox.sandboxId !== authority.sandbox.sandboxId || + receipt.sandbox.driverId !== authority.sandbox.driverId || + receipt.bootstrapIdentity !== authority.bootstrapIdentity || + receipt.authorityFingerprint !== authority.authorityFingerprint || + typeof receipt.recordId !== "string" || + receipt.recordId.length === 0 || + receipt.recordId.includes("\0") || + typeof receipt.recordedAt !== "string" || + !Number.isFinite(recordedAt.getTime()) || + recordedAt.toISOString() !== receipt.recordedAt + ) { + throw new Error( + "Managed bootstrap Docker activation requires the exact durable prepared-authority receipt.", + ); + } +} + +function reconstructDockerBootstrapTransaction( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + replacement: ManagedBootstrapReplacementHandle, + deps: ResolvedDeps, +): DockerBootstrapTransaction { + if ( + replacement.bootstrapIdentity !== handle.bootstrapIdentity || + replacement.originalRuntimeId !== snapshot.runtimeId || + replacement.originalSpecHash !== snapshot.specHash || + replacement.replacementRuntimeId === replacement.originalRuntimeId + ) { + throw new Error( + "Managed bootstrap finalization receipts do not reconstruct one exact Docker transaction.", + ); + } + const transaction = deps.journalStore.load(handle.bootstrapIdentity); + if (!transaction) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: "the durable Docker cutover journal is absent", + }); + } + assertDockerBootstrapTransactionAuthority(transaction, handle, snapshot, null, replacement); + return transaction; +} + +function cleanupUnjournaledPreparedContainer( + input: { + readonly snapshot: ManagedBootstrapObservedSnapshot; + readonly preparedRuntimeId: string; + readonly stagingName: string; + }, + deps: ResolvedDeps, +): void { + if (!FULL_CONTAINER_ID_RE.test(input.preparedRuntimeId)) return; + const original = inspectExact(input.snapshot.runtimeId, deps); + const originalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(input.snapshot.specCanonicalJson).inspect, + ); + if ( + !isStableRunning(original) || + dockerContainerName(original) !== originalName || + normalizeDockerManagedBootstrapLaunchSpec(original).hash !== input.snapshot.specHash + ) { + throw new Error( + "Managed bootstrap cannot clean an unjournaled replacement after original drift.", + ); + } + const prepared = tryInspectExact(input.preparedRuntimeId, deps); + if (!prepared) return; + if ( + String(prepared.Id ?? "").toLowerCase() !== input.preparedRuntimeId || + dockerContainerName(prepared) !== input.stagingName || + !isExplicitlyStopped(prepared) + ) { + throw new Error( + "Managed bootstrap refused cleanup because the unjournaled prepared runtime changed.", + ); + } + const removed = deps.dockerRm(input.preparedRuntimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(input.preparedRuntimeId, deps) !== "absent" + ) { + throw new Error( + `Managed bootstrap could not remove its unjournaled prepared runtime: ${ + commandDetail(removed) || "Docker removal failed" + }`, + ); + } +} + +function resolvePreparedRollbackAuthority(input: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot; + readonly prepared: ManagedBootstrapPreparedReplacementHandle | null; + readonly durablePreparation: ManagedBootstrapDurablePreparationReceipt | null; +}): DockerBootstrapTransaction | null { + if (input.durablePreparation && !input.prepared) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: input.handle.bootstrapIdentity, + runtimeId: input.snapshot.runtimeId, + detail: "durable prepared authority is present without its exact prepared handle", + }); + } + if (!input.prepared) return null; + const authority = transactionFromPreparedAuthority(input.handle, input.snapshot, input.prepared); + if (input.durablePreparation) { + assertDurablePreparationAuthority( + input.handle, + input.snapshot, + input.prepared, + input.durablePreparation, + ); + } + return authority; +} + +export function createDockerManagedBootstrapAdapter( + dependencies: DockerManagedBootstrapDeps = {}, +): DockerManagedBootstrapAdapter { + const deps = resolveDeps(dependencies); + const committedTransactions = new Set(); + const rollbackTombstones = new Map(); + const completedRollback = ( + handle: ManagedBootstrapHeldWorkloadHandle, + alreadyRolledBack: boolean, + ): ManagedBootstrapFinalizationReceipt => { + const receipt = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: true, + alreadyRolledBack, + finalizedAt: deps.now().toISOString(), + } satisfies ManagedBootstrapFinalizationReceipt); + rollbackTombstones.set(handle.bootstrapIdentity, { + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + handle.plan.image.repository, + handle.plan.image.manifestDigest, + ), + receipt, + }); + return receipt; + }; + const priorRollback = ( + handle: ManagedBootstrapHeldWorkloadHandle, + ): ManagedBootstrapFinalizationReceipt | null => { + const tombstone = rollbackTombstones.get(handle.bootstrapIdentity); + if (!tombstone) return null; + const receipt = tombstone.receipt; + if ( + receipt.sandbox.sandboxName !== handle.sandbox.sandboxName || + receipt.sandbox.sandboxId !== handle.sandbox.sandboxId || + receipt.sandbox.driverId !== handle.sandbox.driverId || + tombstone.profileFingerprint !== handle.plan.profile.fingerprint || + tombstone.imageReference !== + expectedImageReference(handle.plan.image.repository, handle.plan.image.manifestDigest) + ) { + throw new Error("Managed bootstrap rollback tombstone does not match its durable identity."); + } + return Object.freeze({ + ...receipt, + alreadyRolledBack: true, + }); + }; + const rollbackBootstrapNow = ({ + handle, + snapshot, + prepared, + durablePreparation, + replacement, + sharedStateAlreadyRolledBack = false, + }: { + readonly handle: ManagedBootstrapHeldWorkloadHandle; + readonly snapshot: ManagedBootstrapObservedSnapshot | null; + readonly prepared: ManagedBootstrapPreparedReplacementHandle | null; + readonly durablePreparation: ManagedBootstrapDurablePreparationReceipt | null; + readonly replacement: ManagedBootstrapReplacementHandle | null; + readonly sharedStateAlreadyRolledBack?: boolean; + }): ManagedBootstrapFinalizationReceipt => { + const finalized = priorRollback(handle); + if (finalized) return finalized; + const journal = deps.journalStore.load(handle.bootstrapIdentity); + if ( + committedTransactions.has(handle.bootstrapIdentity) || + journal?.phase === "shared-state-committed" + ) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: handle.bootstrapIdentity, + cleanupRuntimeId: journal?.originalRuntimeId ?? snapshot?.runtimeId ?? "unknown", + detail: + "rollback is no longer legal after the durable Docker commit fence; retry commit finalization", + }); + } + if (!snapshot) { + if (journal || prepared || durablePreparation || replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal?.originalRuntimeId ?? prepared?.originalRuntimeId ?? "unknown", + detail: "Docker replacement authority exists without its observed snapshot", + }); + } + retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps); + return completedRollback(handle, false); + } + + const preparedAuthority = resolvePreparedRollbackAuthority({ + handle, + snapshot, + prepared, + durablePreparation, + }); + + if (!journal) { + const originalPresence = probeExactDockerContainerAbsence(snapshot.runtimeId, deps); + if (originalPresence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: snapshot.runtimeId, + detail: "the original runtime presence is unknown and no durable journal is available", + }); + } + if (originalPresence === "absent") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: handle.bootstrapIdentity, + cleanupRuntimeId: snapshot.runtimeId, + detail: + "rollback is forbidden because the exact original is absent after journal retirement", + }); + } + const original = inspectExact(snapshot.runtimeId, deps); + const expectedOriginalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, + ); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(original); + if ( + dockerContainerName(original) !== expectedOriginalName || + original.State?.Running !== true || + normalized.hash !== snapshot.specHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: snapshot.runtimeId, + detail: "the journal is absent and the exact original is not a proven restored workload", + }); + } + if (preparedAuthority) { + const observedPrepared = inspectTransactionRuntime( + preparedAuthority, + preparedAuthority.replacementRuntimeId, + deps, + ); + if (observedPrepared) { + assertExplicitlyStopped(observedPrepared, "prepared replacement"); + if ( + dockerContainerName(observedPrepared) !== preparedAuthority.replacementStagingName || + normalizeDockerManagedBootstrapLaunchSpec(observedPrepared).canonicalJson !== + prepared?.preparedSpecCanonicalJson + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: preparedAuthority.replacementRuntimeId, + detail: "the unjournaled prepared runtime changed before exact cleanup", + }); + } + removeExactReplacement(preparedAuthority, observedPrepared, deps); + } + } else if (replacement) { + const replacementPresence = probeExactDockerContainerAbsence( + replacement.replacementRuntimeId, + deps, + ); + if (replacementPresence !== "absent") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: + replacementPresence === "present" + ? "the replacement still exists without durable journal authority" + : "replacement absence is unknown without durable journal authority", + }); + } + } + retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, snapshot.runtimeId); + return completedRollback(handle, true); + } + + if (!preparedAuthority || !durablePreparation) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker cutover lacks its coordinator-recorded prepared authority", + }); + } + const stagedJournal = Object.freeze({ ...journal, phase: "staged" as const }); + if (!sameDockerBootstrapJournal(stagedJournal, preparedAuthority)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker cutover changed its prepared rollback authority", + }); + } + assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); + const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); + if (!original) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "the exact rollback original is absent", + }); + } + assertTransactionOriginal(journal, original); + const observedReplacement = inspectTransactionRuntime( + journal, + journal.replacementRuntimeId, + deps, + ); + + if (journal.phase === "staged") { + assertStableRunning(original, "staged original"); + if (observedReplacement) { + assertExplicitlyStopped(observedReplacement, "staged replacement"); + } + if ( + dockerContainerName(original) !== journal.originalName || + (observedReplacement !== null && + dockerContainerName(observedReplacement) !== journal.replacementStagingName) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "staged transaction runtime state does not match its pre-cutover fence", + }); + } + if (observedReplacement) { + removeExactReplacement(journal, observedReplacement, deps); + } + removeDockerBootstrapJournalDurably(journal, deps); + retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, journal.originalRuntimeId); + return completedRollback(handle, false); + } + + if (journal.phase !== "cutover" && journal.phase !== "rollback-authorized") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: journal.bootstrapIdentity, + cleanupRuntimeId: journal.originalRuntimeId, + detail: "rollback is forbidden by the durable Docker commit phase", + }); + } + + const originalNameNow = dockerContainerName(original); + const replacementNameNow = observedReplacement + ? dockerContainerName(observedReplacement) + : null; + const originalAtTargetRecoverable = + originalNameNow === journal.originalName && + (isStableRunning(original) || isExplicitlyStopped(original)); + const originalAtBackupRecoverable = + originalNameNow === journal.backupName && isExplicitlyStopped(original); + const replacementAtStagingRecoverable = + replacementNameNow === journal.replacementStagingName && + observedReplacement !== null && + isExplicitlyStopped(observedReplacement); + const replacementAtTargetRecoverable = + replacementNameNow === journal.originalName && + observedReplacement !== null && + (isStableRunning(observedReplacement) || isExplicitlyStopped(observedReplacement)); + const validCutoverState = + (originalAtTargetRecoverable && replacementAtStagingRecoverable) || + (originalAtBackupRecoverable && replacementAtStagingRecoverable) || + (originalAtBackupRecoverable && replacementAtTargetRecoverable); + let activeJournal = journal; + + if (journal.phase === "cutover") { + if ( + (!sharedStateAlreadyRolledBack && (!observedReplacement || !validCutoverState)) || + (sharedStateAlreadyRolledBack && + (observedReplacement !== null || + !( + (originalNameNow === journal.backupName && isExplicitlyStopped(original)) || + originalAtTargetRecoverable + ))) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: + observedReplacement === null && !sharedStateAlreadyRolledBack + ? "the exact replacement disappeared before rollback authorization was durable" + : "cutover runtime names or states do not match a recoverable phase", + }); + } + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable transaction authority changed before rollback authorization", + }); + } + + let sharedStatus: "committed" | "none" | "pending" = "none"; + const sharedTransaction = managedSharedStateTransaction( + handle, + journal.replacementRuntimeId, + journal.runtimeImageContentId, + ); + if (!sharedStateAlreadyRolledBack) { + sharedStatus = probeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + profileFingerprint: journal.profileFingerprint, + }, + deps, + ); + if (sharedStatus === "committed") { + const committedJournal = transitionDockerBootstrapJournalDurably( + journal, + "shared-state-committed", + deps, + ); + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: committedJournal.bootstrapIdentity, + cleanupRuntimeId: committedJournal.originalRuntimeId, + detail: "image-owned shared state is durably committed; rollback is no longer legal", + }); + } + } + activeJournal = transitionDockerBootstrapJournalDurably(journal, "rollback-authorized", deps); + if (!sharedStateAlreadyRolledBack && sharedStatus === "pending") { + finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: false, + retainContainerAfterRollback: true, + }, + deps, + ); + } + } else { + if (!originalAtTargetRecoverable && !originalAtBackupRecoverable) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "rollback-authorized original runtime state is not recoverable", + }); + } + if ( + observedReplacement && + originalNameNow === journal.originalName && + replacementNameNow === journal.originalName + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "both transaction runtimes claim the authoritative workload name", + }); + } + if (observedReplacement) { + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "rollback authorization changed before replacement cleanup", + }); + } + const sharedTransaction = managedSharedStateTransaction( + handle, + journal.replacementRuntimeId, + journal.runtimeImageContentId, + ); + const sharedStatus = probeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + profileFingerprint: journal.profileFingerprint, + }, + deps, + ); + if (sharedStatus === "committed") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: + "shared state became committed after rollback authorization; no mutation was attempted", + }); + } + if (sharedStatus === "pending") { + finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: false, + retainContainerAfterRollback: true, + }, + deps, + ); + } + } + } + + const beforeRestore = deps.journalStore.load(activeJournal.bootstrapIdentity); + if (!beforeRestore || !sameDockerBootstrapJournal(beforeRestore, activeJournal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "durable transaction authority changed before original restoration", + }); + } + restoreOriginal(activeJournal, deps); + const restored = inspectExact(activeJournal.originalRuntimeId, deps); + assertStableRunning(restored, "restored workload"); + if ( + dockerContainerName(restored) !== activeJournal.originalName || + normalizeDockerManagedBootstrapLaunchSpec(restored).hash !== activeJournal.originalSpecHash + ) { + throw new Error("Managed bootstrap Docker rollback did not restore its exact original."); + } + removeDockerBootstrapJournalDurably(activeJournal, deps); + retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, activeJournal.originalRuntimeId); + return completedRollback(handle, false); + }; + const commitBootstrapNow = ( + receipt: ManagedBootstrapCompletionReceipt, + transaction: DockerBootstrapTransaction, + input: { + readonly sharedStateStatus: "committed" | "none"; + readonly sharedStateTransaction: ReturnType; + }, + ): void => { + if (committedTransactions.has(receipt.bootstrapIdentity)) return; + if ( + transaction.phase !== "shared-state-committed" || + transaction.replacementRuntimeId !== receipt.runtimeId || + transaction.originalSpecHash !== receipt.originalSpecHash || + transaction.replacementSpecHash !== receipt.replacementSpecHash + ) { + throw new Error("Managed bootstrap Docker commit receipt does not match its commit fence."); + } + const current = deps.journalStore.load(transaction.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, transaction)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "durable commit authority changed before exact cleanup", + }); + } + + const replacement = inspectTransactionRuntime( + transaction, + transaction.replacementRuntimeId, + deps, + ); + if (!replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.replacementRuntimeId, + detail: "the exact committed replacement is absent", + }); + } + assertTransactionReplacement(transaction, replacement); + if ( + dockerContainerName(replacement) !== transaction.originalName || + replacement.State?.Running !== true + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.replacementRuntimeId, + detail: "the exact replacement is not running under the authoritative workload name", + }); + } + + const original = inspectTransactionRuntime(transaction, transaction.originalRuntimeId, deps); + if (original) { + assertTransactionOriginal(transaction, original); + if (dockerContainerName(original) !== transaction.backupName) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "the exact rollback backup is not quiescent under its durable backup name", + }); + } + assertExplicitlyStopped(original, "commit rollback backup"); + const beforeRemove = deps.journalStore.load(transaction.bootstrapIdentity); + if (!beforeRemove || !sameDockerBootstrapJournal(beforeRemove, transaction)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: transaction.bootstrapIdentity, + runtimeId: transaction.originalRuntimeId, + detail: "durable commit authority changed before exact rollback-backup removal", + }); + } + const removed = deps.dockerRm(transaction.originalRuntimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(transaction.originalRuntimeId, deps) !== "absent" + ) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: receipt.bootstrapIdentity, + cleanupRuntimeId: transaction.originalRuntimeId, + detail: `${commandDetail(removed) || "Docker removal failed"}; exact backup absence was not proven`, + }); + } + } + + if (input.sharedStateStatus === "committed") { + try { + clearDockerManagedStartupSharedStateCommitReceipt(input.sharedStateTransaction, deps); + } catch (error) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: receipt.bootstrapIdentity, + cleanupRuntimeId: transaction.replacementRuntimeId, + detail: `exact rollback backup is absent, but its image-owned commit receipt could not be retired: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + } + } + removeDockerBootstrapJournalDurably(transaction, deps); + committedTransactions.add(receipt.bootstrapIdentity); + }; + const finalizeBootstrap = async ( + input: Parameters[0], + ): Promise => { + if (input.outcome === "rollback") { + return rollbackBootstrapNow(input); + } + const { completion, durablePreparation, handle, prepared, replacement, snapshot } = input; + if (!completion || !snapshot || !prepared || !durablePreparation || !replacement) { + throw new Error("Managed bootstrap commit requires one complete cutover receipt."); + } + const preparedAuthority = transactionFromPreparedAuthority(handle, snapshot, prepared); + assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); + const sharedTransaction = managedSharedStateTransaction( + handle, + replacement.replacementRuntimeId, + replacement.runtimeImageContentId, + ); + let sharedStatus = probeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + profileFingerprint: completion.profileFingerprint, + }, + deps, + ); + let journal = deps.journalStore.load(handle.bootstrapIdentity); + + if (!journal) { + if (committedTransactions.has(completion.bootstrapIdentity)) { + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + } + const originalPresence = probeExactDockerContainerAbsence(snapshot.runtimeId, deps); + if (originalPresence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: snapshot.runtimeId, + detail: "the retired-journal commit cannot prove exact backup absence", + }); + } + if (originalPresence !== "absent" || sharedStatus !== "none") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: + originalPresence === "absent" ? replacement.replacementRuntimeId : snapshot.runtimeId, + detail: + "the durable journal is absent before both exact backup and shared commit receipt retirement were proven", + }); + } + const committedReplacement = inspectExact(replacement.replacementRuntimeId, deps); + assertStableRunning(committedReplacement, "committed replacement"); + const originalName = dockerContainerName( + parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, + ); + if ( + dockerContainerName(committedReplacement) !== originalName || + normalizeDockerManagedBootstrapLaunchSpec(committedReplacement).hash !== + replacement.replacementSpecHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: "the retired-journal replacement does not match the exact completion receipt", + }); + } + committedTransactions.add(completion.bootstrapIdentity); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + } + + if ( + !sameDockerBootstrapJournal( + Object.freeze({ ...journal, phase: "staged" as const }), + preparedAuthority, + ) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker commit changed its prepared rollback authority", + }); + } + assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); + if (journal.phase === "staged" || journal.phase === "rollback-authorized") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: `commit is forbidden from durable journal phase ${journal.phase}`, + }); + } + if (!completion.transactionPending && sharedStatus !== "none") { + throw new Error( + "Managed bootstrap image completion disagrees with shared-state transaction status.", + ); + } + + if (journal.phase === "cutover") { + if (completion.transactionPending && sharedStatus === "none") { + throw new Error( + "Managed bootstrap image completion lost its shared-state receipt before the durable commit fence.", + ); + } + if (sharedStatus === "pending") { + let outcome; + try { + outcome = finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: true, + retainContainerAfterRollback: true, + }, + deps, + ); + } catch (error) { + if (error instanceof DockerManagedStartupSharedStateCommitIndeterminateError) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: completion.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: error.message, + }); + } + throw error; + } + if (!outcome.supervisorReady) { + const failure = + outcome.failure ?? new Error("Managed bootstrap shared-state commit did not complete."); + try { + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: + "durable authority changed after shared-state rollback and before restoration", + }); + } + transitionDockerBootstrapJournalDurably(journal, "rollback-authorized", deps); + await rollbackBootstrapNow({ + handle, + snapshot, + prepared, + durablePreparation, + replacement, + sharedStateAlreadyRolledBack: true, + }); + } catch (rollbackError) { + attachManagedBootstrapRollbackError(failure, rollbackError); + } + throw failure; + } + sharedStatus = "committed"; + } + journal = transitionDockerBootstrapJournalDurably(journal, "shared-state-committed", deps); + } else if (completion.transactionPending && sharedStatus === "pending") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "shared state is pending after the durable Docker commit fence", + }); + } + + commitBootstrapNow(completion, journal, { + sharedStateStatus: sharedStatus === "committed" ? "committed" : "none", + sharedStateTransaction: sharedTransaction, + }); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + }; + return { + async createHeldWorkload(input) { + if ( + input.plan.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + input.plan.driverId !== DOCKER_DRIVER_ID || + input.request.agent !== input.plan.profile.agent || + input.request.profileFingerprint !== input.plan.profile.fingerprint + ) { + throw new Error("Managed bootstrap Docker create plan does not match its root request."); + } + const bootstrapIdentity = input.bootstrapIdentity ?? deps.createBootstrapIdentity(); + assertManagedBootstrapIdentity(bootstrapIdentity); + const heldWorkloadArgv = renderManagedBootstrapHeldCommand( + input.request, + bootstrapIdentity, + input.plan.intendedWorkloadArgv, + ); + const createReceipt = await input.launch({ heldWorkloadArgv, bootstrapIdentity }); + if ( + createReceipt.ready !== true || + createReceipt.sandbox.sandboxName !== input.plan.sandboxName || + createReceipt.sandbox.driverId !== input.plan.driverId || + !createReceipt.sandbox.sandboxId + ) { + throw new Error( + "Managed bootstrap Docker create did not return one Ready durable sandbox identity.", + ); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: Object.freeze({ ...createReceipt.sandbox }), + bootstrapIdentity, + heldWorkloadArgv, + intendedWorkloadArgv: Object.freeze([...input.plan.intendedWorkloadArgv]), + plan: input.plan, + createReceipt, + }); + }, + + async cleanupIncompleteCreate(input) { + const { sandbox, runtimeId } = resolveIncompleteCreateSandbox(input, deps); + retainOwnedWorkloadForOwnerCleanup(sandbox, deps, runtimeId); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox, + bootstrapIdentity: input.bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: true, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + }); + }, + + async discoverHeldWorkload( + input: ManagedBootstrapDiscoveryInput, + ): Promise { + if (input.sandbox.driverId !== DOCKER_DRIVER_ID) { + throw new Error("Managed bootstrap Docker adapter received another runtime driver."); + } + const query = queryOpenShellDockerSandboxContainers(input.sandbox.sandboxName, deps); + if (!query.ok) { + throw new Error(`Managed bootstrap Docker discovery failed: ${query.error}`); + } + if (query.ids.length !== 1) { + throw new Error( + `Managed bootstrap requires exactly one labeled Docker workload after Ready; found ${String( + query.ids.length, + )}.`, + ); + } + const runtimeId = String(query.ids[0] ?? "").toLowerCase(); + const inspect = inspectExact(runtimeId, deps); + assertStableRunning(inspect, "held workload"); + assertRootSupervisor(inspect); + assertImage(inspect, input.expectedImage, deps); + assertMetadata(inspect, input.sandbox, input.metadata); + assertBootstrapIdentityInObservedHold(inspect, input.bootstrapIdentity); + return Object.freeze({ + sandbox: input.sandbox, + runtimeId, + bootstrapIdentity: input.bootstrapIdentity, + }); + }, + + async inspectHeldWorkload({ handle, discovered }) { + if ( + discovered.bootstrapIdentity !== handle.bootstrapIdentity || + discovered.sandbox.sandboxId !== handle.sandbox.sandboxId || + discovered.sandbox.driverId !== handle.sandbox.driverId + ) { + throw new Error("Managed bootstrap Docker identity changed before inspection."); + } + const first = inspectExact(discovered.runtimeId, deps); + assertStableRunning(first, "held workload"); + assertRootSupervisor(first); + assertNoRootProcessInjectionEnvironment(first.Config?.Env); + const runtimeImageContentId = assertImage(first, handle.plan.image, deps); + assertMetadata(first, handle.sandbox, handle.plan.metadata); + assertHeldCommand(first, handle.heldWorkloadArgv, handle.bootstrapIdentity); + const firstNormalized = normalizeDockerManagedBootstrapLaunchSpec(first); + const inspect = inspectExact(discovered.runtimeId, deps); + assertStableRunning(inspect, "held workload"); + assertRootSupervisor(inspect); + assertNoRootProcessInjectionEnvironment(inspect.Config?.Env); + if (assertImage(inspect, handle.plan.image, deps) !== runtimeImageContentId) { + throw new Error("Managed bootstrap Docker image content changed during stable capture."); + } + assertMetadata(inspect, handle.sandbox, handle.plan.metadata); + assertHeldCommand(inspect, handle.heldWorkloadArgv, handle.bootstrapIdentity); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(inspect); + if ( + normalized.hash !== firstNormalized.hash || + normalized.canonicalJson !== firstNormalized.canonicalJson + ) { + throw new Error("Managed bootstrap Docker launch spec changed during stable capture."); + } + const supervisorArgv = exactSupervisorArgv(inspect); + if (!exactArrayEqual(supervisorArgv, handle.plan.expectedSupervisorArgv)) { + throw new Error("Managed bootstrap Docker supervisor argv changed before replacement."); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: discovered.runtimeId, + bootstrapIdentity: handle.bootstrapIdentity, + image: handle.plan.image, + runtimeImageContentId, + specHash: normalized.hash, + specCanonicalJson: normalized.canonicalJson, + agentIdentity: Object.freeze({ ...handle.plan.agentIdentity }), + supervisorArgv, + heldWorkloadArgv: handle.heldWorkloadArgv, + metadata: handle.plan.metadata, + }); + }, + + async prepareBootstrapReplacement({ handle, snapshot, request, replacementOptions }) { + if ( + snapshot.bootstrapIdentity !== handle.bootstrapIdentity || + !FULL_CONTAINER_ID_RE.test(snapshot.runtimeId) || + request.agent !== handle.plan.profile.agent || + request.profileFingerprint !== handle.plan.profile.fingerprint + ) { + throw new Error("Managed bootstrap Docker replacement identities do not match."); + } + const parsed = parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson); + const normalizedOriginal = normalizeDockerManagedBootstrapLaunchSpec(parsed.inspect); + if (normalizedOriginal.hash !== snapshot.specHash) { + throw new Error("Managed bootstrap Docker replacement snapshot is not exact."); + } + assertNoRootProcessInjectionEnvironment(parsed.inspect.Config?.Env); + if (parsed.inspect.HostConfig?.ReadonlyRootfs === true) { + throw new Error( + "Managed bootstrap cannot stage its root-owned request in a read-only root filesystem.", + ); + } + const plan = replacementPlan(replacementOptions); + const originalName = dockerContainerName(parsed.inspect); + const backupContainerName = backupName(originalName, handle.bootstrapIdentity); + const stagingName = replacementStagingName(originalName, handle.bootstrapIdentity); + const existingJournal = deps.journalStore.load(handle.bootstrapIdentity); + if (existingJournal) { + assertDockerBootstrapTransactionAuthority(existingJournal, handle, snapshot); + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: existingJournal.bootstrapIdentity, + runtimeId: existingJournal.replacementRuntimeId, + detail: `preparation requires rollback or commit from durable phase ${existingJournal.phase}`, + }); + } + const trampolineCommand = replacementCommand(handle, snapshot); + const cloneArgs = buildDockerGpuCloneRunArgs(parsed.inspect, plan.mode, { + image: expectedImageReference(snapshot.image.repository, snapshot.image.manifestDigest), + openshellSandboxCommand: handle.intendedWorkloadArgv, + requiredUlimits: plan.requiredUlimits, + extraGroupGids: plan.extraGroupGids, + containerEntrypoint: MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, + containerCommand: trampolineCommand, + containerName: stagingName, + }); + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + + let requestFile = ""; + let replacementRuntimeId = ""; + let stagedAuthority: DockerBootstrapTransaction | null = null; + try { + const created = deps.dockerRun(["create", ...cloneArgs], options); + const returnedRuntimeId = String(created.stdout ?? "") + .trim() + .toLowerCase(); + let createdInspect: DockerContainerInspect; + if (FULL_CONTAINER_ID_RE.test(returnedRuntimeId)) { + replacementRuntimeId = returnedRuntimeId; + createdInspect = inspectExact(replacementRuntimeId, deps); + } else { + try { + createdInspect = inspectDockerContainerReference(stagingName, deps); + } catch (lookupError) { + throw new Error( + "Managed bootstrap could not prove a stopped Docker replacement after create: " + + (commandDetail(created) || + (lookupError instanceof Error ? lookupError.message : String(lookupError))), + ); + } + replacementRuntimeId = String(createdInspect.Id ?? "").toLowerCase(); + } + if ( + !FULL_CONTAINER_ID_RE.test(replacementRuntimeId) || + dockerContainerName(createdInspect) !== stagingName + ) { + throw new Error( + "Managed bootstrap Docker create did not resolve one stopped identity-bound staging container.", + ); + } + assertExplicitlyStopped(createdInspect, "created replacement"); + const createdImageContentId = assertImage(createdInspect, snapshot.image, deps); + if (createdImageContentId !== snapshot.runtimeImageContentId) { + throw new Error( + "Managed bootstrap Docker replacement resolved a different image content ID.", + ); + } + assertMetadata(createdInspect, handle.sandbox, snapshot.metadata); + assertRootSupervisor(createdInspect); + const intendedSandboxCommand = openshellSandboxCommandEnvValue(handle.intendedWorkloadArgv); + if (!intendedSandboxCommand) { + throw new Error( + "Managed bootstrap Docker replacement requires one bounded intended workload argv.", + ); + } + assertReplacementBoundary(createdInspect, handle, snapshot); + const expectedActivatedSpecHash = assertReplacementMatchesIntent( + snapshot.specCanonicalJson, + createdInspect, + originalName, + plan, + intendedSandboxCommand, + ); + const preparedSpec = normalizeDockerManagedBootstrapLaunchSpec(createdInspect); + const expectedActivatedSpec = normalizeDockerManagedBootstrapLaunchSpec({ + ...createdInspect, + Name: `/${originalName}`, + }); + if (expectedActivatedSpec.hash !== expectedActivatedSpecHash) { + throw new Error("Managed bootstrap Docker expected activation spec is inconsistent."); + } + stagedAuthority = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + phase: "staged", + bootstrapIdentity: handle.bootstrapIdentity, + sandbox: Object.freeze({ ...handle.sandbox }), + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + snapshot.image.repository, + snapshot.image.manifestDigest, + ), + runtimeImageContentId: snapshot.runtimeImageContentId, + originalRuntimeId: snapshot.runtimeId, + replacementRuntimeId, + originalName, + replacementStagingName: stagingName, + backupName: backupContainerName, + originalSpecHash: snapshot.specHash, + replacementSpecHash: expectedActivatedSpecHash, + }); + + requestFile = writeProtectedEnvelope(handle.bootstrapIdentity, request); + const copied = deps.dockerRun( + ["cp", requestFile, replacementRuntimeId + ":" + MANAGED_BOOTSTRAP_REQUEST_FILE], + options, + ); + assertZero( + copied, + "Managed bootstrap could not stage its protected root-owned 0400 envelope", + ); + + const originalBeforeJournal = inspectExact(snapshot.runtimeId, deps); + assertStableRunning(originalBeforeJournal, "pre-journal original"); + if ( + dockerContainerName(originalBeforeJournal) !== originalName || + normalizeDockerManagedBootstrapLaunchSpec(originalBeforeJournal).hash !== + snapshot.specHash + ) { + throw new Error( + "Managed bootstrap Docker original changed while the replacement was staged.", + ); + } + const replacementBeforeJournal = inspectExact(replacementRuntimeId, deps); + assertTransactionReplacement(stagedAuthority, replacementBeforeJournal); + const observedPreparedSpec = + normalizeDockerManagedBootstrapLaunchSpec(replacementBeforeJournal); + const observedActivatedSpec = normalizeDockerManagedBootstrapLaunchSpec({ + ...replacementBeforeJournal, + Name: `/${originalName}`, + }); + if ( + dockerContainerName(replacementBeforeJournal) !== stagingName || + observedPreparedSpec.canonicalJson !== preparedSpec.canonicalJson || + observedActivatedSpec.canonicalJson !== expectedActivatedSpec.canonicalJson + ) { + throw new Error("Managed bootstrap Docker replacement changed before durable staging."); + } + assertExplicitlyStopped(replacementBeforeJournal, "pre-journal replacement"); + + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + originalRuntimeId: snapshot.runtimeId, + preparedRuntimeId: replacementRuntimeId, + image: snapshot.image, + runtimeImageContentId: snapshot.runtimeImageContentId, + originalSpecHash: snapshot.specHash, + preparedSpecHash: preparedSpec.hash, + preparedSpecCanonicalJson: preparedSpec.canonicalJson, + expectedActivatedSpecHash, + expectedActivatedSpecCanonicalJson: expectedActivatedSpec.canonicalJson, + profileFingerprint: handle.plan.profile.fingerprint, + rollbackAuthority: serializeDockerManagedBootstrapJournal(stagedAuthority), + }); + } catch (error) { + let rollbackError: unknown = null; + try { + const durable = deps.journalStore.load(handle.bootstrapIdentity); + if (!durable) { + cleanupUnjournaledPreparedContainer( + { snapshot, preparedRuntimeId: replacementRuntimeId, stagingName }, + deps, + ); + } + } catch (cleanupError) { + rollbackError = cleanupError; + } + const failure = error instanceof Error ? error : new Error(String(error)); + if (rollbackError) attachManagedBootstrapRollbackError(failure, rollbackError); + throw failure; + } finally { + if (requestFile) cleanupTempDir(requestFile, REQUEST_TEMP_PREFIX); + } + }, + async activateBootstrapReplacement({ handle, snapshot, prepared, durablePreparation }) { + const authority = transactionFromPreparedAuthority(handle, snapshot, prepared); + assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); + const existingJournal = deps.journalStore.load(handle.bootstrapIdentity); + if (existingJournal) { + assertDockerBootstrapTransactionAuthority(existingJournal, handle, snapshot, prepared); + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: existingJournal.bootstrapIdentity, + runtimeId: existingJournal.replacementRuntimeId, + detail: `activation requires rollback or commit from durable phase ${existingJournal.phase}`, + }); + } + const options = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + try { + const originalBeforeJournal = inspectExact(snapshot.runtimeId, deps); + const preparedBeforeJournal = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionOriginal(authority, originalBeforeJournal); + assertTransactionReplacement(authority, preparedBeforeJournal); + assertStableRunning(originalBeforeJournal, "pre-activation original"); + assertExplicitlyStopped(preparedBeforeJournal, "pre-activation replacement"); + if ( + dockerContainerName(originalBeforeJournal) !== authority.originalName || + dockerContainerName(preparedBeforeJournal) !== authority.replacementStagingName || + normalizeDockerManagedBootstrapLaunchSpec(preparedBeforeJournal).canonicalJson !== + prepared.preparedSpecCanonicalJson + ) { + throw new Error( + "Managed bootstrap Docker prepared runtimes changed before durable activation.", + ); + } + + let journal = createDockerBootstrapJournalDurably(authority, deps); + const originalAtFence = inspectExact(snapshot.runtimeId, deps); + const replacementAtFence = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionOriginal(journal, originalAtFence); + assertTransactionReplacement(journal, replacementAtFence); + if ( + dockerContainerName(originalAtFence) !== journal.originalName || + originalAtFence.State?.Running !== true || + dockerContainerName(replacementAtFence) !== journal.replacementStagingName || + normalizeDockerManagedBootstrapLaunchSpec(replacementAtFence).canonicalJson !== + prepared.preparedSpecCanonicalJson + ) { + throw new Error( + "Managed bootstrap Docker staged runtimes changed before the cutover fence.", + ); + } + assertExplicitlyStopped(replacementAtFence, "staged replacement"); + journal = transitionDockerBootstrapJournalDurably(journal, "cutover", deps); + + const stopped = deps.dockerStop(snapshot.runtimeId, { + ...options, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + const afterStop = inspectExact(snapshot.runtimeId, deps); + assertTransactionOriginal(journal, afterStop); + if (dockerContainerName(afterStop) !== journal.originalName) { + throw new Error( + "Managed bootstrap could not prove its exact original stopped after Docker stop: " + + (commandDetail(stopped) || "state did not reach stopped"), + ); + } + assertExplicitlyStopped(afterStop, "stopped original"); + + const renamedOriginal = deps.dockerRename(snapshot.runtimeId, journal.backupName, options); + const afterOriginalRename = inspectExact(snapshot.runtimeId, deps); + assertTransactionOriginal(journal, afterOriginalRename); + if (dockerContainerName(afterOriginalRename) !== journal.backupName) { + throw new Error( + "Managed bootstrap could not prove its exact original backup rename: " + + (commandDetail(renamedOriginal) || "name did not reach backup"), + ); + } + assertExplicitlyStopped(afterOriginalRename, "renamed rollback backup"); + + const renamedReplacement = deps.dockerRename( + prepared.preparedRuntimeId, + journal.originalName, + options, + ); + const afterReplacementRename = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionReplacement(journal, afterReplacementRename); + if (dockerContainerName(afterReplacementRename) !== journal.originalName) { + throw new Error( + "Managed bootstrap could not prove its exact replacement cutover rename: " + + (commandDetail(renamedReplacement) || "name did not reach target"), + ); + } + + const started = deps.dockerStart(prepared.preparedRuntimeId, options); + const running = inspectExact(prepared.preparedRuntimeId, deps); + assertTransactionReplacement(journal, running); + const runningSpec = normalizeDockerManagedBootstrapLaunchSpec(running); + if ( + dockerContainerName(running) !== journal.originalName || + running.State?.Running !== true || + running.State.Paused === true || + running.State.Restarting === true || + running.State.Dead === true || + runningSpec.canonicalJson !== prepared.expectedActivatedSpecCanonicalJson + ) { + throw new Error( + "Managed bootstrap could not prove its exact replacement running after Docker start: " + + (commandDetail(started) || "state did not reach running"), + ); + } + assertReplacementBoundary(running, handle, snapshot); + const preservedOriginal = inspectExact(snapshot.runtimeId, deps); + assertTransactionOriginal(journal, preservedOriginal); + if (dockerContainerName(preservedOriginal) !== journal.backupName) { + throw new Error("Managed bootstrap Docker rollback backup changed during cutover."); + } + assertExplicitlyStopped(preservedOriginal, "preserved rollback backup"); + + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + originalRuntimeId: snapshot.runtimeId, + replacementRuntimeId: prepared.preparedRuntimeId, + image: snapshot.image, + runtimeImageContentId: snapshot.runtimeImageContentId, + originalSpecHash: snapshot.specHash, + replacementSpecHash: prepared.expectedActivatedSpecHash, + replacementSpecCanonicalJson: prepared.expectedActivatedSpecCanonicalJson, + profileFingerprint: handle.plan.profile.fingerprint, + }); + } catch (error) { + let rollbackError: unknown = null; + try { + if (!deps.journalStore.load(handle.bootstrapIdentity)) { + cleanupUnjournaledPreparedContainer( + { + snapshot, + preparedRuntimeId: prepared.preparedRuntimeId, + stagingName: authority.replacementStagingName, + }, + deps, + ); + } + } catch (cleanupError) { + rollbackError = cleanupError; + } + const failure = error instanceof Error ? error : new Error(String(error)); + if (rollbackError) attachManagedBootstrapRollbackError(failure, rollbackError); + throw failure; + } + }, + async awaitBootstrap({ handle, snapshot, replacement, timeoutSecs }) { + if ( + replacement.bootstrapIdentity !== handle.bootstrapIdentity || + replacement.originalRuntimeId !== snapshot.runtimeId || + replacement.profileFingerprint !== handle.plan.profile.fingerprint + ) { + throw new Error("Managed bootstrap Docker completion identities do not match."); + } + const journal = reconstructDockerBootstrapTransaction(handle, snapshot, replacement, deps); + if (journal.phase !== "cutover") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: `bootstrap completion is invalid from durable journal phase ${journal.phase}`, + }); + } + assertCompletedCutoverRuntimeState(journal, deps); + const before = inspectExact(replacement.replacementRuntimeId, deps); + assertStableRunning(before, "replacement"); + const beforeImageContentId = assertImage(before, replacement.image, deps); + if (beforeImageContentId !== replacement.runtimeImageContentId) { + throw new Error("Managed bootstrap Docker replacement image content changed."); + } + assertReplacementBoundary(before, handle, snapshot); + if (!waitForOpenShellSupervisorReconnect(handle.sandbox.sandboxName, timeoutSecs, deps)) { + throw new Error("Managed bootstrap Docker supervisor did not reconnect."); + } + const afterWaitJournal = deps.journalStore.load(journal.bootstrapIdentity); + if (!afterWaitJournal || !sameDockerBootstrapJournal(afterWaitJournal, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable transaction authority changed while awaiting bootstrap", + }); + } + assertCompletedCutoverRuntimeState(afterWaitJournal, deps); + const after = inspectExact(replacement.replacementRuntimeId, deps); + assertStableRunning(after, "completed replacement"); + if (assertImage(after, replacement.image, deps) !== replacement.runtimeImageContentId) { + throw new Error("Managed bootstrap Docker completed image content changed."); + } + assertReplacementBoundary(after, handle, snapshot); + const normalized = normalizeDockerManagedBootstrapLaunchSpec(after); + if (normalized.hash !== replacement.replacementSpecHash) { + throw new Error("Managed bootstrap Docker replacement changed during bootstrap."); + } + const imageCompletion = readProtectedImageCompletion(replacement.replacementRuntimeId, deps); + if ( + imageCompletion.bootstrapIdentity !== replacement.bootstrapIdentity || + imageCompletion.agent !== handle.plan.profile.agent || + imageCompletion.profileFingerprint !== replacement.profileFingerprint + ) { + throw new Error( + "Managed bootstrap Docker image completion identities do not match the transaction.", + ); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: replacement.replacementRuntimeId, + image: replacement.image, + runtimeImageContentId: replacement.runtimeImageContentId, + originalSpecHash: replacement.originalSpecHash, + replacementSpecHash: replacement.replacementSpecHash, + profileFingerprint: replacement.profileFingerprint, + bootstrapIdentity: replacement.bootstrapIdentity, + transactionPending: imageCompletion.transactionPending, + completedAt: deps.now().toISOString(), + }); + }, + + finalizeBootstrap, + }; +} diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.ts b/src/lib/onboard/openshell-docker-sandbox-containers.ts index a8934027fd6..94c067f027c 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.ts @@ -7,6 +7,7 @@ import type { DockerGpuPatchDeps } from "./docker-gpu-patch-types"; export const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; export const OPENSHELL_MANAGED_BY_VALUE = "openshell"; export const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; +export const OPENSHELL_SANDBOX_ID_LABEL = "openshell.ai/sandbox-id"; const DOCKER_SANDBOX_QUERY_TIMEOUT_MS = 30_000; diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 94d235cad52..48e889d78c1 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -122,7 +122,10 @@ describe("runtime provider central source boundary", () => { expect(bootstrapProtocolPaths).toEqual([ "src/lib/onboard/managed-bootstrap/adapter.ts", "src/lib/onboard/managed-bootstrap/docker-journal.ts", + "src/lib/onboard/managed-bootstrap/docker-shared-state.ts", "src/lib/onboard/managed-bootstrap/docker-spec.ts", + "src/lib/onboard/managed-bootstrap/docker-test-fixture.ts", + "src/lib/onboard/managed-bootstrap/docker.ts", "src/lib/onboard/managed-bootstrap/envelope.ts", "src/lib/onboard/managed-bootstrap/index.ts", ]); diff --git a/tsconfig.src.json b/tsconfig.src.json index da1287ae436..13c12fb1310 100644 --- a/tsconfig.src.json +++ b/tsconfig.src.json @@ -16,5 +16,10 @@ "types": ["node"] }, "include": ["src/**/*.ts"], - "exclude": ["node_modules", "nemoclaw", "src/**/*.test.ts"] + "exclude": [ + "node_modules", + "nemoclaw", + "src/**/*.test.ts", + "src/**/*-test-fixture.ts" + ] } From 206228343ae124563be7d13b4dd620e9628f9db0 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 08:36:59 -0700 Subject: [PATCH 104/117] test(onboard): linearize helper environment assertions Signed-off-by: Aaron Erickson --- .../docker-shared-state.test.ts | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts index 740eed20f72..26a5fd58d76 100644 --- a/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts @@ -49,19 +49,26 @@ function nodeHelperCalls(deps: DockerGpuPatchDeps): readonly (readonly string[]) .filter((args) => args.includes("/usr/local/bin/node")); } -function expectCleanNodeHelper(args: readonly string[]): void { +function expectLoaderEnvironmentNeutralized(args: readonly string[]): void { expect(args).toEqual(expect.arrayContaining([...LOADER_ENV_OVERRIDES])); expect(args).not.toContain("BASH_FUNC_*"); +} + +function expectCleanRunNodeHelper(args: readonly string[]): void { + expectLoaderEnvironmentNeutralized(args); + const nodeIndex = args.indexOf("/usr/local/bin/node"); + expect(nodeIndex).toBeGreaterThan(0); + const entrypointIndex = args.indexOf("--entrypoint"); + expect(args[entrypointIndex + 1]).toBe(CLEAN_NODE_COMMAND[0]); + expect(args.slice(nodeIndex - CLEAN_NODE_COMMAND.length + 2, nodeIndex + 1)).toEqual( + CLEAN_NODE_COMMAND.slice(1), + ); +} + +function expectCleanExecNodeHelper(args: readonly string[]): void { + expectLoaderEnvironmentNeutralized(args); const nodeIndex = args.indexOf("/usr/local/bin/node"); expect(nodeIndex).toBeGreaterThan(0); - if (args[0] === "run") { - const entrypointIndex = args.indexOf("--entrypoint"); - expect(args[entrypointIndex + 1]).toBe(CLEAN_NODE_COMMAND[0]); - expect(args.slice(nodeIndex - CLEAN_NODE_COMMAND.length + 2, nodeIndex + 1)).toEqual( - CLEAN_NODE_COMMAND.slice(1), - ); - return; - } expect(args.slice(nodeIndex - CLEAN_NODE_COMMAND.length + 1, nodeIndex + 1)).toEqual( CLEAN_NODE_COMMAND, ); @@ -84,7 +91,8 @@ describe("Docker managed-bootstrap shared-state helper environment", () => { expect(helpers.some((args) => args.includes("--shared-state-transaction-status"))).toBe(true); expect(helpers.some((args) => args.includes("--commit-shared-state-transaction"))).toBe(true); expect(helpers).not.toHaveLength(0); - helpers.forEach(expectCleanNodeHelper); + helpers.filter((args) => args[0] === "run").forEach(expectCleanRunNodeHelper); + helpers.filter((args) => args[0] === "exec").forEach(expectCleanExecNodeHelper); }); it("clears arbitrary image environment before the immutable rollback helper", () => { @@ -102,7 +110,7 @@ describe("Docker managed-bootstrap shared-state helper environment", () => { const helpers = nodeHelperCalls(fake.deps); expect(helpers).toHaveLength(1); expect(helpers[0]).toContain("--rollback-shared-state-transaction"); - expectCleanNodeHelper(helpers[0]!); + expectCleanRunNodeHelper(helpers[0]!); }); it("clears arbitrary container environment before the durable receipt-clear helper", () => { @@ -112,6 +120,6 @@ describe("Docker managed-bootstrap shared-state helper environment", () => { const helpers = nodeHelperCalls(fake.deps); expect(helpers).toHaveLength(1); expect(helpers[0]).toContain("--clear-shared-state-commit-receipt"); - expectCleanNodeHelper(helpers[0]!); + expectCleanExecNodeHelper(helpers[0]!); }); }); From 52e7144914569b31d494b9ef73bbd4019087f19e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 08:41:50 -0700 Subject: [PATCH 105/117] feat(onboard): bind managed create to runtime providers Signed-off-by: Aaron Erickson --- ci/source-shape-test-budget.json | 2 +- docs/inference/verify-inference-route.mdx | 5 +- docs/reference/commands.mdx | 5 +- docs/reference/troubleshooting.mdx | 12 +- .../adapters/openshell/sandbox-identity.ts | 16 + src/lib/onboard.ts | 7 +- .../sandbox-gpu-create-flow.ts | 2 + .../docker-gpu-local-inference.test.ts | 100 +++-- src/lib/onboard/docker-gpu-local-inference.ts | 87 ++++- .../docker-gpu-route-consumers.test.ts | 10 +- ...ocker-gpu-sandbox-create-lifecycle.test.ts | 95 ++++- src/lib/onboard/docker-gpu-sandbox-create.ts | 349 +++++++++++++----- ...ker-startup-command-sandbox-create.test.ts | 103 +++++- src/lib/onboard/managed-bootstrap/README.md | 64 ++-- .../managed-bootstrap/docker-runtime.ts | 294 +++++++++++++++ src/lib/onboard/managed-bootstrap/index.ts | 4 + .../managed-bootstrap/runtime-create.ts | 145 ++++++++ src/lib/onboard/runtime-provider/contract.ts | 13 +- src/lib/onboard/runtime-provider/registry.ts | 3 +- .../runtime-provider-contract.test.ts | 76 ++++ src/lib/onboard/sandbox-create-launch.test.ts | 46 +++ src/lib/onboard/sandbox-create-launch.ts | 29 +- .../onboard/sandbox-gpu-create-flow.test.ts | 154 ++++++++ src/lib/onboard/sandbox-gpu-create-flow.ts | 116 ++++-- .../onboard/sandbox-gpu-create-run-attempt.ts | 302 +++++++++++---- test/onboard-prepared-build-context.test.ts | 9 +- test/onboard-sandbox-recreation.test.ts | 5 + test/onboard-terminal-dashboard.test.ts | 9 +- test/runtime-provider-source-shape.test.ts | 33 +- 29 files changed, 1784 insertions(+), 311 deletions(-) create mode 100644 src/lib/onboard/managed-bootstrap/docker-runtime.ts create mode 100644 src/lib/onboard/managed-bootstrap/runtime-create.ts diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 8e473b6d06a..c5f7af36e78 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -528,7 +528,7 @@ }, { "file": "test/runtime-provider-source-shape.test.ts", - "test": "keeps production activation paths disconnected from managed bootstrap", + "test": "keeps production activation paths disconnected from driver bootstrap adapters", "category": "security" }, { diff --git a/docs/inference/verify-inference-route.mdx b/docs/inference/verify-inference-route.mdx index b13874e8184..f2c480eafc4 100644 --- a/docs/inference/verify-inference-route.mdx +++ b/docs/inference/verify-inference-route.mdx @@ -43,7 +43,10 @@ Restore the configured endpoint or proxy, run `$$nemoclaw onboard --resume` to c For local Ollama and vLLM on Docker GPU sandboxes using the compatibility route, onboarding performs an additional check after the sandbox becomes ready. It requests `https://inference.local/v1/models` from inside the sandbox and accepts only a 2xx response. -When this check fails, onboarding reports the endpoint and local-provider recovery steps before the first agent prompt. +If this check fails after compatibility recreation, onboarding prints failure diagnostics and attempts to restore the pre-patch container before it exits. +If that rollback fails, onboarding reports that the pre-patch container was not restored and prints container-cleanup guidance. +The local-provider failure output includes the endpoint and recovery steps before the first agent prompt. +GPU-proof diagnostics are captured before rollback and can also print cleanup guidance before the final container state is known, so inspect the sandbox and its labeled Docker containers before running a deletion command. NVIDIA NIM and other compatible endpoints receive their provider validation during onboarding but do not receive this local-provider post-ready check. For those routes, continue to the final route check, then use the status command and a short agent request after onboarding. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index b7d6dbb3fcd..eb7ac46f20a 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -813,7 +813,10 @@ On ordinary native Linux, the compatibility path uses an available NVIDIA CDI sp On Docker Desktop WSL, the compatibility path skips CDI and tries Docker `--gpus all` before the NVIDIA runtime. On Jetson/Tegra hosts, the compatibility path uses the NVIDIA runtime and adds eligible host group IDs for the supported GPU device nodes. These include selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices. -If the compatibility attempt fails, onboarding keeps its diagnostics and the failed sandbox in place and prints a manual cleanup command. +After compatibility recreation starts, onboarding keeps the pre-patch container as a rollback backup until the replacement passes the Ready, GPU, and applicable local-inference checks. +If a later check fails, onboarding prints failure diagnostics and attempts to restore the pre-patch container. +If rollback fails, onboarding reports that the pre-patch container was not restored and prints container-cleanup guidance. +GPU-proof diagnostics are captured before rollback and can print that guidance before the final container state is known, so inspect the sandbox and its labeled Docker containers before running a deletion command. Prerequisites: diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 514f5ea50e4..5c72b91f47a 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2542,20 +2542,26 @@ Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses t #### Common compatibility-path recovery -If the compatibility attempt fails on any host, onboarding leaves the failed sandbox and diagnostic bundle in place so you can inspect the OpenShell and Docker state. +After compatibility recreation starts, onboarding keeps the pre-patch container as a rollback backup until the replacement passes the Ready, direct GPU, and applicable local-inference checks. +If a later check fails, onboarding prints failure diagnostics and attempts to restore the pre-patch container before it exits. +When rollback succeeds, the pre-patch sandbox remains available. +When rollback fails, onboarding reports that the pre-patch container was not restored and prints container-cleanup guidance. +GPU-proof diagnostics are captured before rollback and can print that guidance before the final container state is known. +Inspect the sandbox and its labeled Docker containers before running a deletion command. Starting with NemoClaw v0.0.43, the standard installer handles the `/proc//task//comm` permission case during this patch path. If an older release fails direct GPU proof with that path and `Permission denied`, upgrade NemoClaw and rerun onboarding. -The output includes a cleanup command such as: +When inspection confirms that the failed sandbox remains, delete it with a command such as: ```bash openshell sandbox delete ``` -Fix the NVIDIA Container Toolkit or CDI configuration reported in the diagnostics, clean up the failed sandbox, then rerun onboarding. +Fix the NVIDIA Container Toolkit or CDI configuration reported in the diagnostics. +Run the deletion command only after confirming that the pre-patch sandbox was not restored, then rerun onboarding. If you do not need GPU access inside the sandbox, rerun with `--no-sandbox-gpu`. If sandbox creation fails with `CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all`, the OpenShell gateway tried `docker create --device nvidia.com/gpu=all` and Docker could not resolve the CDI spec. diff --git a/src/lib/adapters/openshell/sandbox-identity.ts b/src/lib/adapters/openshell/sandbox-identity.ts index 1820a8f8f7d..dbcc47696c4 100644 --- a/src/lib/adapters/openshell/sandbox-identity.ts +++ b/src/lib/adapters/openshell/sandbox-identity.ts @@ -14,3 +14,19 @@ export function parseOpenShellSandboxId(output: string): string | null { ? (matches[0] as string) : null; } + +export function resolveOpenShellSandboxId( + sandboxName: string, + runCaptureOpenshell: (args: string[], options?: Record) => string, +): string { + const output = runCaptureOpenshell(["sandbox", "get", sandboxName], { + ignoreError: false, + }); + const sandboxId = parseOpenShellSandboxId(output); + if (!sandboxId) { + throw new Error( + `OpenShell sandbox '${sandboxName}' did not return one exact durable sandbox ID.`, + ); + } + return sandboxId; +} diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index f07ded4ddee..b87721a9405 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2709,7 +2709,7 @@ async function createSandboxWithBaseImageResolution( recreateRuntime.advance("creating"); const { createResult, - dockerGpuCreatePatch, + runtimePatch, route: selectedGpuRoute, firstCreateOutput, registryImageRef, @@ -2765,7 +2765,7 @@ async function createSandboxWithBaseImageResolution( } if (effectiveSandboxGpuConfig.sandboxGpuEnabled) { - dockerGpuLocalInference.verifyGpuSandboxLocalInferenceAfterReady( + await dockerGpuLocalInference.verifyGpuSandboxLocalInferenceAndCommitAfterReady( effectiveSandboxGpuConfig, provider, { @@ -2773,11 +2773,10 @@ async function createSandboxWithBaseImageResolution( dockerDriverGateway, selectedRoute: selectedGpuRoute, verifyDirectSandboxGpu, - verifyGpuOrExit: dockerGpuCreatePatch.verifyGpuOrExit, - selectedMode: dockerGpuCreatePatch.selectedMode, runCaptureOpenshell, log: console.log, }, + runtimePatch, ); } diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index 7061252f12c..eebacbacd3d 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -64,8 +64,10 @@ export function createGpuPatchFixture() { maybeApplyDuringCreate: vi.fn(), createFailureMessage: vi.fn(() => null), exitOnPatchError: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), ensureApplied: vi.fn(), waitForSupervisorReconnectIfNeeded: vi.fn(), + commitAfterReady: vi.fn(), selectedMode: vi.fn(() => null), printReadinessFailureIfEnabled: vi.fn(), verifyGpuOrExit: vi.fn(() => VERIFIED_GPU_PROOF), diff --git a/src/lib/onboard/docker-gpu-local-inference.test.ts b/src/lib/onboard/docker-gpu-local-inference.test.ts index 60dfe28c639..3f0f6180ec4 100644 --- a/src/lib/onboard/docker-gpu-local-inference.test.ts +++ b/src/lib/onboard/docker-gpu-local-inference.test.ts @@ -11,6 +11,7 @@ import { shouldUseDockerGpuPatchHostNetwork, verifyDockerGpuSandboxLocalInference, verifyGpuSandboxAfterReady, + verifyGpuSandboxLocalInferenceAndCommitAfterReady, } from "./docker-gpu-local-inference"; const HOST_NETWORK_ENV = { @@ -311,10 +312,10 @@ describe("verifyGpuSandboxAfterReady", () => { }; } - it("runs the GPU proof and the runtime inference gate when the patch is active", () => { + it("runs the GPU proof and the runtime inference gate when the patch is active", async () => { const log = vi.fn(); const verifyDirectSandboxGpu = vi.fn(); - verifyGpuSandboxAfterReady( + await verifyGpuSandboxAfterReady( GPU_CONFIG, "vllm-local", baseOptions({ @@ -327,12 +328,12 @@ describe("verifyGpuSandboxAfterReady", () => { expect(log).toHaveBeenCalledWith(expect.stringContaining("reached local inference")); }); - it("captures the CUDA-usability proof onto the config for status persistence (#4231)", () => { + it("captures the CUDA-usability proof onto the config for status persistence (#4231)", async () => { const proof = { status: "verified" as const, cudaVerified: true, at: "t" }; const config: { sandboxGpuEnabled: boolean; sandboxGpuProof?: typeof proof | null } = { sandboxGpuEnabled: true, }; - verifyGpuSandboxAfterReady( + await verifyGpuSandboxAfterReady( config, "vllm-local", baseOptions({ @@ -343,45 +344,86 @@ describe("verifyGpuSandboxAfterReady", () => { expect(config.sandboxGpuProof).toEqual(proof); }); - it("does not duplicate proof diagnostics when Docker GPU patch verifier handles them", () => { + it("does not duplicate proof diagnostics when Docker GPU patch verifier handles them", async () => { const proofError = new Error("process.exit"); const verifyGpuOrExit = vi.fn(() => { throw proofError; }); const logError = vi.fn(); - expect(() => + await expect( verifyGpuSandboxAfterReady( GPU_CONFIG, "ollama-local", baseOptions({ verifyGpuOrExit, logError }), ), - ).toThrow(proofError); + ).rejects.toBe(proofError); expect(logError).not.toHaveBeenCalled(); }); - it("routes failure diagnostics through the provided error sink and exits", () => { + it("routes failure diagnostics through the provided error sink and throws for rollback", async () => { const logError = vi.fn(); - const exitSpy = vi.spyOn(process, "exit").mockImplementation((() => { - throw new Error("process.exit"); - }) as never); - try { - expect(() => - verifyGpuSandboxAfterReady( - GPU_CONFIG, - "ollama-local", - baseOptions({ - logError, - deps: { execInSandbox: execEmitting("HTTP_000"), sleep: vi.fn() }, - }), - ), - ).toThrow("process.exit"); - expect(exitSpy).toHaveBeenCalledWith(1); - expect(logError).toHaveBeenCalledWith( - expect.stringContaining("Local inference reachability check failed"), - ); - } finally { - exitSpy.mockRestore(); - } + await expect( + verifyGpuSandboxAfterReady( + GPU_CONFIG, + "ollama-local", + baseOptions({ + logError, + deps: { execInSandbox: execEmitting("HTTP_000"), sleep: vi.fn() }, + }), + ), + ).rejects.toThrow("GPU sandbox local inference reachability failed"); + expect(logError).toHaveBeenCalledWith( + expect.stringContaining("Local inference reachability check failed"), + ); + }); +}); + +describe("verifyGpuSandboxLocalInferenceAndCommitAfterReady", () => { + function options() { + return { + ...gpuPatchOptions(), + verifyDirectSandboxGpu: vi.fn(), + runCaptureOpenshell: vi.fn(() => ""), + log: vi.fn(), + }; + } + + it("commits only after local-inference reachability returns HTTP 2xx", async () => { + const runtimePatch = { + commitAfterReady: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + }; + await verifyGpuSandboxLocalInferenceAndCommitAfterReady( + GPU_CONFIG, + "ollama-local", + { + ...options(), + deps: { execInSandbox: execEmitting("HTTP_200"), sleep: vi.fn() }, + }, + runtimePatch, + ); + expect(runtimePatch.commitAfterReady).toHaveBeenCalledOnce(); + expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + }); + + it("rolls back before propagating an inference verification failure", async () => { + const runtimePatch = { + commitAfterReady: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + }; + await expect( + verifyGpuSandboxLocalInferenceAndCommitAfterReady( + GPU_CONFIG, + "ollama-local", + { + ...options(), + deps: { execInSandbox: execEmitting("HTTP_000"), sleep: vi.fn() }, + }, + runtimePatch, + ), + ).rejects.toThrow("GPU sandbox local inference reachability failed"); + expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce(); + expect(runtimePatch.commitAfterReady).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index da58f8c44d8..496ee830448 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -11,6 +11,7 @@ import { import type { DockerGpuPatchMode } from "./docker-gpu-patch-types"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; +import type { ManagedBootstrapRuntimePatch } from "./managed-bootstrap/runtime-create"; import { executeSandboxCommandForVerification } from "./sandbox-verification-exec"; const { @@ -385,9 +386,9 @@ export type GpuSandboxAfterReadyOptions = { verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult; verifyGpuOrExit?: ( verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult, - ) => SandboxGpuProofResult; + ) => Promise; reportGpuProofFailure?: boolean; - selectedMode: () => DockerGpuPatchMode | null; + selectedMode: ManagedBootstrapRuntimePatch["selectedMode"]; runCaptureOpenshell: (args: string[], opts?: Record) => string; env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform; @@ -396,33 +397,47 @@ export type GpuSandboxAfterReadyOptions = { deps?: DockerGpuSandboxInferenceVerifyDeps; }; +function asDockerGpuPatchMode( + selected: ReturnType, +): DockerGpuPatchMode | null { + if (!selected || !["gpus", "nvidia-runtime", "cdi", "startup-command"].includes(selected.kind)) { + return null; + } + return { + kind: selected.kind as DockerGpuPatchMode["kind"], + label: selected.label, + device: selected.device, + args: [...selected.args], + }; +} + /** * Post-readiness GPU sandbox verification orchestrator (kept out of the * ~12k-line onboard.ts entrypoint per the codebase-growth guardrail). Runs the * direct GPU proof, then — only when the Docker GPU patch is active for a local * inference provider — gates on local inference reachability from the sandbox - * runtime (#4509). Exits the process with actionable output if either proof - * fails. + * runtime (#4509). Throws with actionable output if either proof fails so the + * caller can complete rollback before selecting a terminal exit status. */ -export function verifyGpuSandboxAfterReady( +export async function verifyGpuSandboxAfterReady( config: DockerGpuLocalInferenceConfig, provider: string | null | undefined, options: GpuSandboxAfterReadyOptions, -): void { - verifyGpuSandboxAccessAfterReady(config, options); +): Promise { + await verifyGpuSandboxAccessAfterReady(config, options); verifyGpuSandboxLocalInferenceAfterReady(config, provider, options); } -export function verifyGpuSandboxAccessAfterReady( +export async function verifyGpuSandboxAccessAfterReady( config: DockerGpuLocalInferenceConfig, options: GpuSandboxAfterReadyOptions, -): SandboxGpuProofResult { +): Promise { try { // Capture the CUDA-usability proof result and write it back onto the shared // config so onboarding can persist it to the registry and `status` can // report proven usability rather than mere configuration (#4231). const proof = options.verifyGpuOrExit - ? options.verifyGpuOrExit(options.verifyDirectSandboxGpu) + ? await options.verifyGpuOrExit(options.verifyDirectSandboxGpu) : options.verifyDirectSandboxGpu(options.sandboxName); config.sandboxGpuProof = proof; return proof; @@ -431,11 +446,16 @@ export function verifyGpuSandboxAccessAfterReady( // prints the richer Error-phase / patched-container diagnostics before // rethrowing. Avoid a second generic proof-failure block in that path. if (!options.verifyGpuOrExit && options.reportGpuProofFailure !== false) { - printDockerGpuProofFailure(options.sandboxName, error, options.selectedMode(), { - runCaptureOpenshell: options.runCaptureOpenshell, - additionalSummaryLines: adaptDockerGpuRouteForPatch(options.selectedRoute) - .additionalSummaryLines, - }); + printDockerGpuProofFailure( + options.sandboxName, + error, + asDockerGpuPatchMode(options.selectedMode()), + { + runCaptureOpenshell: options.runCaptureOpenshell, + additionalSummaryLines: adaptDockerGpuRouteForPatch(options.selectedRoute) + .additionalSummaryLines, + }, + ); } throw error; } @@ -444,7 +464,7 @@ export function verifyGpuSandboxAccessAfterReady( export function verifyGpuSandboxLocalInferenceAfterReady( config: DockerGpuLocalInferenceConfig, provider: string | null | undefined, - options: GpuSandboxAfterReadyOptions, + options: Omit, ): void { if (options.selectedRoute !== "compatibility") return; const verification = verifyDockerGpuSandboxLocalInference(config, provider, { @@ -469,6 +489,39 @@ export function verifyGpuSandboxLocalInferenceAfterReady( verification, options.logError ?? ((message) => console.error(message)), ); - process.exit(1); + throw new Error( + `GPU sandbox local inference reachability failed for ${verification.endpoint}.`, + ); + } +} + +/** + * Keep the managed create transaction reversible until the sandbox's real + * local-inference reachability check returns HTTP 2xx. Rollback failures are + * attached to the original verification failure so callers retain both pieces + * of evidence. + */ +export async function verifyGpuSandboxLocalInferenceAndCommitAfterReady( + config: DockerGpuLocalInferenceConfig, + provider: string | null | undefined, + options: Omit, + runtimePatch: Pick< + ManagedBootstrapRuntimePatch, + "commitAfterReady" | "rollbackManagedStartupAfterCreateFailure" + >, +): Promise { + try { + verifyGpuSandboxLocalInferenceAfterReady(config, provider, options); + await runtimePatch.commitAfterReady(); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + try { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); + } catch (rollbackError) { + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + throw failure; } } diff --git a/src/lib/onboard/docker-gpu-route-consumers.test.ts b/src/lib/onboard/docker-gpu-route-consumers.test.ts index f2454e5d45d..b11215d340e 100644 --- a/src/lib/onboard/docker-gpu-route-consumers.test.ts +++ b/src/lib/onboard/docker-gpu-route-consumers.test.ts @@ -137,7 +137,7 @@ describe("selected route consumers", () => { expect(reverifyBridgeReachability).not.toHaveBeenCalled(); }); - it("skips compatibility-only inference gates after native wins", () => { + it("skips compatibility-only inference gates after native wins", async () => { const execInSandbox = vi.fn(); expect( verifyDockerGpuSandboxLocalInference(GPU_CONFIG, "ollama-local", { @@ -149,7 +149,7 @@ describe("selected route consumers", () => { ).toEqual({ status: "skipped", reason: "not-docker-gpu-patch" }); const verifyDirectSandboxGpu = vi.fn(); - verifyGpuSandboxAfterReady(GPU_CONFIG, "ollama-local", { + await verifyGpuSandboxAfterReady(GPU_CONFIG, "ollama-local", { sandboxName: "alpha", dockerDriverGateway: true, selectedRoute: "native", @@ -162,11 +162,11 @@ describe("selected route consumers", () => { expect(execInSandbox).not.toHaveBeenCalled(); }); - it("defers native proof diagnostics while automatic fallback owns recovery", () => { + it("defers native proof diagnostics while automatic fallback owns recovery", async () => { const proofError = new Error("native CUDA proof failed"); const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); try { - expect(() => + await expect( verifyGpuSandboxAfterReady(GPU_CONFIG, "ollama-local", { sandboxName: "alpha", dockerDriverGateway: true, @@ -178,7 +178,7 @@ describe("selected route consumers", () => { selectedMode: () => null, runCaptureOpenshell: vi.fn(() => ""), }), - ).toThrow(proofError); + ).rejects.toThrow(proofError); expect(consoleError).not.toHaveBeenCalled(); } finally { consoleError.mockRestore(); diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index fb94eba175e..6ddea611aa9 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -42,7 +42,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { vi.restoreAllMocks(); }); - it("defers backup removal until waitForSupervisorReconnectIfNeeded sees supervisorReady=true", () => { + it("retains the backup after reconnect and removes it only after post-Ready commit", async () => { const deps = makeDeps(); const result = deferredCreateResult(); const recreatePatch = vi.fn(() => result); @@ -84,13 +84,57 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.waitForSupervisorReconnectIfNeeded(); expect(waitForSupervisor).toHaveBeenCalledTimes(1); + expect(finalizeBackup).not.toHaveBeenCalled(); + + await patch.commitAfterReady(); expect(finalizeBackup).toHaveBeenCalledTimes(1); expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: true }, deps); expect(capturePreRollbackDiagnostics).not.toHaveBeenCalled(); expect(onPatchFailureExit).not.toHaveBeenCalled(); }); - it("refuses compatibility success when the backup container cannot be removed", () => { + it("reports a failed post-Ready rollback instead of treating it as restored", async () => { + const deps = makeDeps(); + const result = deferredCreateResult(); + const finalizeBackup = vi.fn(() => ({ + backupRemoved: false, + rolledBack: false, + })); + const onPatchFailureExit = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => ["existing-container"]), + recreatePatch: vi.fn(() => result), + waitForSupervisor: vi.fn(() => true), + finalizeBackup, + onPatchFailureExit, + }, + }); + + patch.maybeApplyDuringCreate(); + patch.waitForSupervisorReconnectIfNeeded(); + await patch.rollbackManagedStartupAfterCreateFailure(); + + expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: false }, deps); + expect(onPatchFailureExit).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + message: expect.stringContaining("pre-patch container was not restored"), + }), + expect.objectContaining({ + context: expect.objectContaining({ + backupContainerName: result.backupContainerName, + rolledBack: false, + }), + }), + ); + }); + + it("refuses compatibility success when the backup container cannot be removed", async () => { const deps = makeDeps(); const result = deferredCreateResult(); const onPatchFailureExit = vi.fn(); @@ -113,11 +157,14 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); patch.waitForSupervisorReconnectIfNeeded(); + expect(onPatchFailureExit).not.toHaveBeenCalled(); + + await patch.commitAfterReady(); expect(onPatchFailureExit).toHaveBeenCalledOnce(); expect(onPatchFailureExit.mock.calls[0]?.[1]).toEqual( expect.objectContaining({ - message: expect.stringContaining("backup container"), + message: expect.stringContaining("rollback backup"), }), ); expect(onPatchFailureExit.mock.calls[0]?.[2]).toEqual( @@ -245,7 +292,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(onPatchFailureExit).not.toHaveBeenCalled(); }); - it("records patchError when recreate throws and exitOnPatchError reports it via printDockerGpuPatchFailureAndExit", () => { + it("records patchError when recreate throws and exitOnPatchError reports it via printDockerGpuPatchFailureAndExit", async () => { const deps = makeDeps(); const recreatePatch = vi.fn(() => { throw new Error("docker rename failed"); @@ -271,7 +318,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); expect(patch.createFailureMessage()).toMatch(/Docker GPU patch failed/); - patch.exitOnPatchError(); + await patch.exitOnPatchError(); expect(onPatchFailureExit).toHaveBeenCalledTimes(1); // Supervisor wait must be skipped because needsSupervisorWait stayed false. patch.waitForSupervisorReconnectIfNeeded(); @@ -279,7 +326,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(finalizeBackup).not.toHaveBeenCalled(); }); - it("hard-stops a structured failed GPU proof on the compatibility route", () => { + it("hard-stops a structured failed GPU proof on the compatibility route", async () => { const deps = makeDeps(); const patch = createDockerGpuSandboxCreatePatch({ route: "compatibility", @@ -291,7 +338,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { }, }); - expect(() => + await expect( patch.verifyGpuOrExit(() => ({ status: "failed", cudaVerified: false, @@ -299,6 +346,38 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { detail: "No devices were found", at: "2026-07-07T00:00:00.000Z", })), - ).toThrow("Sandbox GPU proof returned failed status: nvidia-smi when available"); + ).rejects.toThrow("Sandbox GPU proof returned failed status: nvidia-smi when available"); + }); + + it("reports a failed rollback after GPU-proof diagnostics", async () => { + const deps = makeDeps(); + const result = deferredCreateResult(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => ["existing-container"]), + recreatePatch: vi.fn(() => result), + waitForSupervisor: vi.fn(() => true), + finalizeBackup: vi.fn(() => ({ + backupRemoved: false, + rolledBack: false, + })), + }, + }); + + patch.maybeApplyDuringCreate(); + patch.waitForSupervisorReconnectIfNeeded(); + + await expect( + patch.verifyGpuOrExit(() => { + throw new Error("nvidia-smi failed"); + }), + ).rejects.toThrow("nvidia-smi failed"); + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("pre-patch container was not restored"), + ); }); }); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index b683cbb1c77..52c15d0b5be 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -42,7 +42,7 @@ export { type DockerGpuSandboxCreateDeps = Pick< DockerGpuPatchDeps, - "runOpenshell" | "runCaptureOpenshell" | "sleep" | "dockerCapture" + "runOpenshell" | "runCaptureOpenshell" | "sleep" | "dockerCapture" | "dockerRun" | "dockerStop" >; type WaitSupervisorFn = typeof waitForOpenShellSupervisorReconnect; @@ -61,6 +61,11 @@ type PatchFailureExitFn = ( type DockerGpuSandboxCreatePatchOptions = { route: SelectedDockerGpuRoute; persistStartupCommand?: boolean; + /** + * A managed bootstrap owns the one permitted recreation after Ready. Keep + * route diagnostics/proof active without running the legacy recreator. + */ + externalRecreation?: boolean; sandboxName: string; gpuDevice?: string | null; openshellSandboxCommand?: readonly string[] | null; @@ -91,12 +96,27 @@ type DockerGpuSandboxCreatePatchOptions = { }; }; +export interface DockerManagedBootstrapDeferredCutover { + readonly selectedMode: DockerGpuPatchMode; + readonly failureContext: DockerGpuPatchFailureContext; + rollback(): Promise; + commit(): Promise; +} + export type DockerGpuSandboxCreatePatch = { maybeApplyDuringCreate: () => void; createFailureMessage: () => string | null; - exitOnPatchError: () => void; - ensureApplied: () => void; + exitOnPatchError: () => Promise; + attachManagedBootstrapCutover: (cutover: DockerManagedBootstrapDeferredCutover) => void; + rollbackManagedStartupAfterCreateFailure: () => Promise; + ensureApplied: () => Promise; waitForSupervisorReconnectIfNeeded: () => void; + /** + * Commit an attached managed cutover or remove a legacy recreation backup. + * Call only after authoritative Ready and the required GPU and applicable + * local-inference checks pass. + */ + commitAfterReady: () => Promise; selectedMode: () => DockerGpuPatchMode | null; /** * Print the Docker GPU readiness-failure block (including the Error-phase @@ -106,14 +126,14 @@ export type DockerGpuSandboxCreatePatch = { printReadinessFailureIfEnabled: () => void; /** * Run the GPU proof while distinguishing "sandbox in terminal phase" from - * "proof failed inside a live sandbox". Calls `process.exit(1)` for the - * former and rethrows after printing diagnostics for the latter so the - * onboarding flow surfaces the right failure cause (#4316). Returns the - * CUDA-usability proof result on success so callers can persist it (#4231). + * "proof failed inside a live sandbox". Awaits rollback and throws after + * printing diagnostics so the onboarding flow can select the terminal exit + * status without racing the rollback (#4316). Returns the CUDA-usability + * proof result on success so callers can persist it (#4231). */ verifyGpuOrExit: ( verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult, - ) => SandboxGpuProofResult; + ) => Promise; }; export function createDockerGpuSandboxCreatePatch( @@ -121,8 +141,12 @@ export function createDockerGpuSandboxCreatePatch( ): DockerGpuSandboxCreatePatch { const routeAdapter = adaptDockerGpuRouteForPatch(options.route); let result: DockerGpuPatchResult | null = null; + let managedBootstrapCutover: DockerManagedBootstrapDeferredCutover | null = null; let patchError: unknown = null; let needsSupervisorWait = false; + let cutoverFinalized = false; + let cutoverFinalization: Promise | null = null; + let cutoverFinalizationOutcome: "commit" | "rollback" | null = null; const findContainerIds = options.overrides?.findContainerIds ?? findOpenShellDockerSandboxContainerIds; @@ -145,7 +169,10 @@ export function createDockerGpuSandboxCreatePatch( backend: options.backend, dockerDesktopWsl: options.dockerDesktopWsl ?? isDockerDesktopWslRuntime(), }; - const patchEnabled = routeAdapter.enabled || options.persistStartupCommand === true; + const recreationEnabled = + options.externalRecreation !== true && + (routeAdapter.enabled || options.persistStartupCommand === true); + const patchEnabled = recreationEnabled; const patchTarget = routeAdapter.enabled ? "NVIDIA GPU access" : "restart-safe startup"; const recreateSelectedPatch = createDockerSandboxRecreator({ gpuEnabled: routeAdapter.enabled, @@ -156,21 +183,92 @@ export function createDockerGpuSandboxCreatePatch( recreateStartup: recreateStartupPatch, }); + const applyPatch = (deps: DockerGpuPatchDeps): void => { + if (!recreationEnabled) return; + result = recreateSelectedPatch(false, deps); + needsSupervisorWait = true; + console.log(` ✓ Docker container mode selected: ${result.mode.label}`); + }; + + const rollbackAfterFailure = async (): Promise => { + if (cutoverFinalized || (!managedBootstrapCutover && !result)) return null; + if (cutoverFinalization) { + try { + if (cutoverFinalizationOutcome !== "rollback") { + throw new Error("Managed startup rollback raced an in-progress commit finalization."); + } + await cutoverFinalization; + return null; + } catch (error) { + return error instanceof Error ? error : new Error(String(error)); + } + } + const finalization = (async () => { + await managedBootstrapCutover?.rollback(); + const finalizeOutcome = result + ? finalizeBackup({ result, supervisorReady: false }, options.deps) + : null; + cutoverFinalized = true; + needsSupervisorWait = false; + if (finalizeOutcome && !finalizeOutcome.rolledBack) { + throw new Error( + "Docker container rollback failed; the pre-patch container was not restored.", + ); + } + })(); + cutoverFinalization = finalization; + cutoverFinalizationOutcome = "rollback"; + try { + await finalization; + return null; + } catch (error) { + return error instanceof Error ? error : new Error(String(error)); + } finally { + if (!cutoverFinalized) { + cutoverFinalization = null; + cutoverFinalizationOutcome = null; + } + } + }; + + const reportPatchErrorAndExit = async (): Promise => { + if (!patchError) return; + const rollbackError = await rollbackAfterFailure(); + if (rollbackError) { + patchError = new Error( + `${patchError instanceof Error ? patchError.message : String(patchError)}; managed startup rollback failed: ${rollbackError.message}`, + ); + } + onPatchFailureExit(options.sandboxName, patchError, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }); + }; + const selectedMode = (): DockerGpuPatchMode | null => + managedBootstrapCutover?.selectedMode ?? result?.mode ?? null; + const failureContext = (): DockerGpuPatchFailureContext => + managedBootstrapCutover?.failureContext ?? buildFailureContext(options.sandboxName, result); + return { maybeApplyDuringCreate() { if (!patchEnabled || result || patchError) return; const containerIds = findContainerIds(options.sandboxName); if (containerIds.length === 0) return; + if (containerIds.length !== 1) { + patchError = new Error( + `Docker recreation observed ${String(containerIds.length)} matching containers; refusing an ambiguous replacement.`, + ); + return; + } console.log( ` OpenShell Docker container detected; recreating it with ${patchTarget} before readiness wait...`, ); try { - result = recreateSelectedPatch(false, { + applyPatch({ runCaptureOpenshell: options.deps.runCaptureOpenshell, sleep: options.deps.sleep, }); - needsSupervisorWait = true; - console.log(` ✓ Docker container mode selected: ${result.mode.label}`); } catch (error) { patchError = error; } @@ -183,33 +281,44 @@ export function createDockerGpuSandboxCreatePatch( : "Docker startup-command patch failed while OpenShell sandbox create was still waiting."; }, - exitOnPatchError() { - if (!patchError) return; - onPatchFailureExit(options.sandboxName, patchError, { + async exitOnPatchError() { + await reportPatchErrorAndExit(); + }, + + attachManagedBootstrapCutover(cutover) { + if (managedBootstrapCutover || result || cutoverFinalized) { + throw new Error("Managed bootstrap cutover may be attached exactly once."); + } + managedBootstrapCutover = cutover; + }, + + async rollbackManagedStartupAfterCreateFailure() { + const rollbackError = await rollbackAfterFailure(); + if (!rollbackError) return; + onPatchFailureExit(options.sandboxName, rollbackError, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: { + ...failureContext(), + rolledBack: false, + }, }); }, - ensureApplied() { + async ensureApplied() { if (!patchEnabled || result) return; console.log(` Recreating OpenShell Docker sandbox container with ${patchTarget}...`); try { - result = recreateSelectedPatch(false, options.deps); - needsSupervisorWait = true; - console.log(` ✓ Docker container mode selected: ${result.mode.label}`); + applyPatch(options.deps); } catch (error) { - onPatchFailureExit(options.sandboxName, error, { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - }); + patchError = error; + await reportPatchErrorAndExit(); } }, waitForSupervisorReconnectIfNeeded() { - if (!needsSupervisorWait) return; + if (!needsSupervisorWait || cutoverFinalized) return; const supervisorReconnectTimeoutSecs = getDockerGpuSupervisorReconnectTimeoutSecs( options.timeoutSecs, ); @@ -221,14 +330,17 @@ export function createDockerGpuSandboxCreatePatch( supervisorReconnectTimeoutSecs, { runOpenshell: options.deps.runOpenshell, - // Pass `runCaptureOpenshell` so the supervisor-reconnect wait can - // short-circuit on a terminal sandbox phase instead of burning - // the full reconnect timeout window when the patched container - // crashed on startup (#4316). runCaptureOpenshell: options.deps.runCaptureOpenshell, sleep: options.deps.sleep, }, ); + if (supervisorReady) { + // Reconnect completes the legacy recreation check. Keep its rollback + // backup until the caller accepts authoritative Ready and the required + // GPU checks. + needsSupervisorWait = false; + return; + } if (!supervisorReady && result) { try { captureFailedClone(options.sandboxName, result, options.deps); @@ -239,40 +351,13 @@ export function createDockerGpuSandboxCreatePatch( } } const finalizeOutcome = result - ? finalizeBackup({ result, supervisorReady }, options.deps) + ? finalizeBackup({ result, supervisorReady: false }, options.deps) : null; - if (supervisorReady) { - if (finalizeOutcome && !finalizeOutcome.backupRemoved) { - onPatchFailureExit( - options.sandboxName, - new Error( - "OpenShell supervisor reconnected, but the recreated backup container could not be removed.", - ), - { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - context: { - sandboxName: options.sandboxName, - oldContainerId: result?.oldContainerId, - newContainerId: result?.newContainerId, - backupContainerName: result?.backupContainerName, - selectedMode: result?.mode ?? null, - rolledBack: false, - }, - }, - ); - } - return; - } - const failureMessage = (() => { - if (!finalizeOutcome) { - return "OpenShell supervisor did not reconnect to the recreated container."; - } - return finalizeOutcome.rolledBack - ? "OpenShell supervisor did not reconnect to the recreated container; pre-patch sandbox restored." - : "OpenShell supervisor did not reconnect to the recreated container and rollback failed; pre-patch sandbox was NOT restored."; - })(); + cutoverFinalized = true; + needsSupervisorWait = false; + const failureMessage = finalizeOutcome?.rolledBack + ? "OpenShell supervisor did not reconnect to the recreated container; pre-patch sandbox restored." + : "OpenShell supervisor did not reconnect to the recreated container and rollback failed; pre-patch sandbox was NOT restored."; onPatchFailureExit(options.sandboxName, new Error(failureMessage), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, @@ -288,21 +373,108 @@ export function createDockerGpuSandboxCreatePatch( }); }, + async commitAfterReady() { + if (cutoverFinalized || (!managedBootstrapCutover && !result)) return; + if (needsSupervisorWait) { + const error = new Error( + "Managed startup cannot commit before the recreated OpenShell supervisor reconnects.", + ); + const rollbackError = await rollbackAfterFailure(); + onPatchFailureExit( + options.sandboxName, + rollbackError + ? new Error(`${error.message} Rollback failed: ${rollbackError.message}`) + : error, + { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }, + ); + return; + } + if (cutoverFinalization) { + if (cutoverFinalizationOutcome !== "commit") { + throw new Error("Managed startup commit raced an in-progress rollback finalization."); + } + await cutoverFinalization; + return; + } + const finalization = (async () => { + if (managedBootstrapCutover) { + try { + await managedBootstrapCutover.commit(); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + let rollbackError: Error | null = null; + try { + await managedBootstrapCutover.rollback(); + cutoverFinalized = true; + needsSupervisorWait = false; + } catch (rollbackFailure) { + rollbackError = + rollbackFailure instanceof Error + ? rollbackFailure + : new Error(String(rollbackFailure)); + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + onPatchFailureExit(options.sandboxName, failure, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: { + ...failureContext(), + rolledBack: rollbackError === null, + }, + }); + return; + } + } + const finalizeOutcome = result + ? finalizeBackup({ result, supervisorReady: true }, options.deps) + : null; + cutoverFinalized = true; + if (!finalizeOutcome || finalizeOutcome.backupRemoved) return; + onPatchFailureExit( + options.sandboxName, + new Error("Managed startup passed Ready, but its rollback backup could not be removed."), + { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: failureContext(), + }, + ); + })(); + cutoverFinalization = finalization; + cutoverFinalizationOutcome = "commit"; + try { + await finalization; + } finally { + if (!cutoverFinalized) { + cutoverFinalization = null; + cutoverFinalizationOutcome = null; + } + } + }, + selectedMode() { - return result?.mode ?? null; + return selectedMode(); }, printReadinessFailureIfEnabled() { if (!routeAdapter.enabled) return; - printDockerGpuReadinessFailure(options.sandboxName, result?.mode ?? null, { + printDockerGpuReadinessFailure(options.sandboxName, selectedMode(), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, - context: buildFailureContext(options.sandboxName, result), + context: failureContext(), additionalSummaryLines: routeAdapter.additionalSummaryLines, }); }, - verifyGpuOrExit(verifyDirectSandboxGpu) { + async verifyGpuOrExit(verifyDirectSandboxGpu) { // Before issuing GPU proof commands through `openshell sandbox exec`, // confirm the sandbox is still in a live phase. A sandbox that // transitioned to Error after the readiness wait succeeded (e.g. the @@ -312,7 +484,7 @@ export function createDockerGpuSandboxCreatePatch( // container/Error-phase classification instead of running the proof // (#4316). const sandboxName = options.sandboxName; - const failureContext = buildFailureContext(sandboxName, result); + const currentFailureContext = failureContext(); if (routeAdapter.enabled && options.deps.runCaptureOpenshell) { const list = options.deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true, @@ -321,20 +493,23 @@ export function createDockerGpuSandboxCreatePatch( if (phase) { console.error(""); console.error(` Skipping GPU proof: sandbox '${sandboxName}' is in ${phase} phase.`); - printDockerGpuProofFailure( - sandboxName, - new Error( - `Sandbox '${sandboxName}' entered ${phase} phase after readiness; GPU proof skipped.`, - ), - result?.mode ?? null, - { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - context: failureContext, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - }, + const failure = new Error( + `Sandbox '${sandboxName}' entered ${phase} phase after readiness; GPU proof skipped.`, ); - process.exit(1); + printDockerGpuProofFailure(sandboxName, failure, selectedMode(), { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + context: currentFailureContext, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }); + const rollbackError = await rollbackAfterFailure(); + if (rollbackError) { + console.error(` ${rollbackError.message}`); + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + throw failure; } } try { @@ -346,13 +521,21 @@ export function createDockerGpuSandboxCreatePatch( } return proof; } catch (error) { - printDockerGpuProofFailure(sandboxName, error, result?.mode ?? null, { + const failure = error instanceof Error ? error : new Error(String(error)); + printDockerGpuProofFailure(sandboxName, failure, selectedMode(), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, - context: routeAdapter.enabled ? failureContext : null, + context: routeAdapter.enabled ? currentFailureContext : null, additionalSummaryLines: routeAdapter.additionalSummaryLines, }); - throw error; + const rollbackError = await rollbackAfterFailure(); + if (rollbackError) { + console.error(` ${rollbackError.message}`); + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + throw failure; } }, }; diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts index 17cd02f6432..ec0ad3cb306 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts @@ -68,7 +68,7 @@ describe("Docker startup-command sandbox creation", () => { vi.restoreAllMocks(); }); - it("uses the startup-command recreation path with DCode's exact resource limits", () => { + it("uses the startup-command recreation path with DCode's exact resource limits", async () => { const dockerCaptureOutput: Record = { ps: "old-container-id\n", inspect: JSON.stringify([inspectFixture()]), @@ -102,7 +102,7 @@ describe("Docker startup-command sandbox creation", () => { }, }); - patch.ensureApplied(); + await patch.ensureApplied(); expect(recreatePatch).not.toHaveBeenCalled(); expect(dockerRunDetached.mock.calls[0]?.[0]).toEqual( @@ -156,7 +156,102 @@ describe("Docker startup-command sandbox creation", () => { expect(context.rolledBack).toBe(true); }); - it("reports startup-command creation failures through the composed patch boundary", () => { + it("defers a driver-owned managed cutover until the authoritative caller commits", async () => { + const deps = makeDeps(); + let releaseCommit = () => {}; + const commit = vi.fn( + () => + new Promise((resolve) => { + releaseCommit = resolve; + }), + ); + const rollback = vi.fn(async () => {}); + const patch = createDockerGpuSandboxCreatePatch({ + route: "native", + externalRecreation: true, + sandboxName: "alpha", + timeoutSecs: 60, + deps, + }); + patch.attachManagedBootstrapCutover({ + selectedMode: { + kind: "startup-command", + label: "managed bootstrap", + device: "", + args: [], + }, + failureContext: { sandboxName: "alpha" }, + commit, + rollback, + }); + patch.maybeApplyDuringCreate(); + await patch.ensureApplied(); + patch.waitForSupervisorReconnectIfNeeded(); + expect(commit).not.toHaveBeenCalled(); + const firstCommit = patch.commitAfterReady(); + const duplicateCommit = patch.commitAfterReady(); + expect(commit).toHaveBeenCalledOnce(); + expect(rollback).not.toHaveBeenCalled(); + releaseCommit(); + await Promise.all([firstCommit, duplicateCommit]); + }); + + it("rolls back a driver-owned cutover before reporting commit failure", async () => { + const deps = makeDeps(); + const events: string[] = []; + const commit = vi.fn(async () => { + events.push("commit"); + throw new Error("receipt validation failed"); + }); + const rollback = vi.fn(async () => { + events.push("rollback"); + }); + const onPatchFailureExit = vi.fn(() => { + events.push("exit"); + }); + const patch = createDockerGpuSandboxCreatePatch({ + route: "native", + externalRecreation: true, + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { onPatchFailureExit }, + }); + patch.attachManagedBootstrapCutover({ + selectedMode: { + kind: "startup-command", + label: "managed bootstrap", + device: "", + args: [], + }, + failureContext: { + sandboxName: "alpha", + oldContainerId: "held-container", + newContainerId: "replacement-container", + }, + commit, + rollback, + }); + + await patch.commitAfterReady(); + + expect(events).toEqual(["commit", "rollback", "exit"]); + expect(onPatchFailureExit).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ message: "receipt validation failed" }), + expect.objectContaining({ + context: expect.objectContaining({ + oldContainerId: "held-container", + newContainerId: "replacement-container", + rolledBack: true, + }), + }), + ); + await patch.rollbackManagedStartupAfterCreateFailure(); + expect(rollback).toHaveBeenCalledOnce(); + }); + + it("reports startup-command creation failures through the composed patch boundary", async () => { const deps = makeDeps(); const onPatchFailureExit = vi.fn(); const patch = createDockerGpuSandboxCreatePatch({ @@ -177,7 +272,7 @@ describe("Docker startup-command sandbox creation", () => { patch.maybeApplyDuringCreate(); expect(patch.createFailureMessage()).toMatch(/startup-command patch failed/); - patch.exitOnPatchError(); + await patch.exitOnPatchError(); expect(onPatchFailureExit).toHaveBeenCalledWith( "alpha", expect.objectContaining({ message: "startup recreate failed" }), diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 717fcaa048b..63a790993f5 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -3,9 +3,11 @@ # Managed bootstrap protocol -This directory defines a dormant, driver-neutral transaction contract and its -first driver adapter. It does not register a runtime provider or change sandbox -creation, onboarding, snapshot, clone, or restore behavior. +This directory defines a dormant, driver-neutral transaction contract, its +first driver adapter, and an injectable sandbox-create lifecycle. Production +runtime bundles still report bootstrap as unsupported, so the candidate +lifecycle remains inert. The shared finalization path keeps existing Docker +recreation reversible through later Ready, GPU, and local-inference checks. The protocol binds one random bootstrap identity to: @@ -66,10 +68,12 @@ including its supervisor environment, to immutable prepared authority before activation. The native boundary introduces no driver-specific environment policy. -The first Docker-specific groundwork defines a private, monotonic cutover -journal and a canonical launch-spec normalizer. Each surface is independently -validated and remains dormant: no registered runtime provider imports either -module, and neither changes sandbox creation or lifecycle behavior. +The Docker-specific layers define a private, monotonic cutover journal, a +canonical launch-spec normalizer, and an injectable provider create lifecycle. +The candidate provider surface composes these layers without registering them +in a production runtime bundle. It remains inert, while the shared finalization +surface extends rollback ownership for the existing Docker compatibility and +startup recreation paths. The Docker adapter creates and validates a stopped replacement under an identity-derived staging name while the original remains running. It stages the @@ -88,34 +92,28 @@ must also inject the selected gateway's canonical state root. ## Architectural disposition -The coordinator deliberately lands as a dormant trust-boundary slice before a -provider or image activates it. This keeps the driver-neutral transaction -authority review separate from the first driver implementation instead of -making that implementation the de facto central contract. The coordinator -module remains cohesive because its receipt shapes, normalization, state -transitions, and rollback proofs form one authority boundary; provider-specific -logic must live outside it rather than growing this file. +The runtime-provider bundle is the only bootstrap registration boundary. The +candidate Docker surface owns create routing, replacement construction, +native-to-compatibility fallback evidence, and deferred commit or rollback. +Central onboarding accepts that provider-neutral surface without a Docker or +Podman selection branch. Tests register an MXC-style surface through the same +bundle and render held launches for OpenClaw, Hermes, and LangChain Deep Agents +Code. -This is executable, bounded groundwork rather than an untested placeholder. -`adapter.test.ts` drives prepare, durable record, activation, finalization, and -failure rollback for OpenClaw, Hermes, and LangChain Deep Agents Code through an -MXC-named fake driver. `runtime-provider-source-shape.test.ts` separately -inventories the protocol, provider, and image-packaging surfaces and proves that -production activation cannot import or package the protocol yet. The later -activation slice must add a registered-provider contract test for the same -transaction before removing those dormancy assertions. +The coordinator remains the driver-neutral transaction authority: its receipt +shapes, normalization, state transitions, and rollback proofs form one cohesive +boundary, while provider-specific routing and runtime operations stay outside +it. The candidate composition is executable, bounded groundwork rather than an +untested placeholder, but no registered provider selects it yet. The native entrypoint source is intentionally not compiled into production -artifacts, and neither image-owned source is packaged or selected yet. No -production TypeScript module imports this protocol or the Docker adapter. The +artifacts, and neither image-owned source is packaged or selected yet. The current image definitions do not package `nemoclaw-managed-startup-hold`, `managed-startup-image-runtime.cjs`, or the shared-state bootstrap modes consumed -by the adapter. A later provider integration must compile and verify the -freestanding entrypoint natively for amd64 and arm64 in every agent image. It -must add those prerequisites together with their image-runtime bootstrap modes -and wire the coordinator and Docker adapter into create as one boundary. The -same contract is exercised for OpenClaw, Hermes, and Deep Agents Code without a -provider-specific central switch. The remaining integration and qualification -work is tracked in [epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) -and its linked implementation stack. Until that complete boundary lands, every -registered runtime provider keeps its bootstrap surface unsupported. +by the adapter. Later persistence and qualification slices must compile and +verify the freestanding entrypoint for amd64 and arm64 in every agent image, add +the image-runtime prerequisites, and provide the canonical durable authority +store. The remaining integration and qualification work is tracked in +[epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744). Until that complete +boundary passes protected E2E, every production runtime provider keeps +bootstrap unsupported. diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.ts new file mode 100644 index 00000000000..21932a2207f --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.ts @@ -0,0 +1,294 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { RuntimeProviderBootstrapSurface } from "../runtime-provider/contract"; +import { detectTegraDeviceGroupGids } from "../docker-gpu-jetson-groups"; +import { buildDockerGpuMode, selectDockerGpuPatchMode } from "../docker-gpu-patch-mode"; +import type { DockerGpuPatchMode } from "../docker-gpu-patch-types"; +import { renderCompatibilityFallbackCreateArgs } from "../docker-gpu-route"; +import { + createDockerGpuSandboxCreatePatch, + isDockerDesktopWslRuntime, +} from "../docker-gpu-sandbox-create"; +import { + isImmutableDockerImageId, + queryOpenShellDockerSandboxContainers, + queryOpenShellDockerSandboxRuntimeSnapshot, +} from "../openshell-docker-sandbox-containers"; +import * as sandboxGpuCreateAttempt from "../sandbox-gpu-create-attempt"; +import { + activateManagedBootstrapSequence, + finalizeManagedBootstrapSequence, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + prepareManagedBootstrapSequence, +} from "./adapter"; +import { createDockerManagedBootstrapAdapter } from "./docker"; +import type { + ManagedBootstrapRuntimeCompatibilityLaunchInput, + ManagedBootstrapRuntimeCreateLaunchResult, + ManagedBootstrapRuntimeCreateLifecycle, + ManagedBootstrapRuntimeCreateLifecycleInput, + ManagedBootstrapRuntimeOnboardRoutingInput, +} from "./runtime-create"; + +type SupportedBootstrapSurface = Extract< + RuntimeProviderBootstrapSurface, + { readonly supported: true } +>; + +function dockerReplacementOptions( + mode: DockerGpuPatchMode, + input: ManagedBootstrapRuntimeCreateLifecycleInput, +) { + const backend = input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic"; + return { + values: { + gpuModeArgs: [...mode.args], + gpuModeDevice: mode.device, + gpuModeKind: mode.kind, + gpuModeLabel: mode.label, + requiredUlimits: input.requiredLimits.map( + (limit) => `${limit.name}=${limit.soft}:${limit.hard}`, + ), + extraGroupGids: + backend === "jetson" && input.route === "compatibility" ? detectTegraDeviceGroupGids() : [], + }, + }; +} + +function selectedDockerMode( + input: ManagedBootstrapRuntimeCreateLifecycleInput, + dockerDesktopWsl: boolean | undefined, +): DockerGpuPatchMode { + const backend = input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic"; + if (input.route !== "compatibility" || !input.sandboxGpuConfig.sandboxGpuEnabled) { + return buildDockerGpuMode("startup-command"); + } + const selection = selectDockerGpuPatchMode( + { + image: `${input.image.repository}@${input.image.manifestDigest}`, + device: input.sandboxGpuConfig.sandboxGpuDevice, + backend, + dockerDesktopWsl, + }, + input.dependencies, + ); + if (selection.mode) return selection.mode; + throw new Error( + backend === "jetson" + ? "Docker did not accept the Jetson NVIDIA runtime GPU mode for managed bootstrap." + : "Docker did not accept a compatibility GPU mode for managed bootstrap.", + ); +} + +function createDockerLifecycle( + providerId: string, + input: ManagedBootstrapRuntimeCreateLifecycleInput, +): ManagedBootstrapRuntimeCreateLifecycle { + if (input.providerId !== providerId) { + throw new Error( + `Managed bootstrap provider '${providerId}' cannot run authority for '${input.providerId}'.`, + ); + } + const dockerDesktopWsl = + input.route === "compatibility" ? isDockerDesktopWslRuntime() : undefined; + const mode = selectedDockerMode(input, dockerDesktopWsl); + const backend = input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic"; + const persistStartupCommand = + input.persistStartupCommand && (input.route !== "native" || input.requiredLimits.length > 0); + const patch = createDockerGpuSandboxCreatePatch({ + route: input.route, + persistStartupCommand, + externalRecreation: true, + sandboxName: input.sandboxName, + gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, + openshellSandboxCommand: input.heldWorkloadArgv, + requiredUlimits: input.requiredLimits, + timeoutSecs: input.timeoutSecs, + backend, + dockerDesktopWsl, + deps: input.dependencies, + ...(input.onPatchFailure + ? { + overrides: { + onPatchFailureExit: (_sandboxName: string, error: unknown) => + input.onPatchFailure?.(error), + }, + } + : {}), + }); + const adapter = input.adapterOverride ?? createDockerManagedBootstrapAdapter(input.dependencies); + const createPlan = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandboxName: input.sandboxName, + driverId: providerId, + image: input.image, + profile: { + agent: input.request.agent, + fingerprint: input.request.profileFingerprint, + }, + agentIdentity: input.agentIdentity, + intendedWorkloadArgv: input.intendedWorkloadArgv, + expectedSupervisorArgv: input.expectedSupervisorArgv, + metadata: {}, + } as const; + const replacementOptions = dockerReplacementOptions(mode, input); + + return { + launchArgv: input.launchArgv, + patch, + async prepareNetwork() { + if (input.route !== "compatibility") return; + const { enforceDockerGpuPatchPreserveNetwork } = await import( + "../docker-gpu-local-inference" + ); + await enforceDockerGpuPatchPreserveNetwork( + input.network.inferenceProvider, + input.sandboxGpuConfig, + { + dockerDriverGateway: input.network.dockerDriverGateway, + selectedRoute: input.route, + gatewayPort: input.network.gatewayPort, + log: console.log, + }, + ); + }, + async runCreate( + launch: (input: { + readonly heldWorkloadArgv: readonly string[]; + readonly bootstrapIdentity: string; + }) => Promise>, + ): Promise { + const launchState: { value?: ManagedBootstrapRuntimeCreateLaunchResult } = {}; + const prepared = await prepareManagedBootstrapSequence(adapter, { + create: { + bootstrapIdentity: input.bootstrapIdentity, + plan: createPlan, + request: input.request, + launch: async (launchInput) => { + const launched = await launch(launchInput); + launchState.value = launched; + return launched.receipt; + }, + }, + request: input.request, + replacementOptions, + }); + const activated = await activateManagedBootstrapSequence(adapter, { + transaction: prepared, + authorityStore: input.authorityStore, + timeoutSecs: input.timeoutSecs, + }); + const launched = launchState.value; + if (!launched) { + await finalizeManagedBootstrapSequence(adapter, { + outcome: "rollback", + transaction: activated, + }); + throw new Error("Managed bootstrap did not return its OpenShell create receipt."); + } + let finalized = false; + patch.attachManagedBootstrapCutover({ + selectedMode: mode, + failureContext: { + sandboxName: input.sandboxName, + oldContainerId: activated.snapshot.runtimeId, + newContainerId: activated.replacement.replacementRuntimeId, + backupContainerName: null, + selectedMode: mode, + }, + async rollback() { + if (finalized) return; + await finalizeManagedBootstrapSequence(adapter, { + outcome: "rollback", + transaction: activated, + }); + finalized = true; + }, + async commit() { + if (finalized) return; + await finalizeManagedBootstrapSequence(adapter, { + outcome: "commit", + transaction: activated, + }); + finalized = true; + }, + }); + return launched.value; + }, + }; +} + +function createDockerOnboardRouting(input: ManagedBootstrapRuntimeOnboardRoutingInput) { + const baseline = input.nativeFallbackEnabled + ? queryOpenShellDockerSandboxContainers(input.sandboxName) + : null; + const inspectNativeRuntime = () => { + const snapshot = queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName); + return snapshot.ok + ? { + imageId: snapshot.imageId, + bookkeepingImageRef: snapshot.bookkeepingImageRef, + stateError: snapshot.stateError, + nativeGpuAttachmentState: snapshot.nativeGpuAttachmentState, + } + : null; + }; + return { + nativeFallbackHasCleanBaseline: baseline?.ok === true && baseline.ids.length === 0, + inspectNativeRuntime, + isNativeCreateRoutingFailure: (output: string, sawProgress: boolean): boolean => + sandboxGpuCreateAttempt.isNativeGpuCreateRoutingFailure(output, { sawProgress }), + isTrustedNativeRuntimeError: (error: string): boolean => + sandboxGpuCreateAttempt.isTrustedNativeGpuRuntimeError(error), + isNativeReadinessRoutingFailure: (failure: { + readonly failurePhase: string | null; + readonly runtimeError: string; + }): boolean => sandboxGpuCreateAttempt.isNativeGpuReadinessRoutingFailure(failure), + prepareCompatibilityLaunch: ( + compatibility: ManagedBootstrapRuntimeCompatibilityLaunchInput, + ) => { + const runtime = compatibility.runtimeSnapshot; + const imageId = + runtime?.imageId ?? + (compatibility.prebuildImageId && isImmutableDockerImageId(compatibility.prebuildImageId) + ? compatibility.prebuildImageId.toLowerCase() + : null); + let registryImageRef = compatibility.currentRegistryImageRef; + if ( + !registryImageRef && + runtime?.bookkeepingImageRef && + !isImmutableDockerImageId(runtime.bookkeepingImageRef) + ) { + registryImageRef = runtime.bookkeepingImageRef; + } + const createArgs = renderCompatibilityFallbackCreateArgs(compatibility.createArgs, { + imageRef: imageId, + allowUnbuiltSource: compatibility.allowUnbuiltSource, + compatibilityPolicyPath: compatibility.compatibilityPolicyPath, + }); + return { + createArgv: input.openshellArgv([ + "sandbox", + "create", + ...createArgs, + "--", + ...compatibility.startupCommand, + ]), + registryImageRef, + }; + }, + }; +} + +/** Candidate Docker surface. Production activation remains a later qualification slice. */ +export function createDockerManagedBootstrapSurface( + providerId = "docker", +): SupportedBootstrapSurface { + return { + providerId, + supported: true, + createLifecycle: (input) => createDockerLifecycle(providerId, input), + createOnboardRouting: createDockerOnboardRouting, + }; +} diff --git a/src/lib/onboard/managed-bootstrap/index.ts b/src/lib/onboard/managed-bootstrap/index.ts index 17099572608..c55768afe02 100644 --- a/src/lib/onboard/managed-bootstrap/index.ts +++ b/src/lib/onboard/managed-bootstrap/index.ts @@ -20,3 +20,7 @@ export { serializeManagedBootstrapEnvelope, serializeManagedBootstrapImageCompletion, } from "./envelope"; +export type { + ManagedBootstrapRuntimeCreateLifecycle, + ManagedBootstrapRuntimePatch, +} from "./runtime-create"; diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.ts b/src/lib/onboard/managed-bootstrap/runtime-create.ts new file mode 100644 index 00000000000..9154b0de44b --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/runtime-create.ts @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxGpuProofResult } from "../../state/registry"; +import type { ManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import type { SandboxGpuConfig } from "../sandbox-gpu-mode"; +import type { + ManagedBootstrapAdapter, + ManagedBootstrapAgentIdentity, + ManagedBootstrapAuthorityStore, + ManagedBootstrapCreateReceipt, + ManagedBootstrapImageIdentity, +} from "./adapter"; + +export interface ManagedBootstrapRuntimeCommandResult { + readonly status?: number | null; + readonly stdout?: string | Buffer | null; + readonly stderr?: string | Buffer | null; + readonly error?: Error | null; +} + +export interface ManagedBootstrapRuntimeDependencies { + readonly runCaptureOpenshell?: (args: string[], options?: Record) => string; + readonly runOpenshell?: ( + args: string[], + options?: Record, + ) => ManagedBootstrapRuntimeCommandResult; + readonly sleep?: (seconds: number) => void; +} + +export type ManagedBootstrapRuntimeRoute = "none" | "native" | "compatibility"; + +export interface ManagedBootstrapRuntimeLimit { + readonly name: string; + readonly soft: number; + readonly hard: number; +} + +/** Provider-neutral lifecycle surface consumed by sandbox-create coordinators. */ +export interface ManagedBootstrapRuntimePatch { + maybeApplyDuringCreate(): void | Promise; + createFailureMessage(): string | null; + exitOnPatchError(): void | Promise; + rollbackManagedStartupAfterCreateFailure(): void | Promise; + ensureApplied(): void | Promise; + waitForSupervisorReconnectIfNeeded(): void | Promise; + commitAfterReady(): void | Promise; + selectedMode(): { + readonly kind: string; + readonly label: string; + readonly device: string; + readonly args: readonly string[]; + } | null; + printReadinessFailureIfEnabled(): void; + verifyGpuOrExit( + verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult, + ): Promise; +} + +export interface ManagedBootstrapRuntimeCreateLifecycleInput { + readonly providerId: string; + readonly bootstrapIdentity: string; + readonly request: ManagedStartupRootApplyRequest; + readonly image: ManagedBootstrapImageIdentity; + readonly agentIdentity: ManagedBootstrapAgentIdentity; + readonly intendedWorkloadArgv: readonly string[]; + readonly expectedSupervisorArgv: readonly string[]; + readonly launchArgv: readonly string[]; + readonly heldWorkloadArgv: readonly string[]; + readonly authorityStore: ManagedBootstrapAuthorityStore; + readonly adapterOverride?: ManagedBootstrapAdapter; + readonly route: ManagedBootstrapRuntimeRoute; + readonly persistStartupCommand: boolean; + readonly sandboxName: string; + readonly sandboxGpuConfig: SandboxGpuConfig; + readonly requiredLimits: readonly ManagedBootstrapRuntimeLimit[]; + readonly timeoutSecs: number; + readonly onPatchFailure?: (error: unknown) => never; + readonly network: { + readonly inferenceProvider: string; + readonly dockerDriverGateway: boolean; + readonly gatewayPort: number; + }; + readonly dependencies: ManagedBootstrapRuntimeDependencies; +} + +export interface ManagedBootstrapRuntimeCreateLaunchResult { + readonly value: T; + readonly receipt: ManagedBootstrapCreateReceipt; +} + +export interface ManagedBootstrapRuntimeCreateLifecycle { + readonly launchArgv: readonly string[]; + readonly patch: ManagedBootstrapRuntimePatch; + prepareNetwork(): Promise; + runCreate( + launch: (input: { + readonly heldWorkloadArgv: readonly string[]; + readonly bootstrapIdentity: string; + }) => Promise>, + ): Promise; +} + +export interface ManagedBootstrapRuntimeSnapshot { + readonly imageId: string | null; + readonly bookkeepingImageRef: string | null; + readonly stateError: string; + readonly nativeGpuAttachmentState: "present" | "absent" | "unknown"; +} + +export interface ManagedBootstrapRuntimeCompatibilityLaunchInput { + readonly createArgs: readonly string[]; + readonly currentRegistryImageRef: string | null; + readonly prebuildImageId: string | null; + readonly allowUnbuiltSource: boolean; + readonly compatibilityPolicyPath: string; + readonly startupCommand: readonly string[]; + readonly runtimeSnapshot: ManagedBootstrapRuntimeSnapshot | null; +} + +export interface ManagedBootstrapRuntimeCompatibilityLaunch { + readonly createArgv: readonly string[]; + readonly registryImageRef: string | null; +} + +/** Provider-owned native-to-compatibility evidence and launch preparation. */ +export interface ManagedBootstrapRuntimeOnboardRouting { + readonly nativeFallbackHasCleanBaseline: boolean; + inspectNativeRuntime(): ManagedBootstrapRuntimeSnapshot | null; + isNativeCreateRoutingFailure(output: string, sawProgress: boolean): boolean; + isTrustedNativeRuntimeError(error: string): boolean; + isNativeReadinessRoutingFailure(input: { + readonly failurePhase: string | null; + readonly runtimeError: string; + }): boolean; + prepareCompatibilityLaunch( + input: ManagedBootstrapRuntimeCompatibilityLaunchInput, + ): ManagedBootstrapRuntimeCompatibilityLaunch; +} + +export interface ManagedBootstrapRuntimeOnboardRoutingInput { + readonly sandboxName: string; + readonly openshellArgv: (args: string[]) => string[]; + readonly nativeFallbackEnabled: boolean; +} diff --git a/src/lib/onboard/runtime-provider/contract.ts b/src/lib/onboard/runtime-provider/contract.ts index 8888d95f83a..8938b6244ff 100644 --- a/src/lib/onboard/runtime-provider/contract.ts +++ b/src/lib/onboard/runtime-provider/contract.ts @@ -3,6 +3,12 @@ import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; import type { ManagedImageSelectionPolicy } from "../workload/source"; +import type { + ManagedBootstrapRuntimeCreateLifecycle, + ManagedBootstrapRuntimeCreateLifecycleInput, + ManagedBootstrapRuntimeOnboardRouting, + ManagedBootstrapRuntimeOnboardRoutingInput, +} from "../managed-bootstrap/runtime-create"; export const RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION = 1 as const; export const RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION = 1 as const; @@ -261,7 +267,12 @@ export type RuntimeProviderMutationAuthoritySurface = export type RuntimeProviderBootstrapSurface = | RuntimeProviderSupportedSurface<{ - prepare(sandbox: SandboxEntry): unknown; + createLifecycle( + input: ManagedBootstrapRuntimeCreateLifecycleInput, + ): ManagedBootstrapRuntimeCreateLifecycle; + createOnboardRouting( + input: ManagedBootstrapRuntimeOnboardRoutingInput, + ): ManagedBootstrapRuntimeOnboardRouting; }> | RuntimeProviderUnsupportedSurface; diff --git a/src/lib/onboard/runtime-provider/registry.ts b/src/lib/onboard/runtime-provider/registry.ts index 332792d2dc2..1d0a31ae6aa 100644 --- a/src/lib/onboard/runtime-provider/registry.ts +++ b/src/lib/onboard/runtime-provider/registry.ts @@ -337,7 +337,8 @@ function validateMutationAuthoritySurface( function validateBootstrapSurface(surface: Record): void { if (surface.supported === true) { - requireFunction(surface, "prepare", "bootstrap"); + requireFunction(surface, "createLifecycle", "bootstrap"); + requireFunction(surface, "createOnboardRouting", "bootstrap"); } } diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index bf2a47fe4b4..3ac4da9e4fc 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -21,6 +21,7 @@ import { loadAgent } from "../../agent/defs"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; import { cloneSandboxWorkloadReceipt } from "../../state/registry/workload"; import { MANAGED_IMAGE_REPOSITORIES } from "../managed-image/contract"; +import { createDockerManagedBootstrapSurface } from "../managed-bootstrap/docker-runtime"; import { encodeManagedStartupProfile, type ManagedStartupProfile, @@ -145,6 +146,28 @@ describe("RuntimeProviderBundle registry contract", () => { } }); + it("validates the dormant Docker bootstrap candidate through the same bundle registry", () => { + const docker = createDockerRuntimeProviderBundle(); + const providers = createRuntimeProviderBundleRegistry([ + [ + "docker", + { + ...docker, + bootstrap: createDockerManagedBootstrapSurface(), + }, + ], + ]); + + expect(providers.docker?.bootstrap).toMatchObject({ + providerId: "docker", + supported: true, + }); + expect(CURRENT_RUNTIME_PROVIDER_BUNDLES.docker?.bootstrap).toMatchObject({ + providerId: "docker", + supported: false, + }); + }); + it("deeply clones and freezes every registered nested value", () => { const source = mxcBundle(); const registry = createRuntimeProviderBundleRegistry([["mxc", source]]); @@ -170,6 +193,59 @@ describe("RuntimeProviderBundle registry contract", () => { }).toThrow(TypeError); }); + it("registers an MXC-style managed-bootstrap provider through the bundle surface", () => { + const bundle = mxcBundle(); + const createLifecycle = vi.fn(() => ({ + launchArgv: ["mxc", "create"], + patch: { + maybeApplyDuringCreate: vi.fn(), + createFailureMessage: vi.fn(() => null), + exitOnPatchError: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + ensureApplied: vi.fn(), + waitForSupervisorReconnectIfNeeded: vi.fn(), + commitAfterReady: vi.fn(), + selectedMode: vi.fn(() => null), + printReadinessFailureIfEnabled: vi.fn(), + verifyGpuOrExit: vi.fn(async (verify) => verify("alpha")), + }, + prepareNetwork: vi.fn(async () => undefined), + runCreate: vi.fn(), + })); + const createOnboardRouting = vi.fn(() => ({ + nativeFallbackHasCleanBaseline: false, + inspectNativeRuntime: vi.fn(() => null), + isNativeCreateRoutingFailure: vi.fn(() => false), + isTrustedNativeRuntimeError: vi.fn(() => false), + isNativeReadinessRoutingFailure: vi.fn(() => false), + prepareCompatibilityLaunch: vi.fn(() => ({ createArgv: [], registryImageRef: null })), + })); + const providers = createRuntimeProviderBundleRegistry([ + [ + "mxc", + replaceSurface(bundle, "bootstrap", { + providerId: "mxc", + supported: true, + createLifecycle, + createOnboardRouting, + }), + ], + ]); + const registered = providers.mxc!; + expectSupportedSurface(registered.bootstrap); + + const routing = registered.bootstrap.createOnboardRouting({ + sandboxName: "alpha", + openshellArgv: (args) => args, + nativeFallbackEnabled: false, + }); + + expect(registered.identity.id).toBe("mxc"); + expect(routing.nativeFallbackHasCleanBaseline).toBe(false); + expect(createOnboardRouting).toHaveBeenCalledOnce(); + expect(createLifecycle).not.toHaveBeenCalled(); + }); + it("rejects an omitted managed platform without changing legacy receipt acceptance", () => { const { platform: _omittedPlatform, ...managedWithoutPlatform } = MANAGED_RECEIPT; const persistedManaged = cloneSandboxWorkloadReceipt(managedWithoutPlatform); diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index 3a650fc9c11..8e822e0ce65 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -8,9 +8,12 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; import { loadAgent } from "../agent/defs"; import { SANDBOX_BUILD_CONTEXT_PREFIX } from "../sandbox/build-context"; import { createOpenshellCliHelpers } from "./openshell-cli"; +import { encodeManagedStartupProfile } from "./managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; import { buildSandboxRuntimeEnvArgs, prepareSandboxCreateLaunch, @@ -104,6 +107,49 @@ describe("buildSandboxRuntimeEnvArgs", () => { }); describe("prepareSandboxCreateLaunch", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("renders one identity-bound held launch for %s without exposing the startup profile", (agentName) => { + const request = createManagedStartupRootApplyRequest({ + agent: agentName, + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agentName)), + }); + const result = prepareSandboxCreateLaunch({ + agent: loadAgent(agentName), + chatUiUrl: "", + createArgs: ["--name", `${agentName}-sandbox`], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: disabledHermesDashboardState, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + openshellArgv: (args) => ["openshell", ...args], + buildEnv: () => ({}), + managedStartupRootApplyRequest: request, + }); + + expect(result.intendedSandboxStartupCommand).toEqual([ + "env", + ...result.envArgs, + "nemoclaw-start", + ]); + expect(result.managedBootstrapIdentity).toMatch(/^[a-f0-9]{64}$/u); + expect(result.sandboxStartupCommand).toEqual([ + ...result.intendedSandboxStartupCommand.slice(0, -1), + "/usr/local/bin/nemoclaw-managed-startup-hold", + "--agent", + agentName, + "--profile-fingerprint", + request.profileFingerprint, + "--bootstrap-identity", + result.managedBootstrapIdentity, + ]); + expect(result.createArgv.join("\n")).not.toContain(request.encodedProfile); + }); + it("builds the sandbox create command and runtime env envelope", () => { const openshellShellCommand = vi.fn((args: string[]) => `openshell ${args.join(" ")}`); const result = prepareSandboxCreateLaunch({ diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index 1f3aa3cd01b..09e703f6d4b 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -9,6 +9,11 @@ import { appendExtraPlaceholderKeysEnvArg } from "./extra-placeholder-keys"; import type { HermesDashboardOnboardState } from "./hermes-dashboard"; import { appendHermesDashboardEnvArgs } from "./hermes-dashboard"; import { appendHostProxyEnvArgs } from "./host-proxy-env"; +import { + createManagedBootstrapIdentity, + renderManagedBootstrapHeldCommand, +} from "./managed-bootstrap/adapter"; +import type { ManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; import { appendOpenClawRuntimeEnvArgs } from "./openclaw-runtime-env"; import { prebuildSandboxImageIfEligible, @@ -57,6 +62,8 @@ export interface SandboxCreateLaunchInput { openshellShellCommand: OpenshellShellCommand; openshellArgv?: OpenshellArgv; buildEnv?(): Record; + /** Dormant until a complete runtime bundle and durable authority store are selected. */ + managedStartupRootApplyRequest?: ManagedStartupRootApplyRequest | null; } export interface SandboxCreateLaunch { @@ -66,6 +73,9 @@ export interface SandboxCreateLaunch { envArgs: string[]; sandboxEnv: Record; sandboxStartupCommand: string[]; + intendedSandboxStartupCommand: string[]; + managedBootstrapIdentity: string | null; + managedStartupRootApplyRequest: ManagedStartupRootApplyRequest | null; } export interface SandboxCreateLaunchWithPrebuildInput extends SandboxCreateLaunchInput { @@ -203,7 +213,21 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San // from openshell because bash returns the status of the last pipeline // command (awk, always 0) unless pipefail is set. Removing the pipe // lets the real exit code flow through to run(). - const sandboxStartupCommand = ["env", ...envArgs, "nemoclaw-start"]; + const intendedSandboxStartupCommand = ["env", ...envArgs, "nemoclaw-start"]; + const managedStartupRootApplyRequest = input.managedStartupRootApplyRequest ?? null; + const managedBootstrapIdentity = managedStartupRootApplyRequest + ? createManagedBootstrapIdentity() + : null; + const sandboxStartupCommand = + managedStartupRootApplyRequest && managedBootstrapIdentity + ? [ + ...renderManagedBootstrapHeldCommand( + managedStartupRootApplyRequest, + managedBootstrapIdentity, + intendedSandboxStartupCommand, + ), + ] + : intendedSandboxStartupCommand; const openshellArgs = ["sandbox", "create", ...input.createArgs, "--", ...sandboxStartupCommand]; const createCommand = renderSandboxCreateCommand( input.createArgs, @@ -221,6 +245,9 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San envArgs, sandboxEnv, sandboxStartupCommand, + intendedSandboxStartupCommand, + managedBootstrapIdentity, + managedStartupRootApplyRequest, }; } diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index c7671ff4e60..720306078f4 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -3,6 +3,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { createInMemoryRuntimeProviderBundle } from "../../../test/helpers/runtime-provider-bundle"; + const mocks = vi.hoisted(() => ({ streamSandboxCreate: vi.fn(), waitForCreatedSandboxReadyWithTrace: vi.fn(), @@ -59,11 +62,23 @@ import { setupGpuFlowMocks, VERIFIED_GPU_PROOF as VERIFIED_PROOF, } from "./__test-helpers__/sandbox-gpu-create-flow"; +import type { + ManagedBootstrapRuntimeCreateLifecycleInput, + ManagedBootstrapRuntimePatch, +} from "./managed-bootstrap/runtime-create"; +import { encodeManagedStartupProfile } from "./managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; +import { prepareSandboxCreateLaunch } from "./sandbox-create-launch"; import { runSandboxGpuCreateFlow, type SandboxGpuCreateFlowDeps, type SandboxGpuCreateFlowInput, } from "./sandbox-gpu-create-flow"; +import type { + RuntimeProviderBootstrapSurface, + RuntimeProviderBundle, +} from "./runtime-provider/contract"; +import { createRuntimeProviderBundleRegistry } from "./runtime-provider/registry"; const FAILED_PROOF: SandboxGpuProofResult = { status: "failed", @@ -150,6 +165,145 @@ function createSourceInput(): SandboxGpuCreateFlowInput { beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); +describe("runSandboxGpuCreateFlow provider-owned managed create", () => { + it("runs an MXC-style bundle without a Docker branch in central orchestration", async () => { + const input = createInput(); + input.sandboxGpuConfig = { + mode: "0", + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }; + input.gpuRoutePlan = "none"; + input.initialGpuRoute = "none"; + const request = createManagedStartupRootApplyRequest({ + agent: "openclaw", + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")), + }); + const launch = prepareSandboxCreateLaunch({ + agent: null, + sandboxName: "alpha", + chatUiUrl: "", + createArgs: ["--name", "alpha"], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: { config: null, enabled: false }, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + openshellArgv: (args) => ["openshell", ...args], + buildEnv: () => ({}), + managedStartupRootApplyRequest: request, + }); + input.createArgv = launch.createArgv; + input.sandboxEnv = launch.sandboxEnv; + input.sandboxStartupCommand = launch.sandboxStartupCommand; + const patch = createPatch() as unknown as ManagedBootstrapRuntimePatch; + const createLifecycle = vi.fn( + (lifecycleInput: ManagedBootstrapRuntimeCreateLifecycleInput) => ({ + launchArgv: ["mxc-launch", ...lifecycleInput.launchArgv.slice(1)], + patch, + prepareNetwork: vi.fn(async () => undefined), + runCreate: async ( + start: (held: { + readonly heldWorkloadArgv: readonly string[]; + readonly bootstrapIdentity: string; + }) => Promise<{ readonly value: T }>, + ): Promise => + ( + await start({ + heldWorkloadArgv: lifecycleInput.heldWorkloadArgv, + bootstrapIdentity: lifecycleInput.bootstrapIdentity, + }) + ).value, + }), + ); + const source = createInMemoryRuntimeProviderBundle({ + providerId: "mxc", + workloadProfile: { + support: null, + hostArchitectures: [], + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: true, + }, + }); + const registered = createRuntimeProviderBundleRegistry([ + [ + "mxc", + { + ...source, + bootstrap: { + providerId: "mxc", + supported: true, + createLifecycle, + createOnboardRouting: vi.fn(() => ({ + nativeFallbackHasCleanBaseline: false, + inspectNativeRuntime: vi.fn(() => null), + isNativeCreateRoutingFailure: vi.fn(() => false), + isTrustedNativeRuntimeError: vi.fn(() => false), + isNativeReadinessRoutingFailure: vi.fn(() => false), + prepareCompatibilityLaunch: vi.fn(() => ({ + createArgv: [], + registryImageRef: null, + })), + })), + }, + }, + ], + ]); + const runtimeProvider = registered.mxc as RuntimeProviderBundle & { + readonly bootstrap: Extract; + }; + input.managedBootstrap = { + bootstrapIdentity: launch.managedBootstrapIdentity!, + runtimeProvider, + authorityStore: { + async recordPreparedAuthority(authority) { + return { + schemaVersion: 1, + sandbox: authority.sandbox, + bootstrapIdentity: authority.bootstrapIdentity, + authorityFingerprint: authority.authorityFingerprint, + recordId: "mxc-record-alpha", + recordedAt: "2026-07-31T00:00:00.000Z", + }; + }, + }, + request, + image: { + repository: "registry.example/nemoclaw-openclaw", + manifestDigest: `sha256:${"d".repeat(64)}`, + }, + agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + intendedWorkloadArgv: launch.intendedSandboxStartupCommand, + expectedSupervisorArgv: ["/mxc/supervisor"], + }; + const deps = createDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementation((args) => + args[1] === "get" ? "ID: mxc-alpha\n" : "alpha Ready", + ); + + const result = await runSandboxGpuCreateFlow(input, deps); + + expect(result).toMatchObject({ route: "none", runtimePatch: patch }); + expect(createLifecycle).toHaveBeenCalledWith( + expect.objectContaining({ providerId: "mxc", route: "none" }), + ); + expect(mocks.streamSandboxCreate).toHaveBeenCalledWith( + "mxc-launch", + input.createArgv.slice(1), + input.sandboxEnv, + expect.anything(), + ); + expect(mocks.createDockerGpuSandboxCreatePatch).not.toHaveBeenCalled(); + expect(mocks.queryOpenShellDockerSandboxContainers).not.toHaveBeenCalled(); + expect(mocks.queryOpenShellDockerSandboxRuntimeSnapshot).not.toHaveBeenCalled(); + expect(mocks.enforceDockerGpuPatchPreserveNetwork).not.toHaveBeenCalled(); + }); +}); + describe("runSandboxGpuCreateFlow proof authorization", () => { it("does not retry compatibility when the native proof throws an exec/policy error (#6110)", async () => { const deps = createDeps(); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 98574891e0b..878363389e9 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -10,12 +10,23 @@ import type { DockerGpuPatchDeps, DockerUlimit } from "./docker-gpu-patch-types" import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { renderCompatibilityFallbackCreateArgs } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; -import type { DockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +import type { + ManagedBootstrapAdapter, + ManagedBootstrapAgentIdentity, + ManagedBootstrapAuthorityStore, + ManagedBootstrapImageIdentity, +} from "./managed-bootstrap/adapter"; +import type { ManagedBootstrapRuntimePatch } from "./managed-bootstrap/runtime-create"; +import type { ManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; import { isImmutableDockerImageId } from "./openshell-docker-sandbox-containers"; import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; import { createSandboxGpuCreateAttemptRunner } from "./sandbox-gpu-create-run-attempt"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; import type { SandboxPrebuildResult } from "./sandbox-prebuild"; +import type { + RuntimeProviderBootstrapSurface, + RuntimeProviderBundle, +} from "./runtime-provider/contract"; import { addTraceEvent } from "./tracing"; export { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; @@ -41,6 +52,18 @@ export interface SandboxGpuCreateFlowInput { restoreBackupPath: string | null; terminalAgent: boolean; persistStartupCommand?: boolean; + managedBootstrap?: { + readonly bootstrapIdentity: string; + readonly runtimeProvider: RuntimeProviderBundle & { + readonly bootstrap: Extract; + }; + readonly authorityStore: ManagedBootstrapAuthorityStore; + readonly request: ManagedStartupRootApplyRequest; + readonly image: ManagedBootstrapImageIdentity; + readonly agentIdentity: ManagedBootstrapAgentIdentity; + readonly intendedWorkloadArgv: readonly string[]; + readonly expectedSupervisorArgv: readonly string[]; + } | null; requiredUlimits?: readonly DockerUlimit[] | null; } @@ -50,11 +73,13 @@ export interface SandboxGpuCreateFlowDeps { sleep: Sleep; openshellArgv(args: string[]): string[]; verifyDirectSandboxGpu(sandboxName: string): SandboxGpuProofResult; + /** Production callers omit this factory and use the runtime provider's adapter. */ + createManagedBootstrapAdapter?: () => ManagedBootstrapAdapter; } export interface SandboxGpuCreateFlowResult { createResult: StreamSandboxCreateResult; - dockerGpuCreatePatch: DockerGpuSandboxCreatePatch; + runtimePatch: ManagedBootstrapRuntimePatch; route: SelectedDockerGpuRoute; firstCreateOutput: string; /** Mutable tag/reference retained only for registry and image-GC bookkeeping. */ @@ -105,46 +130,65 @@ export async function runSandboxGpuCreateFlow( throw new Error("Compatibility retry policy was not materialized."); } const nativeRuntimeSnapshot = attemptRunner.state.nativeRuntimeSnapshot; - const prebuildImageId = input.prebuild.imageId; - const imageId = - nativeRuntimeSnapshot?.imageId ?? - (prebuildImageId && isImmutableDockerImageId(prebuildImageId) - ? prebuildImageId.toLowerCase() - : null); - if ( - !registryImageRef && - nativeRuntimeSnapshot?.bookkeepingImageRef && - !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) - ) { - registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; + if (attemptRunner.managedRouting) { + const prepared = attemptRunner.managedRouting.prepareCompatibilityLaunch({ + createArgs: input.prebuild.createArgs, + currentRegistryImageRef: registryImageRef, + prebuildImageId: input.prebuild.imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + startupCommand: input.sandboxStartupCommand, + runtimeSnapshot: nativeRuntimeSnapshot, + }); + attemptRunner.state.compatibilityArgv = [...prepared.createArgv]; + registryImageRef = prepared.registryImageRef; + } else { + const prebuildImageId = input.prebuild.imageId; + const imageId = + nativeRuntimeSnapshot?.imageId ?? + (prebuildImageId && isImmutableDockerImageId(prebuildImageId) + ? prebuildImageId.toLowerCase() + : null); + if ( + !registryImageRef && + nativeRuntimeSnapshot?.bookkeepingImageRef && + !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) + ) { + registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; + } + const compatibilityArgs = renderCompatibilityFallbackCreateArgs( + input.prebuild.createArgs, + { + imageRef: imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + }, + ); + attemptRunner.state.compatibilityArgv = deps.openshellArgv([ + "sandbox", + "create", + ...compatibilityArgs, + "--", + ...input.sandboxStartupCommand, + ]); } - const compatibilityArgs = renderCompatibilityFallbackCreateArgs(input.prebuild.createArgs, { - imageRef: imageId, - allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, - compatibilityPolicyPath: input.compatibilityPolicyPath, - }); - attemptRunner.state.compatibilityArgv = deps.openshellArgv([ - "sandbox", - "create", - ...compatibilityArgs, - "--", - ...input.sandboxStartupCommand, - ]); if (attemptRunner.state.compatibilityArgv.length === 0) { throw new Error("Compatibility sandbox create executable is missing."); } }, activateCompatibilityAttempt: async () => { - await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( - input.provider, - input.sandboxGpuConfig, - { - dockerDriverGateway: input.dockerDriverGateway, - selectedRoute: "compatibility", - gatewayPort: input.gatewayPort, - log: console.log, - }, - ); + if (!input.managedBootstrap) { + await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( + input.provider, + input.sandboxGpuConfig, + { + dockerDriverGateway: input.dockerDriverGateway, + selectedRoute: "compatibility", + gatewayPort: input.gatewayPort, + log: console.log, + }, + ); + } input.sandboxGpuConfig.sandboxGpuProof = null; }, traceEvent: addTraceEvent, diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 726758cc0ac..8d33c1a40d4 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { printSandboxCreateRecoveryHints } from "../build-context"; +import { resolveOpenShellSandboxId } from "../adapters/openshell/sandbox-identity"; import { getSandboxDeleteOutcome } from "../domain/sandbox/destroy"; import { streamSandboxCreate } from "../sandbox/create-stream"; import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-ready-gate"; @@ -13,8 +14,8 @@ import { reportSandboxCreateFailure } from "./created-sandbox-failure"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +import type { ManagedBootstrapRuntimeSnapshot } from "./managed-bootstrap/runtime-create"; import { - type OpenShellDockerSandboxRuntimeSnapshotQuery, queryOpenShellDockerSandboxContainers, queryOpenShellDockerSandboxRuntimeSnapshot, } from "./openshell-docker-sandbox-containers"; @@ -28,7 +29,7 @@ import * as sandboxGpuPreflight from "./sandbox-gpu-preflight"; import * as sandboxReadinessTracing from "./sandbox-readiness-tracing"; import { addTraceEvent } from "./tracing"; -type NativeRuntimeSnapshot = Extract; +type NativeRuntimeSnapshot = ManagedBootstrapRuntimeSnapshot; export type SandboxGpuCreateAttemptState = { firstCreateOutput: string; @@ -41,6 +42,12 @@ export type SandboxGpuCreateAttemptState = { // Ready row. Require one confirmation poll before advancing to the GPU proof. const COMPATIBILITY_STABLE_READY_POLLS = 2; +class ManagedBootstrapCreateStreamFailure extends Error { + constructor(readonly result: Awaited>) { + super("Managed bootstrap held workload did not complete its create stream."); + } +} + export function createSandboxGpuCreateAttemptRunner( input: SandboxGpuCreateFlowInput, deps: SandboxGpuCreateFlowDeps, @@ -51,13 +58,26 @@ export function createSandboxGpuCreateAttemptRunner( allowUnbuiltCompatibilitySource: false, nativeRuntimeSnapshot: null, }; + const managedRouting = input.managedBootstrap?.runtimeProvider.bootstrap.createOnboardRouting({ + sandboxName: input.sandboxName, + openshellArgv: deps.openshellArgv, + nativeFallbackEnabled: + input.initialGpuRoute === "native" && input.gpuRoutePlan === "native-with-fallback", + }); const nativeFallbackBaseline = - input.initialGpuRoute === "native" && input.gpuRoutePlan === "native-with-fallback" + !managedRouting && + input.initialGpuRoute === "native" && + input.gpuRoutePlan === "native-with-fallback" ? queryOpenShellDockerSandboxContainers(input.sandboxName) : null; const nativeFallbackHasCleanBaseline = - nativeFallbackBaseline?.ok === true && nativeFallbackBaseline.ids.length === 0; - const inspectNativeRuntime = () => queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName); + managedRouting?.nativeFallbackHasCleanBaseline ?? + (nativeFallbackBaseline?.ok === true && nativeFallbackBaseline.ids.length === 0); + const inspectNativeRuntime = (): NativeRuntimeSnapshot | null => { + if (managedRouting) return managedRouting.inspectNativeRuntime(); + const snapshot = queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName); + return snapshot.ok ? snapshot : null; + }; const runAttempt = async (route: SelectedDockerGpuRoute) => { const compatibility = route === "compatibility"; @@ -70,73 +90,199 @@ export function createSandboxGpuCreateAttemptRunner( ); } const hasRequiredUlimits = (input.requiredUlimits?.length ?? 0) > 0; - const dockerGpuCreatePatch = createDockerGpuSandboxCreatePatch({ - route, - // The startup clone preserves native CDI devices, so DCode can apply its - // exact required limits without replacing the native GPU envelope. - // Other native routes are not swapped solely to persist a command. - persistStartupCommand: - input.persistStartupCommand === true && (route !== "native" || hasRequiredUlimits), - sandboxName: input.sandboxName, - gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, - openshellSandboxCommand: input.sandboxStartupCommand, - requiredUlimits: input.requiredUlimits, - timeoutSecs: input.sandboxReadyTimeoutSecs, - backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", - deps, - }); + const managedBootstrap = input.managedBootstrap ?? null; const attemptArgv = state.compatibilityArgv ?? input.createArgv; - const [createExecutable, ...createExecutableArgs] = attemptArgv; + const managedLifecycle = managedBootstrap + ? managedBootstrap.runtimeProvider.bootstrap.createLifecycle({ + providerId: managedBootstrap.runtimeProvider.identity.id, + bootstrapIdentity: managedBootstrap.bootstrapIdentity, + request: managedBootstrap.request, + image: managedBootstrap.image, + agentIdentity: managedBootstrap.agentIdentity, + intendedWorkloadArgv: managedBootstrap.intendedWorkloadArgv, + expectedSupervisorArgv: managedBootstrap.expectedSupervisorArgv, + launchArgv: attemptArgv, + heldWorkloadArgv: input.sandboxStartupCommand, + authorityStore: managedBootstrap.authorityStore, + ...(deps.createManagedBootstrapAdapter + ? { adapterOverride: deps.createManagedBootstrapAdapter() } + : {}), + route, + persistStartupCommand: input.persistStartupCommand === true, + sandboxName: input.sandboxName, + sandboxGpuConfig: input.sandboxGpuConfig, + requiredLimits: input.requiredUlimits ?? [], + timeoutSecs: input.sandboxReadyTimeoutSecs, + network: { + inferenceProvider: input.provider, + dockerDriverGateway: input.dockerDriverGateway, + gatewayPort: input.gatewayPort, + }, + dependencies: { + runCaptureOpenshell: deps.runCaptureOpenshell, + runOpenshell: deps.runOpenshell, + sleep: deps.sleep, + }, + }) + : null; + const runtimePatch = + managedLifecycle?.patch ?? + createDockerGpuSandboxCreatePatch({ + route, + // The startup clone preserves native CDI devices, so DCode can apply its + // exact required limits without replacing the native GPU envelope. + // Other native routes are not swapped solely to persist a command. + persistStartupCommand: + input.persistStartupCommand === true && (route !== "native" || hasRequiredUlimits), + externalRecreation: false, + sandboxName: input.sandboxName, + gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, + openshellSandboxCommand: input.sandboxStartupCommand, + requiredUlimits: input.requiredUlimits, + timeoutSecs: input.sandboxReadyTimeoutSecs, + backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", + deps, + }); + await managedLifecycle?.prepareNetwork(); + const [createExecutable, ...createExecutableArgs] = managedLifecycle?.launchArgv ?? attemptArgv; if (!createExecutable) throw new Error("Sandbox create executable is missing."); - const createResult = await streamSandboxCreate( - createExecutable, - createExecutableArgs, - input.sandboxEnv, - { + const streamCreate = () => + streamSandboxCreate(createExecutable, createExecutableArgs, input.sandboxEnv, { readyCheck: () => { const list = deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); return isSandboxReady(list, input.sandboxName); }, - onPoll: () => dockerGpuCreatePatch.maybeApplyDuringCreate(), + onPoll: () => runtimePatch.maybeApplyDuringCreate(), readyCheckOutputPatterns: getReadyCheckOutputPatternsForAgent( input.terminalAgent, input.sandboxEnv, ), - failureCheck: dockerGpuCreatePatch.createFailureMessage, + failureCheck: runtimePatch.createFailureMessage, traceEvent: addTraceEvent, initialPhase: compatibility && (input.prebuild.imageRef || state.compatibilityArgv) ? "create" : undefined, - }, - ); + }); + let createResult: Awaited>; + let managedIncompleteCreateRecovered = false; + if (managedBootstrap && managedLifecycle) { + try { + createResult = await managedLifecycle.runCreate( + async ({ heldWorkloadArgv, bootstrapIdentity }) => { + if ( + bootstrapIdentity !== managedBootstrap.bootstrapIdentity || + heldWorkloadArgv.length !== input.sandboxStartupCommand.length || + heldWorkloadArgv.some((value, index) => value !== input.sandboxStartupCommand[index]) + ) { + throw new Error( + "Managed bootstrap launch does not match the rendered identity-bound hold.", + ); + } + const result = await streamCreate(); + const createFailure = + result.status === 0 ? null : classifySandboxCreateFailure(result.output); + if (result.status !== 0 && createFailure?.kind !== "sandbox_create_incomplete") { + throw new ManagedBootstrapCreateStreamFailure(result); + } + if (createFailure?.kind === "sandbox_create_incomplete") { + const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ + sandboxName: input.sandboxName, + timeoutSecs: input.sandboxReadyTimeoutSecs, + runCaptureOpenshell: deps.runCaptureOpenshell, + isSandboxReady, + getSandboxFailurePhase, + stableReadyPolls: 1, + sleep: deps.sleep, + }); + if (!readiness.ready) { + throw new Error( + `Managed bootstrap incomplete create did not reach authoritative Ready state (${readiness.reason}).`, + ); + } + } else { + const list = deps.runCaptureOpenshell(["sandbox", "list"], { + ignoreError: true, + }); + if (!isSandboxReady(list, input.sandboxName)) { + throw new Error( + "Managed bootstrap create completed without an authoritative Ready sandbox.", + ); + } + } + let sandboxId: string; + try { + sandboxId = resolveOpenShellSandboxId(input.sandboxName, deps.runCaptureOpenshell); + } catch (error) { + throw new Error( + createFailure?.kind === "sandbox_create_incomplete" + ? "Managed bootstrap incomplete create did not return one exact durable sandbox identity after Ready." + : "Managed bootstrap create did not return one exact durable sandbox identity after Ready.", + { cause: error }, + ); + } + managedIncompleteCreateRecovered = createFailure?.kind === "sandbox_create_incomplete"; + return { + value: result, + receipt: { + sandbox: { + sandboxName: input.sandboxName, + sandboxId, + driverId: managedBootstrap.runtimeProvider.identity.id, + }, + ready: true, + readyAt: new Date().toISOString(), + }, + }; + }, + ); + } catch (error) { + if (!(error instanceof ManagedBootstrapCreateStreamFailure)) throw error; + createResult = error.result; + } + } else { + createResult = await streamCreate(); + } if (!state.firstCreateOutput) state.firstCreateOutput = createResult.output; - dockerGpuCreatePatch.exitOnPatchError(); + await runtimePatch.exitOnPatchError(); if (createResult.status !== 0) { const failure = classifySandboxCreateFailure(createResult.output); if (failure.kind === "sandbox_create_incomplete") { console.warn(""); - console.warn( - ` Create stream exited with code ${createResult.status} after sandbox was created.`, - ); - console.warn(" Checking whether the sandbox reaches Ready state..."); + if (managedIncompleteCreateRecovered) { + console.warn( + ` Create stream exited with code ${createResult.status}; the exact durable sandbox reached Ready, and onboarding is continuing with final checks.`, + ); + } else { + console.warn( + ` Create stream exited with code ${createResult.status} after sandbox was created.`, + ); + console.warn(" Checking whether the sandbox reaches Ready state..."); + } } else if ( route === "native" && input.gpuRoutePlan === "native-with-fallback" && nativeFallbackHasCleanBaseline && (() => { if ( - sandboxGpuCreateAttempt.isNativeGpuCreateRoutingFailure(createResult.output, { - sawProgress: createResult.sawProgress, - }) + managedRouting + ? managedRouting.isNativeCreateRoutingFailure( + createResult.output, + createResult.sawProgress, + ) + : sandboxGpuCreateAttempt.isNativeGpuCreateRoutingFailure(createResult.output, { + sawProgress: createResult.sawProgress, + }) ) { state.allowUnbuiltCompatibilitySource = input.prebuild.imageRef === null; return true; } const snapshot = inspectNativeRuntime(); if ( - snapshot.ok && - sandboxGpuCreateAttempt.isTrustedNativeGpuRuntimeError(snapshot.stateError) + snapshot && + (managedRouting + ? managedRouting.isTrustedNativeRuntimeError(snapshot.stateError) + : sandboxGpuCreateAttempt.isTrustedNativeGpuRuntimeError(snapshot.stateError)) ) { state.nativeRuntimeSnapshot = snapshot; return true; @@ -144,6 +290,7 @@ export function createSandboxGpuCreateAttemptRunner( return false; })() ) { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); return { ok: false, route, @@ -152,6 +299,7 @@ export function createSandboxGpuCreateAttemptRunner( fallbackEligible: true, } as const; } else { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); reportSandboxCreateFailure( { sandboxName: input.sandboxName, @@ -171,8 +319,8 @@ export function createSandboxGpuCreateAttemptRunner( ); } } - dockerGpuCreatePatch.ensureApplied(); - dockerGpuCreatePatch.waitForSupervisorReconnectIfNeeded(); + await runtimePatch.ensureApplied(); + await runtimePatch.waitForSupervisorReconnectIfNeeded(); console.log(" Waiting for sandbox to become ready..."); const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ sandboxName: input.sandboxName, @@ -197,13 +345,19 @@ export function createSandboxGpuCreateAttemptRunner( const runtimeSnapshot = canClassifyNativeReadiness ? inspectNativeRuntime() : null; if ( canClassifyNativeReadiness && - runtimeSnapshot?.ok && - sandboxGpuCreateAttempt.isNativeGpuReadinessRoutingFailure({ - failurePhase: readiness.failurePhase, - runtimeError: runtimeSnapshot.stateError, - }) + runtimeSnapshot && + (managedRouting + ? managedRouting.isNativeReadinessRoutingFailure({ + failurePhase: readiness.failurePhase, + runtimeError: runtimeSnapshot.stateError, + }) + : sandboxGpuCreateAttempt.isNativeGpuReadinessRoutingFailure({ + failurePhase: readiness.failurePhase, + runtimeError: runtimeSnapshot.stateError, + })) ) { state.nativeRuntimeSnapshot = runtimeSnapshot; + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); return { ok: false, route, @@ -214,10 +368,11 @@ export function createSandboxGpuCreateAttemptRunner( fallbackEligible: true, } as const; } + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); printSandboxCreateFailureDiagnostics(input.sandboxName, { backupPath: input.restoreBackupPath, }); - if (compatibility) dockerGpuCreatePatch.printReadinessFailureIfEnabled(); + if (compatibility) runtimePatch.printReadinessFailureIfEnabled(); else { const deletion = deps.runOpenshell(["sandbox", "delete", input.sandboxName], { ignoreError: true, @@ -240,27 +395,32 @@ export function createSandboxGpuCreateAttemptRunner( route === "native" && input.gpuRoutePlan === "native-with-fallback" && nativeFallbackHasCleanBaseline; - const proof: SandboxGpuProofResult = dockerGpuLocalInference.verifyGpuSandboxAccessAfterReady( - input.sandboxGpuConfig, - { - sandboxName: input.sandboxName, - dockerDriverGateway: input.dockerDriverGateway, - selectedRoute: route, - verifyDirectSandboxGpu: deps.verifyDirectSandboxGpu, - verifyGpuOrExit: deferNativeProofFailure - ? undefined - : dockerGpuCreatePatch.verifyGpuOrExit, - reportGpuProofFailure: !deferNativeProofFailure, - selectedMode: dockerGpuCreatePatch.selectedMode, - runCaptureOpenshell: deps.runCaptureOpenshell, - log: console.log, - }, - ); + let proof: SandboxGpuProofResult; + try { + proof = await dockerGpuLocalInference.verifyGpuSandboxAccessAfterReady( + input.sandboxGpuConfig, + { + sandboxName: input.sandboxName, + dockerDriverGateway: input.dockerDriverGateway, + selectedRoute: route, + verifyDirectSandboxGpu: deps.verifyDirectSandboxGpu, + verifyGpuOrExit: deferNativeProofFailure ? undefined : runtimePatch.verifyGpuOrExit, + reportGpuProofFailure: !deferNativeProofFailure, + selectedMode: runtimePatch.selectedMode, + runCaptureOpenshell: deps.runCaptureOpenshell, + log: console.log, + }, + ); + } catch (error) { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); + throw error; + } if (deferNativeProofFailure && proof.status === "failed") { if (sandboxGpuPreflight.isExplicitNvidiaSmiDriverProofFailure(proof)) { const snapshot = inspectNativeRuntime(); - if (snapshot.ok && snapshot.nativeGpuAttachmentState === "absent") { + if (snapshot?.nativeGpuAttachmentState === "absent") { state.nativeRuntimeSnapshot = snapshot; + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); return { ok: false, route, @@ -272,6 +432,7 @@ export function createSandboxGpuCreateAttemptRunner( } as const; } } + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); console.error(""); console.error(" Native sandbox GPU proof failed."); console.error( @@ -283,15 +444,22 @@ export function createSandboxGpuCreateAttemptRunner( process.exit(1); } if (proof.status === "failed") { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); throw new Error("Sandbox GPU proof returned failed status."); } } + // GPU-enabled cutover stays reversible until the caller also proves the + // configured host-local inference path. Non-GPU workloads have completed + // their final authoritative Ready gate here. + if (!input.sandboxGpuConfig.sandboxGpuEnabled) { + await runtimePatch.commitAfterReady(); + } return { ok: true, route, - value: { createResult, dockerGpuCreatePatch }, + value: { createResult, runtimePatch }, } as const; }; - return { state, runAttempt }; + return { state, managedRouting, runAttempt }; } diff --git a/test/onboard-prepared-build-context.test.ts b/test/onboard-prepared-build-context.test.ts index 0d91d612ae4..5babec1765c 100644 --- a/test/onboard-prepared-build-context.test.ts +++ b/test/onboard-prepared-build-context.test.ts @@ -98,12 +98,15 @@ let stageCalls = 0; dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch = () => ({ maybeApplyDuringCreate: () => {}, createFailureMessage: () => null, - exitOnPatchError: () => {}, - ensureApplied: () => {}, + exitOnPatchError: async () => {}, + attachManagedBootstrapCutover: () => {}, + rollbackManagedStartupAfterCreateFailure: async () => {}, + ensureApplied: async () => {}, waitForSupervisorReconnectIfNeeded: () => {}, + commitAfterReady: async () => {}, selectedMode: () => null, printReadinessFailureIfEnabled: () => {}, - verifyGpuOrExit: (verify) => verify(sandboxName), + verifyGpuOrExit: async (verify) => verify(sandboxName), }); buildContextStage.stageCreateSandboxBuildContext = () => { diff --git a/test/onboard-sandbox-recreation.test.ts b/test/onboard-sandbox-recreation.test.ts index 11b35a58b6e..c0a4eb2f6f3 100644 --- a/test/onboard-sandbox-recreation.test.ts +++ b/test/onboard-sandbox-recreation.test.ts @@ -1203,6 +1203,7 @@ const fs = require("node:fs"); const commands = []; let sandboxListCalls = 0; +let dockerPsCalls = 0; const keepAlive = setInterval(() => {}, 1000); runner.run = (command, opts = {}) => { _deleted = _deleted || _n(command).includes("sandbox delete"); @@ -1210,6 +1211,10 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { + if (_n(command).startsWith("docker ps -a --no-trunc ")) { + dockerPsCalls += 1; + if (dockerPsCalls === 1) return "a".repeat(64); + } if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) { sandboxListCalls += 1; diff --git a/test/onboard-terminal-dashboard.test.ts b/test/onboard-terminal-dashboard.test.ts index 61baa83310a..0dd2686eadd 100644 --- a/test/onboard-terminal-dashboard.test.ts +++ b/test/onboard-terminal-dashboard.test.ts @@ -67,12 +67,15 @@ const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, "") dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch = () => ({ maybeApplyDuringCreate: () => {}, createFailureMessage: () => null, - exitOnPatchError: () => {}, - ensureApplied: () => {}, + exitOnPatchError: async () => {}, + attachManagedBootstrapCutover: () => {}, + rollbackManagedStartupAfterCreateFailure: async () => {}, + ensureApplied: async () => {}, waitForSupervisorReconnectIfNeeded: () => {}, + commitAfterReady: async () => {}, selectedMode: () => null, printReadinessFailureIfEnabled: () => {}, - verifyGpuOrExit: (verify) => verify(sandboxName), + verifyGpuOrExit: async (verify) => verify(sandboxName), }); agentOnboard.createAgentSandbox = () => { diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 48e889d78c1..9045d5faad4 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -28,8 +28,8 @@ let activationPaths: string[] = []; let providerPaths: string[] = []; let dockerfilePaths: string[] = []; let packagingPaths: string[] = []; -const bootstrapLoad = - /(?:from\s*|import\s*|import\s*\(\s*|require\s*\(\s*)["'][^"']*managed-bootstrap/iu; +const driverBootstrapLoad = + /(?:from\s*|import\s*|import\s*\(\s*|require\s*\(\s*)["'][^"']*managed-bootstrap\/(?:docker|docker-journal|docker-runtime)/iu; const packagedBootstrapAsset = /(?:nemoclaw-managed-bootstrap|managed-bootstrap-trampoline|managed-startup-image-runtime\.cjs|nemoclaw-managed-startup-hold)/u; @@ -70,7 +70,7 @@ beforeAll(() => { }); describe("runtime provider central source boundary", () => { - // source-shape-contract: compatibility -- Migrated lifecycle and mutation consumers must stay provider-neutral while production selection excludes unqualified future providers and managed-bootstrap dependencies + // source-shape-contract: compatibility -- Migrated lifecycle and mutation consumers must stay provider-neutral while production selection excludes unqualified future providers and driver-specific bootstrap dependencies it("keeps migrated provider identities and implementations behind the one bundle composition", () => { const driverNeutralActions = { "actions/inference-set.ts": read("src/lib/actions/inference-set.ts"), @@ -114,7 +114,12 @@ describe("runtime provider central source boundary", () => { expect(driverNeutralActions["actions/sandbox/start.ts"]).toMatch( /resolved\.lifecycle\.verifyStarted\(/u, ); - expect(Object.values(providerContract).join("\n")).not.toMatch(/managed-bootstrap/u); + expect(providerContract.contract).toMatch( + /import type[\s\S]*from ["']\.\.\/managed-bootstrap\/runtime-create["']/u, + ); + expect( + [providerContract.current, providerContract.docker, providerContract.registry].join("\n"), + ).not.toMatch(/managed-bootstrap/u); expect(providerContract.current).not.toMatch(/\b(?:podman|mxc)\b/iu); }); @@ -122,12 +127,14 @@ describe("runtime provider central source boundary", () => { expect(bootstrapProtocolPaths).toEqual([ "src/lib/onboard/managed-bootstrap/adapter.ts", "src/lib/onboard/managed-bootstrap/docker-journal.ts", + "src/lib/onboard/managed-bootstrap/docker-runtime.ts", "src/lib/onboard/managed-bootstrap/docker-shared-state.ts", "src/lib/onboard/managed-bootstrap/docker-spec.ts", "src/lib/onboard/managed-bootstrap/docker-test-fixture.ts", "src/lib/onboard/managed-bootstrap/docker.ts", "src/lib/onboard/managed-bootstrap/envelope.ts", "src/lib/onboard/managed-bootstrap/index.ts", + "src/lib/onboard/managed-bootstrap/runtime-create.ts", ]); }); @@ -167,19 +174,25 @@ describe("runtime provider central source boundary", () => { expect(bootstrapProtocolSource).not.toMatch(/\b(?:docker|podman|openshell|mxc)\b/iu); }); - // source-shape-contract: security -- Production onboarding must not activate managed bootstrap until a complete provider image and rollback implementation lands together - it("keeps production activation paths disconnected from managed bootstrap", () => { + // source-shape-contract: security -- Production onboarding may consume the provider-neutral create contract but cannot select a driver-specific bootstrap implementation + it("keeps production activation paths disconnected from driver bootstrap adapters", () => { const onboardEntry = read("src/lib/onboard.ts"); const activationSource = activationPaths.map(read).join("\n"); - expect(onboardEntry).not.toMatch(bootstrapLoad); - expect(activationSource).not.toMatch(bootstrapLoad); + expect(onboardEntry).not.toMatch(driverBootstrapLoad); + expect(activationSource).not.toMatch(driverBootstrapLoad); }); // source-shape-contract: security -- Registered runtime providers must remain bootstrap-unsupported until their complete transaction implementations are qualified it("keeps registered providers bootstrap-unsupported", () => { const dockerProvider = read("src/lib/onboard/runtime-provider/docker.ts"); - const providerSource = providerPaths.map(read).join("\n"); - expect(providerSource).not.toMatch(/managed-bootstrap/iu); + const providerImplementationSource = providerPaths + .filter((path) => path !== "src/lib/onboard/runtime-provider/contract.ts") + .map(read) + .join("\n"); + expect(dockerProvider).not.toMatch( + /(?:from\s+["'][^"']*managed-bootstrap|require\([^)]*managed-bootstrap)/u, + ); + expect(providerImplementationSource).not.toMatch(/managed-bootstrap/iu); expect(dockerProvider.match(/bootstrap:\s*unsupported\(/gu)).toHaveLength(2); expect(dockerProvider.match(/recovery:\s*unsupported\(/gu)).toHaveLength(2); }); From 7ed953c7d0934b4c8781b125a4ce57757591781d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 10:59:46 -0700 Subject: [PATCH 106/117] feat(onboard): bind managed create to runtime providers Signed-off-by: Aaron Erickson --- ci/source-shape-test-budget.json | 2 +- docs/inference/verify-inference-route.mdx | 10 +- docs/reference/commands.mdx | 5 +- docs/reference/troubleshooting.mdx | 14 +- .../adapters/openshell/sandbox-identity.ts | 16 + src/lib/onboard.ts | 7 +- .../sandbox-gpu-create-flow.ts | 2 + .../docker-gpu-local-inference.test.ts | 100 +++-- src/lib/onboard/docker-gpu-local-inference.ts | 87 ++++- .../docker-gpu-route-consumers.test.ts | 10 +- ...ocker-gpu-sandbox-create-lifecycle.test.ts | 95 ++++- src/lib/onboard/docker-gpu-sandbox-create.ts | 349 +++++++++++++----- ...ker-startup-command-sandbox-create.test.ts | 103 +++++- src/lib/onboard/managed-bootstrap/README.md | 71 ++-- .../managed-bootstrap/docker-runtime.ts | 294 +++++++++++++++ src/lib/onboard/managed-bootstrap/index.ts | 4 + .../managed-bootstrap/runtime-create.ts | 145 ++++++++ src/lib/onboard/runtime-provider/contract.ts | 13 +- src/lib/onboard/runtime-provider/registry.ts | 3 +- .../runtime-provider-contract.test.ts | 76 ++++ src/lib/onboard/sandbox-create-launch.test.ts | 46 +++ src/lib/onboard/sandbox-create-launch.ts | 29 +- .../onboard/sandbox-gpu-create-flow.test.ts | 154 ++++++++ src/lib/onboard/sandbox-gpu-create-flow.ts | 116 ++++-- .../onboard/sandbox-gpu-create-run-attempt.ts | 302 +++++++++++---- test/onboard-prepared-build-context.test.ts | 9 +- test/onboard-sandbox-recreation.test.ts | 5 + test/onboard-terminal-dashboard.test.ts | 9 +- test/runtime-provider-source-shape.test.ts | 77 +++- 29 files changed, 1841 insertions(+), 312 deletions(-) create mode 100644 src/lib/onboard/managed-bootstrap/docker-runtime.ts create mode 100644 src/lib/onboard/managed-bootstrap/runtime-create.ts diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 8e473b6d06a..c5f7af36e78 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -528,7 +528,7 @@ }, { "file": "test/runtime-provider-source-shape.test.ts", - "test": "keeps production activation paths disconnected from managed bootstrap", + "test": "keeps production activation paths disconnected from driver bootstrap adapters", "category": "security" }, { diff --git a/docs/inference/verify-inference-route.mdx b/docs/inference/verify-inference-route.mdx index b13874e8184..b260592517c 100644 --- a/docs/inference/verify-inference-route.mdx +++ b/docs/inference/verify-inference-route.mdx @@ -41,11 +41,15 @@ Restore the configured endpoint or proxy, run `$$nemoclaw onboard --resume` to c ## Understand Local Provider Post-Ready Checks -For local Ollama and vLLM on Docker GPU sandboxes using the compatibility route, onboarding performs an additional check after the sandbox becomes ready. +For local Ollama, local vLLM, and local NVIDIA NIM on Docker GPU sandboxes using the compatibility route, onboarding performs an additional check after the sandbox becomes ready. +Local NIM uses the `vllm-local` route, so it receives the same reversible post-ready check as local vLLM. It requests `https://inference.local/v1/models` from inside the sandbox and accepts only a 2xx response. -When this check fails, onboarding reports the endpoint and local-provider recovery steps before the first agent prompt. +If this check fails after compatibility recreation, onboarding prints failure diagnostics and attempts to restore the pre-patch container before it exits. +If that rollback fails, onboarding reports that the pre-patch container was not restored and prints container-cleanup guidance. +The local-provider failure output includes the endpoint and recovery steps before the first agent prompt. +GPU-proof diagnostics are captured before rollback and can also print cleanup guidance before the final container state is known, so inspect the sandbox and its labeled Docker containers before running a deletion command. -NVIDIA NIM and other compatible endpoints receive their provider validation during onboarding but do not receive this local-provider post-ready check. +Remote NVIDIA NIM and other compatible endpoints receive their provider validation during onboarding but do not receive this local-provider post-ready check. For those routes, continue to the final route check, then use the status command and a short agent request after onboarding. ## Understand Final Route Checks diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 7d231cb4331..ceb977e9b7b 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -813,7 +813,10 @@ On ordinary native Linux, the compatibility path uses an available NVIDIA CDI sp On Docker Desktop WSL, the compatibility path skips CDI and tries Docker `--gpus all` before the NVIDIA runtime. On Jetson/Tegra hosts, the compatibility path uses the NVIDIA runtime and adds eligible host group IDs for the supported GPU device nodes. These include selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices. -If the compatibility attempt fails, onboarding keeps its diagnostics and the failed sandbox in place and prints a manual cleanup command. +After compatibility recreation starts, onboarding keeps the pre-patch container as a rollback backup until the replacement passes the Ready, GPU, and applicable local-inference checks. +If a later check fails, onboarding prints failure diagnostics and attempts to restore the pre-patch container. +If rollback fails, onboarding reports that the pre-patch container was not restored and prints container-cleanup guidance. +GPU-proof diagnostics are captured before rollback and can print that guidance before the final container state is known, so inspect the sandbox and its labeled Docker containers before running a deletion command. Prerequisites: diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index e8d7228b931..51fb92cd978 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2506,7 +2506,7 @@ Identify the matching path before applying the recovery guidance. | --- | --- | --- | | Native `--gpu` is rejected, host runtime evidence identifies GPU injection failure, or an explicit driver proof fails and host configuration confirms no GPU attachment | Ordinary Linux native attempt | The default native-only route stops. Retry with `NEMOCLAW_DOCKER_GPU_PATCH=fallback` only if you explicitly accept one bounded compatibility retry, or use `=1` to select compatibility before creation. | | `Cleanup could not be proven safe` | Native-to-compatibility handoff | Run the printed sandbox deletion command, verify both the gateway row and OpenShell-managed Docker containers labeled for that sandbox are absent, then rerun onboarding. | -| The patched container exits or the compatibility attempt fails | Compatibility recreation | Inspect the saved diagnostics, repair the NVIDIA Container Toolkit/CDI configuration, clean up the failed sandbox, and rerun onboarding. | +| The patched container exits or the compatibility attempt fails | Compatibility recreation | Inspect the saved diagnostics and the rollback outcome, then repair the NVIDIA Container Toolkit/CDI configuration. Keep the sandbox when the pre-patch container was restored; delete it only after inspection confirms restoration failed. Then rerun onboarding. | | A recreated container inherits only a loopback DNS stub and no usable upstream | Compatibility DNS fallback | Repair the host's `systemd-resolved` upstream configuration, then rerun onboarding. | For bridge-networked compatibility recreation without an explicit container DNS setting, NemoClaw selects a usable IPv4 upstream from `systemd-resolved` and probes that exact `--dns` path before it stops the original container. @@ -2552,20 +2552,26 @@ Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses t #### Common compatibility-path recovery -If the compatibility attempt fails on any host, onboarding leaves the failed sandbox and diagnostic bundle in place so you can inspect the OpenShell and Docker state. +After compatibility recreation starts, onboarding keeps the pre-patch container as a rollback backup until the replacement passes the Ready, direct GPU, and applicable local-inference checks. +If a later check fails, onboarding prints failure diagnostics and attempts to restore the pre-patch container before it exits. +When rollback succeeds, the pre-patch sandbox remains available. +When rollback fails, onboarding reports that the pre-patch container was not restored and prints container-cleanup guidance. +GPU-proof diagnostics are captured before rollback and can print that guidance before the final container state is known. +Inspect the sandbox and its labeled Docker containers before running a deletion command. Starting with NemoClaw v0.0.43, the standard installer handles the `/proc//task//comm` permission case during this patch path. If an older release fails direct GPU proof with that path and `Permission denied`, upgrade NemoClaw and rerun onboarding. -The output includes a cleanup command such as: +When inspection confirms that the failed sandbox remains, delete it with a command such as: ```bash openshell sandbox delete ``` -Fix the NVIDIA Container Toolkit or CDI configuration reported in the diagnostics, clean up the failed sandbox, then rerun onboarding. +Fix the NVIDIA Container Toolkit or CDI configuration reported in the diagnostics. +Run the deletion command only after confirming that the pre-patch sandbox was not restored, then rerun onboarding. If you do not need GPU access inside the sandbox, rerun with `--no-sandbox-gpu`. If sandbox creation fails with `CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all`, the OpenShell gateway tried `docker create --device nvidia.com/gpu=all` and Docker could not resolve the CDI spec. diff --git a/src/lib/adapters/openshell/sandbox-identity.ts b/src/lib/adapters/openshell/sandbox-identity.ts index 1820a8f8f7d..dbcc47696c4 100644 --- a/src/lib/adapters/openshell/sandbox-identity.ts +++ b/src/lib/adapters/openshell/sandbox-identity.ts @@ -14,3 +14,19 @@ export function parseOpenShellSandboxId(output: string): string | null { ? (matches[0] as string) : null; } + +export function resolveOpenShellSandboxId( + sandboxName: string, + runCaptureOpenshell: (args: string[], options?: Record) => string, +): string { + const output = runCaptureOpenshell(["sandbox", "get", sandboxName], { + ignoreError: false, + }); + const sandboxId = parseOpenShellSandboxId(output); + if (!sandboxId) { + throw new Error( + `OpenShell sandbox '${sandboxName}' did not return one exact durable sandbox ID.`, + ); + } + return sandboxId; +} diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index d2c07e3e537..b625740adb2 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2708,7 +2708,7 @@ async function createSandboxWithBaseImageResolution( recreateRuntime.advance("creating"); const { createResult, - dockerGpuCreatePatch, + runtimePatch, route: selectedGpuRoute, firstCreateOutput, registryImageRef, @@ -2764,7 +2764,7 @@ async function createSandboxWithBaseImageResolution( } if (effectiveSandboxGpuConfig.sandboxGpuEnabled) { - dockerGpuLocalInference.verifyGpuSandboxLocalInferenceAfterReady( + await dockerGpuLocalInference.verifyGpuSandboxLocalInferenceAndCommitAfterReady( effectiveSandboxGpuConfig, provider, { @@ -2772,11 +2772,10 @@ async function createSandboxWithBaseImageResolution( dockerDriverGateway, selectedRoute: selectedGpuRoute, verifyDirectSandboxGpu, - verifyGpuOrExit: dockerGpuCreatePatch.verifyGpuOrExit, - selectedMode: dockerGpuCreatePatch.selectedMode, runCaptureOpenshell, log: console.log, }, + runtimePatch, ); } diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index 7061252f12c..eebacbacd3d 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -64,8 +64,10 @@ export function createGpuPatchFixture() { maybeApplyDuringCreate: vi.fn(), createFailureMessage: vi.fn(() => null), exitOnPatchError: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), ensureApplied: vi.fn(), waitForSupervisorReconnectIfNeeded: vi.fn(), + commitAfterReady: vi.fn(), selectedMode: vi.fn(() => null), printReadinessFailureIfEnabled: vi.fn(), verifyGpuOrExit: vi.fn(() => VERIFIED_GPU_PROOF), diff --git a/src/lib/onboard/docker-gpu-local-inference.test.ts b/src/lib/onboard/docker-gpu-local-inference.test.ts index 60dfe28c639..3f0f6180ec4 100644 --- a/src/lib/onboard/docker-gpu-local-inference.test.ts +++ b/src/lib/onboard/docker-gpu-local-inference.test.ts @@ -11,6 +11,7 @@ import { shouldUseDockerGpuPatchHostNetwork, verifyDockerGpuSandboxLocalInference, verifyGpuSandboxAfterReady, + verifyGpuSandboxLocalInferenceAndCommitAfterReady, } from "./docker-gpu-local-inference"; const HOST_NETWORK_ENV = { @@ -311,10 +312,10 @@ describe("verifyGpuSandboxAfterReady", () => { }; } - it("runs the GPU proof and the runtime inference gate when the patch is active", () => { + it("runs the GPU proof and the runtime inference gate when the patch is active", async () => { const log = vi.fn(); const verifyDirectSandboxGpu = vi.fn(); - verifyGpuSandboxAfterReady( + await verifyGpuSandboxAfterReady( GPU_CONFIG, "vllm-local", baseOptions({ @@ -327,12 +328,12 @@ describe("verifyGpuSandboxAfterReady", () => { expect(log).toHaveBeenCalledWith(expect.stringContaining("reached local inference")); }); - it("captures the CUDA-usability proof onto the config for status persistence (#4231)", () => { + it("captures the CUDA-usability proof onto the config for status persistence (#4231)", async () => { const proof = { status: "verified" as const, cudaVerified: true, at: "t" }; const config: { sandboxGpuEnabled: boolean; sandboxGpuProof?: typeof proof | null } = { sandboxGpuEnabled: true, }; - verifyGpuSandboxAfterReady( + await verifyGpuSandboxAfterReady( config, "vllm-local", baseOptions({ @@ -343,45 +344,86 @@ describe("verifyGpuSandboxAfterReady", () => { expect(config.sandboxGpuProof).toEqual(proof); }); - it("does not duplicate proof diagnostics when Docker GPU patch verifier handles them", () => { + it("does not duplicate proof diagnostics when Docker GPU patch verifier handles them", async () => { const proofError = new Error("process.exit"); const verifyGpuOrExit = vi.fn(() => { throw proofError; }); const logError = vi.fn(); - expect(() => + await expect( verifyGpuSandboxAfterReady( GPU_CONFIG, "ollama-local", baseOptions({ verifyGpuOrExit, logError }), ), - ).toThrow(proofError); + ).rejects.toBe(proofError); expect(logError).not.toHaveBeenCalled(); }); - it("routes failure diagnostics through the provided error sink and exits", () => { + it("routes failure diagnostics through the provided error sink and throws for rollback", async () => { const logError = vi.fn(); - const exitSpy = vi.spyOn(process, "exit").mockImplementation((() => { - throw new Error("process.exit"); - }) as never); - try { - expect(() => - verifyGpuSandboxAfterReady( - GPU_CONFIG, - "ollama-local", - baseOptions({ - logError, - deps: { execInSandbox: execEmitting("HTTP_000"), sleep: vi.fn() }, - }), - ), - ).toThrow("process.exit"); - expect(exitSpy).toHaveBeenCalledWith(1); - expect(logError).toHaveBeenCalledWith( - expect.stringContaining("Local inference reachability check failed"), - ); - } finally { - exitSpy.mockRestore(); - } + await expect( + verifyGpuSandboxAfterReady( + GPU_CONFIG, + "ollama-local", + baseOptions({ + logError, + deps: { execInSandbox: execEmitting("HTTP_000"), sleep: vi.fn() }, + }), + ), + ).rejects.toThrow("GPU sandbox local inference reachability failed"); + expect(logError).toHaveBeenCalledWith( + expect.stringContaining("Local inference reachability check failed"), + ); + }); +}); + +describe("verifyGpuSandboxLocalInferenceAndCommitAfterReady", () => { + function options() { + return { + ...gpuPatchOptions(), + verifyDirectSandboxGpu: vi.fn(), + runCaptureOpenshell: vi.fn(() => ""), + log: vi.fn(), + }; + } + + it("commits only after local-inference reachability returns HTTP 2xx", async () => { + const runtimePatch = { + commitAfterReady: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + }; + await verifyGpuSandboxLocalInferenceAndCommitAfterReady( + GPU_CONFIG, + "ollama-local", + { + ...options(), + deps: { execInSandbox: execEmitting("HTTP_200"), sleep: vi.fn() }, + }, + runtimePatch, + ); + expect(runtimePatch.commitAfterReady).toHaveBeenCalledOnce(); + expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + }); + + it("rolls back before propagating an inference verification failure", async () => { + const runtimePatch = { + commitAfterReady: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + }; + await expect( + verifyGpuSandboxLocalInferenceAndCommitAfterReady( + GPU_CONFIG, + "ollama-local", + { + ...options(), + deps: { execInSandbox: execEmitting("HTTP_000"), sleep: vi.fn() }, + }, + runtimePatch, + ), + ).rejects.toThrow("GPU sandbox local inference reachability failed"); + expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce(); + expect(runtimePatch.commitAfterReady).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index da58f8c44d8..496ee830448 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -11,6 +11,7 @@ import { import type { DockerGpuPatchMode } from "./docker-gpu-patch-types"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; +import type { ManagedBootstrapRuntimePatch } from "./managed-bootstrap/runtime-create"; import { executeSandboxCommandForVerification } from "./sandbox-verification-exec"; const { @@ -385,9 +386,9 @@ export type GpuSandboxAfterReadyOptions = { verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult; verifyGpuOrExit?: ( verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult, - ) => SandboxGpuProofResult; + ) => Promise; reportGpuProofFailure?: boolean; - selectedMode: () => DockerGpuPatchMode | null; + selectedMode: ManagedBootstrapRuntimePatch["selectedMode"]; runCaptureOpenshell: (args: string[], opts?: Record) => string; env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform; @@ -396,33 +397,47 @@ export type GpuSandboxAfterReadyOptions = { deps?: DockerGpuSandboxInferenceVerifyDeps; }; +function asDockerGpuPatchMode( + selected: ReturnType, +): DockerGpuPatchMode | null { + if (!selected || !["gpus", "nvidia-runtime", "cdi", "startup-command"].includes(selected.kind)) { + return null; + } + return { + kind: selected.kind as DockerGpuPatchMode["kind"], + label: selected.label, + device: selected.device, + args: [...selected.args], + }; +} + /** * Post-readiness GPU sandbox verification orchestrator (kept out of the * ~12k-line onboard.ts entrypoint per the codebase-growth guardrail). Runs the * direct GPU proof, then — only when the Docker GPU patch is active for a local * inference provider — gates on local inference reachability from the sandbox - * runtime (#4509). Exits the process with actionable output if either proof - * fails. + * runtime (#4509). Throws with actionable output if either proof fails so the + * caller can complete rollback before selecting a terminal exit status. */ -export function verifyGpuSandboxAfterReady( +export async function verifyGpuSandboxAfterReady( config: DockerGpuLocalInferenceConfig, provider: string | null | undefined, options: GpuSandboxAfterReadyOptions, -): void { - verifyGpuSandboxAccessAfterReady(config, options); +): Promise { + await verifyGpuSandboxAccessAfterReady(config, options); verifyGpuSandboxLocalInferenceAfterReady(config, provider, options); } -export function verifyGpuSandboxAccessAfterReady( +export async function verifyGpuSandboxAccessAfterReady( config: DockerGpuLocalInferenceConfig, options: GpuSandboxAfterReadyOptions, -): SandboxGpuProofResult { +): Promise { try { // Capture the CUDA-usability proof result and write it back onto the shared // config so onboarding can persist it to the registry and `status` can // report proven usability rather than mere configuration (#4231). const proof = options.verifyGpuOrExit - ? options.verifyGpuOrExit(options.verifyDirectSandboxGpu) + ? await options.verifyGpuOrExit(options.verifyDirectSandboxGpu) : options.verifyDirectSandboxGpu(options.sandboxName); config.sandboxGpuProof = proof; return proof; @@ -431,11 +446,16 @@ export function verifyGpuSandboxAccessAfterReady( // prints the richer Error-phase / patched-container diagnostics before // rethrowing. Avoid a second generic proof-failure block in that path. if (!options.verifyGpuOrExit && options.reportGpuProofFailure !== false) { - printDockerGpuProofFailure(options.sandboxName, error, options.selectedMode(), { - runCaptureOpenshell: options.runCaptureOpenshell, - additionalSummaryLines: adaptDockerGpuRouteForPatch(options.selectedRoute) - .additionalSummaryLines, - }); + printDockerGpuProofFailure( + options.sandboxName, + error, + asDockerGpuPatchMode(options.selectedMode()), + { + runCaptureOpenshell: options.runCaptureOpenshell, + additionalSummaryLines: adaptDockerGpuRouteForPatch(options.selectedRoute) + .additionalSummaryLines, + }, + ); } throw error; } @@ -444,7 +464,7 @@ export function verifyGpuSandboxAccessAfterReady( export function verifyGpuSandboxLocalInferenceAfterReady( config: DockerGpuLocalInferenceConfig, provider: string | null | undefined, - options: GpuSandboxAfterReadyOptions, + options: Omit, ): void { if (options.selectedRoute !== "compatibility") return; const verification = verifyDockerGpuSandboxLocalInference(config, provider, { @@ -469,6 +489,39 @@ export function verifyGpuSandboxLocalInferenceAfterReady( verification, options.logError ?? ((message) => console.error(message)), ); - process.exit(1); + throw new Error( + `GPU sandbox local inference reachability failed for ${verification.endpoint}.`, + ); + } +} + +/** + * Keep the managed create transaction reversible until the sandbox's real + * local-inference reachability check returns HTTP 2xx. Rollback failures are + * attached to the original verification failure so callers retain both pieces + * of evidence. + */ +export async function verifyGpuSandboxLocalInferenceAndCommitAfterReady( + config: DockerGpuLocalInferenceConfig, + provider: string | null | undefined, + options: Omit, + runtimePatch: Pick< + ManagedBootstrapRuntimePatch, + "commitAfterReady" | "rollbackManagedStartupAfterCreateFailure" + >, +): Promise { + try { + verifyGpuSandboxLocalInferenceAfterReady(config, provider, options); + await runtimePatch.commitAfterReady(); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + try { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); + } catch (rollbackError) { + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + throw failure; } } diff --git a/src/lib/onboard/docker-gpu-route-consumers.test.ts b/src/lib/onboard/docker-gpu-route-consumers.test.ts index f2454e5d45d..b11215d340e 100644 --- a/src/lib/onboard/docker-gpu-route-consumers.test.ts +++ b/src/lib/onboard/docker-gpu-route-consumers.test.ts @@ -137,7 +137,7 @@ describe("selected route consumers", () => { expect(reverifyBridgeReachability).not.toHaveBeenCalled(); }); - it("skips compatibility-only inference gates after native wins", () => { + it("skips compatibility-only inference gates after native wins", async () => { const execInSandbox = vi.fn(); expect( verifyDockerGpuSandboxLocalInference(GPU_CONFIG, "ollama-local", { @@ -149,7 +149,7 @@ describe("selected route consumers", () => { ).toEqual({ status: "skipped", reason: "not-docker-gpu-patch" }); const verifyDirectSandboxGpu = vi.fn(); - verifyGpuSandboxAfterReady(GPU_CONFIG, "ollama-local", { + await verifyGpuSandboxAfterReady(GPU_CONFIG, "ollama-local", { sandboxName: "alpha", dockerDriverGateway: true, selectedRoute: "native", @@ -162,11 +162,11 @@ describe("selected route consumers", () => { expect(execInSandbox).not.toHaveBeenCalled(); }); - it("defers native proof diagnostics while automatic fallback owns recovery", () => { + it("defers native proof diagnostics while automatic fallback owns recovery", async () => { const proofError = new Error("native CUDA proof failed"); const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); try { - expect(() => + await expect( verifyGpuSandboxAfterReady(GPU_CONFIG, "ollama-local", { sandboxName: "alpha", dockerDriverGateway: true, @@ -178,7 +178,7 @@ describe("selected route consumers", () => { selectedMode: () => null, runCaptureOpenshell: vi.fn(() => ""), }), - ).toThrow(proofError); + ).rejects.toThrow(proofError); expect(consoleError).not.toHaveBeenCalled(); } finally { consoleError.mockRestore(); diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index fb94eba175e..6ddea611aa9 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -42,7 +42,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { vi.restoreAllMocks(); }); - it("defers backup removal until waitForSupervisorReconnectIfNeeded sees supervisorReady=true", () => { + it("retains the backup after reconnect and removes it only after post-Ready commit", async () => { const deps = makeDeps(); const result = deferredCreateResult(); const recreatePatch = vi.fn(() => result); @@ -84,13 +84,57 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.waitForSupervisorReconnectIfNeeded(); expect(waitForSupervisor).toHaveBeenCalledTimes(1); + expect(finalizeBackup).not.toHaveBeenCalled(); + + await patch.commitAfterReady(); expect(finalizeBackup).toHaveBeenCalledTimes(1); expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: true }, deps); expect(capturePreRollbackDiagnostics).not.toHaveBeenCalled(); expect(onPatchFailureExit).not.toHaveBeenCalled(); }); - it("refuses compatibility success when the backup container cannot be removed", () => { + it("reports a failed post-Ready rollback instead of treating it as restored", async () => { + const deps = makeDeps(); + const result = deferredCreateResult(); + const finalizeBackup = vi.fn(() => ({ + backupRemoved: false, + rolledBack: false, + })); + const onPatchFailureExit = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => ["existing-container"]), + recreatePatch: vi.fn(() => result), + waitForSupervisor: vi.fn(() => true), + finalizeBackup, + onPatchFailureExit, + }, + }); + + patch.maybeApplyDuringCreate(); + patch.waitForSupervisorReconnectIfNeeded(); + await patch.rollbackManagedStartupAfterCreateFailure(); + + expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: false }, deps); + expect(onPatchFailureExit).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + message: expect.stringContaining("pre-patch container was not restored"), + }), + expect.objectContaining({ + context: expect.objectContaining({ + backupContainerName: result.backupContainerName, + rolledBack: false, + }), + }), + ); + }); + + it("refuses compatibility success when the backup container cannot be removed", async () => { const deps = makeDeps(); const result = deferredCreateResult(); const onPatchFailureExit = vi.fn(); @@ -113,11 +157,14 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); patch.waitForSupervisorReconnectIfNeeded(); + expect(onPatchFailureExit).not.toHaveBeenCalled(); + + await patch.commitAfterReady(); expect(onPatchFailureExit).toHaveBeenCalledOnce(); expect(onPatchFailureExit.mock.calls[0]?.[1]).toEqual( expect.objectContaining({ - message: expect.stringContaining("backup container"), + message: expect.stringContaining("rollback backup"), }), ); expect(onPatchFailureExit.mock.calls[0]?.[2]).toEqual( @@ -245,7 +292,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(onPatchFailureExit).not.toHaveBeenCalled(); }); - it("records patchError when recreate throws and exitOnPatchError reports it via printDockerGpuPatchFailureAndExit", () => { + it("records patchError when recreate throws and exitOnPatchError reports it via printDockerGpuPatchFailureAndExit", async () => { const deps = makeDeps(); const recreatePatch = vi.fn(() => { throw new Error("docker rename failed"); @@ -271,7 +318,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); expect(patch.createFailureMessage()).toMatch(/Docker GPU patch failed/); - patch.exitOnPatchError(); + await patch.exitOnPatchError(); expect(onPatchFailureExit).toHaveBeenCalledTimes(1); // Supervisor wait must be skipped because needsSupervisorWait stayed false. patch.waitForSupervisorReconnectIfNeeded(); @@ -279,7 +326,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(finalizeBackup).not.toHaveBeenCalled(); }); - it("hard-stops a structured failed GPU proof on the compatibility route", () => { + it("hard-stops a structured failed GPU proof on the compatibility route", async () => { const deps = makeDeps(); const patch = createDockerGpuSandboxCreatePatch({ route: "compatibility", @@ -291,7 +338,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { }, }); - expect(() => + await expect( patch.verifyGpuOrExit(() => ({ status: "failed", cudaVerified: false, @@ -299,6 +346,38 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { detail: "No devices were found", at: "2026-07-07T00:00:00.000Z", })), - ).toThrow("Sandbox GPU proof returned failed status: nvidia-smi when available"); + ).rejects.toThrow("Sandbox GPU proof returned failed status: nvidia-smi when available"); + }); + + it("reports a failed rollback after GPU-proof diagnostics", async () => { + const deps = makeDeps(); + const result = deferredCreateResult(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => ["existing-container"]), + recreatePatch: vi.fn(() => result), + waitForSupervisor: vi.fn(() => true), + finalizeBackup: vi.fn(() => ({ + backupRemoved: false, + rolledBack: false, + })), + }, + }); + + patch.maybeApplyDuringCreate(); + patch.waitForSupervisorReconnectIfNeeded(); + + await expect( + patch.verifyGpuOrExit(() => { + throw new Error("nvidia-smi failed"); + }), + ).rejects.toThrow("nvidia-smi failed"); + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("pre-patch container was not restored"), + ); }); }); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index b683cbb1c77..52c15d0b5be 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -42,7 +42,7 @@ export { type DockerGpuSandboxCreateDeps = Pick< DockerGpuPatchDeps, - "runOpenshell" | "runCaptureOpenshell" | "sleep" | "dockerCapture" + "runOpenshell" | "runCaptureOpenshell" | "sleep" | "dockerCapture" | "dockerRun" | "dockerStop" >; type WaitSupervisorFn = typeof waitForOpenShellSupervisorReconnect; @@ -61,6 +61,11 @@ type PatchFailureExitFn = ( type DockerGpuSandboxCreatePatchOptions = { route: SelectedDockerGpuRoute; persistStartupCommand?: boolean; + /** + * A managed bootstrap owns the one permitted recreation after Ready. Keep + * route diagnostics/proof active without running the legacy recreator. + */ + externalRecreation?: boolean; sandboxName: string; gpuDevice?: string | null; openshellSandboxCommand?: readonly string[] | null; @@ -91,12 +96,27 @@ type DockerGpuSandboxCreatePatchOptions = { }; }; +export interface DockerManagedBootstrapDeferredCutover { + readonly selectedMode: DockerGpuPatchMode; + readonly failureContext: DockerGpuPatchFailureContext; + rollback(): Promise; + commit(): Promise; +} + export type DockerGpuSandboxCreatePatch = { maybeApplyDuringCreate: () => void; createFailureMessage: () => string | null; - exitOnPatchError: () => void; - ensureApplied: () => void; + exitOnPatchError: () => Promise; + attachManagedBootstrapCutover: (cutover: DockerManagedBootstrapDeferredCutover) => void; + rollbackManagedStartupAfterCreateFailure: () => Promise; + ensureApplied: () => Promise; waitForSupervisorReconnectIfNeeded: () => void; + /** + * Commit an attached managed cutover or remove a legacy recreation backup. + * Call only after authoritative Ready and the required GPU and applicable + * local-inference checks pass. + */ + commitAfterReady: () => Promise; selectedMode: () => DockerGpuPatchMode | null; /** * Print the Docker GPU readiness-failure block (including the Error-phase @@ -106,14 +126,14 @@ export type DockerGpuSandboxCreatePatch = { printReadinessFailureIfEnabled: () => void; /** * Run the GPU proof while distinguishing "sandbox in terminal phase" from - * "proof failed inside a live sandbox". Calls `process.exit(1)` for the - * former and rethrows after printing diagnostics for the latter so the - * onboarding flow surfaces the right failure cause (#4316). Returns the - * CUDA-usability proof result on success so callers can persist it (#4231). + * "proof failed inside a live sandbox". Awaits rollback and throws after + * printing diagnostics so the onboarding flow can select the terminal exit + * status without racing the rollback (#4316). Returns the CUDA-usability + * proof result on success so callers can persist it (#4231). */ verifyGpuOrExit: ( verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult, - ) => SandboxGpuProofResult; + ) => Promise; }; export function createDockerGpuSandboxCreatePatch( @@ -121,8 +141,12 @@ export function createDockerGpuSandboxCreatePatch( ): DockerGpuSandboxCreatePatch { const routeAdapter = adaptDockerGpuRouteForPatch(options.route); let result: DockerGpuPatchResult | null = null; + let managedBootstrapCutover: DockerManagedBootstrapDeferredCutover | null = null; let patchError: unknown = null; let needsSupervisorWait = false; + let cutoverFinalized = false; + let cutoverFinalization: Promise | null = null; + let cutoverFinalizationOutcome: "commit" | "rollback" | null = null; const findContainerIds = options.overrides?.findContainerIds ?? findOpenShellDockerSandboxContainerIds; @@ -145,7 +169,10 @@ export function createDockerGpuSandboxCreatePatch( backend: options.backend, dockerDesktopWsl: options.dockerDesktopWsl ?? isDockerDesktopWslRuntime(), }; - const patchEnabled = routeAdapter.enabled || options.persistStartupCommand === true; + const recreationEnabled = + options.externalRecreation !== true && + (routeAdapter.enabled || options.persistStartupCommand === true); + const patchEnabled = recreationEnabled; const patchTarget = routeAdapter.enabled ? "NVIDIA GPU access" : "restart-safe startup"; const recreateSelectedPatch = createDockerSandboxRecreator({ gpuEnabled: routeAdapter.enabled, @@ -156,21 +183,92 @@ export function createDockerGpuSandboxCreatePatch( recreateStartup: recreateStartupPatch, }); + const applyPatch = (deps: DockerGpuPatchDeps): void => { + if (!recreationEnabled) return; + result = recreateSelectedPatch(false, deps); + needsSupervisorWait = true; + console.log(` ✓ Docker container mode selected: ${result.mode.label}`); + }; + + const rollbackAfterFailure = async (): Promise => { + if (cutoverFinalized || (!managedBootstrapCutover && !result)) return null; + if (cutoverFinalization) { + try { + if (cutoverFinalizationOutcome !== "rollback") { + throw new Error("Managed startup rollback raced an in-progress commit finalization."); + } + await cutoverFinalization; + return null; + } catch (error) { + return error instanceof Error ? error : new Error(String(error)); + } + } + const finalization = (async () => { + await managedBootstrapCutover?.rollback(); + const finalizeOutcome = result + ? finalizeBackup({ result, supervisorReady: false }, options.deps) + : null; + cutoverFinalized = true; + needsSupervisorWait = false; + if (finalizeOutcome && !finalizeOutcome.rolledBack) { + throw new Error( + "Docker container rollback failed; the pre-patch container was not restored.", + ); + } + })(); + cutoverFinalization = finalization; + cutoverFinalizationOutcome = "rollback"; + try { + await finalization; + return null; + } catch (error) { + return error instanceof Error ? error : new Error(String(error)); + } finally { + if (!cutoverFinalized) { + cutoverFinalization = null; + cutoverFinalizationOutcome = null; + } + } + }; + + const reportPatchErrorAndExit = async (): Promise => { + if (!patchError) return; + const rollbackError = await rollbackAfterFailure(); + if (rollbackError) { + patchError = new Error( + `${patchError instanceof Error ? patchError.message : String(patchError)}; managed startup rollback failed: ${rollbackError.message}`, + ); + } + onPatchFailureExit(options.sandboxName, patchError, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }); + }; + const selectedMode = (): DockerGpuPatchMode | null => + managedBootstrapCutover?.selectedMode ?? result?.mode ?? null; + const failureContext = (): DockerGpuPatchFailureContext => + managedBootstrapCutover?.failureContext ?? buildFailureContext(options.sandboxName, result); + return { maybeApplyDuringCreate() { if (!patchEnabled || result || patchError) return; const containerIds = findContainerIds(options.sandboxName); if (containerIds.length === 0) return; + if (containerIds.length !== 1) { + patchError = new Error( + `Docker recreation observed ${String(containerIds.length)} matching containers; refusing an ambiguous replacement.`, + ); + return; + } console.log( ` OpenShell Docker container detected; recreating it with ${patchTarget} before readiness wait...`, ); try { - result = recreateSelectedPatch(false, { + applyPatch({ runCaptureOpenshell: options.deps.runCaptureOpenshell, sleep: options.deps.sleep, }); - needsSupervisorWait = true; - console.log(` ✓ Docker container mode selected: ${result.mode.label}`); } catch (error) { patchError = error; } @@ -183,33 +281,44 @@ export function createDockerGpuSandboxCreatePatch( : "Docker startup-command patch failed while OpenShell sandbox create was still waiting."; }, - exitOnPatchError() { - if (!patchError) return; - onPatchFailureExit(options.sandboxName, patchError, { + async exitOnPatchError() { + await reportPatchErrorAndExit(); + }, + + attachManagedBootstrapCutover(cutover) { + if (managedBootstrapCutover || result || cutoverFinalized) { + throw new Error("Managed bootstrap cutover may be attached exactly once."); + } + managedBootstrapCutover = cutover; + }, + + async rollbackManagedStartupAfterCreateFailure() { + const rollbackError = await rollbackAfterFailure(); + if (!rollbackError) return; + onPatchFailureExit(options.sandboxName, rollbackError, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: { + ...failureContext(), + rolledBack: false, + }, }); }, - ensureApplied() { + async ensureApplied() { if (!patchEnabled || result) return; console.log(` Recreating OpenShell Docker sandbox container with ${patchTarget}...`); try { - result = recreateSelectedPatch(false, options.deps); - needsSupervisorWait = true; - console.log(` ✓ Docker container mode selected: ${result.mode.label}`); + applyPatch(options.deps); } catch (error) { - onPatchFailureExit(options.sandboxName, error, { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - }); + patchError = error; + await reportPatchErrorAndExit(); } }, waitForSupervisorReconnectIfNeeded() { - if (!needsSupervisorWait) return; + if (!needsSupervisorWait || cutoverFinalized) return; const supervisorReconnectTimeoutSecs = getDockerGpuSupervisorReconnectTimeoutSecs( options.timeoutSecs, ); @@ -221,14 +330,17 @@ export function createDockerGpuSandboxCreatePatch( supervisorReconnectTimeoutSecs, { runOpenshell: options.deps.runOpenshell, - // Pass `runCaptureOpenshell` so the supervisor-reconnect wait can - // short-circuit on a terminal sandbox phase instead of burning - // the full reconnect timeout window when the patched container - // crashed on startup (#4316). runCaptureOpenshell: options.deps.runCaptureOpenshell, sleep: options.deps.sleep, }, ); + if (supervisorReady) { + // Reconnect completes the legacy recreation check. Keep its rollback + // backup until the caller accepts authoritative Ready and the required + // GPU checks. + needsSupervisorWait = false; + return; + } if (!supervisorReady && result) { try { captureFailedClone(options.sandboxName, result, options.deps); @@ -239,40 +351,13 @@ export function createDockerGpuSandboxCreatePatch( } } const finalizeOutcome = result - ? finalizeBackup({ result, supervisorReady }, options.deps) + ? finalizeBackup({ result, supervisorReady: false }, options.deps) : null; - if (supervisorReady) { - if (finalizeOutcome && !finalizeOutcome.backupRemoved) { - onPatchFailureExit( - options.sandboxName, - new Error( - "OpenShell supervisor reconnected, but the recreated backup container could not be removed.", - ), - { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - context: { - sandboxName: options.sandboxName, - oldContainerId: result?.oldContainerId, - newContainerId: result?.newContainerId, - backupContainerName: result?.backupContainerName, - selectedMode: result?.mode ?? null, - rolledBack: false, - }, - }, - ); - } - return; - } - const failureMessage = (() => { - if (!finalizeOutcome) { - return "OpenShell supervisor did not reconnect to the recreated container."; - } - return finalizeOutcome.rolledBack - ? "OpenShell supervisor did not reconnect to the recreated container; pre-patch sandbox restored." - : "OpenShell supervisor did not reconnect to the recreated container and rollback failed; pre-patch sandbox was NOT restored."; - })(); + cutoverFinalized = true; + needsSupervisorWait = false; + const failureMessage = finalizeOutcome?.rolledBack + ? "OpenShell supervisor did not reconnect to the recreated container; pre-patch sandbox restored." + : "OpenShell supervisor did not reconnect to the recreated container and rollback failed; pre-patch sandbox was NOT restored."; onPatchFailureExit(options.sandboxName, new Error(failureMessage), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, @@ -288,21 +373,108 @@ export function createDockerGpuSandboxCreatePatch( }); }, + async commitAfterReady() { + if (cutoverFinalized || (!managedBootstrapCutover && !result)) return; + if (needsSupervisorWait) { + const error = new Error( + "Managed startup cannot commit before the recreated OpenShell supervisor reconnects.", + ); + const rollbackError = await rollbackAfterFailure(); + onPatchFailureExit( + options.sandboxName, + rollbackError + ? new Error(`${error.message} Rollback failed: ${rollbackError.message}`) + : error, + { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }, + ); + return; + } + if (cutoverFinalization) { + if (cutoverFinalizationOutcome !== "commit") { + throw new Error("Managed startup commit raced an in-progress rollback finalization."); + } + await cutoverFinalization; + return; + } + const finalization = (async () => { + if (managedBootstrapCutover) { + try { + await managedBootstrapCutover.commit(); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + let rollbackError: Error | null = null; + try { + await managedBootstrapCutover.rollback(); + cutoverFinalized = true; + needsSupervisorWait = false; + } catch (rollbackFailure) { + rollbackError = + rollbackFailure instanceof Error + ? rollbackFailure + : new Error(String(rollbackFailure)); + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + onPatchFailureExit(options.sandboxName, failure, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: { + ...failureContext(), + rolledBack: rollbackError === null, + }, + }); + return; + } + } + const finalizeOutcome = result + ? finalizeBackup({ result, supervisorReady: true }, options.deps) + : null; + cutoverFinalized = true; + if (!finalizeOutcome || finalizeOutcome.backupRemoved) return; + onPatchFailureExit( + options.sandboxName, + new Error("Managed startup passed Ready, but its rollback backup could not be removed."), + { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: failureContext(), + }, + ); + })(); + cutoverFinalization = finalization; + cutoverFinalizationOutcome = "commit"; + try { + await finalization; + } finally { + if (!cutoverFinalized) { + cutoverFinalization = null; + cutoverFinalizationOutcome = null; + } + } + }, + selectedMode() { - return result?.mode ?? null; + return selectedMode(); }, printReadinessFailureIfEnabled() { if (!routeAdapter.enabled) return; - printDockerGpuReadinessFailure(options.sandboxName, result?.mode ?? null, { + printDockerGpuReadinessFailure(options.sandboxName, selectedMode(), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, - context: buildFailureContext(options.sandboxName, result), + context: failureContext(), additionalSummaryLines: routeAdapter.additionalSummaryLines, }); }, - verifyGpuOrExit(verifyDirectSandboxGpu) { + async verifyGpuOrExit(verifyDirectSandboxGpu) { // Before issuing GPU proof commands through `openshell sandbox exec`, // confirm the sandbox is still in a live phase. A sandbox that // transitioned to Error after the readiness wait succeeded (e.g. the @@ -312,7 +484,7 @@ export function createDockerGpuSandboxCreatePatch( // container/Error-phase classification instead of running the proof // (#4316). const sandboxName = options.sandboxName; - const failureContext = buildFailureContext(sandboxName, result); + const currentFailureContext = failureContext(); if (routeAdapter.enabled && options.deps.runCaptureOpenshell) { const list = options.deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true, @@ -321,20 +493,23 @@ export function createDockerGpuSandboxCreatePatch( if (phase) { console.error(""); console.error(` Skipping GPU proof: sandbox '${sandboxName}' is in ${phase} phase.`); - printDockerGpuProofFailure( - sandboxName, - new Error( - `Sandbox '${sandboxName}' entered ${phase} phase after readiness; GPU proof skipped.`, - ), - result?.mode ?? null, - { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - context: failureContext, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - }, + const failure = new Error( + `Sandbox '${sandboxName}' entered ${phase} phase after readiness; GPU proof skipped.`, ); - process.exit(1); + printDockerGpuProofFailure(sandboxName, failure, selectedMode(), { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + context: currentFailureContext, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }); + const rollbackError = await rollbackAfterFailure(); + if (rollbackError) { + console.error(` ${rollbackError.message}`); + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + throw failure; } } try { @@ -346,13 +521,21 @@ export function createDockerGpuSandboxCreatePatch( } return proof; } catch (error) { - printDockerGpuProofFailure(sandboxName, error, result?.mode ?? null, { + const failure = error instanceof Error ? error : new Error(String(error)); + printDockerGpuProofFailure(sandboxName, failure, selectedMode(), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, - context: routeAdapter.enabled ? failureContext : null, + context: routeAdapter.enabled ? currentFailureContext : null, additionalSummaryLines: routeAdapter.additionalSummaryLines, }); - throw error; + const rollbackError = await rollbackAfterFailure(); + if (rollbackError) { + console.error(` ${rollbackError.message}`); + ( + failure as Error & { managedBootstrapRollbackError?: unknown } + ).managedBootstrapRollbackError = rollbackError; + } + throw failure; } }, }; diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts index 17cd02f6432..ec0ad3cb306 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts @@ -68,7 +68,7 @@ describe("Docker startup-command sandbox creation", () => { vi.restoreAllMocks(); }); - it("uses the startup-command recreation path with DCode's exact resource limits", () => { + it("uses the startup-command recreation path with DCode's exact resource limits", async () => { const dockerCaptureOutput: Record = { ps: "old-container-id\n", inspect: JSON.stringify([inspectFixture()]), @@ -102,7 +102,7 @@ describe("Docker startup-command sandbox creation", () => { }, }); - patch.ensureApplied(); + await patch.ensureApplied(); expect(recreatePatch).not.toHaveBeenCalled(); expect(dockerRunDetached.mock.calls[0]?.[0]).toEqual( @@ -156,7 +156,102 @@ describe("Docker startup-command sandbox creation", () => { expect(context.rolledBack).toBe(true); }); - it("reports startup-command creation failures through the composed patch boundary", () => { + it("defers a driver-owned managed cutover until the authoritative caller commits", async () => { + const deps = makeDeps(); + let releaseCommit = () => {}; + const commit = vi.fn( + () => + new Promise((resolve) => { + releaseCommit = resolve; + }), + ); + const rollback = vi.fn(async () => {}); + const patch = createDockerGpuSandboxCreatePatch({ + route: "native", + externalRecreation: true, + sandboxName: "alpha", + timeoutSecs: 60, + deps, + }); + patch.attachManagedBootstrapCutover({ + selectedMode: { + kind: "startup-command", + label: "managed bootstrap", + device: "", + args: [], + }, + failureContext: { sandboxName: "alpha" }, + commit, + rollback, + }); + patch.maybeApplyDuringCreate(); + await patch.ensureApplied(); + patch.waitForSupervisorReconnectIfNeeded(); + expect(commit).not.toHaveBeenCalled(); + const firstCommit = patch.commitAfterReady(); + const duplicateCommit = patch.commitAfterReady(); + expect(commit).toHaveBeenCalledOnce(); + expect(rollback).not.toHaveBeenCalled(); + releaseCommit(); + await Promise.all([firstCommit, duplicateCommit]); + }); + + it("rolls back a driver-owned cutover before reporting commit failure", async () => { + const deps = makeDeps(); + const events: string[] = []; + const commit = vi.fn(async () => { + events.push("commit"); + throw new Error("receipt validation failed"); + }); + const rollback = vi.fn(async () => { + events.push("rollback"); + }); + const onPatchFailureExit = vi.fn(() => { + events.push("exit"); + }); + const patch = createDockerGpuSandboxCreatePatch({ + route: "native", + externalRecreation: true, + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { onPatchFailureExit }, + }); + patch.attachManagedBootstrapCutover({ + selectedMode: { + kind: "startup-command", + label: "managed bootstrap", + device: "", + args: [], + }, + failureContext: { + sandboxName: "alpha", + oldContainerId: "held-container", + newContainerId: "replacement-container", + }, + commit, + rollback, + }); + + await patch.commitAfterReady(); + + expect(events).toEqual(["commit", "rollback", "exit"]); + expect(onPatchFailureExit).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ message: "receipt validation failed" }), + expect.objectContaining({ + context: expect.objectContaining({ + oldContainerId: "held-container", + newContainerId: "replacement-container", + rolledBack: true, + }), + }), + ); + await patch.rollbackManagedStartupAfterCreateFailure(); + expect(rollback).toHaveBeenCalledOnce(); + }); + + it("reports startup-command creation failures through the composed patch boundary", async () => { const deps = makeDeps(); const onPatchFailureExit = vi.fn(); const patch = createDockerGpuSandboxCreatePatch({ @@ -177,7 +272,7 @@ describe("Docker startup-command sandbox creation", () => { patch.maybeApplyDuringCreate(); expect(patch.createFailureMessage()).toMatch(/startup-command patch failed/); - patch.exitOnPatchError(); + await patch.exitOnPatchError(); expect(onPatchFailureExit).toHaveBeenCalledWith( "alpha", expect.objectContaining({ message: "startup recreate failed" }), diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index a102dc031e7..159ab401e86 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -3,9 +3,11 @@ # Managed bootstrap protocol -This directory defines a dormant, driver-neutral transaction contract and its -first driver adapter. It does not register a runtime provider, activate managed -bootstrap, or change the current user-visible lifecycle paths. +This directory defines a dormant, driver-neutral transaction contract, its +first driver adapter, and an injectable sandbox-create lifecycle. Production +runtime bundles still report bootstrap as unsupported, so the candidate +lifecycle remains inert. The shared finalization path keeps existing Docker +recreation reversible through later Ready, GPU, and local-inference checks. The protocol binds one random bootstrap identity to: @@ -66,10 +68,12 @@ including its supervisor environment, to immutable prepared authority before activation. The native boundary introduces no driver-specific environment policy. -The first Docker-specific groundwork defines a private, monotonic cutover -journal and a canonical launch-spec normalizer. Each surface is independently -validated and remains dormant: no registered runtime provider imports either -module, and neither changes sandbox creation or lifecycle behavior. +The Docker-specific layers define a private, monotonic cutover journal, a +canonical launch-spec normalizer, and an injectable provider create lifecycle. +The candidate provider surface composes these layers without registering them +in a production runtime bundle. It remains inert, while the shared finalization +surface extends rollback ownership for the existing Docker compatibility and +startup recreation paths. The Docker adapter creates and validates a stopped replacement under an identity-derived staging name while the original remains running. It stages the @@ -89,35 +93,40 @@ Activation must also inject the selected gateway's canonical state root. ## Architectural disposition -The coordinator deliberately lands as a dormant trust-boundary slice before a -provider or image activates it. This keeps the driver-neutral transaction -authority review separate from the first driver implementation instead of -making that implementation the de facto central contract. The coordinator -module remains cohesive because its receipt shapes, normalization, state -transitions, and rollback proofs form one authority boundary; provider-specific -logic must live outside it rather than growing this file. +The runtime-provider bundle is the only bootstrap registration boundary. The +candidate Docker surface owns create routing, replacement construction, +native-to-compatibility fallback evidence, and deferred commit or rollback. +Central onboarding accepts that provider-neutral surface without a Docker or +Podman selection branch. Tests register an MXC-style surface through the same +bundle and render held launches for OpenClaw, Hermes, and LangChain Deep Agents +Code. + +The coordinator remains the driver-neutral transaction authority: its receipt +shapes, normalization, state transitions, and rollback proofs form one cohesive +boundary, while provider-specific routing and runtime operations stay outside +it. This is executable, bounded groundwork rather than an untested placeholder. `adapter.test.ts` drives prepare, durable record, activation, finalization, and failure rollback for OpenClaw, Hermes, and LangChain Deep Agents Code through an -MXC-named fake driver. `runtime-provider-source-shape.test.ts` separately -inventories the protocol, provider, and image-packaging surfaces and proves that -production activation does not import or install the protocol into a runtime -image yet. The later activation slice must add a registered-provider contract -test for the same transaction before removing those dormancy assertions. +MXC-named fake driver. `runtime-provider-contract.test.ts` registers both the +dormant Docker candidate and an MXC-style bootstrap surface through the same +provider bundle contract without changing the production registry. +`runtime-provider-source-shape.test.ts` separately inventories the protocol, +provider, and image-packaging surfaces and proves that production activation +does not select a driver-specific bootstrap implementation. The native entrypoint source is intentionally not compiled into production artifacts, and neither image-owned source is installed or selected in a runtime -image yet. No production activation or provider module outside this dormant -directory imports the protocol or Docker adapter. The current image definitions -do not package `nemoclaw-managed-startup-hold`, +image yet. Production onboarding imports only the provider-neutral create +contract; no activation path or registered provider imports or selects the +driver-specific Docker candidate. The current image definitions do not package +`nemoclaw-managed-startup-hold`, `managed-startup-image-runtime.cjs`, or the shared-state bootstrap modes consumed -by the adapter. A later provider integration must compile and verify the -freestanding entrypoint natively for amd64 and arm64 in every agent image. It -must add those prerequisites together with their image-runtime bootstrap modes -and wire the coordinator and Docker adapter into create as one boundary. The -same contract is exercised for OpenClaw, Hermes, and Deep Agents Code without a -provider-specific central switch. The remaining integration and qualification -work is tracked in [epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) -and its linked implementation stack. Until that complete boundary lands, every -registered runtime provider keeps its bootstrap surface unsupported. +by the adapter. Later persistence and qualification slices must compile and +verify the freestanding entrypoint for amd64 and arm64 in every agent image, add +the image-runtime prerequisites, and provide the canonical durable authority +store. The remaining integration and qualification work is tracked in +[epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744). Until that complete +boundary passes protected E2E, every production runtime provider keeps +bootstrap unsupported. diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.ts new file mode 100644 index 00000000000..6da0f754f1b --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.ts @@ -0,0 +1,294 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { detectTegraDeviceGroupGids } from "../docker-gpu-jetson-groups"; +import { buildDockerGpuMode, selectDockerGpuPatchMode } from "../docker-gpu-patch-mode"; +import type { DockerGpuPatchMode } from "../docker-gpu-patch-types"; +import { renderCompatibilityFallbackCreateArgs } from "../docker-gpu-route"; +import { + createDockerGpuSandboxCreatePatch, + isDockerDesktopWslRuntime, +} from "../docker-gpu-sandbox-create"; +import { + isImmutableDockerImageId, + queryOpenShellDockerSandboxContainers, + queryOpenShellDockerSandboxRuntimeSnapshot, +} from "../openshell-docker-sandbox-containers"; +import type { RuntimeProviderBootstrapSurface } from "../runtime-provider/contract"; +import * as sandboxGpuCreateAttempt from "../sandbox-gpu-create-attempt"; +import { + activateManagedBootstrapSequence, + finalizeManagedBootstrapSequence, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + prepareManagedBootstrapSequence, +} from "./adapter"; +import { createDockerManagedBootstrapAdapter } from "./docker"; +import type { + ManagedBootstrapRuntimeCompatibilityLaunchInput, + ManagedBootstrapRuntimeCreateLaunchResult, + ManagedBootstrapRuntimeCreateLifecycle, + ManagedBootstrapRuntimeCreateLifecycleInput, + ManagedBootstrapRuntimeOnboardRoutingInput, +} from "./runtime-create"; + +type SupportedBootstrapSurface = Extract< + RuntimeProviderBootstrapSurface, + { readonly supported: true } +>; + +function dockerReplacementOptions( + mode: DockerGpuPatchMode, + input: ManagedBootstrapRuntimeCreateLifecycleInput, +) { + const backend = input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic"; + return { + values: { + gpuModeArgs: [...mode.args], + gpuModeDevice: mode.device, + gpuModeKind: mode.kind, + gpuModeLabel: mode.label, + requiredUlimits: input.requiredLimits.map( + (limit) => `${limit.name}=${limit.soft}:${limit.hard}`, + ), + extraGroupGids: + backend === "jetson" && input.route === "compatibility" ? detectTegraDeviceGroupGids() : [], + }, + }; +} + +function selectedDockerMode( + input: ManagedBootstrapRuntimeCreateLifecycleInput, + dockerDesktopWsl: boolean | undefined, +): DockerGpuPatchMode { + const backend = input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic"; + if (input.route !== "compatibility" || !input.sandboxGpuConfig.sandboxGpuEnabled) { + return buildDockerGpuMode("startup-command"); + } + const selection = selectDockerGpuPatchMode( + { + image: `${input.image.repository}@${input.image.manifestDigest}`, + device: input.sandboxGpuConfig.sandboxGpuDevice, + backend, + dockerDesktopWsl, + }, + input.dependencies, + ); + if (selection.mode) return selection.mode; + throw new Error( + backend === "jetson" + ? "Docker did not accept the Jetson NVIDIA runtime GPU mode for managed bootstrap." + : "Docker did not accept a compatibility GPU mode for managed bootstrap.", + ); +} + +function createDockerLifecycle( + providerId: string, + input: ManagedBootstrapRuntimeCreateLifecycleInput, +): ManagedBootstrapRuntimeCreateLifecycle { + if (input.providerId !== providerId) { + throw new Error( + `Managed bootstrap provider '${providerId}' cannot run authority for '${input.providerId}'.`, + ); + } + const dockerDesktopWsl = + input.route === "compatibility" ? isDockerDesktopWslRuntime() : undefined; + const mode = selectedDockerMode(input, dockerDesktopWsl); + const backend = input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic"; + const persistStartupCommand = + input.persistStartupCommand && (input.route !== "native" || input.requiredLimits.length > 0); + const patch = createDockerGpuSandboxCreatePatch({ + route: input.route, + persistStartupCommand, + externalRecreation: true, + sandboxName: input.sandboxName, + gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, + openshellSandboxCommand: input.heldWorkloadArgv, + requiredUlimits: input.requiredLimits, + timeoutSecs: input.timeoutSecs, + backend, + dockerDesktopWsl, + deps: input.dependencies, + ...(input.onPatchFailure + ? { + overrides: { + onPatchFailureExit: (_sandboxName: string, error: unknown) => + input.onPatchFailure?.(error), + }, + } + : {}), + }); + const adapter = input.adapterOverride ?? createDockerManagedBootstrapAdapter(input.dependencies); + const createPlan = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandboxName: input.sandboxName, + driverId: providerId, + image: input.image, + profile: { + agent: input.request.agent, + fingerprint: input.request.profileFingerprint, + }, + agentIdentity: input.agentIdentity, + intendedWorkloadArgv: input.intendedWorkloadArgv, + expectedSupervisorArgv: input.expectedSupervisorArgv, + metadata: {}, + } as const; + const replacementOptions = dockerReplacementOptions(mode, input); + + return { + launchArgv: input.launchArgv, + patch, + async prepareNetwork() { + if (input.route !== "compatibility") return; + const { enforceDockerGpuPatchPreserveNetwork } = await import( + "../docker-gpu-local-inference" + ); + await enforceDockerGpuPatchPreserveNetwork( + input.network.inferenceProvider, + input.sandboxGpuConfig, + { + dockerDriverGateway: input.network.gatewayUsesContainerBridge, + selectedRoute: input.route, + gatewayPort: input.network.gatewayPort, + log: console.log, + }, + ); + }, + async runCreate( + launch: (input: { + readonly heldWorkloadArgv: readonly string[]; + readonly bootstrapIdentity: string; + }) => Promise>, + ): Promise { + const launchState: { value?: ManagedBootstrapRuntimeCreateLaunchResult } = {}; + const prepared = await prepareManagedBootstrapSequence(adapter, { + create: { + bootstrapIdentity: input.bootstrapIdentity, + plan: createPlan, + request: input.request, + launch: async (launchInput) => { + const launched = await launch(launchInput); + launchState.value = launched; + return launched.receipt; + }, + }, + request: input.request, + replacementOptions, + }); + const activated = await activateManagedBootstrapSequence(adapter, { + transaction: prepared, + authorityStore: input.authorityStore, + timeoutSecs: input.timeoutSecs, + }); + const launched = launchState.value; + if (!launched) { + await finalizeManagedBootstrapSequence(adapter, { + outcome: "rollback", + transaction: activated, + }); + throw new Error("Managed bootstrap did not return its OpenShell create receipt."); + } + let finalized = false; + patch.attachManagedBootstrapCutover({ + selectedMode: mode, + failureContext: { + sandboxName: input.sandboxName, + oldContainerId: activated.snapshot.runtimeId, + newContainerId: activated.replacement.replacementRuntimeId, + backupContainerName: null, + selectedMode: mode, + }, + async rollback() { + if (finalized) return; + await finalizeManagedBootstrapSequence(adapter, { + outcome: "rollback", + transaction: activated, + }); + finalized = true; + }, + async commit() { + if (finalized) return; + await finalizeManagedBootstrapSequence(adapter, { + outcome: "commit", + transaction: activated, + }); + finalized = true; + }, + }); + return launched.value; + }, + }; +} + +function createDockerOnboardRouting(input: ManagedBootstrapRuntimeOnboardRoutingInput) { + const baseline = input.nativeFallbackEnabled + ? queryOpenShellDockerSandboxContainers(input.sandboxName) + : null; + const inspectNativeRuntime = () => { + const snapshot = queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName); + return snapshot.ok + ? { + imageId: snapshot.imageId, + bookkeepingImageRef: snapshot.bookkeepingImageRef, + stateError: snapshot.stateError, + nativeGpuAttachmentState: snapshot.nativeGpuAttachmentState, + } + : null; + }; + return { + nativeFallbackHasCleanBaseline: baseline?.ok === true && baseline.ids.length === 0, + inspectNativeRuntime, + isNativeCreateRoutingFailure: (output: string, sawProgress: boolean): boolean => + sandboxGpuCreateAttempt.isNativeGpuCreateRoutingFailure(output, { sawProgress }), + isTrustedNativeRuntimeError: (error: string): boolean => + sandboxGpuCreateAttempt.isTrustedNativeGpuRuntimeError(error), + isNativeReadinessRoutingFailure: (failure: { + readonly failurePhase: string | null; + readonly runtimeError: string; + }): boolean => sandboxGpuCreateAttempt.isNativeGpuReadinessRoutingFailure(failure), + prepareCompatibilityLaunch: ( + compatibility: ManagedBootstrapRuntimeCompatibilityLaunchInput, + ) => { + const runtime = compatibility.runtimeSnapshot; + const imageId = + runtime?.imageId ?? + (compatibility.prebuildImageId && isImmutableDockerImageId(compatibility.prebuildImageId) + ? compatibility.prebuildImageId.toLowerCase() + : null); + let registryImageRef = compatibility.currentRegistryImageRef; + if ( + !registryImageRef && + runtime?.bookkeepingImageRef && + !isImmutableDockerImageId(runtime.bookkeepingImageRef) + ) { + registryImageRef = runtime.bookkeepingImageRef; + } + const createArgs = renderCompatibilityFallbackCreateArgs(compatibility.createArgs, { + imageRef: imageId, + allowUnbuiltSource: compatibility.allowUnbuiltSource, + compatibilityPolicyPath: compatibility.compatibilityPolicyPath, + }); + return { + createArgv: input.openshellArgv([ + "sandbox", + "create", + ...createArgs, + "--", + ...compatibility.startupCommand, + ]), + registryImageRef, + }; + }, + }; +} + +/** Candidate Docker surface. Production activation remains a later qualification slice. */ +export function createDockerManagedBootstrapSurface( + providerId = "docker", +): SupportedBootstrapSurface { + return { + providerId, + supported: true, + createLifecycle: (input) => createDockerLifecycle(providerId, input), + createOnboardRouting: createDockerOnboardRouting, + }; +} diff --git a/src/lib/onboard/managed-bootstrap/index.ts b/src/lib/onboard/managed-bootstrap/index.ts index 17099572608..c55768afe02 100644 --- a/src/lib/onboard/managed-bootstrap/index.ts +++ b/src/lib/onboard/managed-bootstrap/index.ts @@ -20,3 +20,7 @@ export { serializeManagedBootstrapEnvelope, serializeManagedBootstrapImageCompletion, } from "./envelope"; +export type { + ManagedBootstrapRuntimeCreateLifecycle, + ManagedBootstrapRuntimePatch, +} from "./runtime-create"; diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.ts b/src/lib/onboard/managed-bootstrap/runtime-create.ts new file mode 100644 index 00000000000..198d91710f0 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/runtime-create.ts @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxGpuProofResult } from "../../state/registry"; +import type { ManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import type { SandboxGpuConfig } from "../sandbox-gpu-mode"; +import type { + ManagedBootstrapAdapter, + ManagedBootstrapAgentIdentity, + ManagedBootstrapAuthorityStore, + ManagedBootstrapCreateReceipt, + ManagedBootstrapImageIdentity, +} from "./adapter"; + +export interface ManagedBootstrapRuntimeCommandResult { + readonly status?: number | null; + readonly stdout?: string | Buffer | null; + readonly stderr?: string | Buffer | null; + readonly error?: Error | null; +} + +export interface ManagedBootstrapRuntimeDependencies { + readonly runCaptureOpenshell?: (args: string[], options?: Record) => string; + readonly runOpenshell?: ( + args: string[], + options?: Record, + ) => ManagedBootstrapRuntimeCommandResult; + readonly sleep?: (seconds: number) => void; +} + +export type ManagedBootstrapRuntimeRoute = "none" | "native" | "compatibility"; + +export interface ManagedBootstrapRuntimeLimit { + readonly name: string; + readonly soft: number; + readonly hard: number; +} + +/** Provider-neutral lifecycle surface consumed by sandbox-create coordinators. */ +export interface ManagedBootstrapRuntimePatch { + maybeApplyDuringCreate(): void | Promise; + createFailureMessage(): string | null; + exitOnPatchError(): void | Promise; + rollbackManagedStartupAfterCreateFailure(): void | Promise; + ensureApplied(): void | Promise; + waitForSupervisorReconnectIfNeeded(): void | Promise; + commitAfterReady(): void | Promise; + selectedMode(): { + readonly kind: string; + readonly label: string; + readonly device: string; + readonly args: readonly string[]; + } | null; + printReadinessFailureIfEnabled(): void; + verifyGpuOrExit( + verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult, + ): Promise; +} + +export interface ManagedBootstrapRuntimeCreateLifecycleInput { + readonly providerId: string; + readonly bootstrapIdentity: string; + readonly request: ManagedStartupRootApplyRequest; + readonly image: ManagedBootstrapImageIdentity; + readonly agentIdentity: ManagedBootstrapAgentIdentity; + readonly intendedWorkloadArgv: readonly string[]; + readonly expectedSupervisorArgv: readonly string[]; + readonly launchArgv: readonly string[]; + readonly heldWorkloadArgv: readonly string[]; + readonly authorityStore: ManagedBootstrapAuthorityStore; + readonly adapterOverride?: ManagedBootstrapAdapter; + readonly route: ManagedBootstrapRuntimeRoute; + readonly persistStartupCommand: boolean; + readonly sandboxName: string; + readonly sandboxGpuConfig: SandboxGpuConfig; + readonly requiredLimits: readonly ManagedBootstrapRuntimeLimit[]; + readonly timeoutSecs: number; + readonly onPatchFailure?: (error: unknown) => never; + readonly network: { + readonly inferenceProvider: string; + readonly gatewayUsesContainerBridge: boolean; + readonly gatewayPort: number; + }; + readonly dependencies: ManagedBootstrapRuntimeDependencies; +} + +export interface ManagedBootstrapRuntimeCreateLaunchResult { + readonly value: T; + readonly receipt: ManagedBootstrapCreateReceipt; +} + +export interface ManagedBootstrapRuntimeCreateLifecycle { + readonly launchArgv: readonly string[]; + readonly patch: ManagedBootstrapRuntimePatch; + prepareNetwork(): Promise; + runCreate( + launch: (input: { + readonly heldWorkloadArgv: readonly string[]; + readonly bootstrapIdentity: string; + }) => Promise>, + ): Promise; +} + +export interface ManagedBootstrapRuntimeSnapshot { + readonly imageId: string | null; + readonly bookkeepingImageRef: string | null; + readonly stateError: string; + readonly nativeGpuAttachmentState: "present" | "absent" | "unknown"; +} + +export interface ManagedBootstrapRuntimeCompatibilityLaunchInput { + readonly createArgs: readonly string[]; + readonly currentRegistryImageRef: string | null; + readonly prebuildImageId: string | null; + readonly allowUnbuiltSource: boolean; + readonly compatibilityPolicyPath: string; + readonly startupCommand: readonly string[]; + readonly runtimeSnapshot: ManagedBootstrapRuntimeSnapshot | null; +} + +export interface ManagedBootstrapRuntimeCompatibilityLaunch { + readonly createArgv: readonly string[]; + readonly registryImageRef: string | null; +} + +/** Provider-owned native-to-compatibility evidence and launch preparation. */ +export interface ManagedBootstrapRuntimeOnboardRouting { + readonly nativeFallbackHasCleanBaseline: boolean; + inspectNativeRuntime(): ManagedBootstrapRuntimeSnapshot | null; + isNativeCreateRoutingFailure(output: string, sawProgress: boolean): boolean; + isTrustedNativeRuntimeError(error: string): boolean; + isNativeReadinessRoutingFailure(input: { + readonly failurePhase: string | null; + readonly runtimeError: string; + }): boolean; + prepareCompatibilityLaunch( + input: ManagedBootstrapRuntimeCompatibilityLaunchInput, + ): ManagedBootstrapRuntimeCompatibilityLaunch; +} + +export interface ManagedBootstrapRuntimeOnboardRoutingInput { + readonly sandboxName: string; + readonly openshellArgv: (args: string[]) => string[]; + readonly nativeFallbackEnabled: boolean; +} diff --git a/src/lib/onboard/runtime-provider/contract.ts b/src/lib/onboard/runtime-provider/contract.ts index 8888d95f83a..5e806862404 100644 --- a/src/lib/onboard/runtime-provider/contract.ts +++ b/src/lib/onboard/runtime-provider/contract.ts @@ -2,6 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; +import type { + ManagedBootstrapRuntimeCreateLifecycle, + ManagedBootstrapRuntimeCreateLifecycleInput, + ManagedBootstrapRuntimeOnboardRouting, + ManagedBootstrapRuntimeOnboardRoutingInput, +} from "../managed-bootstrap/runtime-create"; import type { ManagedImageSelectionPolicy } from "../workload/source"; export const RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION = 1 as const; @@ -261,7 +267,12 @@ export type RuntimeProviderMutationAuthoritySurface = export type RuntimeProviderBootstrapSurface = | RuntimeProviderSupportedSurface<{ - prepare(sandbox: SandboxEntry): unknown; + createLifecycle( + input: ManagedBootstrapRuntimeCreateLifecycleInput, + ): ManagedBootstrapRuntimeCreateLifecycle; + createOnboardRouting( + input: ManagedBootstrapRuntimeOnboardRoutingInput, + ): ManagedBootstrapRuntimeOnboardRouting; }> | RuntimeProviderUnsupportedSurface; diff --git a/src/lib/onboard/runtime-provider/registry.ts b/src/lib/onboard/runtime-provider/registry.ts index 332792d2dc2..1d0a31ae6aa 100644 --- a/src/lib/onboard/runtime-provider/registry.ts +++ b/src/lib/onboard/runtime-provider/registry.ts @@ -337,7 +337,8 @@ function validateMutationAuthoritySurface( function validateBootstrapSurface(surface: Record): void { if (surface.supported === true) { - requireFunction(surface, "prepare", "bootstrap"); + requireFunction(surface, "createLifecycle", "bootstrap"); + requireFunction(surface, "createOnboardRouting", "bootstrap"); } } diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index bf2a47fe4b4..b5628f99b78 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -20,6 +20,7 @@ import { stopSandbox } from "../../actions/sandbox/stop"; import { loadAgent } from "../../agent/defs"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; import { cloneSandboxWorkloadReceipt } from "../../state/registry/workload"; +import { createDockerManagedBootstrapSurface } from "../managed-bootstrap/docker-runtime"; import { MANAGED_IMAGE_REPOSITORIES } from "../managed-image/contract"; import { encodeManagedStartupProfile, @@ -145,6 +146,28 @@ describe("RuntimeProviderBundle registry contract", () => { } }); + it("validates the dormant Docker bootstrap candidate through the same bundle registry", () => { + const docker = createDockerRuntimeProviderBundle(); + const providers = createRuntimeProviderBundleRegistry([ + [ + "docker", + { + ...docker, + bootstrap: createDockerManagedBootstrapSurface(), + }, + ], + ]); + + expect(providers.docker?.bootstrap).toMatchObject({ + providerId: "docker", + supported: true, + }); + expect(CURRENT_RUNTIME_PROVIDER_BUNDLES.docker?.bootstrap).toMatchObject({ + providerId: "docker", + supported: false, + }); + }); + it("deeply clones and freezes every registered nested value", () => { const source = mxcBundle(); const registry = createRuntimeProviderBundleRegistry([["mxc", source]]); @@ -170,6 +193,59 @@ describe("RuntimeProviderBundle registry contract", () => { }).toThrow(TypeError); }); + it("registers an MXC-style managed-bootstrap provider through the bundle surface", () => { + const bundle = mxcBundle(); + const createLifecycle = vi.fn(() => ({ + launchArgv: ["mxc", "create"], + patch: { + maybeApplyDuringCreate: vi.fn(), + createFailureMessage: vi.fn(() => null), + exitOnPatchError: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + ensureApplied: vi.fn(), + waitForSupervisorReconnectIfNeeded: vi.fn(), + commitAfterReady: vi.fn(), + selectedMode: vi.fn(() => null), + printReadinessFailureIfEnabled: vi.fn(), + verifyGpuOrExit: vi.fn(async (verify) => verify("alpha")), + }, + prepareNetwork: vi.fn(async () => undefined), + runCreate: vi.fn(), + })); + const createOnboardRouting = vi.fn(() => ({ + nativeFallbackHasCleanBaseline: false, + inspectNativeRuntime: vi.fn(() => null), + isNativeCreateRoutingFailure: vi.fn(() => false), + isTrustedNativeRuntimeError: vi.fn(() => false), + isNativeReadinessRoutingFailure: vi.fn(() => false), + prepareCompatibilityLaunch: vi.fn(() => ({ createArgv: [], registryImageRef: null })), + })); + const providers = createRuntimeProviderBundleRegistry([ + [ + "mxc", + replaceSurface(bundle, "bootstrap", { + providerId: "mxc", + supported: true, + createLifecycle, + createOnboardRouting, + }), + ], + ]); + const registered = providers.mxc!; + expectSupportedSurface(registered.bootstrap); + + const routing = registered.bootstrap.createOnboardRouting({ + sandboxName: "alpha", + openshellArgv: (args) => args, + nativeFallbackEnabled: false, + }); + + expect(registered.identity.id).toBe("mxc"); + expect(routing.nativeFallbackHasCleanBaseline).toBe(false); + expect(createOnboardRouting).toHaveBeenCalledOnce(); + expect(createLifecycle).not.toHaveBeenCalled(); + }); + it("rejects an omitted managed platform without changing legacy receipt acceptance", () => { const { platform: _omittedPlatform, ...managedWithoutPlatform } = MANAGED_RECEIPT; const persistedManaged = cloneSandboxWorkloadReceipt(managedWithoutPlatform); diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index 3a650fc9c11..dec7913e208 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -8,8 +8,11 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; import { loadAgent } from "../agent/defs"; import { SANDBOX_BUILD_CONTEXT_PREFIX } from "../sandbox/build-context"; +import { encodeManagedStartupProfile } from "./managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; import { createOpenshellCliHelpers } from "./openshell-cli"; import { buildSandboxRuntimeEnvArgs, @@ -104,6 +107,49 @@ describe("buildSandboxRuntimeEnvArgs", () => { }); describe("prepareSandboxCreateLaunch", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("renders one identity-bound held launch for %s without exposing the startup profile", (agentName) => { + const request = createManagedStartupRootApplyRequest({ + agent: agentName, + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agentName)), + }); + const result = prepareSandboxCreateLaunch({ + agent: loadAgent(agentName), + chatUiUrl: "", + createArgs: ["--name", `${agentName}-sandbox`], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: disabledHermesDashboardState, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + openshellArgv: (args) => ["openshell", ...args], + buildEnv: () => ({}), + managedStartupRootApplyRequest: request, + }); + + expect(result.intendedSandboxStartupCommand).toEqual([ + "env", + ...result.envArgs, + "nemoclaw-start", + ]); + expect(result.managedBootstrapIdentity).toMatch(/^[a-f0-9]{64}$/u); + expect(result.sandboxStartupCommand).toEqual([ + ...result.intendedSandboxStartupCommand.slice(0, -1), + "/usr/local/bin/nemoclaw-managed-startup-hold", + "--agent", + agentName, + "--profile-fingerprint", + request.profileFingerprint, + "--bootstrap-identity", + result.managedBootstrapIdentity, + ]); + expect(result.createArgv.join("\n")).not.toContain(request.encodedProfile); + }); + it("builds the sandbox create command and runtime env envelope", () => { const openshellShellCommand = vi.fn((args: string[]) => `openshell ${args.join(" ")}`); const result = prepareSandboxCreateLaunch({ diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index 1f3aa3cd01b..09e703f6d4b 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -9,6 +9,11 @@ import { appendExtraPlaceholderKeysEnvArg } from "./extra-placeholder-keys"; import type { HermesDashboardOnboardState } from "./hermes-dashboard"; import { appendHermesDashboardEnvArgs } from "./hermes-dashboard"; import { appendHostProxyEnvArgs } from "./host-proxy-env"; +import { + createManagedBootstrapIdentity, + renderManagedBootstrapHeldCommand, +} from "./managed-bootstrap/adapter"; +import type { ManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; import { appendOpenClawRuntimeEnvArgs } from "./openclaw-runtime-env"; import { prebuildSandboxImageIfEligible, @@ -57,6 +62,8 @@ export interface SandboxCreateLaunchInput { openshellShellCommand: OpenshellShellCommand; openshellArgv?: OpenshellArgv; buildEnv?(): Record; + /** Dormant until a complete runtime bundle and durable authority store are selected. */ + managedStartupRootApplyRequest?: ManagedStartupRootApplyRequest | null; } export interface SandboxCreateLaunch { @@ -66,6 +73,9 @@ export interface SandboxCreateLaunch { envArgs: string[]; sandboxEnv: Record; sandboxStartupCommand: string[]; + intendedSandboxStartupCommand: string[]; + managedBootstrapIdentity: string | null; + managedStartupRootApplyRequest: ManagedStartupRootApplyRequest | null; } export interface SandboxCreateLaunchWithPrebuildInput extends SandboxCreateLaunchInput { @@ -203,7 +213,21 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San // from openshell because bash returns the status of the last pipeline // command (awk, always 0) unless pipefail is set. Removing the pipe // lets the real exit code flow through to run(). - const sandboxStartupCommand = ["env", ...envArgs, "nemoclaw-start"]; + const intendedSandboxStartupCommand = ["env", ...envArgs, "nemoclaw-start"]; + const managedStartupRootApplyRequest = input.managedStartupRootApplyRequest ?? null; + const managedBootstrapIdentity = managedStartupRootApplyRequest + ? createManagedBootstrapIdentity() + : null; + const sandboxStartupCommand = + managedStartupRootApplyRequest && managedBootstrapIdentity + ? [ + ...renderManagedBootstrapHeldCommand( + managedStartupRootApplyRequest, + managedBootstrapIdentity, + intendedSandboxStartupCommand, + ), + ] + : intendedSandboxStartupCommand; const openshellArgs = ["sandbox", "create", ...input.createArgs, "--", ...sandboxStartupCommand]; const createCommand = renderSandboxCreateCommand( input.createArgs, @@ -221,6 +245,9 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San envArgs, sandboxEnv, sandboxStartupCommand, + intendedSandboxStartupCommand, + managedBootstrapIdentity, + managedStartupRootApplyRequest, }; } diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index c7671ff4e60..8602f2e51f5 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -3,6 +3,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { createInMemoryRuntimeProviderBundle } from "../../../test/helpers/runtime-provider-bundle"; + const mocks = vi.hoisted(() => ({ streamSandboxCreate: vi.fn(), waitForCreatedSandboxReadyWithTrace: vi.fn(), @@ -59,6 +62,18 @@ import { setupGpuFlowMocks, VERIFIED_GPU_PROOF as VERIFIED_PROOF, } from "./__test-helpers__/sandbox-gpu-create-flow"; +import type { + ManagedBootstrapRuntimeCreateLifecycleInput, + ManagedBootstrapRuntimePatch, +} from "./managed-bootstrap/runtime-create"; +import { encodeManagedStartupProfile } from "./managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; +import type { + RuntimeProviderBootstrapSurface, + RuntimeProviderBundle, +} from "./runtime-provider/contract"; +import { createRuntimeProviderBundleRegistry } from "./runtime-provider/registry"; +import { prepareSandboxCreateLaunch } from "./sandbox-create-launch"; import { runSandboxGpuCreateFlow, type SandboxGpuCreateFlowDeps, @@ -150,6 +165,145 @@ function createSourceInput(): SandboxGpuCreateFlowInput { beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); +describe("runSandboxGpuCreateFlow provider-owned managed create", () => { + it("runs an MXC-style bundle without a Docker branch in central orchestration", async () => { + const input = createInput(); + input.sandboxGpuConfig = { + mode: "0", + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }; + input.gpuRoutePlan = "none"; + input.initialGpuRoute = "none"; + const request = createManagedStartupRootApplyRequest({ + agent: "openclaw", + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")), + }); + const launch = prepareSandboxCreateLaunch({ + agent: null, + sandboxName: "alpha", + chatUiUrl: "", + createArgs: ["--name", "alpha"], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: { config: null, enabled: false }, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + openshellArgv: (args) => ["openshell", ...args], + buildEnv: () => ({}), + managedStartupRootApplyRequest: request, + }); + input.createArgv = launch.createArgv; + input.sandboxEnv = launch.sandboxEnv; + input.sandboxStartupCommand = launch.sandboxStartupCommand; + const patch = createPatch() as unknown as ManagedBootstrapRuntimePatch; + const createLifecycle = vi.fn( + (lifecycleInput: ManagedBootstrapRuntimeCreateLifecycleInput) => ({ + launchArgv: ["mxc-launch", ...lifecycleInput.launchArgv.slice(1)], + patch, + prepareNetwork: vi.fn(async () => undefined), + runCreate: async ( + start: (held: { + readonly heldWorkloadArgv: readonly string[]; + readonly bootstrapIdentity: string; + }) => Promise<{ readonly value: T }>, + ): Promise => + ( + await start({ + heldWorkloadArgv: lifecycleInput.heldWorkloadArgv, + bootstrapIdentity: lifecycleInput.bootstrapIdentity, + }) + ).value, + }), + ); + const source = createInMemoryRuntimeProviderBundle({ + providerId: "mxc", + workloadProfile: { + support: null, + hostArchitectures: [], + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: true, + }, + }); + const registered = createRuntimeProviderBundleRegistry([ + [ + "mxc", + { + ...source, + bootstrap: { + providerId: "mxc", + supported: true, + createLifecycle, + createOnboardRouting: vi.fn(() => ({ + nativeFallbackHasCleanBaseline: false, + inspectNativeRuntime: vi.fn(() => null), + isNativeCreateRoutingFailure: vi.fn(() => false), + isTrustedNativeRuntimeError: vi.fn(() => false), + isNativeReadinessRoutingFailure: vi.fn(() => false), + prepareCompatibilityLaunch: vi.fn(() => ({ + createArgv: [], + registryImageRef: null, + })), + })), + }, + }, + ], + ]); + const runtimeProvider = registered.mxc as RuntimeProviderBundle & { + readonly bootstrap: Extract; + }; + input.managedBootstrap = { + bootstrapIdentity: launch.managedBootstrapIdentity!, + runtimeProvider, + authorityStore: { + async recordPreparedAuthority(authority) { + return { + schemaVersion: 1, + sandbox: authority.sandbox, + bootstrapIdentity: authority.bootstrapIdentity, + authorityFingerprint: authority.authorityFingerprint, + recordId: "mxc-record-alpha", + recordedAt: "2026-07-31T00:00:00.000Z", + }; + }, + }, + request, + image: { + repository: "registry.example/nemoclaw-openclaw", + manifestDigest: `sha256:${"d".repeat(64)}`, + }, + agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + intendedWorkloadArgv: launch.intendedSandboxStartupCommand, + expectedSupervisorArgv: ["/mxc/supervisor"], + }; + const deps = createDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementation((args) => + args[1] === "get" ? "ID: mxc-alpha\n" : "alpha Ready", + ); + + const result = await runSandboxGpuCreateFlow(input, deps); + + expect(result).toMatchObject({ route: "none", runtimePatch: patch }); + expect(createLifecycle).toHaveBeenCalledWith( + expect.objectContaining({ providerId: "mxc", route: "none" }), + ); + expect(mocks.streamSandboxCreate).toHaveBeenCalledWith( + "mxc-launch", + input.createArgv.slice(1), + input.sandboxEnv, + expect.anything(), + ); + expect(mocks.createDockerGpuSandboxCreatePatch).not.toHaveBeenCalled(); + expect(mocks.queryOpenShellDockerSandboxContainers).not.toHaveBeenCalled(); + expect(mocks.queryOpenShellDockerSandboxRuntimeSnapshot).not.toHaveBeenCalled(); + expect(mocks.enforceDockerGpuPatchPreserveNetwork).not.toHaveBeenCalled(); + }); +}); + describe("runSandboxGpuCreateFlow proof authorization", () => { it("does not retry compatibility when the native proof throws an exec/policy error (#6110)", async () => { const deps = createDeps(); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 98574891e0b..48acddb658d 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -10,8 +10,19 @@ import type { DockerGpuPatchDeps, DockerUlimit } from "./docker-gpu-patch-types" import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { renderCompatibilityFallbackCreateArgs } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; -import type { DockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +import type { + ManagedBootstrapAdapter, + ManagedBootstrapAgentIdentity, + ManagedBootstrapAuthorityStore, + ManagedBootstrapImageIdentity, +} from "./managed-bootstrap/adapter"; +import type { ManagedBootstrapRuntimePatch } from "./managed-bootstrap/runtime-create"; +import type { ManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; import { isImmutableDockerImageId } from "./openshell-docker-sandbox-containers"; +import type { + RuntimeProviderBootstrapSurface, + RuntimeProviderBundle, +} from "./runtime-provider/contract"; import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; import { createSandboxGpuCreateAttemptRunner } from "./sandbox-gpu-create-run-attempt"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; @@ -41,6 +52,18 @@ export interface SandboxGpuCreateFlowInput { restoreBackupPath: string | null; terminalAgent: boolean; persistStartupCommand?: boolean; + managedBootstrap?: { + readonly bootstrapIdentity: string; + readonly runtimeProvider: RuntimeProviderBundle & { + readonly bootstrap: Extract; + }; + readonly authorityStore: ManagedBootstrapAuthorityStore; + readonly request: ManagedStartupRootApplyRequest; + readonly image: ManagedBootstrapImageIdentity; + readonly agentIdentity: ManagedBootstrapAgentIdentity; + readonly intendedWorkloadArgv: readonly string[]; + readonly expectedSupervisorArgv: readonly string[]; + } | null; requiredUlimits?: readonly DockerUlimit[] | null; } @@ -50,11 +73,13 @@ export interface SandboxGpuCreateFlowDeps { sleep: Sleep; openshellArgv(args: string[]): string[]; verifyDirectSandboxGpu(sandboxName: string): SandboxGpuProofResult; + /** Production callers omit this factory and use the runtime provider's adapter. */ + createManagedBootstrapAdapter?: () => ManagedBootstrapAdapter; } export interface SandboxGpuCreateFlowResult { createResult: StreamSandboxCreateResult; - dockerGpuCreatePatch: DockerGpuSandboxCreatePatch; + runtimePatch: ManagedBootstrapRuntimePatch; route: SelectedDockerGpuRoute; firstCreateOutput: string; /** Mutable tag/reference retained only for registry and image-GC bookkeeping. */ @@ -105,46 +130,65 @@ export async function runSandboxGpuCreateFlow( throw new Error("Compatibility retry policy was not materialized."); } const nativeRuntimeSnapshot = attemptRunner.state.nativeRuntimeSnapshot; - const prebuildImageId = input.prebuild.imageId; - const imageId = - nativeRuntimeSnapshot?.imageId ?? - (prebuildImageId && isImmutableDockerImageId(prebuildImageId) - ? prebuildImageId.toLowerCase() - : null); - if ( - !registryImageRef && - nativeRuntimeSnapshot?.bookkeepingImageRef && - !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) - ) { - registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; + if (attemptRunner.managedRouting) { + const prepared = attemptRunner.managedRouting.prepareCompatibilityLaunch({ + createArgs: input.prebuild.createArgs, + currentRegistryImageRef: registryImageRef, + prebuildImageId: input.prebuild.imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + startupCommand: input.sandboxStartupCommand, + runtimeSnapshot: nativeRuntimeSnapshot, + }); + attemptRunner.state.compatibilityArgv = [...prepared.createArgv]; + registryImageRef = prepared.registryImageRef; + } else { + const prebuildImageId = input.prebuild.imageId; + const imageId = + nativeRuntimeSnapshot?.imageId ?? + (prebuildImageId && isImmutableDockerImageId(prebuildImageId) + ? prebuildImageId.toLowerCase() + : null); + if ( + !registryImageRef && + nativeRuntimeSnapshot?.bookkeepingImageRef && + !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) + ) { + registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; + } + const compatibilityArgs = renderCompatibilityFallbackCreateArgs( + input.prebuild.createArgs, + { + imageRef: imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + }, + ); + attemptRunner.state.compatibilityArgv = deps.openshellArgv([ + "sandbox", + "create", + ...compatibilityArgs, + "--", + ...input.sandboxStartupCommand, + ]); } - const compatibilityArgs = renderCompatibilityFallbackCreateArgs(input.prebuild.createArgs, { - imageRef: imageId, - allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, - compatibilityPolicyPath: input.compatibilityPolicyPath, - }); - attemptRunner.state.compatibilityArgv = deps.openshellArgv([ - "sandbox", - "create", - ...compatibilityArgs, - "--", - ...input.sandboxStartupCommand, - ]); if (attemptRunner.state.compatibilityArgv.length === 0) { throw new Error("Compatibility sandbox create executable is missing."); } }, activateCompatibilityAttempt: async () => { - await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( - input.provider, - input.sandboxGpuConfig, - { - dockerDriverGateway: input.dockerDriverGateway, - selectedRoute: "compatibility", - gatewayPort: input.gatewayPort, - log: console.log, - }, - ); + if (!input.managedBootstrap) { + await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( + input.provider, + input.sandboxGpuConfig, + { + dockerDriverGateway: input.dockerDriverGateway, + selectedRoute: "compatibility", + gatewayPort: input.gatewayPort, + log: console.log, + }, + ); + } input.sandboxGpuConfig.sandboxGpuProof = null; }, traceEvent: addTraceEvent, diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 726758cc0ac..8eeb4a8aac0 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { resolveOpenShellSandboxId } from "../adapters/openshell/sandbox-identity"; import { printSandboxCreateRecoveryHints } from "../build-context"; import { getSandboxDeleteOutcome } from "../domain/sandbox/destroy"; import { streamSandboxCreate } from "../sandbox/create-stream"; @@ -13,8 +14,8 @@ import { reportSandboxCreateFailure } from "./created-sandbox-failure"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +import type { ManagedBootstrapRuntimeSnapshot } from "./managed-bootstrap/runtime-create"; import { - type OpenShellDockerSandboxRuntimeSnapshotQuery, queryOpenShellDockerSandboxContainers, queryOpenShellDockerSandboxRuntimeSnapshot, } from "./openshell-docker-sandbox-containers"; @@ -28,7 +29,7 @@ import * as sandboxGpuPreflight from "./sandbox-gpu-preflight"; import * as sandboxReadinessTracing from "./sandbox-readiness-tracing"; import { addTraceEvent } from "./tracing"; -type NativeRuntimeSnapshot = Extract; +type NativeRuntimeSnapshot = ManagedBootstrapRuntimeSnapshot; export type SandboxGpuCreateAttemptState = { firstCreateOutput: string; @@ -41,6 +42,12 @@ export type SandboxGpuCreateAttemptState = { // Ready row. Require one confirmation poll before advancing to the GPU proof. const COMPATIBILITY_STABLE_READY_POLLS = 2; +class ManagedBootstrapCreateStreamFailure extends Error { + constructor(readonly result: Awaited>) { + super("Managed bootstrap held workload did not complete its create stream."); + } +} + export function createSandboxGpuCreateAttemptRunner( input: SandboxGpuCreateFlowInput, deps: SandboxGpuCreateFlowDeps, @@ -51,13 +58,26 @@ export function createSandboxGpuCreateAttemptRunner( allowUnbuiltCompatibilitySource: false, nativeRuntimeSnapshot: null, }; + const managedRouting = input.managedBootstrap?.runtimeProvider.bootstrap.createOnboardRouting({ + sandboxName: input.sandboxName, + openshellArgv: deps.openshellArgv, + nativeFallbackEnabled: + input.initialGpuRoute === "native" && input.gpuRoutePlan === "native-with-fallback", + }); const nativeFallbackBaseline = - input.initialGpuRoute === "native" && input.gpuRoutePlan === "native-with-fallback" + !managedRouting && + input.initialGpuRoute === "native" && + input.gpuRoutePlan === "native-with-fallback" ? queryOpenShellDockerSandboxContainers(input.sandboxName) : null; const nativeFallbackHasCleanBaseline = - nativeFallbackBaseline?.ok === true && nativeFallbackBaseline.ids.length === 0; - const inspectNativeRuntime = () => queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName); + managedRouting?.nativeFallbackHasCleanBaseline ?? + (nativeFallbackBaseline?.ok === true && nativeFallbackBaseline.ids.length === 0); + const inspectNativeRuntime = (): NativeRuntimeSnapshot | null => { + if (managedRouting) return managedRouting.inspectNativeRuntime(); + const snapshot = queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName); + return snapshot.ok ? snapshot : null; + }; const runAttempt = async (route: SelectedDockerGpuRoute) => { const compatibility = route === "compatibility"; @@ -70,73 +90,199 @@ export function createSandboxGpuCreateAttemptRunner( ); } const hasRequiredUlimits = (input.requiredUlimits?.length ?? 0) > 0; - const dockerGpuCreatePatch = createDockerGpuSandboxCreatePatch({ - route, - // The startup clone preserves native CDI devices, so DCode can apply its - // exact required limits without replacing the native GPU envelope. - // Other native routes are not swapped solely to persist a command. - persistStartupCommand: - input.persistStartupCommand === true && (route !== "native" || hasRequiredUlimits), - sandboxName: input.sandboxName, - gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, - openshellSandboxCommand: input.sandboxStartupCommand, - requiredUlimits: input.requiredUlimits, - timeoutSecs: input.sandboxReadyTimeoutSecs, - backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", - deps, - }); + const managedBootstrap = input.managedBootstrap ?? null; const attemptArgv = state.compatibilityArgv ?? input.createArgv; - const [createExecutable, ...createExecutableArgs] = attemptArgv; + const managedLifecycle = managedBootstrap + ? managedBootstrap.runtimeProvider.bootstrap.createLifecycle({ + providerId: managedBootstrap.runtimeProvider.identity.id, + bootstrapIdentity: managedBootstrap.bootstrapIdentity, + request: managedBootstrap.request, + image: managedBootstrap.image, + agentIdentity: managedBootstrap.agentIdentity, + intendedWorkloadArgv: managedBootstrap.intendedWorkloadArgv, + expectedSupervisorArgv: managedBootstrap.expectedSupervisorArgv, + launchArgv: attemptArgv, + heldWorkloadArgv: input.sandboxStartupCommand, + authorityStore: managedBootstrap.authorityStore, + ...(deps.createManagedBootstrapAdapter + ? { adapterOverride: deps.createManagedBootstrapAdapter() } + : {}), + route, + persistStartupCommand: input.persistStartupCommand === true, + sandboxName: input.sandboxName, + sandboxGpuConfig: input.sandboxGpuConfig, + requiredLimits: input.requiredUlimits ?? [], + timeoutSecs: input.sandboxReadyTimeoutSecs, + network: { + inferenceProvider: input.provider, + gatewayUsesContainerBridge: input.dockerDriverGateway, + gatewayPort: input.gatewayPort, + }, + dependencies: { + runCaptureOpenshell: deps.runCaptureOpenshell, + runOpenshell: deps.runOpenshell, + sleep: deps.sleep, + }, + }) + : null; + const runtimePatch = + managedLifecycle?.patch ?? + createDockerGpuSandboxCreatePatch({ + route, + // The startup clone preserves native CDI devices, so DCode can apply its + // exact required limits without replacing the native GPU envelope. + // Other native routes are not swapped solely to persist a command. + persistStartupCommand: + input.persistStartupCommand === true && (route !== "native" || hasRequiredUlimits), + externalRecreation: false, + sandboxName: input.sandboxName, + gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, + openshellSandboxCommand: input.sandboxStartupCommand, + requiredUlimits: input.requiredUlimits, + timeoutSecs: input.sandboxReadyTimeoutSecs, + backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", + deps, + }); + await managedLifecycle?.prepareNetwork(); + const [createExecutable, ...createExecutableArgs] = managedLifecycle?.launchArgv ?? attemptArgv; if (!createExecutable) throw new Error("Sandbox create executable is missing."); - const createResult = await streamSandboxCreate( - createExecutable, - createExecutableArgs, - input.sandboxEnv, - { + const streamCreate = () => + streamSandboxCreate(createExecutable, createExecutableArgs, input.sandboxEnv, { readyCheck: () => { const list = deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); return isSandboxReady(list, input.sandboxName); }, - onPoll: () => dockerGpuCreatePatch.maybeApplyDuringCreate(), + onPoll: () => runtimePatch.maybeApplyDuringCreate(), readyCheckOutputPatterns: getReadyCheckOutputPatternsForAgent( input.terminalAgent, input.sandboxEnv, ), - failureCheck: dockerGpuCreatePatch.createFailureMessage, + failureCheck: runtimePatch.createFailureMessage, traceEvent: addTraceEvent, initialPhase: compatibility && (input.prebuild.imageRef || state.compatibilityArgv) ? "create" : undefined, - }, - ); + }); + let createResult: Awaited>; + let managedIncompleteCreateRecovered = false; + if (managedBootstrap && managedLifecycle) { + try { + createResult = await managedLifecycle.runCreate( + async ({ heldWorkloadArgv, bootstrapIdentity }) => { + if ( + bootstrapIdentity !== managedBootstrap.bootstrapIdentity || + heldWorkloadArgv.length !== input.sandboxStartupCommand.length || + heldWorkloadArgv.some((value, index) => value !== input.sandboxStartupCommand[index]) + ) { + throw new Error( + "Managed bootstrap launch does not match the rendered identity-bound hold.", + ); + } + const result = await streamCreate(); + const createFailure = + result.status === 0 ? null : classifySandboxCreateFailure(result.output); + if (result.status !== 0 && createFailure?.kind !== "sandbox_create_incomplete") { + throw new ManagedBootstrapCreateStreamFailure(result); + } + if (createFailure?.kind === "sandbox_create_incomplete") { + const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ + sandboxName: input.sandboxName, + timeoutSecs: input.sandboxReadyTimeoutSecs, + runCaptureOpenshell: deps.runCaptureOpenshell, + isSandboxReady, + getSandboxFailurePhase, + stableReadyPolls: 1, + sleep: deps.sleep, + }); + if (!readiness.ready) { + throw new Error( + `Managed bootstrap incomplete create did not reach authoritative Ready state (${readiness.reason}).`, + ); + } + } else { + const list = deps.runCaptureOpenshell(["sandbox", "list"], { + ignoreError: true, + }); + if (!isSandboxReady(list, input.sandboxName)) { + throw new Error( + "Managed bootstrap create completed without an authoritative Ready sandbox.", + ); + } + } + let sandboxId: string; + try { + sandboxId = resolveOpenShellSandboxId(input.sandboxName, deps.runCaptureOpenshell); + } catch (error) { + throw new Error( + createFailure?.kind === "sandbox_create_incomplete" + ? "Managed bootstrap incomplete create did not return one exact durable sandbox identity after Ready." + : "Managed bootstrap create did not return one exact durable sandbox identity after Ready.", + { cause: error }, + ); + } + managedIncompleteCreateRecovered = createFailure?.kind === "sandbox_create_incomplete"; + return { + value: result, + receipt: { + sandbox: { + sandboxName: input.sandboxName, + sandboxId, + driverId: managedBootstrap.runtimeProvider.identity.id, + }, + ready: true, + readyAt: new Date().toISOString(), + }, + }; + }, + ); + } catch (error) { + if (!(error instanceof ManagedBootstrapCreateStreamFailure)) throw error; + createResult = error.result; + } + } else { + createResult = await streamCreate(); + } if (!state.firstCreateOutput) state.firstCreateOutput = createResult.output; - dockerGpuCreatePatch.exitOnPatchError(); + await runtimePatch.exitOnPatchError(); if (createResult.status !== 0) { const failure = classifySandboxCreateFailure(createResult.output); if (failure.kind === "sandbox_create_incomplete") { console.warn(""); - console.warn( - ` Create stream exited with code ${createResult.status} after sandbox was created.`, - ); - console.warn(" Checking whether the sandbox reaches Ready state..."); + if (managedIncompleteCreateRecovered) { + console.warn( + ` Create stream exited with code ${createResult.status}; the exact durable sandbox reached Ready, and onboarding is continuing with final checks.`, + ); + } else { + console.warn( + ` Create stream exited with code ${createResult.status} after sandbox was created.`, + ); + console.warn(" Checking whether the sandbox reaches Ready state..."); + } } else if ( route === "native" && input.gpuRoutePlan === "native-with-fallback" && nativeFallbackHasCleanBaseline && (() => { if ( - sandboxGpuCreateAttempt.isNativeGpuCreateRoutingFailure(createResult.output, { - sawProgress: createResult.sawProgress, - }) + managedRouting + ? managedRouting.isNativeCreateRoutingFailure( + createResult.output, + createResult.sawProgress, + ) + : sandboxGpuCreateAttempt.isNativeGpuCreateRoutingFailure(createResult.output, { + sawProgress: createResult.sawProgress, + }) ) { state.allowUnbuiltCompatibilitySource = input.prebuild.imageRef === null; return true; } const snapshot = inspectNativeRuntime(); if ( - snapshot.ok && - sandboxGpuCreateAttempt.isTrustedNativeGpuRuntimeError(snapshot.stateError) + snapshot && + (managedRouting + ? managedRouting.isTrustedNativeRuntimeError(snapshot.stateError) + : sandboxGpuCreateAttempt.isTrustedNativeGpuRuntimeError(snapshot.stateError)) ) { state.nativeRuntimeSnapshot = snapshot; return true; @@ -144,6 +290,7 @@ export function createSandboxGpuCreateAttemptRunner( return false; })() ) { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); return { ok: false, route, @@ -152,6 +299,7 @@ export function createSandboxGpuCreateAttemptRunner( fallbackEligible: true, } as const; } else { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); reportSandboxCreateFailure( { sandboxName: input.sandboxName, @@ -171,8 +319,8 @@ export function createSandboxGpuCreateAttemptRunner( ); } } - dockerGpuCreatePatch.ensureApplied(); - dockerGpuCreatePatch.waitForSupervisorReconnectIfNeeded(); + await runtimePatch.ensureApplied(); + await runtimePatch.waitForSupervisorReconnectIfNeeded(); console.log(" Waiting for sandbox to become ready..."); const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ sandboxName: input.sandboxName, @@ -197,13 +345,19 @@ export function createSandboxGpuCreateAttemptRunner( const runtimeSnapshot = canClassifyNativeReadiness ? inspectNativeRuntime() : null; if ( canClassifyNativeReadiness && - runtimeSnapshot?.ok && - sandboxGpuCreateAttempt.isNativeGpuReadinessRoutingFailure({ - failurePhase: readiness.failurePhase, - runtimeError: runtimeSnapshot.stateError, - }) + runtimeSnapshot && + (managedRouting + ? managedRouting.isNativeReadinessRoutingFailure({ + failurePhase: readiness.failurePhase, + runtimeError: runtimeSnapshot.stateError, + }) + : sandboxGpuCreateAttempt.isNativeGpuReadinessRoutingFailure({ + failurePhase: readiness.failurePhase, + runtimeError: runtimeSnapshot.stateError, + })) ) { state.nativeRuntimeSnapshot = runtimeSnapshot; + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); return { ok: false, route, @@ -214,10 +368,11 @@ export function createSandboxGpuCreateAttemptRunner( fallbackEligible: true, } as const; } + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); printSandboxCreateFailureDiagnostics(input.sandboxName, { backupPath: input.restoreBackupPath, }); - if (compatibility) dockerGpuCreatePatch.printReadinessFailureIfEnabled(); + if (compatibility) runtimePatch.printReadinessFailureIfEnabled(); else { const deletion = deps.runOpenshell(["sandbox", "delete", input.sandboxName], { ignoreError: true, @@ -240,27 +395,32 @@ export function createSandboxGpuCreateAttemptRunner( route === "native" && input.gpuRoutePlan === "native-with-fallback" && nativeFallbackHasCleanBaseline; - const proof: SandboxGpuProofResult = dockerGpuLocalInference.verifyGpuSandboxAccessAfterReady( - input.sandboxGpuConfig, - { - sandboxName: input.sandboxName, - dockerDriverGateway: input.dockerDriverGateway, - selectedRoute: route, - verifyDirectSandboxGpu: deps.verifyDirectSandboxGpu, - verifyGpuOrExit: deferNativeProofFailure - ? undefined - : dockerGpuCreatePatch.verifyGpuOrExit, - reportGpuProofFailure: !deferNativeProofFailure, - selectedMode: dockerGpuCreatePatch.selectedMode, - runCaptureOpenshell: deps.runCaptureOpenshell, - log: console.log, - }, - ); + let proof: SandboxGpuProofResult; + try { + proof = await dockerGpuLocalInference.verifyGpuSandboxAccessAfterReady( + input.sandboxGpuConfig, + { + sandboxName: input.sandboxName, + dockerDriverGateway: input.dockerDriverGateway, + selectedRoute: route, + verifyDirectSandboxGpu: deps.verifyDirectSandboxGpu, + verifyGpuOrExit: deferNativeProofFailure ? undefined : runtimePatch.verifyGpuOrExit, + reportGpuProofFailure: !deferNativeProofFailure, + selectedMode: runtimePatch.selectedMode, + runCaptureOpenshell: deps.runCaptureOpenshell, + log: console.log, + }, + ); + } catch (error) { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); + throw error; + } if (deferNativeProofFailure && proof.status === "failed") { if (sandboxGpuPreflight.isExplicitNvidiaSmiDriverProofFailure(proof)) { const snapshot = inspectNativeRuntime(); - if (snapshot.ok && snapshot.nativeGpuAttachmentState === "absent") { + if (snapshot?.nativeGpuAttachmentState === "absent") { state.nativeRuntimeSnapshot = snapshot; + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); return { ok: false, route, @@ -272,6 +432,7 @@ export function createSandboxGpuCreateAttemptRunner( } as const; } } + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); console.error(""); console.error(" Native sandbox GPU proof failed."); console.error( @@ -283,15 +444,22 @@ export function createSandboxGpuCreateAttemptRunner( process.exit(1); } if (proof.status === "failed") { + await runtimePatch.rollbackManagedStartupAfterCreateFailure(); throw new Error("Sandbox GPU proof returned failed status."); } } + // GPU-enabled cutover stays reversible until the caller also proves the + // configured host-local inference path. Non-GPU workloads have completed + // their final authoritative Ready gate here. + if (!input.sandboxGpuConfig.sandboxGpuEnabled) { + await runtimePatch.commitAfterReady(); + } return { ok: true, route, - value: { createResult, dockerGpuCreatePatch }, + value: { createResult, runtimePatch }, } as const; }; - return { state, runAttempt }; + return { state, managedRouting, runAttempt }; } diff --git a/test/onboard-prepared-build-context.test.ts b/test/onboard-prepared-build-context.test.ts index 0d91d612ae4..5babec1765c 100644 --- a/test/onboard-prepared-build-context.test.ts +++ b/test/onboard-prepared-build-context.test.ts @@ -98,12 +98,15 @@ let stageCalls = 0; dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch = () => ({ maybeApplyDuringCreate: () => {}, createFailureMessage: () => null, - exitOnPatchError: () => {}, - ensureApplied: () => {}, + exitOnPatchError: async () => {}, + attachManagedBootstrapCutover: () => {}, + rollbackManagedStartupAfterCreateFailure: async () => {}, + ensureApplied: async () => {}, waitForSupervisorReconnectIfNeeded: () => {}, + commitAfterReady: async () => {}, selectedMode: () => null, printReadinessFailureIfEnabled: () => {}, - verifyGpuOrExit: (verify) => verify(sandboxName), + verifyGpuOrExit: async (verify) => verify(sandboxName), }); buildContextStage.stageCreateSandboxBuildContext = () => { diff --git a/test/onboard-sandbox-recreation.test.ts b/test/onboard-sandbox-recreation.test.ts index 11b35a58b6e..c0a4eb2f6f3 100644 --- a/test/onboard-sandbox-recreation.test.ts +++ b/test/onboard-sandbox-recreation.test.ts @@ -1203,6 +1203,7 @@ const fs = require("node:fs"); const commands = []; let sandboxListCalls = 0; +let dockerPsCalls = 0; const keepAlive = setInterval(() => {}, 1000); runner.run = (command, opts = {}) => { _deleted = _deleted || _n(command).includes("sandbox delete"); @@ -1210,6 +1211,10 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { + if (_n(command).startsWith("docker ps -a --no-trunc ")) { + dockerPsCalls += 1; + if (dockerPsCalls === 1) return "a".repeat(64); + } if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) { sandboxListCalls += 1; diff --git a/test/onboard-terminal-dashboard.test.ts b/test/onboard-terminal-dashboard.test.ts index 61baa83310a..0dd2686eadd 100644 --- a/test/onboard-terminal-dashboard.test.ts +++ b/test/onboard-terminal-dashboard.test.ts @@ -67,12 +67,15 @@ const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, "") dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch = () => ({ maybeApplyDuringCreate: () => {}, createFailureMessage: () => null, - exitOnPatchError: () => {}, - ensureApplied: () => {}, + exitOnPatchError: async () => {}, + attachManagedBootstrapCutover: () => {}, + rollbackManagedStartupAfterCreateFailure: async () => {}, + ensureApplied: async () => {}, waitForSupervisorReconnectIfNeeded: () => {}, + commitAfterReady: async () => {}, selectedMode: () => null, printReadinessFailureIfEnabled: () => {}, - verifyGpuOrExit: (verify) => verify(sandboxName), + verifyGpuOrExit: async (verify) => verify(sandboxName), }); agentOnboard.createAgentSandbox = () => { diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 48e889d78c1..d4f3f74de5b 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -28,11 +28,19 @@ let activationPaths: string[] = []; let providerPaths: string[] = []; let dockerfilePaths: string[] = []; let packagingPaths: string[] = []; -const bootstrapLoad = - /(?:from\s*|import\s*|import\s*\(\s*|require\s*\(\s*)["'][^"']*managed-bootstrap/iu; +const managedBootstrapLoad = + /(?:from\s*|import\s*|import\s*\(\s*|require\s*\(\s*)["']([^"']*managed-bootstrap(?:\/[^"']*)?)["']/giu; +const allowedManagedBootstrapLoad = + /\/managed-bootstrap\/(?:adapter|runtime-create)(?:\.[cm]?[jt]s)?$/u; const packagedBootstrapAsset = /(?:nemoclaw-managed-bootstrap|managed-bootstrap-trampoline|managed-startup-image-runtime\.cjs|nemoclaw-managed-startup-hold)/u; +function disallowedManagedBootstrapLoads(source: string): string[] { + return [...source.matchAll(managedBootstrapLoad)] + .map((match) => match[1] ?? "") + .filter((specifier) => !allowedManagedBootstrapLoad.test(specifier)); +} + beforeAll(() => { productionPaths = trackedPaths( "src/lib/onboard.ts", @@ -70,7 +78,7 @@ beforeAll(() => { }); describe("runtime provider central source boundary", () => { - // source-shape-contract: compatibility -- Migrated lifecycle and mutation consumers must stay provider-neutral while production selection excludes unqualified future providers and managed-bootstrap dependencies + // source-shape-contract: compatibility -- Migrated lifecycle and mutation consumers must stay provider-neutral while production selection excludes unqualified future providers and driver-specific bootstrap dependencies it("keeps migrated provider identities and implementations behind the one bundle composition", () => { const driverNeutralActions = { "actions/inference-set.ts": read("src/lib/actions/inference-set.ts"), @@ -114,7 +122,12 @@ describe("runtime provider central source boundary", () => { expect(driverNeutralActions["actions/sandbox/start.ts"]).toMatch( /resolved\.lifecycle\.verifyStarted\(/u, ); - expect(Object.values(providerContract).join("\n")).not.toMatch(/managed-bootstrap/u); + expect(providerContract.contract).toMatch( + /import type[\s\S]*from ["']\.\.\/managed-bootstrap\/runtime-create["']/u, + ); + expect( + [providerContract.current, providerContract.docker, providerContract.registry].join("\n"), + ).not.toMatch(/managed-bootstrap/u); expect(providerContract.current).not.toMatch(/\b(?:podman|mxc)\b/iu); }); @@ -122,12 +135,14 @@ describe("runtime provider central source boundary", () => { expect(bootstrapProtocolPaths).toEqual([ "src/lib/onboard/managed-bootstrap/adapter.ts", "src/lib/onboard/managed-bootstrap/docker-journal.ts", + "src/lib/onboard/managed-bootstrap/docker-runtime.ts", "src/lib/onboard/managed-bootstrap/docker-shared-state.ts", "src/lib/onboard/managed-bootstrap/docker-spec.ts", "src/lib/onboard/managed-bootstrap/docker-test-fixture.ts", "src/lib/onboard/managed-bootstrap/docker.ts", "src/lib/onboard/managed-bootstrap/envelope.ts", "src/lib/onboard/managed-bootstrap/index.ts", + "src/lib/onboard/managed-bootstrap/runtime-create.ts", ]); }); @@ -160,26 +175,68 @@ describe("runtime provider central source boundary", () => { read("src/lib/onboard/managed-bootstrap/envelope.ts"), read("src/lib/onboard/managed-bootstrap/index.ts"), ].join("\n"); + const runtimeCreateContract = read("src/lib/onboard/managed-bootstrap/runtime-create.ts"); expect(bootstrapProtocolSource).not.toMatch(/from\s+["'][^"']*(?:docker|podman)[^"']*["']/iu); expect(bootstrapProtocolSource).not.toMatch( /(?:driverId|providerId)\s*(?:===|!==)\s*["'](?:docker|podman)["']/iu, ); expect(bootstrapProtocolSource).not.toMatch(/\b(?:docker|podman|openshell|mxc)\b/iu); + expect(runtimeCreateContract).not.toMatch(/\b(?:docker|podman|mxc)\b/iu); + expect(runtimeCreateContract).not.toMatch( + /(?:driverId|providerId)\s*(?:===|!==)\s*["'][^"']+["']/iu, + ); }); - // source-shape-contract: security -- Production onboarding must not activate managed bootstrap until a complete provider image and rollback implementation lands together - it("keeps production activation paths disconnected from managed bootstrap", () => { + // source-shape-contract: security -- Production onboarding may consume the provider-neutral create contract but cannot select a driver-specific bootstrap implementation + it("keeps production activation paths disconnected from driver bootstrap adapters", () => { const onboardEntry = read("src/lib/onboard.ts"); const activationSource = activationPaths.map(read).join("\n"); - expect(onboardEntry).not.toMatch(bootstrapLoad); - expect(activationSource).not.toMatch(bootstrapLoad); + expect(disallowedManagedBootstrapLoads(onboardEntry)).toEqual([]); + expect(disallowedManagedBootstrapLoads(activationSource)).toEqual([]); + expect( + disallowedManagedBootstrapLoads( + [ + 'import type { Contract } from "../managed-bootstrap/adapter";', + 'import type { Lifecycle } from "../../managed-bootstrap/runtime-create.mts";', + ].join("\n"), + ), + ).toEqual([]); + expect( + disallowedManagedBootstrapLoads( + [ + 'import "../managed-bootstrap";', + 'import "../managed-bootstrap/index";', + 'import "../managed-bootstrap/docker-runtime";', + 'await import("../managed-bootstrap/podman-runtime");', + 'require("../managed-bootstrap/mxc-runtime");', + 'export { provider } from "../managed-bootstrap/future-provider";', + 'import type { Fake } from "../fake-managed-bootstrap/adapter";', + 'import type { Nested } from "../managed-bootstrap/docker/adapter";', + ].join("\n"), + ), + ).toEqual([ + "../managed-bootstrap", + "../managed-bootstrap/index", + "../managed-bootstrap/docker-runtime", + "../managed-bootstrap/podman-runtime", + "../managed-bootstrap/mxc-runtime", + "../managed-bootstrap/future-provider", + "../fake-managed-bootstrap/adapter", + "../managed-bootstrap/docker/adapter", + ]); }); // source-shape-contract: security -- Registered runtime providers must remain bootstrap-unsupported until their complete transaction implementations are qualified it("keeps registered providers bootstrap-unsupported", () => { const dockerProvider = read("src/lib/onboard/runtime-provider/docker.ts"); - const providerSource = providerPaths.map(read).join("\n"); - expect(providerSource).not.toMatch(/managed-bootstrap/iu); + const providerImplementationSource = providerPaths + .filter((path) => path !== "src/lib/onboard/runtime-provider/contract.ts") + .map(read) + .join("\n"); + expect(dockerProvider).not.toMatch( + /(?:from\s+["'][^"']*managed-bootstrap|require\([^)]*managed-bootstrap)/u, + ); + expect(providerImplementationSource).not.toMatch(/managed-bootstrap/iu); expect(dockerProvider.match(/bootstrap:\s*unsupported\(/gu)).toHaveLength(2); expect(dockerProvider.match(/recovery:\s*unsupported\(/gu)).toHaveLength(2); }); From 34b43764dd37885b88d1b931861cdf6de92b508b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 11:28:59 -0700 Subject: [PATCH 107/117] test(docs): cover local NIM post-ready wording Signed-off-by: Aaron Erickson --- test/inference-options-docs.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/inference-options-docs.test.ts b/test/inference-options-docs.test.ts index 59bab6d063e..0a453be5640 100644 --- a/test/inference-options-docs.test.ts +++ b/test/inference-options-docs.test.ts @@ -428,7 +428,7 @@ describe("inference setup navigation", () => { expect(getSandboxRuntimeInferenceEndpoint("nvidia-nim")).toBeNull(); expect(getSandboxRuntimeInferenceEndpoint("compatible-endpoint")).toBeNull(); expect(section).toContain( - "For local Ollama and vLLM on Docker GPU sandboxes using the compatibility route", + "For local Ollama, local vLLM, and local NVIDIA NIM on Docker GPU sandboxes using the compatibility route", ); expect(section).toContain("NVIDIA NIM and other compatible endpoints"); }); From 1c4933cd19c77eefeeb78524efb4b81e5b5301e1 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 11:38:14 -0700 Subject: [PATCH 108/117] feat(onboard): persist managed bootstrap finalization Bind Docker transaction and finalization authority to exact runtime identities. Persist image-owned shared-state commit receipts for later lifecycle reconciliation. Harden durable receipt recovery and order-independent authority comparisons. Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/README.md | 25 +- src/lib/onboard/managed-bootstrap/adapter.ts | 8 +- .../managed-bootstrap/docker-journal.test.ts | 182 +++++++ .../managed-bootstrap/docker-journal.ts | 461 ++++++++++++++++- .../managed-bootstrap/docker-shared-state.ts | 3 +- .../managed-bootstrap/docker-test-fixture.ts | 48 +- .../onboard/managed-bootstrap/docker.test.ts | 83 +++- src/lib/onboard/managed-bootstrap/docker.ts | 325 ++++++++---- ...d-startup-shared-state-transaction.test.ts | 251 +++++++++- .../onboard/managed-startup/image-runtime.ts | 53 +- .../shared-state-transaction.ts | 466 +++++++++++++++++- 11 files changed, 1784 insertions(+), 121 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 159ab401e86..f25082b0a76 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -85,11 +85,26 @@ and sandbox ID and then enter the destructive cutover. Post-cutover rollback publishes `rollback-authorized` before exact replacement deletion; pre-cutover staged cleanup removes only the exact prepared replacement without that journal transition. Commit publishes `shared-state-committed` before exact backup -deletion. Cleanup is bound to full runtime IDs. Mutable OpenShell names are read -only to detect ownership reuse, and unsafe name-only deletion returns a typed -retention error. The dormant adapter assumes the protocol's single coordinator; -multi-process lease/arbitration remains an explicit production-activation gate. -Activation must also inject the selected gateway's canonical state root. +deletion. Cleanup is bound to full runtime IDs. Its private state root retains +enumerable, versioned unfinished records containing the provider and sandbox +identities, plan and profile +fingerprints, exact original and replacement IDs, rollback target, and phase. +Exact commit and cleanup receipts are durable terminal records, so adapter +recreation does not depend on process-local transaction sets or tombstone maps. +The image-owned shared-state transaction uses the same identity-bound model: a +commit atomically moves its pending manifest and backups into a durable receipt +namespace, compacts that state to an exact commit receipt, and rejects rollback +when a later image-runtime invocation reads that receipt. The provider may +retire that receipt only after it proves the external rollback backup is gone, +so that this receipt does not block the next bootstrap attempt. +Enumeration reconstructs only unfinished records. The bounded +[3.12b recovery slice](https://github.com/NVIDIA/NemoClaw/issues/7744) owns phase +reconciliation and cross-surface resume or rollback. The adapter reads mutable +OpenShell names only to detect ownership reuse. Unsafe name-only deletion returns +a typed retention error. The dormant adapter assumes the protocol's single +coordinator; multi-process lease/arbitration remains an explicit +production-activation gate. Activation must also inject the selected gateway's +canonical state root. ## Architectural disposition diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index 415fced86cd..ee1a834985f 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -933,13 +933,15 @@ function normalizePreparedReplacement( }); } +export function createManagedBootstrapPlanFingerprint(plan: ManagedBootstrapExpectedPlan): string { + return createHash("sha256").update(canonicalJson(plan), "utf8").digest("hex"); +} + export function createManagedBootstrapPreparedAuthority( transaction: ManagedBootstrapPreparedTransaction, ): ManagedBootstrapPreparedAuthority { const { handle, snapshot, prepared } = transaction; - const planFingerprint = createHash("sha256") - .update(canonicalJson(handle.plan), "utf8") - .digest("hex"); + const planFingerprint = createManagedBootstrapPlanFingerprint(handle.plan); const bound = Object.freeze({ schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, phase: "prepared" as const, diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index 7ed0d8bba6a..0f3e98e1352 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -9,10 +9,15 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapFinalizationRecord, type DockerManagedBootstrapJournal, + parseDockerManagedBootstrapFinalizationRecord, parseDockerManagedBootstrapJournal, + sameDockerManagedBootstrapReceipt, + serializeDockerManagedBootstrapFinalizationRecord, serializeDockerManagedBootstrapJournal, } from "./docker-journal"; @@ -22,11 +27,13 @@ const journal = Object.freeze({ schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, phase: "staged", bootstrapIdentity: IDENTITY, + providerId: "docker", sandbox: { sandboxName: "alpha", sandboxId: "sandbox-alpha", driverId: "docker", }, + planFingerprint: "9".repeat(64), profileFingerprint: "2".repeat(64), imageReference: `registry.example/image@sha256:${"3".repeat(64)}`, runtimeImageContentId: `sha256:${"4".repeat(64)}`, @@ -37,7 +44,59 @@ const journal = Object.freeze({ backupName: "openshell-alpha-backup", originalSpecHash: "7".repeat(64), replacementSpecHash: "8".repeat(64), + rollbackTargetRuntimeId: "5".repeat(64), + rollbackTargetSpecHash: "7".repeat(64), + preparationReceipt: { + schemaVersion: 1, + sandbox: { + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + driverId: "docker", + }, + bootstrapIdentity: IDENTITY, + authorityFingerprint: "a".repeat(64), + recordId: "prepared-alpha", + recordedAt: "2026-07-31T19:59:59.000Z", + }, + commitReceipt: null, } satisfies DockerManagedBootstrapJournal); +const finalization = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase: "committed", + bootstrapIdentity: IDENTITY, + providerId: "docker", + sandbox: journal.sandbox, + planFingerprint: journal.planFingerprint, + profileFingerprint: journal.profileFingerprint, + imageReference: journal.imageReference, + commitReceipt: { + schemaVersion: 1, + sandbox: journal.sandbox, + runtimeId: journal.replacementRuntimeId, + image: { + repository: "registry.example/image", + manifestDigest: `sha256:${"3".repeat(64)}` as const, + }, + runtimeImageContentId: journal.runtimeImageContentId, + originalSpecHash: journal.originalSpecHash, + replacementSpecHash: journal.replacementSpecHash, + profileFingerprint: journal.profileFingerprint, + bootstrapIdentity: IDENTITY, + transactionPending: false, + completedAt: "2026-07-31T20:00:00.000Z", + }, + cleanupReceipt: { + schemaVersion: 1, + sandbox: journal.sandbox, + bootstrapIdentity: IDENTITY, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: "2026-07-31T20:00:01.000Z", + }, +} satisfies DockerManagedBootstrapFinalizationRecord); function readPinnedPrivateFile(target: string): { readonly mode: number; readonly text: string } { const descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); @@ -78,6 +137,9 @@ describe("Docker managed bootstrap journal", () => { ); expect(store.transition(IDENTITY, "staged", "cutover").phase).toBe("cutover"); + expect(store.recordCompletion(IDENTITY, finalization.commitReceipt).commitReceipt).toEqual( + finalization.commitReceipt, + ); expect(store.transition(IDENTITY, "cutover", "shared-state-committed").phase).toBe( "shared-state-committed", ); @@ -172,4 +234,124 @@ describe("Docker managed bootstrap journal", () => { serializeDockerManagedBootstrapJournal(Object.freeze({ ...journal, phase: "staged" })), ).toBe(`${JSON.stringify(journal)}\n`); }); + + it("compares equivalent receipts independent of property insertion order", () => { + const preparation = journal.preparationReceipt; + const reorderedPreparation = { + recordedAt: preparation.recordedAt, + recordId: preparation.recordId, + authorityFingerprint: preparation.authorityFingerprint, + bootstrapIdentity: preparation.bootstrapIdentity, + sandbox: { + driverId: preparation.sandbox.driverId, + sandboxId: preparation.sandbox.sandboxId, + sandboxName: preparation.sandbox.sandboxName, + }, + schemaVersion: preparation.schemaVersion, + } satisfies typeof preparation; + expect( + sameDockerManagedBootstrapReceipt("preparation", preparation, reorderedPreparation), + ).toBe(true); + + const completion = finalization.commitReceipt; + const reorderedCompletion = { + completedAt: completion.completedAt, + transactionPending: completion.transactionPending, + bootstrapIdentity: completion.bootstrapIdentity, + profileFingerprint: completion.profileFingerprint, + replacementSpecHash: completion.replacementSpecHash, + originalSpecHash: completion.originalSpecHash, + runtimeImageContentId: completion.runtimeImageContentId, + image: { + manifestDigest: completion.image.manifestDigest, + repository: completion.image.repository, + }, + runtimeId: completion.runtimeId, + sandbox: { + driverId: completion.sandbox.driverId, + sandboxId: completion.sandbox.sandboxId, + sandboxName: completion.sandbox.sandboxName, + }, + schemaVersion: completion.schemaVersion, + } satisfies typeof completion; + expect(sameDockerManagedBootstrapReceipt("completion", completion, reorderedCompletion)).toBe( + true, + ); + }); + + it("enumerates unfinished records and reloads exact terminal receipts from a new journal store", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const first = createFileDockerManagedBootstrapJournalStore(root); + first.create(journal); + expect(first.listUnfinished()).toEqual([journal]); + + first.recordFinalization(finalization); + expect(first.listUnfinished()).toEqual([]); + const restarted = createFileDockerManagedBootstrapJournalStore(root); + expect(restarted.loadFinalization(IDENTITY)).toEqual(finalization); + expect( + parseDockerManagedBootstrapFinalizationRecord( + serializeDockerManagedBootstrapFinalizationRecord(finalization), + ), + ).toEqual(finalization); + expect(() => + restarted.recordFinalization({ + ...finalization, + cleanupReceipt: { ...finalization.cleanupReceipt, finalizedAt: "2026-07-31T20:00:02.000Z" }, + }), + ).toThrow("finalization record changed"); + }); + + it("ignores interrupted atomic-write sidecars during unfinished enumeration", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + const directory = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); + for (const target of [ + `${IDENTITY}.json`, + `${IDENTITY}.json.decision`, + `${IDENTITY}.json.finalized`, + ]) { + fs.writeFileSync(path.join(directory, `.${target}.${process.pid}.abcdef.tmp`), "partial", { + mode: 0o600, + }); + } + + expect(store.listUnfinished()).toEqual([journal]); + }); + + it("reloads the exact completion receipt from a new journal store", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const first = createFileDockerManagedBootstrapJournalStore(root); + first.create(journal); + first.transition(IDENTITY, "staged", "cutover"); + const completed = first.recordCompletion(IDENTITY, finalization.commitReceipt); + expect(completed.commitReceipt).toEqual(finalization.commitReceipt); + + const restarted = createFileDockerManagedBootstrapJournalStore(root); + expect(restarted.listUnfinished()).toEqual([completed]); + expect(restarted.recordCompletion(IDENTITY, finalization.commitReceipt)).toEqual(completed); + expect(() => + restarted.recordCompletion(IDENTITY, { + ...finalization.commitReceipt, + completedAt: "2026-07-31T20:00:02.000Z", + }), + ).toThrow("completion receipt changed"); + }); + + it("fails closed when enumeration encounters an unsupported state entry", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + fs.writeFileSync( + path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, "unexpected.json"), + "{}\n", + { mode: 0o600 }, + ); + expect(() => store.listUnfinished()).toThrow("unsupported entry"); + }); }); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index c6043409d7d..cffa3dc0b8c 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -4,12 +4,19 @@ import fs from "node:fs"; import path from "node:path"; -import type { ManagedBootstrapSandboxIdentity } from "./adapter"; +import type { + ManagedBootstrapCompletionReceipt, + ManagedBootstrapDurablePreparationReceipt, + ManagedBootstrapFinalizationReceipt, + ManagedBootstrapSandboxIdentity, +} from "./adapter"; -export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 1 as const; +export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 2 as const; export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY = "managed-bootstrap"; +export const DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION = 1 as const; const SHA256_RE = /^[a-f0-9]{64}$/u; +const MANIFEST_DIGEST_RE = /^sha256:[a-f0-9]{64}$/u; const MAX_JOURNAL_BYTES = 32 * 1024; const JOURNAL_DIRECTORY_MODE = 0o700; const JOURNAL_FILE_MODE = 0o600; @@ -28,7 +35,9 @@ export interface DockerManagedBootstrapJournal { readonly schemaVersion: typeof DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION; readonly phase: DockerManagedBootstrapJournalPhase; readonly bootstrapIdentity: string; + readonly providerId: string; readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly planFingerprint: string; readonly profileFingerprint: string; readonly imageReference: string; readonly runtimeImageContentId: string; @@ -39,17 +48,41 @@ export interface DockerManagedBootstrapJournal { readonly backupName: string; readonly originalSpecHash: string; readonly replacementSpecHash: string; + readonly rollbackTargetRuntimeId: string; + readonly rollbackTargetSpecHash: string; + readonly preparationReceipt: ManagedBootstrapDurablePreparationReceipt | null; + readonly commitReceipt: ManagedBootstrapCompletionReceipt | null; +} + +export interface DockerManagedBootstrapFinalizationRecord { + readonly schemaVersion: typeof DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION; + readonly phase: "committed" | "rolled-back"; + readonly bootstrapIdentity: string; + readonly providerId: string; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly planFingerprint: string; + readonly profileFingerprint: string; + readonly imageReference: string; + readonly commitReceipt: ManagedBootstrapCompletionReceipt | null; + readonly cleanupReceipt: ManagedBootstrapFinalizationReceipt; } export interface DockerManagedBootstrapJournalStore { create(journal: DockerManagedBootstrapJournal): void; load(bootstrapIdentity: string): DockerManagedBootstrapJournal | null; + listUnfinished(): readonly DockerManagedBootstrapJournal[]; transition( bootstrapIdentity: string, expected: DockerManagedBootstrapJournalPhase, next: DockerManagedBootstrapJournalPhase, ): DockerManagedBootstrapJournal; + recordCompletion( + bootstrapIdentity: string, + receipt: ManagedBootstrapCompletionReceipt, + ): DockerManagedBootstrapJournal; remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]): void; + recordFinalization(record: DockerManagedBootstrapFinalizationRecord): void; + loadFinalization(bootstrapIdentity: string): DockerManagedBootstrapFinalizationRecord | null; } /** @@ -137,15 +170,21 @@ export function normalizeDockerManagedBootstrapJournal( const expectedKeys = [ "backupName", "bootstrapIdentity", + "commitReceipt", "imageReference", "originalName", "originalRuntimeId", "originalSpecHash", "phase", + "planFingerprint", + "preparationReceipt", "profileFingerprint", + "providerId", "replacementRuntimeId", "replacementSpecHash", "replacementStagingName", + "rollbackTargetRuntimeId", + "rollbackTargetSpecHash", "runtimeImageContentId", "sandbox", "schemaVersion", @@ -160,7 +199,9 @@ export function normalizeDockerManagedBootstrapJournal( schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, phase: exactPhase(journal.phase), bootstrapIdentity: exactSha256(journal.bootstrapIdentity, "bootstrap identity"), + providerId: exactString(journal.providerId, "provider ID"), sandbox: exactSandbox(journal.sandbox), + planFingerprint: exactSha256(journal.planFingerprint, "plan fingerprint"), profileFingerprint: exactSha256(journal.profileFingerprint, "profile fingerprint"), imageReference: exactString(journal.imageReference, "image reference"), runtimeImageContentId: exactString(journal.runtimeImageContentId, "runtime image content ID"), @@ -175,6 +216,20 @@ export function normalizeDockerManagedBootstrapJournal( backupName: exactString(journal.backupName, "backup name", 253), originalSpecHash: exactSha256(journal.originalSpecHash, "original spec hash"), replacementSpecHash: exactSha256(journal.replacementSpecHash, "replacement spec hash"), + rollbackTargetRuntimeId: exactSha256( + journal.rollbackTargetRuntimeId, + "rollback target runtime ID", + ), + rollbackTargetSpecHash: exactSha256( + journal.rollbackTargetSpecHash, + "rollback target spec hash", + ), + preparationReceipt: + journal.preparationReceipt === null + ? null + : exactPreparationReceipt(journal.preparationReceipt), + commitReceipt: + journal.commitReceipt === null ? null : exactCompletionReceipt(journal.commitReceipt), } satisfies DockerManagedBootstrapJournal); if (normalized.originalRuntimeId === normalized.replacementRuntimeId) { fail("original and replacement runtime IDs must differ"); @@ -185,6 +240,33 @@ export function normalizeDockerManagedBootstrapJournal( ) { fail("original, staging, and backup names must be distinct"); } + if ( + normalized.providerId !== normalized.sandbox.driverId || + normalized.rollbackTargetRuntimeId !== normalized.originalRuntimeId || + normalized.rollbackTargetSpecHash !== normalized.originalSpecHash + ) { + fail("provider or rollback authority does not match the transaction identity"); + } + if ( + (normalized.preparationReceipt !== null && + (normalized.preparationReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + normalized.preparationReceipt.sandbox.sandboxName !== normalized.sandbox.sandboxName || + normalized.preparationReceipt.sandbox.sandboxId !== normalized.sandbox.sandboxId || + normalized.preparationReceipt.sandbox.driverId !== normalized.sandbox.driverId)) || + (normalized.commitReceipt !== null && + (normalized.commitReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + normalized.commitReceipt.sandbox.sandboxName !== normalized.sandbox.sandboxName || + normalized.commitReceipt.sandbox.sandboxId !== normalized.sandbox.sandboxId || + normalized.commitReceipt.sandbox.driverId !== normalized.sandbox.driverId || + normalized.commitReceipt.runtimeId !== normalized.replacementRuntimeId || + normalized.commitReceipt.profileFingerprint !== normalized.profileFingerprint || + normalized.commitReceipt.originalSpecHash !== normalized.originalSpecHash || + normalized.commitReceipt.replacementSpecHash !== normalized.replacementSpecHash || + `${normalized.commitReceipt.image.repository}@${normalized.commitReceipt.image.manifestDigest}` !== + normalized.imageReference)) + ) { + fail("durable preparation or commit receipt does not match the transaction identity"); + } return normalized; } @@ -220,6 +302,275 @@ export function parseDockerManagedBootstrapJournal(text: string): DockerManagedB return journal; } +function exactTimestamp(value: unknown, label: string): string { + const timestamp = exactString(value, label, 128); + const parsed = new Date(timestamp); + if (!Number.isFinite(parsed.getTime()) || parsed.toISOString() !== timestamp) { + fail(`${label} must be one canonical timestamp`); + } + return timestamp; +} + +function exactBoolean(value: unknown, label: string): boolean { + if (typeof value !== "boolean") fail(`${label} must be boolean`); + return value; +} + +function exactNullableSha256(value: unknown, label: string): string | null { + return value === null ? null : exactSha256(value, label); +} + +function exactImage(value: unknown): ManagedBootstrapCompletionReceipt["image"] { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("completion image identity must be an object"); + } + const image = value as Record; + if (Object.keys(image).sort().join(",") !== "manifestDigest,repository") { + fail("completion image identity schema is invalid"); + } + const manifestDigest = exactString(image.manifestDigest, "completion manifest digest", 128); + if (!MANIFEST_DIGEST_RE.test(manifestDigest)) { + fail("completion manifest digest must be canonical sha256"); + } + return Object.freeze({ + repository: exactString(image.repository, "completion image repository"), + manifestDigest: manifestDigest as `sha256:${string}`, + }); +} + +function exactPreparationReceipt(value: unknown): ManagedBootstrapDurablePreparationReceipt { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("durable preparation receipt must be an object"); + } + const receipt = value as Record; + const expectedKeys = [ + "authorityFingerprint", + "bootstrapIdentity", + "recordId", + "recordedAt", + "sandbox", + "schemaVersion", + ]; + if ( + Object.keys(receipt).sort().join(",") !== expectedKeys.sort().join(",") || + receipt.schemaVersion !== 1 + ) { + fail("durable preparation receipt schema is invalid"); + } + return Object.freeze({ + schemaVersion: 1, + sandbox: exactSandbox(receipt.sandbox), + bootstrapIdentity: exactSha256( + receipt.bootstrapIdentity, + "durable preparation bootstrap identity", + ), + authorityFingerprint: exactSha256( + receipt.authorityFingerprint, + "durable preparation authority fingerprint", + ), + recordId: exactString(receipt.recordId, "durable preparation record ID", 1024), + recordedAt: exactTimestamp(receipt.recordedAt, "durable preparation timestamp"), + }); +} + +function exactCompletionReceipt(value: unknown): ManagedBootstrapCompletionReceipt { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("commit receipt must be an object"); + } + const receipt = value as Record; + const expectedKeys = [ + "bootstrapIdentity", + "completedAt", + "image", + "originalSpecHash", + "profileFingerprint", + "replacementSpecHash", + "runtimeId", + "runtimeImageContentId", + "sandbox", + "schemaVersion", + "transactionPending", + ]; + if ( + Object.keys(receipt).sort().join(",") !== expectedKeys.sort().join(",") || + receipt.schemaVersion !== 1 + ) { + fail("commit receipt schema is invalid"); + } + return Object.freeze({ + schemaVersion: 1, + sandbox: exactSandbox(receipt.sandbox), + runtimeId: exactSha256(receipt.runtimeId, "commit runtime ID"), + image: exactImage(receipt.image), + runtimeImageContentId: exactString( + receipt.runtimeImageContentId, + "commit runtime image content ID", + ), + originalSpecHash: exactSha256(receipt.originalSpecHash, "commit original spec hash"), + replacementSpecHash: exactSha256(receipt.replacementSpecHash, "commit replacement spec hash"), + profileFingerprint: exactSha256(receipt.profileFingerprint, "commit profile fingerprint"), + bootstrapIdentity: exactSha256(receipt.bootstrapIdentity, "commit bootstrap identity"), + transactionPending: exactBoolean(receipt.transactionPending, "commit transaction pending"), + completedAt: exactTimestamp(receipt.completedAt, "commit completion timestamp"), + }); +} + +export function sameDockerManagedBootstrapReceipt( + kind: "preparation", + left: ManagedBootstrapDurablePreparationReceipt, + right: ManagedBootstrapDurablePreparationReceipt, +): boolean; +export function sameDockerManagedBootstrapReceipt( + kind: "completion", + left: ManagedBootstrapCompletionReceipt, + right: ManagedBootstrapCompletionReceipt, +): boolean; +export function sameDockerManagedBootstrapReceipt( + kind: "preparation" | "completion", + left: ManagedBootstrapDurablePreparationReceipt | ManagedBootstrapCompletionReceipt, + right: ManagedBootstrapDurablePreparationReceipt | ManagedBootstrapCompletionReceipt, +): boolean { + if (kind === "preparation") { + return ( + JSON.stringify(exactPreparationReceipt(left)) === + JSON.stringify(exactPreparationReceipt(right)) + ); + } + return ( + JSON.stringify(exactCompletionReceipt(left)) === JSON.stringify(exactCompletionReceipt(right)) + ); +} + +function exactCleanupReceipt(value: unknown): ManagedBootstrapFinalizationReceipt { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("cleanup receipt must be an object"); + } + const receipt = value as Record; + const expectedKeys = [ + "alreadyRolledBack", + "bootstrapIdentity", + "finalizedAt", + "heldWorkloadRemoved", + "outcome", + "restoredRuntimeId", + "restoredSpecHash", + "sandbox", + "schemaVersion", + ]; + if ( + Object.keys(receipt).sort().join(",") !== expectedKeys.sort().join(",") || + receipt.schemaVersion !== 1 || + !["committed", "rolled-back"].includes(String(receipt.outcome)) + ) { + fail("cleanup receipt schema is invalid"); + } + return Object.freeze({ + schemaVersion: 1, + sandbox: exactSandbox(receipt.sandbox), + bootstrapIdentity: exactSha256(receipt.bootstrapIdentity, "cleanup bootstrap identity"), + outcome: receipt.outcome as "committed" | "rolled-back", + restoredRuntimeId: exactNullableSha256(receipt.restoredRuntimeId, "restored runtime ID"), + restoredSpecHash: exactNullableSha256(receipt.restoredSpecHash, "restored spec hash"), + heldWorkloadRemoved: exactBoolean(receipt.heldWorkloadRemoved, "held workload removed"), + alreadyRolledBack: exactBoolean(receipt.alreadyRolledBack, "already rolled back"), + finalizedAt: exactTimestamp(receipt.finalizedAt, "cleanup finalization timestamp"), + }); +} + +export function normalizeDockerManagedBootstrapFinalizationRecord( + value: unknown, +): DockerManagedBootstrapFinalizationRecord { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("finalization record must be an object"); + } + const record = value as Record; + const expectedKeys = [ + "bootstrapIdentity", + "cleanupReceipt", + "commitReceipt", + "imageReference", + "phase", + "planFingerprint", + "profileFingerprint", + "providerId", + "sandbox", + "schemaVersion", + ]; + if ( + Object.keys(record).sort().join(",") !== expectedKeys.sort().join(",") || + record.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION || + !["committed", "rolled-back"].includes(String(record.phase)) + ) { + fail("finalization record schema is invalid"); + } + const phase = record.phase as "committed" | "rolled-back"; + const sandbox = exactSandbox(record.sandbox); + const commitReceipt = + record.commitReceipt === null ? null : exactCompletionReceipt(record.commitReceipt); + const cleanupReceipt = exactCleanupReceipt(record.cleanupReceipt); + const normalized = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase, + bootstrapIdentity: exactSha256(record.bootstrapIdentity, "finalization bootstrap identity"), + providerId: exactString(record.providerId, "finalization provider ID"), + sandbox, + planFingerprint: exactSha256(record.planFingerprint, "finalization plan fingerprint"), + profileFingerprint: exactSha256(record.profileFingerprint, "finalization profile fingerprint"), + imageReference: exactString(record.imageReference, "finalization image reference"), + commitReceipt, + cleanupReceipt, + } satisfies DockerManagedBootstrapFinalizationRecord); + if ( + normalized.providerId !== sandbox.driverId || + normalized.bootstrapIdentity !== cleanupReceipt.bootstrapIdentity || + normalized.phase !== cleanupReceipt.outcome || + JSON.stringify(normalized.sandbox) !== JSON.stringify(cleanupReceipt.sandbox) || + (phase === "committed") !== (commitReceipt !== null) || + (commitReceipt !== null && + (commitReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + commitReceipt.profileFingerprint !== normalized.profileFingerprint || + JSON.stringify(commitReceipt.sandbox) !== JSON.stringify(normalized.sandbox) || + `${commitReceipt.image.repository}@${commitReceipt.image.manifestDigest}` !== + normalized.imageReference)) + ) { + fail("finalization receipts do not match their durable transaction identity"); + } + return normalized; +} + +export function serializeDockerManagedBootstrapFinalizationRecord( + record: DockerManagedBootstrapFinalizationRecord, +): string { + const serialized = `${JSON.stringify(normalizeDockerManagedBootstrapFinalizationRecord(record))}\n`; + if (Buffer.byteLength(serialized, "utf8") > MAX_JOURNAL_BYTES) { + fail("serialized finalization record exceeds its bounded transport"); + } + return serialized; +} + +export function parseDockerManagedBootstrapFinalizationRecord( + text: string, +): DockerManagedBootstrapFinalizationRecord { + if ( + text.length === 0 || + text.includes("\0") || + Buffer.byteLength(text, "utf8") > MAX_JOURNAL_BYTES + ) { + fail("serialized finalization record is empty or too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("serialized finalization record is not valid JSON"); + } + const record = normalizeDockerManagedBootstrapFinalizationRecord(parsed); + if (serializeDockerManagedBootstrapFinalizationRecord(record) !== text) { + fail("serialized finalization record is not canonical"); + } + return record; +} + function assertDirectory(directory: string): void { fs.mkdirSync(directory, { recursive: true, mode: JOURNAL_DIRECTORY_MODE }); const stat = fs.lstatSync(directory); @@ -237,6 +588,10 @@ function decisionPath(target: string): string { return `${target}.decision`; } +function finalizationPath(target: string): string { + return `${target}.finalized`; +} + function sameStableMetadata(left: fs.BigIntStats, right: fs.BigIntStats): boolean { return ( left.dev === right.dev && @@ -361,6 +716,15 @@ function atomicWrite( if (cleanupFailure !== null) throw cleanupFailure.error; } +function sameSerializedJournal( + left: DockerManagedBootstrapJournal, + right: DockerManagedBootstrapJournal, +): boolean { + return ( + serializeDockerManagedBootstrapJournal(left) === serializeDockerManagedBootstrapJournal(right) + ); +} + export function createFileDockerManagedBootstrapJournalStore( stateRoot: string, ): DockerManagedBootstrapJournalStore { @@ -386,9 +750,26 @@ export function createFileDockerManagedBootstrapJournalStore( } return decided; }; + const loadFinalization = ( + bootstrapIdentity: string, + ): DockerManagedBootstrapFinalizationRecord | null => { + assertDirectory(directory); + const contents = readPrivateFile( + finalizationPath(journalPath(directory, bootstrapIdentity)), + "finalization", + ); + return contents === null ? null : parseDockerManagedBootstrapFinalizationRecord(contents); + }; return Object.freeze({ create(journal: DockerManagedBootstrapJournal) { const normalized = normalizeDockerManagedBootstrapJournal(journal); + if ( + normalized.phase !== "staged" || + normalized.preparationReceipt === null || + normalized.commitReceipt !== null + ) { + fail("a new journal requires staged durable preparation authority"); + } assertDirectory(directory); const target = journalPath(directory, normalized.bootstrapIdentity); if (readPrivateFile(decisionPath(target), "decision") !== null) { @@ -397,6 +778,34 @@ export function createFileDockerManagedBootstrapJournalStore( atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(normalized), true); }, load, + listUnfinished() { + assertDirectory(directory); + const identities: string[] = []; + for (const name of fs.readdirSync(directory)) { + const match = name.match(/^([a-f0-9]{64})\.json$/u); + if (match) { + identities.push(match[1]); + continue; + } + if ( + /^\.[a-f0-9]{64}\.json(?:\.decision|\.finalized)?\.[0-9]+\.[a-f0-9]+\.tmp$/u.test(name) || + /^[a-f0-9]{64}\.json\.(?:decision|finalized)$/u.test(name) + ) { + continue; + } + fail(`journal directory contains an unsupported entry: ${name}`); + } + return Object.freeze( + identities + .sort() + .filter((identity) => loadFinalization(identity) === null) + .map((identity) => { + const journal = load(identity); + if (!journal) fail(`enumerated journal ${identity} disappeared`); + return journal; + }), + ); + }, transition( bootstrapIdentity: string, expected: DockerManagedBootstrapJournalPhase, @@ -429,6 +838,33 @@ export function createFileDockerManagedBootstrapJournalStore( atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(updated), false); return updated; }, + recordCompletion( + bootstrapIdentity: string, + receipt: ManagedBootstrapCompletionReceipt, + ): DockerManagedBootstrapJournal { + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const current = load(bootstrapIdentity); + if (!current || current.phase !== "cutover") { + fail(`completion recording requires phase cutover, found ${current?.phase ?? "absent"}`); + } + const updated = normalizeDockerManagedBootstrapJournal({ + ...current, + commitReceipt: receipt, + }); + if (current.commitReceipt !== null) { + if (!sameSerializedJournal(current, updated)) { + fail("completion receipt changed for this bootstrap identity"); + } + return current; + } + atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(updated), false); + const persisted = load(bootstrapIdentity); + if (!persisted || !sameSerializedJournal(persisted, updated)) { + fail("completion receipt was not durably re-readable"); + } + return persisted; + }, remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]) { assertDirectory(directory); const target = journalPath(directory, bootstrapIdentity); @@ -444,5 +880,26 @@ export function createFileDockerManagedBootstrapJournalStore( fs.unlinkSync(target); fsyncDirectory(directory); }, + recordFinalization(record: DockerManagedBootstrapFinalizationRecord) { + const normalized = normalizeDockerManagedBootstrapFinalizationRecord(record); + assertDirectory(directory); + const target = finalizationPath(journalPath(directory, normalized.bootstrapIdentity)); + const serialized = serializeDockerManagedBootstrapFinalizationRecord(normalized); + const existing = readPrivateFile(target, "finalization"); + if (existing !== null) { + if (existing !== serialized) + fail("finalization record changed for this bootstrap identity"); + return; + } + try { + atomicWrite(directory, target, serialized, true); + } catch (error) { + if (readPrivateFile(target, "finalization") !== serialized) throw error; + } + if (readPrivateFile(target, "finalization") !== serialized) { + fail("finalization record was not durably re-readable"); + } + }, + loadFinalization, }); } diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts index 415fe56fb82..a92d2e335ed 100644 --- a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts @@ -17,6 +17,7 @@ import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "../docker-gpu-pat import { MANAGED_STARTUP_RUNTIME_EXECUTABLE } from "../managed-startup/image-runtime"; import { MANAGED_STARTUP_AGENTS, type ManagedStartupAgent } from "../managed-startup/profile"; import { + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, } from "../managed-startup/shared-state-transaction"; @@ -24,8 +25,6 @@ import { isImmutableDockerImageId } from "../openshell-docker-sandbox-containers import { cleanupTempDir, secureTempFile } from "../temp-files"; const RECEIPT_TEMP_PREFIX = "nemoclaw-managed-startup-receipt"; -const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY = - "/var/lib/nemoclaw/managed-startup-shared-state-commit-v1"; const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; const DURABLE_IDENTITY_RE = /^[a-f0-9]{64}$/u; const DOCKER_MUTATION_OPTIONS = { diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index cb069cb5bca..7f64895bd74 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -21,13 +21,18 @@ import { } from "./adapter"; import type { DockerManagedBootstrapDeps } from "./docker"; import { + type DockerManagedBootstrapFinalizationRecord, type DockerManagedBootstrapJournal, DockerManagedBootstrapJournalAcknowledgementLostError, type DockerManagedBootstrapJournalPhase, type DockerManagedBootstrapJournalStore, } from "./docker-journal"; import { normalizeDockerManagedBootstrapLaunchSpec } from "./docker-spec"; -import { parseManagedBootstrapEnvelope } from "./envelope"; +import { + MANAGED_BOOTSTRAP_COMPLETION_FILE, + parseManagedBootstrapEnvelope, + serializeManagedBootstrapImageCompletion, +} from "./envelope"; export const IDENTITY = "1".repeat(64); export const OLD_ID = "2".repeat(64); @@ -206,6 +211,7 @@ export function fixture(options: DockerFixtureOptions = {}) { let original: DockerContainerInspect | null = originalInspect(agentInputs(options.agent)); let replacement: DockerContainerInspect | null = null; let journal: DockerManagedBootstrapJournal | null = null; + let finalization: DockerManagedBootstrapFinalizationRecord | null = null; let sharedState: "committed" | "none" | "pending" = options.sharedState ?? "none"; const events: string[] = []; const lostAcknowledgements = new Set(options.lostAcknowledgements ?? []); @@ -224,6 +230,7 @@ export function fixture(options: DockerFixtureOptions = {}) { } }, load: () => copyJournal(), + listUnfinished: () => (journal && !finalization ? [structuredClone(journal)] : []), transition(_identity, expected, next) { const current = journal !== null && journal.phase === expected @@ -240,6 +247,20 @@ export function fixture(options: DockerFixtureOptions = {}) { } return structuredClone(journal); }, + recordCompletion(_identity, receipt) { + if (!journal || journal.phase !== "cutover") { + throw new Error("completion requires cutover journal"); + } + if ( + journal.commitReceipt !== null && + JSON.stringify(journal.commitReceipt) !== JSON.stringify(receipt) + ) { + throw new Error("completion changed"); + } + journal = { ...journal, commitReceipt: structuredClone(receipt) }; + events.push("journal:completion"); + return structuredClone(journal); + }, remove(_identity, expected) { const current = journal; void (current !== null && expected.includes(current.phase) @@ -253,6 +274,14 @@ export function fixture(options: DockerFixtureOptions = {}) { ); } }, + recordFinalization(value) { + if (finalization && JSON.stringify(finalization) !== JSON.stringify(value)) { + throw new Error("finalization changed"); + } + finalization = structuredClone(value); + events.push(`finalization:${value.phase}`); + }, + loadFinalization: () => (finalization ? structuredClone(finalization) : null), }; const inspect = (reference: string): DockerContainerInspect => { const candidates = [original, replacement].filter( @@ -323,6 +352,20 @@ export function fixture(options: DockerFixtureOptions = {}) { return ok(); }; const copyFromContainer = () => { + if (source === `${NEW_ID}:${MANAGED_BOOTSTRAP_COMPLETION_FILE}`) { + fs.writeFileSync( + destination, + serializeManagedBootstrapImageCompletion({ + bootstrapIdentity: IDENTITY, + agent: options.agent ?? "hermes", + profileFingerprint: agentInputs(options.agent).request.profileFingerprint, + transactionPending: sharedState === "pending", + }), + { mode: 0o444 }, + ); + fs.chmodSync(destination, 0o444); + return ok(); + } const receipt = source.split(":")[1]; const expected = receipt?.includes("shared-state-commit") ? "committed" : "pending"; return sharedState === expected @@ -433,6 +476,9 @@ export function fixture(options: DockerFixtureOptions = {}) { get journal() { return journal; }, + get finalization() { + return finalization; + }, get original() { return original; }, diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 7979330b180..f0391a2417e 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -49,11 +49,23 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.events).not.toContain(`stop:${OLD_ID}`); fake.events.push("authority:recorded"); const durable = durablePreparation(handle, snapshot, prepared); + const reorderedDurable = { + recordedAt: durable.recordedAt, + recordId: durable.recordId, + authorityFingerprint: durable.authorityFingerprint, + bootstrapIdentity: durable.bootstrapIdentity, + sandbox: { + driverId: durable.sandbox.driverId, + sandboxId: durable.sandbox.sandboxId, + sandboxName: durable.sandbox.sandboxName, + }, + schemaVersion: durable.schemaVersion, + } satisfies typeof durable; const replacement = await adapter.activateBootstrapReplacement({ handle, snapshot, prepared, - durablePreparation: durable, + durablePreparation: reorderedDurable, }); const order = fake.events; expect(order).toContain("authority:recorded"); @@ -68,17 +80,70 @@ describe("Docker managed bootstrap adapter", () => { replacementRuntimeId: NEW_ID, }); + const commitReceipt = await adapter.awaitBootstrap({ + handle, + snapshot, + replacement, + timeoutSecs: 1, + }); + const reorderedCommitReceipt = { + completedAt: commitReceipt.completedAt, + transactionPending: commitReceipt.transactionPending, + bootstrapIdentity: commitReceipt.bootstrapIdentity, + profileFingerprint: commitReceipt.profileFingerprint, + replacementSpecHash: commitReceipt.replacementSpecHash, + originalSpecHash: commitReceipt.originalSpecHash, + runtimeImageContentId: commitReceipt.runtimeImageContentId, + image: { + manifestDigest: commitReceipt.image.manifestDigest, + repository: commitReceipt.image.repository, + }, + runtimeId: commitReceipt.runtimeId, + sandbox: { + driverId: commitReceipt.sandbox.driverId, + sandboxId: commitReceipt.sandbox.sandboxId, + sandboxName: commitReceipt.sandbox.sandboxName, + }, + schemaVersion: commitReceipt.schemaVersion, + } satisfies typeof commitReceipt; + expect(fake.events).toContain("journal:completion"); + expect(fake.events).toContain(`start:${NEW_ID}`); + expect(fake.events.indexOf("journal:completion")).toBeGreaterThan( + fake.events.indexOf(`start:${NEW_ID}`), + ); + const finalized = await adapter.finalizeBootstrap({ + outcome: "commit", + handle, + snapshot, + prepared, + durablePreparation: reorderedDurable, + replacement, + completion: reorderedCommitReceipt, + }); + expect(finalized).toMatchObject({ outcome: "committed" }); + expect(fake.events).toContain("journal:shared-state-committed"); + expect(fake.events).toContain(`rm:${OLD_ID}`); + expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( + fake.events.indexOf(`rm:${OLD_ID}`), + ); + expect(fake.journal).toBeNull(); + expect(fake.finalization).toMatchObject({ phase: "committed", commitReceipt }); + expect(fake.sharedState).toBe("none"); + expect(fake.replacement?.Id).toBe(NEW_ID); + + const eventCount = fake.events.length; await expect( - adapter.finalizeBootstrap({ + createDockerManagedBootstrapAdapter(fake.deps).finalizeBootstrap({ outcome: "commit", handle, snapshot, prepared, - durablePreparation: durable, + durablePreparation: reorderedDurable, replacement, - completion: completion(replacement), + completion: reorderedCommitReceipt, }), - ).resolves.toMatchObject({ outcome: "committed" }); + ).resolves.toEqual(finalized); + expect(fake.events).toHaveLength(eventCount); expect(fake.events).toContain("journal:shared-state-committed"); expect(fake.events).toContain(`rm:${OLD_ID}`); expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( @@ -109,6 +174,12 @@ describe("Docker managed bootstrap adapter", () => { prepared, durablePreparation: durable, }); + const commitReceipt = await adapter.awaitBootstrap({ + handle, + snapshot, + replacement, + timeoutSecs: 1, + }); vi.mocked(fake.deps.dockerStop!).mockReturnValue({ status: 1, stderr: "injected quiesce failure", @@ -122,7 +193,7 @@ describe("Docker managed bootstrap adapter", () => { prepared, durablePreparation: durable, replacement, - completion: completion(replacement), + completion: commitReceipt, }), ).rejects.toThrow( /logical commit validation failed: Managed-startup shared-state commit helper failed.*injected commit failure.*new workload could not be quiesced.*injected quiesce failure/u, diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index 03afb2ba280..e633ee9f2ed 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -42,6 +42,7 @@ import { assertManagedBootstrapSafeProcessEnvironmentKey, attachManagedBootstrapRollbackError, createManagedBootstrapIdentity, + createManagedBootstrapPlanFingerprint, createManagedBootstrapPreparedAuthority, MANAGED_BOOTSTRAP_SCHEMA_VERSION, type ManagedBootstrapAdapter, @@ -64,11 +65,15 @@ import { } from "./adapter"; import { createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapFinalizationRecord, type DockerManagedBootstrapJournal, DockerManagedBootstrapJournalAcknowledgementLostError, type DockerManagedBootstrapJournalStore, parseDockerManagedBootstrapJournal, + sameDockerManagedBootstrapReceipt, + serializeDockerManagedBootstrapFinalizationRecord, serializeDockerManagedBootstrapJournal, } from "./docker-journal"; import { @@ -144,12 +149,6 @@ type ResolvedDeps = Required< type DockerBootstrapTransaction = DockerManagedBootstrapJournal; -interface DockerBootstrapRollbackTombstone { - readonly profileFingerprint: string; - readonly imageReference: string; - readonly receipt: ManagedBootstrapFinalizationReceipt; -} - export interface DockerManagedBootstrapAdapter extends ManagedBootstrapAdapter {} function resolveDeps(deps: DockerManagedBootstrapDeps): ResolvedDeps { @@ -1426,6 +1425,26 @@ function sameDockerBootstrapJournal( ); } +function sameDockerBootstrapPreparedAuthority( + left: DockerBootstrapTransaction, + right: DockerBootstrapTransaction, +): boolean { + return sameDockerBootstrapJournal( + Object.freeze({ + ...left, + phase: "staged" as const, + preparationReceipt: null, + commitReceipt: null, + }), + Object.freeze({ + ...right, + phase: "staged" as const, + preparationReceipt: null, + commitReceipt: null, + }), + ); +} + function createDockerBootstrapJournalDurably( journal: DockerBootstrapTransaction, deps: ResolvedDeps, @@ -1467,6 +1486,27 @@ function transitionDockerBootstrapJournalDurably( return persisted; } +function recordDockerBootstrapCompletionDurably( + journal: DockerBootstrapTransaction, + receipt: ManagedBootstrapCompletionReceipt, + deps: ResolvedDeps, +): DockerBootstrapTransaction { + const expected = Object.freeze({ ...journal, commitReceipt: receipt }); + try { + deps.journalStore.recordCompletion(journal.bootstrapIdentity, receipt); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + if (!recovered || !sameDockerBootstrapJournal(recovered, expected)) throw error; + return recovered; + } + const persisted = deps.journalStore.load(journal.bootstrapIdentity); + if (!persisted || !sameDockerBootstrapJournal(persisted, expected)) { + throw new Error("Managed bootstrap Docker completion receipt was not durably re-readable."); + } + return persisted; +} + function removeDockerBootstrapJournalDurably( journal: DockerBootstrapTransaction, deps: ResolvedDeps, @@ -1490,6 +1530,7 @@ function assertDockerBootstrapTransactionAuthority( snapshot: ManagedBootstrapObservedSnapshot, prepared?: ManagedBootstrapPreparedReplacementHandle | null, replacement?: ManagedBootstrapReplacementHandle | null, + durablePreparation?: ManagedBootstrapDurablePreparationReceipt | null, ): void { const originalName = dockerContainerName( parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, @@ -1498,9 +1539,11 @@ function assertDockerBootstrapTransactionAuthority( if ( transaction.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION || transaction.bootstrapIdentity !== handle.bootstrapIdentity || + transaction.providerId !== expectedSandbox.driverId || transaction.sandbox.sandboxName !== expectedSandbox.sandboxName || transaction.sandbox.sandboxId !== expectedSandbox.sandboxId || transaction.sandbox.driverId !== expectedSandbox.driverId || + transaction.planFingerprint !== createManagedBootstrapPlanFingerprint(handle.plan) || transaction.profileFingerprint !== handle.plan.profile.fingerprint || transaction.imageReference !== expectedImageReference(snapshot.image.repository, snapshot.image.manifestDigest) || @@ -1511,6 +1554,22 @@ function assertDockerBootstrapTransactionAuthority( replacementStagingName(originalName, handle.bootstrapIdentity) || transaction.backupName !== backupName(originalName, handle.bootstrapIdentity) || transaction.originalSpecHash !== snapshot.specHash || + transaction.rollbackTargetRuntimeId !== snapshot.runtimeId || + transaction.rollbackTargetSpecHash !== snapshot.specHash || + (transaction.preparationReceipt !== null && + prepared !== undefined && + prepared !== null && + transaction.preparationReceipt.authorityFingerprint !== + createManagedBootstrapPreparedAuthority({ handle, snapshot, prepared }) + .authorityFingerprint) || + (durablePreparation !== undefined && + durablePreparation !== null && + (transaction.preparationReceipt === null || + !sameDockerManagedBootstrapReceipt( + "preparation", + transaction.preparationReceipt, + durablePreparation, + ))) || (prepared !== undefined && prepared !== null && (transaction.originalRuntimeId !== prepared.originalRuntimeId || @@ -1677,8 +1736,64 @@ export function createDockerManagedBootstrapAdapter( dependencies: DockerManagedBootstrapDeps = {}, ): DockerManagedBootstrapAdapter { const deps = resolveDeps(dependencies); - const committedTransactions = new Set(); - const rollbackTombstones = new Map(); + const finalizationRecord = ( + handle: ManagedBootstrapHeldWorkloadHandle, + ): DockerManagedBootstrapFinalizationRecord | null => { + const record = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + if (!record) return null; + if ( + record.providerId !== handle.sandbox.driverId || + record.sandbox.sandboxName !== handle.sandbox.sandboxName || + record.sandbox.sandboxId !== handle.sandbox.sandboxId || + record.sandbox.driverId !== handle.sandbox.driverId || + record.planFingerprint !== createManagedBootstrapPlanFingerprint(handle.plan) || + record.profileFingerprint !== handle.plan.profile.fingerprint || + record.imageReference !== + expectedImageReference(handle.plan.image.repository, handle.plan.image.manifestDigest) + ) { + throw new Error("Managed bootstrap finalization record does not match its durable identity."); + } + return record; + }; + const persistFinalization = ( + handle: ManagedBootstrapHeldWorkloadHandle, + phase: "committed" | "rolled-back", + commitReceipt: ManagedBootstrapCompletionReceipt | null, + cleanupReceipt: ManagedBootstrapFinalizationReceipt, + ): ManagedBootstrapFinalizationReceipt => { + const record = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase, + bootstrapIdentity: handle.bootstrapIdentity, + providerId: handle.sandbox.driverId, + sandbox: handle.sandbox, + planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + handle.plan.image.repository, + handle.plan.image.manifestDigest, + ), + commitReceipt, + cleanupReceipt, + } satisfies DockerManagedBootstrapFinalizationRecord); + const serialized = serializeDockerManagedBootstrapFinalizationRecord(record); + try { + deps.journalStore.recordFinalization(record); + } catch (error) { + const recovered = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + if ( + !recovered || + serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized + ) { + throw error; + } + } + const persisted = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + if (!persisted || serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized) { + throw new Error("Managed bootstrap finalization receipt was not durably re-readable."); + } + return persisted.cleanupReceipt; + }; const completedRollback = ( handle: ManagedBootstrapHeldWorkloadHandle, alreadyRolledBack: boolean, @@ -1694,34 +1809,39 @@ export function createDockerManagedBootstrapAdapter( alreadyRolledBack, finalizedAt: deps.now().toISOString(), } satisfies ManagedBootstrapFinalizationReceipt); - rollbackTombstones.set(handle.bootstrapIdentity, { - profileFingerprint: handle.plan.profile.fingerprint, - imageReference: expectedImageReference( - handle.plan.image.repository, - handle.plan.image.manifestDigest, - ), - receipt, - }); - return receipt; + return persistFinalization(handle, "rolled-back", null, receipt); + }; + const completedCommit = ( + handle: ManagedBootstrapHeldWorkloadHandle, + commitReceipt: ManagedBootstrapCompletionReceipt, + ): ManagedBootstrapFinalizationReceipt => { + const cleanupReceipt = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + } satisfies ManagedBootstrapFinalizationReceipt); + return persistFinalization(handle, "committed", commitReceipt, cleanupReceipt); }; const priorRollback = ( handle: ManagedBootstrapHeldWorkloadHandle, ): ManagedBootstrapFinalizationReceipt | null => { - const tombstone = rollbackTombstones.get(handle.bootstrapIdentity); - if (!tombstone) return null; - const receipt = tombstone.receipt; - if ( - receipt.sandbox.sandboxName !== handle.sandbox.sandboxName || - receipt.sandbox.sandboxId !== handle.sandbox.sandboxId || - receipt.sandbox.driverId !== handle.sandbox.driverId || - tombstone.profileFingerprint !== handle.plan.profile.fingerprint || - tombstone.imageReference !== - expectedImageReference(handle.plan.image.repository, handle.plan.image.manifestDigest) - ) { - throw new Error("Managed bootstrap rollback tombstone does not match its durable identity."); + const finalized = finalizationRecord(handle); + if (!finalized) return null; + if (finalized.phase === "committed") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: handle.bootstrapIdentity, + cleanupRuntimeId: finalized.commitReceipt?.runtimeId ?? "unknown", + detail: "rollback is no longer legal after the durable finalization receipt", + }); } return Object.freeze({ - ...receipt, + ...finalized.cleanupReceipt, alreadyRolledBack: true, }); }; @@ -1743,10 +1863,7 @@ export function createDockerManagedBootstrapAdapter( const finalized = priorRollback(handle); if (finalized) return finalized; const journal = deps.journalStore.load(handle.bootstrapIdentity); - if ( - committedTransactions.has(handle.bootstrapIdentity) || - journal?.phase === "shared-state-committed" - ) { + if (journal?.phase === "shared-state-committed") { throw new ManagedBootstrapDurableCommitCleanupPendingError({ bootstrapIdentity: handle.bootstrapIdentity, cleanupRuntimeId: journal?.originalRuntimeId ?? snapshot?.runtimeId ?? "unknown", @@ -1854,15 +1971,21 @@ export function createDockerManagedBootstrapAdapter( detail: "durable Docker cutover lacks its coordinator-recorded prepared authority", }); } - const stagedJournal = Object.freeze({ ...journal, phase: "staged" as const }); - if (!sameDockerBootstrapJournal(stagedJournal, preparedAuthority)) { + if (!sameDockerBootstrapPreparedAuthority(journal, preparedAuthority)) { throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: handle.bootstrapIdentity, runtimeId: journal.replacementRuntimeId, detail: "durable Docker cutover changed its prepared rollback authority", }); } - assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); + assertDockerBootstrapTransactionAuthority( + journal, + handle, + snapshot, + prepared, + replacement, + durablePreparation, + ); const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); if (!original) { throw new ManagedBootstrapCommitStateIndeterminateError({ @@ -2082,14 +2205,14 @@ export function createDockerManagedBootstrapAdapter( return completedRollback(handle, false); }; const commitBootstrapNow = ( + handle: ManagedBootstrapHeldWorkloadHandle, receipt: ManagedBootstrapCompletionReceipt, transaction: DockerBootstrapTransaction, input: { readonly sharedStateStatus: "committed" | "none"; readonly sharedStateTransaction: ReturnType; }, - ): void => { - if (committedTransactions.has(receipt.bootstrapIdentity)) return; + ): ManagedBootstrapFinalizationReceipt => { if ( transaction.phase !== "shared-state-committed" || transaction.replacementRuntimeId !== receipt.runtimeId || @@ -2181,7 +2304,7 @@ export function createDockerManagedBootstrapAdapter( } } removeDockerBootstrapJournalDurably(transaction, deps); - committedTransactions.add(receipt.bootstrapIdentity); + return completedCommit(handle, receipt); }; const finalizeBootstrap = async ( input: Parameters[0], @@ -2193,6 +2316,21 @@ export function createDockerManagedBootstrapAdapter( if (!completion || !snapshot || !prepared || !durablePreparation || !replacement) { throw new Error("Managed bootstrap commit requires one complete cutover receipt."); } + const finalized = finalizationRecord(handle); + if (finalized) { + if ( + finalized.phase !== "committed" || + !finalized.commitReceipt || + !sameDockerManagedBootstrapReceipt("completion", finalized.commitReceipt, completion) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: "durable finalization cannot change outcome or commit receipt", + }); + } + return finalized.cleanupReceipt; + } const preparedAuthority = transactionFromPreparedAuthority(handle, snapshot, prepared); assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); const sharedTransaction = managedSharedStateTransaction( @@ -2210,19 +2348,6 @@ export function createDockerManagedBootstrapAdapter( let journal = deps.journalStore.load(handle.bootstrapIdentity); if (!journal) { - if (committedTransactions.has(completion.bootstrapIdentity)) { - return Object.freeze({ - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox: handle.sandbox, - bootstrapIdentity: handle.bootstrapIdentity, - outcome: "committed", - restoredRuntimeId: null, - restoredSpecHash: null, - heldWorkloadRemoved: false, - alreadyRolledBack: false, - finalizedAt: deps.now().toISOString(), - }); - } const originalPresence = probeExactDockerContainerAbsence(snapshot.runtimeId, deps); if (originalPresence === "unknown") { throw new ManagedBootstrapCommitStateIndeterminateError({ @@ -2256,33 +2381,34 @@ export function createDockerManagedBootstrapAdapter( detail: "the retired-journal replacement does not match the exact completion receipt", }); } - committedTransactions.add(completion.bootstrapIdentity); - return Object.freeze({ - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox: handle.sandbox, + return completedCommit(handle, completion); + } + + if (!sameDockerBootstrapPreparedAuthority(journal, preparedAuthority)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: handle.bootstrapIdentity, - outcome: "committed", - restoredRuntimeId: null, - restoredSpecHash: null, - heldWorkloadRemoved: false, - alreadyRolledBack: false, - finalizedAt: deps.now().toISOString(), + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker commit changed its prepared rollback authority", }); } - + assertDockerBootstrapTransactionAuthority( + journal, + handle, + snapshot, + prepared, + replacement, + durablePreparation, + ); if ( - !sameDockerBootstrapJournal( - Object.freeze({ ...journal, phase: "staged" as const }), - preparedAuthority, - ) + journal.commitReceipt === null || + !sameDockerManagedBootstrapReceipt("completion", journal.commitReceipt, completion) ) { throw new ManagedBootstrapCommitStateIndeterminateError({ - bootstrapIdentity: handle.bootstrapIdentity, + bootstrapIdentity: journal.bootstrapIdentity, runtimeId: journal.replacementRuntimeId, - detail: "durable Docker commit changed its prepared rollback authority", + detail: "commit requires the exact durable completion receipt", }); } - assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); if (journal.phase === "staged" || journal.phase === "rollback-authorized") { throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: journal.bootstrapIdentity, @@ -2361,21 +2487,10 @@ export function createDockerManagedBootstrapAdapter( }); } - commitBootstrapNow(completion, journal, { + return commitBootstrapNow(handle, completion, journal, { sharedStateStatus: sharedStatus === "committed" ? "committed" : "none", sharedStateTransaction: sharedTransaction, }); - return Object.freeze({ - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox: handle.sandbox, - bootstrapIdentity: handle.bootstrapIdentity, - outcome: "committed", - restoredRuntimeId: null, - restoredSpecHash: null, - heldWorkloadRemoved: false, - alreadyRolledBack: false, - finalizedAt: deps.now().toISOString(), - }); }; return { async createHeldWorkload(input) { @@ -2631,7 +2746,9 @@ export function createDockerManagedBootstrapAdapter( schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, phase: "staged", bootstrapIdentity: handle.bootstrapIdentity, + providerId: handle.sandbox.driverId, sandbox: Object.freeze({ ...handle.sandbox }), + planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), profileFingerprint: handle.plan.profile.fingerprint, imageReference: expectedImageReference( snapshot.image.repository, @@ -2645,6 +2762,10 @@ export function createDockerManagedBootstrapAdapter( backupName: backupContainerName, originalSpecHash: snapshot.specHash, replacementSpecHash: expectedActivatedSpecHash, + rollbackTargetRuntimeId: snapshot.runtimeId, + rollbackTargetSpecHash: snapshot.specHash, + preparationReceipt: null, + commitReceipt: null, }); requestFile = writeProtectedEnvelope(handle.bootstrapIdentity, request); @@ -2724,9 +2845,20 @@ export function createDockerManagedBootstrapAdapter( async activateBootstrapReplacement({ handle, snapshot, prepared, durablePreparation }) { const authority = transactionFromPreparedAuthority(handle, snapshot, prepared); assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); + const durableAuthority = Object.freeze({ + ...authority, + preparationReceipt: durablePreparation, + }); const existingJournal = deps.journalStore.load(handle.bootstrapIdentity); if (existingJournal) { - assertDockerBootstrapTransactionAuthority(existingJournal, handle, snapshot, prepared); + assertDockerBootstrapTransactionAuthority( + existingJournal, + handle, + snapshot, + prepared, + null, + durablePreparation, + ); throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: existingJournal.bootstrapIdentity, runtimeId: existingJournal.replacementRuntimeId, @@ -2756,7 +2888,7 @@ export function createDockerManagedBootstrapAdapter( ); } - let journal = createDockerBootstrapJournalDurably(authority, deps); + let journal = createDockerBootstrapJournalDurably(durableAuthority, deps); const originalAtFence = inspectExact(snapshot.runtimeId, deps); const replacementAtFence = inspectExact(prepared.preparedRuntimeId, deps); assertTransactionOriginal(journal, originalAtFence); @@ -2929,7 +3061,19 @@ export function createDockerManagedBootstrapAdapter( "Managed bootstrap Docker image completion identities do not match the transaction.", ); } - return Object.freeze({ + if (afterWaitJournal.commitReceipt !== null) { + if ( + afterWaitJournal.commitReceipt.transactionPending !== imageCompletion.transactionPending + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: afterWaitJournal.bootstrapIdentity, + runtimeId: afterWaitJournal.replacementRuntimeId, + detail: "durable completion disagrees with the image-owned transaction receipt", + }); + } + return afterWaitJournal.commitReceipt; + } + const completion = Object.freeze({ schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, sandbox: handle.sandbox, runtimeId: replacement.replacementRuntimeId, @@ -2942,6 +3086,15 @@ export function createDockerManagedBootstrapAdapter( transactionPending: imageCompletion.transactionPending, completedAt: deps.now().toISOString(), }); + const completedJournal = recordDockerBootstrapCompletionDurably( + afterWaitJournal, + completion, + deps, + ); + if (completedJournal.commitReceipt === null) { + throw new Error("Managed bootstrap Docker completion receipt disappeared after recording."); + } + return completedJournal.commitReceipt; }, finalizeBootstrap, diff --git a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts index 3022fc71850..6ec1563e73e 100644 --- a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts +++ b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts @@ -10,9 +10,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; import type { SandboxMessagingPlan } from "../messaging/manifest"; import type { ManagedStartupAgent, ManagedStartupProfile } from "./managed-startup/profile"; +import { fingerprintManagedStartupProfile } from "./managed-startup/profile"; import { beginManagedStartupSharedStateTransaction, + clearManagedStartupSharedStateCommitReceipt, commitManagedStartupSharedStateTransaction, + getManagedStartupSharedStateTransactionStatus, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, type ManagedStartupSharedTransactionOptions, rollbackManagedStartupSharedStateTransaction, } from "./managed-startup/shared-state-transaction"; @@ -67,6 +71,13 @@ describe("managed startup shared-state transaction", () => { ); } + function commitReceiptDirectory(): string { + return path.join( + path.dirname(transactionDirectory), + path.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY), + ); + } + it.each([ "openclaw", "hermes", @@ -246,6 +257,192 @@ describe("managed startup shared-state transaction", () => { expect(commitManagedStartupSharedStateTransaction("openclaw", options)).toBe(false); }); + it("fsyncs every transaction namespace before exposing a pending receipt", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "before\n"); + const open = vi.spyOn(fs, "openSync"); + const fsync = vi.spyOn(fs, "fsyncSync"); + + expect( + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options), + ).toBe(true); + + const transactionParent = path.dirname(transactionDirectory); + const backupDirectory = path.join(transactionDirectory, "backups"); + expect(open).toHaveBeenCalledWith(transactionParent, fs.constants.O_RDONLY); + expect(open).toHaveBeenCalledWith(transactionDirectory, fs.constants.O_RDONLY); + expect(open).toHaveBeenCalledWith(backupDirectory, fs.constants.O_RDONLY); + // File contents plus parent, backup, and manifest directory entries all + // reach stable storage before the transaction is returned as pending. + expect(fsync.mock.calls.length).toBeGreaterThanOrEqual(6); + }); + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("persists one exact compact %s bootstrap commit across fresh calls, forbids rollback, and retires it for the next attempt", (agent) => { + const profile = managedStartupE2eProfile(agent); + const bootstrapIdentity = "b".repeat(64); + const nextBootstrapIdentity = "d".repeat(64); + const boundOptions = { ...options, bootstrapIdentity }; + const root = agentRoot(agent); + fs.mkdirSync(root); + const config = path.join( + root, + agent === "openclaw" ? "openclaw.json" : agent === "hermes" ? "config.yaml" : "config.toml", + ); + fs.writeFileSync(config, "before\n"); + + expect(beginManagedStartupSharedStateTransaction(profile, boundOptions)).toBe(true); + fs.writeFileSync(config, "committed\n"); + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("pending"); + expect(() => + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: "e".repeat(64), + bootstrapIdentity, + }, + options, + ), + ).toThrow(/expected agent, profile fingerprint, or bootstrap identity/u); + expect(commitManagedStartupSharedStateTransaction(agent, boundOptions)).toBe(true); + + const receiptDirectory = commitReceiptDirectory(); + const receiptFile = path.join(receiptDirectory, "receipt.json"); + expect(fs.existsSync(transactionDirectory)).toBe(false); + expect(fs.readdirSync(receiptDirectory)).toEqual(["receipt.json"]); + expect(mode(receiptDirectory)).toBe(0o700); + expect(mode(receiptFile)).toBe(0o400); + expect(JSON.parse(fs.readFileSync(receiptFile, "utf8"))).toEqual({ + schemaVersion: 1, + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }); + + // These calls reconstruct state solely from the image-owned receipt. + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("committed"); + expect(commitManagedStartupSharedStateTransaction(agent, boundOptions)).toBe(true); + expect(() => rollbackManagedStartupSharedStateTransaction(agent, boundOptions)).toThrow( + /durably committed and cannot be rolled back/u, + ); + expect(() => + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: "e".repeat(64), + bootstrapIdentity, + }, + options, + ), + ).toThrow(/different bootstrap attempt/u); + expect(fs.readFileSync(config, "utf8")).toBe("committed\n"); + + expect(clearManagedStartupSharedStateCommitReceipt(agent, boundOptions)).toBe(true); + expect(fs.existsSync(receiptDirectory)).toBe(false); + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("none"); + + const nextOptions = { ...options, bootstrapIdentity: nextBootstrapIdentity }; + expect(beginManagedStartupSharedStateTransaction(profile, nextOptions)).toBe(true); + expect(rollbackManagedStartupSharedStateTransaction(agent, nextOptions)).toBe(true); + }); + + it.each([ + "during-compact-receipt-write", + "before-backup-removal", + "during-backup-removal", + "after-backup-removal", + "after-manifest-removal", + ] as const)("recovers an atomically established commit interrupted %s", (interruption) => { + const profile = managedStartupE2eProfile("openclaw"); + const bootstrapIdentity = "b".repeat(64); + const boundOptions = { ...options, bootstrapIdentity }; + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "before\n"); + beginManagedStartupSharedStateTransaction(profile, boundOptions); + fs.writeFileSync(path.join(root, "openclaw.json"), "committed\n"); + + const originalRmSync = fs.rmSync.bind(fs); + const rm = vi.spyOn(fs, "rmSync").mockImplementation((( + target: fs.PathLike, + removeOptions?: fs.RmDirOptions, + ) => + String(target).endsWith(`${path.sep}backups`) + ? (() => { + throw new Error("injected post-rename cleanup interruption"); + })() + : originalRmSync(target, removeOptions)) as typeof fs.rmSync); + expect(() => commitManagedStartupSharedStateTransaction("openclaw", boundOptions)).toThrow( + /injected post-rename cleanup interruption/u, + ); + rm.mockRestore(); + + expect(fs.existsSync(transactionDirectory)).toBe(false); + const committedDirectory = commitReceiptDirectory(); + const backups = path.join(committedDirectory, "backups"); + const manifest = path.join(committedDirectory, "manifest.json"); + const applyInterruption: Record void> = { + "during-compact-receipt-write": () => + fs.renameSync( + path.join(committedDirectory, "receipt.json"), + path.join(committedDirectory, ".receipt.json.1234567890abcdef12345678"), + ), + "before-backup-removal": () => undefined, + "during-backup-removal": () => { + const [firstBackup] = fs.readdirSync(backups); + expect(firstBackup).toBeTruthy(); + fs.unlinkSync(path.join(backups, firstBackup!)); + }, + "after-backup-removal": () => originalRmSync(backups, { force: false, recursive: true }), + "after-manifest-removal": () => fs.unlinkSync(manifest), + }; + applyInterruption[interruption](); + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent: "openclaw", + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("committed"); + expect(commitManagedStartupSharedStateTransaction("openclaw", boundOptions)).toBe(true); + expect(fs.readdirSync(committedDirectory)).toEqual(["receipt.json"]); + expect(clearManagedStartupSharedStateCommitReceipt("openclaw", boundOptions)).toBe(true); + }); + it("resumes the same pending profile idempotently and rejects profile drift", () => { const root = agentRoot("openclaw"); fs.mkdirSync(root); @@ -258,10 +455,62 @@ describe("managed startup shared-state transaction", () => { managedStartupE2eProfile("openclaw", true), options, ), - ).toThrow(/belongs to a different profile/u); + ).toThrow(/belongs to a different agent, profile fingerprint, or bootstrap attempt/u); expect(rollbackManagedStartupSharedStateTransaction("openclaw", options)).toBe(true); }); + it("validates a directly mounted copied receipt under an unchanged 0755 image parent", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "{}\n"); + const profile = managedStartupE2eProfile("openclaw"); + const bootstrapIdentity = "b".repeat(64); + const boundOptions = { ...options, bootstrapIdentity }; + beginManagedStartupSharedStateTransaction(profile, boundOptions); + + const imageParent = path.join(temporaryRoot, "image-var-lib-nemoclaw"); + const copiedReceipt = path.join(imageParent, "managed-startup-shared-state-transaction-v1"); + fs.mkdirSync(imageParent, { mode: 0o755 }); + fs.chmodSync(imageParent, 0o755); + fs.cpSync(transactionDirectory, copiedReceipt, { + recursive: true, + preserveTimestamps: true, + }); + // Node 22.23 normalizes copied directory modes to 0755. Recreate the + // protected modes that the container-copy fixture is intended to model. + fs.chmodSync(copiedReceipt, 0o700); + fs.chmodSync(path.join(copiedReceipt, "backups"), 0o700); + + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent: "openclaw", + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + { + ...boundOptions, + transactionDirectory: copiedReceipt, + }, + ), + ).toBe("pending"); + + fs.chmodSync(imageParent, 0o700); + expect(() => + getManagedStartupSharedStateTransactionStatus( + { + agent: "openclaw", + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + { + ...boundOptions, + transactionDirectory: copiedReceipt, + }, + ), + ).toThrow(/must be .* mode 755/u); + }); + it("rejects planted target and ancestor symlinks before creating a receipt", () => { const outside = path.join(temporaryRoot, "outside"); fs.mkdirSync(outside); diff --git a/src/lib/onboard/managed-startup/image-runtime.ts b/src/lib/onboard/managed-startup/image-runtime.ts index f8e5c7f4edd..898783c7f30 100644 --- a/src/lib/onboard/managed-startup/image-runtime.ts +++ b/src/lib/onboard/managed-startup/image-runtime.ts @@ -35,7 +35,9 @@ import { } from "./root-apply"; import { beginManagedStartupSharedStateTransaction, + clearManagedStartupSharedStateCommitReceipt, commitManagedStartupSharedStateTransaction, + getManagedStartupSharedStateTransactionStatus, MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, rollbackManagedStartupSharedStateTransaction, } from "./shared-state-transaction"; @@ -1540,7 +1542,7 @@ function readCliAgent(argv: readonly string[], expectedLength = 2): string { const index = argv.indexOf("--agent"); if (index < 0 || index + 1 >= argv.length || argv.length !== expectedLength) { fail( - "usage: managed-startup-image-runtime [--apply-root-stdin|--wait-for-completion|--verify-completion|--begin-shared-state-transaction|--commit-shared-state-transaction] --agent ", + "usage: managed-startup-image-runtime [--apply-root-stdin|--wait-for-completion|--verify-completion|--begin-shared-state-transaction|--commit-shared-state-transaction|--clear-shared-state-commit-receipt|--shared-state-transaction-status] --agent ", ); } return argv[index + 1] as string; @@ -1554,6 +1556,14 @@ function readCliFingerprint(argv: readonly string[]): string { return argv[index + 1] as string; } +function readCliBootstrapIdentity(argv: readonly string[]): string { + const index = argv.indexOf("--bootstrap-identity"); + if (index < 0 || index + 1 >= argv.length || !SHA256_RE.test(String(argv[index + 1] ?? ""))) { + fail("managed bootstrap identity argument is missing or invalid"); + } + return argv[index + 1] as string; +} + export async function main(argv: readonly string[] = process.argv.slice(2)): Promise { if (argv.length === 1 && argv[0] === "--internal-write-openclaw-hash") { internalWriteOpenClawHash(); @@ -1600,29 +1610,58 @@ export async function main(argv: readonly string[] = process.argv.slice(2)): Pro return; } if ( - argv.length === 4 && + (argv.length === 4 || argv.length === 6) && argv[0] === "--rollback-shared-state-transaction" && - argv[3] === "--read-only-receipt" + argv[argv.length - 1] === "--read-only-receipt" ) { requireRoot(); - const agent = exactAgent(readCliAgent(argv, 4)); + const agent = exactAgent(readCliAgent(argv, argv.length)); const rolledBack = rollbackManagedStartupSharedStateTransaction(agent, { transactionDirectory: MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, readOnlyReceipt: true, + bootstrapIdentity: argv.length === 6 ? readCliBootstrapIdentity(argv) : null, }); if (!rolledBack) fail("read-only shared-state rollback receipt is missing"); console.log(`[managed-startup] verified and restored ${agent} shared state`); return; } - if (argv.length === 3 && argv[0] === "--commit-shared-state-transaction") { + if ((argv.length === 3 || argv.length === 5) && argv[0] === "--commit-shared-state-transaction") { requireRoot(); - const agent = exactAgent(readCliAgent(argv, 3)); - if (!commitManagedStartupSharedStateTransaction(agent)) { + const agent = exactAgent(readCliAgent(argv, argv.length)); + if ( + !commitManagedStartupSharedStateTransaction(agent, { + bootstrapIdentity: argv.length === 5 ? readCliBootstrapIdentity(argv) : null, + }) + ) { fail("managed startup transaction is missing at commit"); } console.log(`[managed-startup] committed ${agent} shared state`); return; } + if (argv.length === 5 && argv[0] === "--clear-shared-state-commit-receipt") { + requireRoot(); + const agent = exactAgent(readCliAgent(argv, 5)); + const bootstrapIdentity = readCliBootstrapIdentity(argv); + if (!clearManagedStartupSharedStateCommitReceipt(agent, { bootstrapIdentity })) { + fail("managed startup durable commit receipt is missing at cleanup"); + } + console.log(`[managed-startup] cleared ${agent} durable shared-state commit receipt`); + return; + } + if (argv.length === 7 && argv[0] === "--shared-state-transaction-status") { + requireRoot(); + const agent = exactAgent(readCliAgent(argv, 7)); + const profileFingerprint = readCliFingerprint(argv); + const bootstrapIdentity = readCliBootstrapIdentity(argv); + process.stdout.write( + `${getManagedStartupSharedStateTransactionStatus({ + agent, + profileFingerprint, + bootstrapIdentity, + })}\n`, + ); + return; + } const result = await applyManagedStartupImageProfile(readCliAgent(argv)); console.log( result.adapterApplied diff --git a/src/lib/onboard/managed-startup/shared-state-transaction.ts b/src/lib/onboard/managed-startup/shared-state-transaction.ts index d921ec36d89..62894abeb5c 100644 --- a/src/lib/onboard/managed-startup/shared-state-transaction.ts +++ b/src/lib/onboard/managed-startup/shared-state-transaction.ts @@ -20,14 +20,19 @@ const MAX_TRANSACTION_FILES = 128; const MAX_TRANSACTION_FILE_BYTES = 8 * 1024 * 1024; const MAX_TRANSACTION_TOTAL_BYTES = 32 * 1024 * 1024; const MAX_MANIFEST_BYTES = 256 * 1024; +const MAX_COMMIT_RECEIPT_BYTES = 4096; const TRANSACTION_PARENT_DIRECTORY_MODE = 0o755; const TRANSACTION_DIRECTORY_MODE = 0o700; const TRANSACTION_FILE_MODE = 0o400; +const ATOMIC_TEMPORARY_FILE_MODE = 0o600; export const MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY = "/var/lib/nemoclaw/managed-startup-shared-state-transaction-v1"; export const MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY = "/run/nemoclaw/managed-startup-shared-rollback-receipt-v1"; +export const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY = + "/var/lib/nemoclaw/managed-startup-shared-state-commit-v1"; +const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE = "receipt.json"; interface FilePresentReceipt { readonly path: string; @@ -66,13 +71,23 @@ interface TransactionManifest { readonly schemaVersion: typeof TRANSACTION_SCHEMA_VERSION; readonly agent: ManagedStartupAgent; readonly profileFingerprint: string; + readonly bootstrapIdentity: string | null; readonly files: readonly FileReceipt[]; readonly directories: readonly DirectoryReceipt[]; } +interface CommitReceipt { + readonly schemaVersion: typeof TRANSACTION_SCHEMA_VERSION; + readonly agent: ManagedStartupAgent; + readonly profileFingerprint: string; + readonly bootstrapIdentity: string; +} + export interface ManagedStartupSharedTransactionOptions { readonly sandboxRoot?: string; readonly transactionDirectory?: string; + /** Test/helper seam. Production derives the fixed image-owned commit receipt path. */ + readonly commitReceiptDirectory?: string; /** Test seam. Production always retains the root:root defaults. */ readonly trustedUid?: number; /** Test seam. Production always retains the root:root defaults. */ @@ -82,6 +97,8 @@ export interface ManagedStartupSharedTransactionOptions { * so ownership may reflect the Docker CLI user instead of container root. */ readonly readOnlyReceipt?: boolean; + /** One-attempt identity for managed bootstrap; null for legacy root application. */ + readonly bootstrapIdentity?: string | null; } interface ResolvedOptions { @@ -90,9 +107,12 @@ interface ResolvedOptions { readonly transactionDirectory: string; readonly backupDirectory: string; readonly manifestFile: string; + readonly commitReceiptDirectory: string; + readonly commitReceiptFile: string; readonly trustedUid: number; readonly trustedGid: number; readonly readOnlyReceipt: boolean; + readonly bootstrapIdentity: string | null; } interface StableFile { @@ -109,11 +129,28 @@ function resolveOptions(options: ManagedStartupSharedTransactionOptions = {}): R const transactionDirectory = path.resolve( options.transactionDirectory ?? MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, ); + const commitReceiptDirectory = path.resolve( + options.commitReceiptDirectory ?? + (options.transactionDirectory + ? path.join( + path.dirname(transactionDirectory), + path.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY), + ) + : MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY), + ); if ( transactionDirectory === sandboxRoot || - transactionDirectory.startsWith(`${sandboxRoot}${path.sep}`) + transactionDirectory.startsWith(`${sandboxRoot}${path.sep}`) || + commitReceiptDirectory === sandboxRoot || + commitReceiptDirectory.startsWith(`${sandboxRoot}${path.sep}`) || + path.dirname(commitReceiptDirectory) !== path.dirname(transactionDirectory) || + commitReceiptDirectory === transactionDirectory ) { - fail("transaction receipts must not be stored in sandbox-shared state"); + fail("transaction and commit receipts require distinct paths outside sandbox-shared state"); + } + const bootstrapIdentity = options.bootstrapIdentity ?? null; + if (bootstrapIdentity !== null && !/^[a-f0-9]{64}$/u.test(bootstrapIdentity)) { + fail("bootstrap identity must encode 32 lowercase-hex bytes"); } return { sandboxRoot, @@ -121,9 +158,15 @@ function resolveOptions(options: ManagedStartupSharedTransactionOptions = {}): R transactionDirectory, backupDirectory: path.join(transactionDirectory, "backups"), manifestFile: path.join(transactionDirectory, "manifest.json"), + commitReceiptDirectory, + commitReceiptFile: path.join( + commitReceiptDirectory, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE, + ), trustedUid: options.trustedUid ?? 0, trustedGid: options.trustedGid ?? 0, readOnlyReceipt: options.readOnlyReceipt ?? false, + bootstrapIdentity, }; } @@ -482,16 +525,63 @@ function atomicWriteTrustedFile( } } +function fsyncDirectory(directory: string): void { + const descriptor = fs.openSync(directory, fs.constants.O_RDONLY); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + function canonicalManifest(manifest: TransactionManifest): string { return `${JSON.stringify(manifest, null, 2)}\n`; } +function canonicalCommitReceipt(receipt: CommitReceipt): string { + return `${JSON.stringify(receipt, null, 2)}\n`; +} + function requireExactKeys(record: Record, keys: readonly string[]): void { if (Object.keys(record).sort().join(",") !== [...keys].sort().join(",")) { fail("transaction manifest contains unexpected fields"); } } +function parseCommitReceipt(text: string): CommitReceipt { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("commit receipt is not valid JSON"); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + fail("commit receipt must be an object"); + } + const record = parsed as Record; + requireExactKeys(record, ["agent", "bootstrapIdentity", "profileFingerprint", "schemaVersion"]); + if ( + record.schemaVersion !== TRANSACTION_SCHEMA_VERSION || + !["openclaw", "hermes", "langchain-deepagents-code"].includes(String(record.agent)) || + typeof record.profileFingerprint !== "string" || + !/^[a-f0-9]{64}$/u.test(record.profileFingerprint) || + typeof record.bootstrapIdentity !== "string" || + !/^[a-f0-9]{64}$/u.test(record.bootstrapIdentity) + ) { + fail("commit receipt has an invalid envelope"); + } + const receipt: CommitReceipt = { + schemaVersion: TRANSACTION_SCHEMA_VERSION, + agent: record.agent as ManagedStartupAgent, + profileFingerprint: record.profileFingerprint, + bootstrapIdentity: record.bootstrapIdentity, + }; + if (canonicalCommitReceipt(receipt) !== text) { + fail("commit receipt is not canonical"); + } + return receipt; +} + function safeMetadata(value: unknown): value is number { return Number.isSafeInteger(value) && (value as number) >= 0; } @@ -509,6 +599,7 @@ function parseManifest(text: string): TransactionManifest { const record = parsed as Record; requireExactKeys(record, [ "agent", + "bootstrapIdentity", "directories", "files", "profileFingerprint", @@ -519,6 +610,11 @@ function parseManifest(text: string): TransactionManifest { !["openclaw", "hermes", "langchain-deepagents-code"].includes(String(record.agent)) || typeof record.profileFingerprint !== "string" || !/^[a-f0-9]{64}$/u.test(record.profileFingerprint) || + !( + record.bootstrapIdentity === null || + (typeof record.bootstrapIdentity === "string" && + /^[a-f0-9]{64}$/u.test(record.bootstrapIdentity)) + ) || !Array.isArray(record.files) || !Array.isArray(record.directories) || record.files.length > MAX_TRANSACTION_FILES || @@ -613,6 +709,7 @@ function parseManifest(text: string): TransactionManifest { schemaVersion: TRANSACTION_SCHEMA_VERSION, agent: record.agent as ManagedStartupAgent, profileFingerprint: record.profileFingerprint, + bootstrapIdentity: record.bootstrapIdentity as string | null, files, directories, }; @@ -639,9 +736,9 @@ function requireTrustedTransactionPath( } } -function requireReadOnlyReceiptMount(options: ResolvedOptions): void { +function requireReadOnlyReceiptMount(target: string, options: ResolvedOptions): void { if (!options.readOnlyReceipt) return; - const probe = path.join(options.transactionDirectory, ".nemoclaw-write-probe"); + const probe = path.join(target, ".nemoclaw-write-probe"); let descriptor: number | undefined; try { descriptor = fs.openSync( @@ -664,7 +761,7 @@ function loadManifest(options: ResolvedOptions): TransactionManifest | null { requireTransactionBoundaries(options); if (!pathExistsNoFollow(options.transactionDirectory)) return null; requireTrustedTransactionPath(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE, options); - requireReadOnlyReceiptMount(options); + requireReadOnlyReceiptMount(options.transactionDirectory, options); requireTrustedTransactionPath(options.backupDirectory, TRANSACTION_DIRECTORY_MODE, options); requireTrustedTransactionPath(options.manifestFile, TRANSACTION_FILE_MODE, options); const stable = readStableFile(options.manifestFile, MAX_MANIFEST_BYTES); @@ -679,6 +776,59 @@ function loadManifest(options: ResolvedOptions): TransactionManifest | null { return parseManifest(stable.bytes.toString("utf8")); } +function transactionOptionsAt( + options: ResolvedOptions, + transactionDirectory: string, +): ResolvedOptions { + return { + ...options, + transactionDirectory, + backupDirectory: path.join(transactionDirectory, "backups"), + manifestFile: path.join(transactionDirectory, "manifest.json"), + }; +} + +function loadCommitReceipt( + options: ResolvedOptions, +): { readonly receipt: CommitReceipt; readonly compact: boolean } | null { + requireTransactionBoundaries(options); + if (!pathExistsNoFollow(options.commitReceiptDirectory)) return null; + requireTrustedTransactionPath( + options.commitReceiptDirectory, + TRANSACTION_DIRECTORY_MODE, + options, + ); + if (pathExistsNoFollow(options.commitReceiptFile)) { + requireReadOnlyReceiptMount(options.commitReceiptDirectory, options); + requireTrustedTransactionPath(options.commitReceiptFile, TRANSACTION_FILE_MODE, options); + const stable = readStableFile(options.commitReceiptFile, MAX_COMMIT_RECEIPT_BYTES); + if ( + (!options.readOnlyReceipt && + (Number(stable.stat.uid) !== options.trustedUid || + Number(stable.stat.gid) !== options.trustedGid)) || + Number(stable.stat.mode & 0o7777n) !== TRANSACTION_FILE_MODE + ) { + fail("commit receipt ownership changed while it was read"); + } + return { receipt: parseCommitReceipt(stable.bytes.toString("utf8")), compact: true }; + } + const stagedOptions = transactionOptionsAt(options, options.commitReceiptDirectory); + const staged = loadManifest(stagedOptions); + if (!staged || staged.bootstrapIdentity === null) { + fail("durable commit staging receipt is incomplete"); + } + verifyAllBackups(staged.files, stagedOptions); + return { + receipt: { + schemaVersion: TRANSACTION_SCHEMA_VERSION, + agent: staged.agent, + profileFingerprint: staged.profileFingerprint, + bootstrapIdentity: staged.bootstrapIdentity, + }, + compact: false, + }; +} + function verifyBackup(receipt: FilePresentReceipt, options: ResolvedOptions): Buffer { const backupPath = path.join(options.backupDirectory, receipt.backup); requireTrustedTransactionPath(backupPath, TRANSACTION_FILE_MODE, options); @@ -742,11 +892,162 @@ function directoryMatchesReceipt(target: string, receipt: DirectoryPresentReceip function removeTransactionDirectory(options: ResolvedOptions): void { requireTrustedTransactionPath(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE, options); fs.rmSync(options.transactionDirectory, { force: false, recursive: true }); + fsyncDirectory(options.transactionParentDirectory); if (pathExistsNoFollow(options.transactionDirectory)) { fail("transaction directory remained after cleanup"); } } +function assertCommitReceiptMatches( + receipt: CommitReceipt, + expected: { + readonly agent: ManagedStartupAgent; + readonly profileFingerprint?: string; + readonly bootstrapIdentity: string; + }, +): void { + if ( + receipt.agent !== expected.agent || + (expected.profileFingerprint !== undefined && + receipt.profileFingerprint !== expected.profileFingerprint) || + receipt.bootstrapIdentity !== expected.bootstrapIdentity + ) { + fail("durable commit receipt belongs to a different bootstrap attempt"); + } +} + +function loadCommitStagingManifest(options: ResolvedOptions): TransactionManifest | null { + if (!pathExistsNoFollow(options.manifestFile)) return null; + requireTrustedTransactionPath(options.manifestFile, TRANSACTION_FILE_MODE, options); + const stable = readStableFile(options.manifestFile, MAX_MANIFEST_BYTES); + if ( + Number(stable.stat.uid) !== options.trustedUid || + Number(stable.stat.gid) !== options.trustedGid || + Number(stable.stat.mode & 0o7777n) !== TRANSACTION_FILE_MODE + ) { + fail("durable commit staging manifest ownership changed while it was read"); + } + return parseManifest(stable.bytes.toString("utf8")); +} + +function retireInterruptedCommitReceiptWrites( + receipt: CommitReceipt, + options: ResolvedOptions, +): void { + const temporaryPattern = new RegExp( + `^\\.${MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE.replace(".", "\\.")}\\.[a-f0-9]{24}$`, + "u", + ); + for (const entry of fs.readdirSync(options.commitReceiptDirectory)) { + if (!temporaryPattern.test(entry)) continue; + const target = path.join(options.commitReceiptDirectory, entry); + const stat = fs.lstatSync(target); + if ( + stat.isSymbolicLink() || + !stat.isFile() || + stat.nlink !== 1 || + stat.uid !== options.trustedUid || + stat.gid !== options.trustedGid || + ![ATOMIC_TEMPORARY_FILE_MODE, TRANSACTION_FILE_MODE].includes(modeOf(stat)) + ) { + fail("interrupted durable commit receipt write has unsafe metadata"); + } + const stable = readStableFile(target, MAX_COMMIT_RECEIPT_BYTES); + const mode = Number(stable.stat.mode & 0o7777n); + if ( + Number(stable.stat.uid) !== options.trustedUid || + Number(stable.stat.gid) !== options.trustedGid || + ![ATOMIC_TEMPORARY_FILE_MODE, TRANSACTION_FILE_MODE].includes(mode) + ) { + fail("interrupted durable commit receipt write changed during verification"); + } + if (stable.bytes.length > 0) { + let interruptedReceipt: CommitReceipt | null = null; + try { + interruptedReceipt = parseCommitReceipt(stable.bytes.toString("utf8")); + } catch { + // The atomic writer may have crashed after any partial write. The + // trusted 0700 directory and exact random temp-name shape bind this + // artifact to that interrupted write; the established receipt is now + // authoritative. + } + if (interruptedReceipt) assertCommitReceiptMatches(interruptedReceipt, receipt); + } + fs.unlinkSync(target); + fsyncDirectory(options.commitReceiptDirectory); + } +} + +function compactDurableCommitReceipt( + state: { readonly receipt: CommitReceipt; readonly compact: boolean }, + options: ResolvedOptions, +): void { + if (!state.compact) { + atomicWriteTrustedFile( + options.commitReceiptFile, + canonicalCommitReceipt(state.receipt), + TRANSACTION_FILE_MODE, + options.trustedUid, + options.trustedGid, + ); + fsyncDirectory(options.commitReceiptDirectory); + } + retireInterruptedCommitReceiptWrites(state.receipt, options); + const stagedOptions = transactionOptionsAt(options, options.commitReceiptDirectory); + const manifestExists = pathExistsNoFollow(stagedOptions.manifestFile); + const backupsExist = pathExistsNoFollow(stagedOptions.backupDirectory); + const unexpectedBeforeCleanup = fs + .readdirSync(options.commitReceiptDirectory) + .filter( + (entry) => + ![ + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE, + path.basename(stagedOptions.backupDirectory), + path.basename(stagedOptions.manifestFile), + ].includes(entry), + ); + if (unexpectedBeforeCleanup.length !== 0) { + fail("durable commit receipt directory contains unexpected artifacts"); + } + if (manifestExists) { + // The fsynced compact receipt is authoritative after commit. Validate the + // remaining manifest identity without requiring a complete backup tree: + // recursive backup deletion may have been interrupted at any point. + const staged = loadCommitStagingManifest(stagedOptions); + if (!staged || staged.bootstrapIdentity === null) { + fail("durable commit staging receipt disappeared during cleanup"); + } + assertCommitReceiptMatches(state.receipt, { + agent: staged.agent, + profileFingerprint: staged.profileFingerprint, + bootstrapIdentity: staged.bootstrapIdentity, + }); + } + if (backupsExist) { + requireTrustedTransactionPath( + stagedOptions.backupDirectory, + TRANSACTION_DIRECTORY_MODE, + options, + ); + fs.rmSync(stagedOptions.backupDirectory, { force: false, recursive: true }); + fsyncDirectory(options.commitReceiptDirectory); + } + if (manifestExists) { + requireTrustedTransactionPath(stagedOptions.manifestFile, TRANSACTION_FILE_MODE, options); + fs.unlinkSync(stagedOptions.manifestFile); + fsyncDirectory(options.commitReceiptDirectory); + } + const unexpected = fs + .readdirSync(options.commitReceiptDirectory) + .filter((entry) => entry !== MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE); + if (unexpected.length !== 0) { + fail("durable commit receipt directory contains unexpected artifacts"); + } + const verified = loadCommitReceipt(options); + if (!verified?.compact) fail("durable commit receipt did not compact successfully"); + assertCommitReceiptMatches(verified.receipt, state.receipt); +} + export function beginManagedStartupSharedStateTransaction( profile: ManagedStartupProfile, inputOptions: ManagedStartupSharedTransactionOptions = {}, @@ -758,10 +1059,28 @@ export function beginManagedStartupSharedStateTransaction( } requireTransactionBoundaries(options); const profileFingerprint = fingerprintManagedStartupProfile(profile); + const committed = loadCommitReceipt(options); + if (committed) { + if (options.bootstrapIdentity === null) { + fail("a durable managed bootstrap commit receipt already exists"); + } + assertCommitReceiptMatches(committed.receipt, { + agent: profile.agent, + profileFingerprint, + bootstrapIdentity: options.bootstrapIdentity, + }); + fail("this managed bootstrap attempt is already durably committed"); + } const pending = loadManifest(options); if (pending) { - if (pending.agent !== profile.agent || pending.profileFingerprint !== profileFingerprint) { - fail("a pending managed startup transaction belongs to a different profile"); + if ( + pending.agent !== profile.agent || + pending.profileFingerprint !== profileFingerprint || + pending.bootstrapIdentity !== options.bootstrapIdentity + ) { + fail( + "a pending managed startup transaction belongs to a different agent, profile fingerprint, or bootstrap attempt", + ); } verifyAllBackups(pending.files, options); return false; @@ -780,6 +1099,7 @@ export function beginManagedStartupSharedStateTransaction( schemaVersion: TRANSACTION_SCHEMA_VERSION, agent: profile.agent, profileFingerprint, + bootstrapIdentity: options.bootstrapIdentity, files: snapshots.map(({ receipt }) => receipt), directories, }; @@ -801,9 +1121,11 @@ export function beginManagedStartupSharedStateTransaction( }; fs.chownSync(options.transactionDirectory, options.trustedUid, options.trustedGid); fs.chmodSync(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE); + fsyncDirectory(options.transactionParentDirectory); fs.mkdirSync(options.backupDirectory, { mode: TRANSACTION_DIRECTORY_MODE }); fs.chownSync(options.backupDirectory, options.trustedUid, options.trustedGid); fs.chmodSync(options.backupDirectory, TRANSACTION_DIRECTORY_MODE); + fsyncDirectory(options.transactionDirectory); for (const snapshot of snapshots) { if (snapshot.receipt.state !== "file" || snapshot.bytes === null) continue; atomicWriteTrustedFile( @@ -814,6 +1136,7 @@ export function beginManagedStartupSharedStateTransaction( options.trustedGid, ); } + fsyncDirectory(options.backupDirectory); atomicWriteTrustedFile( options.manifestFile, canonicalManifest(manifest), @@ -821,6 +1144,7 @@ export function beginManagedStartupSharedStateTransaction( options.trustedUid, options.trustedGid, ); + fsyncDirectory(options.transactionDirectory); loadManifest(options); } catch (error) { try { @@ -990,11 +1314,25 @@ export function rollbackManagedStartupSharedStateTransaction( ): boolean { const options = resolveOptions(inputOptions); requireTransactionIdentity(options); + const committed = loadCommitReceipt(options); + if (committed) { + if (options.bootstrapIdentity === null) { + fail("shared state is already durably committed"); + } + assertCommitReceiptMatches(committed.receipt, { + agent: expectedAgent, + bootstrapIdentity: options.bootstrapIdentity, + }); + fail("shared state is already durably committed and cannot be rolled back"); + } const manifest = loadManifest(options); if (!manifest) return false; if (manifest.agent !== expectedAgent) { fail(`pending transaction targets ${manifest.agent}, expected ${expectedAgent}`); } + if (manifest.bootstrapIdentity !== options.bootstrapIdentity) { + fail("pending transaction belongs to a different bootstrap attempt"); + } const backups = verifyAllBackups(manifest.files, options); ensureOriginalDirectories(manifest.directories, options); restoreFiles(manifest.files, backups, options); @@ -1015,11 +1353,123 @@ export function commitManagedStartupSharedStateTransaction( if (options.readOnlyReceipt) { fail("cannot commit a read-only rollback receipt"); } + const committed = loadCommitReceipt(options); + if (committed) { + if (options.bootstrapIdentity === null) { + fail("durable commit receipt is missing its expected bootstrap identity"); + } + assertCommitReceiptMatches(committed.receipt, { + agent: expectedAgent, + bootstrapIdentity: options.bootstrapIdentity, + }); + compactDurableCommitReceipt(committed, options); + return true; + } const manifest = loadManifest(options); if (!manifest) return false; if (manifest.agent !== expectedAgent) { fail(`pending transaction targets ${manifest.agent}, expected ${expectedAgent}`); } - removeTransactionDirectory(options); + if (manifest.bootstrapIdentity !== options.bootstrapIdentity) { + fail("pending transaction belongs to a different bootstrap attempt"); + } + if (manifest.bootstrapIdentity === null) { + removeTransactionDirectory(options); + return true; + } + verifyAllBackups(manifest.files, options); + if (pathExistsNoFollow(options.commitReceiptDirectory)) { + fail("durable commit receipt path appeared before transaction commit"); + } + try { + fs.renameSync(options.transactionDirectory, options.commitReceiptDirectory); + fsyncDirectory(options.transactionParentDirectory); + } catch (error) { + fail(`could not atomically establish durable commit state: ${(error as Error).message}`); + } + const renamed = loadCommitReceipt(options); + if (!renamed) fail("durable commit state disappeared after atomic rename"); + assertCommitReceiptMatches(renamed.receipt, { + agent: expectedAgent, + profileFingerprint: manifest.profileFingerprint, + bootstrapIdentity: manifest.bootstrapIdentity, + }); + compactDurableCommitReceipt(renamed, options); return true; } + +/** + * Retire one exact durable bootstrap commit only after the runtime owner has + * proven its external rollback backup is gone. This prevents a completed + * attempt's image-owned receipt from blocking a later bootstrap with a + * different identity in the same persisted workload. + */ +export function clearManagedStartupSharedStateCommitReceipt( + expectedAgent: ManagedStartupAgent, + inputOptions: ManagedStartupSharedTransactionOptions = {}, +): boolean { + const options = resolveOptions(inputOptions); + requireTransactionIdentity(options); + if (options.readOnlyReceipt) { + fail("cannot clear a durable commit from a read-only receipt"); + } + if (options.bootstrapIdentity === null) { + fail("durable commit cleanup requires its bootstrap identity"); + } + const committed = loadCommitReceipt(options); + if (!committed) return false; + assertCommitReceiptMatches(committed.receipt, { + agent: expectedAgent, + bootstrapIdentity: options.bootstrapIdentity, + }); + compactDurableCommitReceipt(committed, options); + requireTrustedTransactionPath( + options.commitReceiptDirectory, + TRANSACTION_DIRECTORY_MODE, + options, + ); + requireTrustedTransactionPath(options.commitReceiptFile, TRANSACTION_FILE_MODE, options); + const entries = fs.readdirSync(options.commitReceiptDirectory); + if (entries.length !== 1 || entries[0] !== MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE) { + fail("durable commit receipt directory contains unexpected artifacts"); + } + fs.rmSync(options.commitReceiptDirectory, { force: false, recursive: true }); + fsyncDirectory(options.transactionParentDirectory); + if (pathExistsNoFollow(options.commitReceiptDirectory)) { + fail("durable commit receipt remained after cleanup"); + } + return true; +} + +export function getManagedStartupSharedStateTransactionStatus( + expected: { + readonly agent: ManagedStartupAgent; + readonly profileFingerprint: string; + readonly bootstrapIdentity: string; + }, + inputOptions: ManagedStartupSharedTransactionOptions = {}, +): "committed" | "none" | "pending" { + const options = resolveOptions({ + ...inputOptions, + bootstrapIdentity: expected.bootstrapIdentity, + }); + requireTransactionIdentity(options); + const manifest = loadManifest(options); + if (manifest) { + if ( + manifest.agent !== expected.agent || + manifest.profileFingerprint !== expected.profileFingerprint || + manifest.bootstrapIdentity !== expected.bootstrapIdentity + ) { + fail( + "pending transaction does not match the expected agent, profile fingerprint, or bootstrap identity", + ); + } + verifyAllBackups(manifest.files, options); + return "pending"; + } + const committed = loadCommitReceipt(options); + if (!committed) return "none"; + assertCommitReceiptMatches(committed.receipt, expected); + return "committed"; +} From bab428daafae8d0a601f28cc5cdb726f41a0d474 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 12:11:22 -0700 Subject: [PATCH 109/117] refactor(onboard): defer bootstrap recovery enumeration Signed-off-by: Aaron Erickson --- .../managed-bootstrap/docker-journal.test.ts | 37 +------------------ .../managed-bootstrap/docker-journal.ts | 29 --------------- .../managed-bootstrap/docker-test-fixture.ts | 1 - 3 files changed, 1 insertion(+), 66 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index 0f3e98e1352..2f7c232f80e 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -279,15 +279,13 @@ describe("Docker managed bootstrap journal", () => { ); }); - it("enumerates unfinished records and reloads exact terminal receipts from a new journal store", () => { + it("reloads exact terminal receipts from a new journal store", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); roots.push(root); const first = createFileDockerManagedBootstrapJournalStore(root); first.create(journal); - expect(first.listUnfinished()).toEqual([journal]); first.recordFinalization(finalization); - expect(first.listUnfinished()).toEqual([]); const restarted = createFileDockerManagedBootstrapJournalStore(root); expect(restarted.loadFinalization(IDENTITY)).toEqual(finalization); expect( @@ -303,25 +301,6 @@ describe("Docker managed bootstrap journal", () => { ).toThrow("finalization record changed"); }); - it("ignores interrupted atomic-write sidecars during unfinished enumeration", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); - roots.push(root); - const store = createFileDockerManagedBootstrapJournalStore(root); - store.create(journal); - const directory = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); - for (const target of [ - `${IDENTITY}.json`, - `${IDENTITY}.json.decision`, - `${IDENTITY}.json.finalized`, - ]) { - fs.writeFileSync(path.join(directory, `.${target}.${process.pid}.abcdef.tmp`), "partial", { - mode: 0o600, - }); - } - - expect(store.listUnfinished()).toEqual([journal]); - }); - it("reloads the exact completion receipt from a new journal store", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); roots.push(root); @@ -332,7 +311,6 @@ describe("Docker managed bootstrap journal", () => { expect(completed.commitReceipt).toEqual(finalization.commitReceipt); const restarted = createFileDockerManagedBootstrapJournalStore(root); - expect(restarted.listUnfinished()).toEqual([completed]); expect(restarted.recordCompletion(IDENTITY, finalization.commitReceipt)).toEqual(completed); expect(() => restarted.recordCompletion(IDENTITY, { @@ -341,17 +319,4 @@ describe("Docker managed bootstrap journal", () => { }), ).toThrow("completion receipt changed"); }); - - it("fails closed when enumeration encounters an unsupported state entry", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); - roots.push(root); - const store = createFileDockerManagedBootstrapJournalStore(root); - store.create(journal); - fs.writeFileSync( - path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, "unexpected.json"), - "{}\n", - { mode: 0o600 }, - ); - expect(() => store.listUnfinished()).toThrow("unsupported entry"); - }); }); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index cffa3dc0b8c..015bda1d12d 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -70,7 +70,6 @@ export interface DockerManagedBootstrapFinalizationRecord { export interface DockerManagedBootstrapJournalStore { create(journal: DockerManagedBootstrapJournal): void; load(bootstrapIdentity: string): DockerManagedBootstrapJournal | null; - listUnfinished(): readonly DockerManagedBootstrapJournal[]; transition( bootstrapIdentity: string, expected: DockerManagedBootstrapJournalPhase, @@ -778,34 +777,6 @@ export function createFileDockerManagedBootstrapJournalStore( atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(normalized), true); }, load, - listUnfinished() { - assertDirectory(directory); - const identities: string[] = []; - for (const name of fs.readdirSync(directory)) { - const match = name.match(/^([a-f0-9]{64})\.json$/u); - if (match) { - identities.push(match[1]); - continue; - } - if ( - /^\.[a-f0-9]{64}\.json(?:\.decision|\.finalized)?\.[0-9]+\.[a-f0-9]+\.tmp$/u.test(name) || - /^[a-f0-9]{64}\.json\.(?:decision|finalized)$/u.test(name) - ) { - continue; - } - fail(`journal directory contains an unsupported entry: ${name}`); - } - return Object.freeze( - identities - .sort() - .filter((identity) => loadFinalization(identity) === null) - .map((identity) => { - const journal = load(identity); - if (!journal) fail(`enumerated journal ${identity} disappeared`); - return journal; - }), - ); - }, transition( bootstrapIdentity: string, expected: DockerManagedBootstrapJournalPhase, diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index 7f64895bd74..088a3ed7a05 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -230,7 +230,6 @@ export function fixture(options: DockerFixtureOptions = {}) { } }, load: () => copyJournal(), - listUnfinished: () => (journal && !finalization ? [structuredClone(journal)] : []), transition(_identity, expected, next) { const current = journal !== null && journal.phase === expected From a3571b2fb794a6bf082f05146fb24096d448548e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 12:17:37 -0700 Subject: [PATCH 110/117] docs(onboard): bind recovery enumeration to its consumer Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index f25082b0a76..b5f8434f79f 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -86,8 +86,8 @@ publishes `rollback-authorized` before exact replacement deletion; pre-cutover staged cleanup removes only the exact prepared replacement without that journal transition. Commit publishes `shared-state-committed` before exact backup deletion. Cleanup is bound to full runtime IDs. Its private state root retains -enumerable, versioned unfinished records containing the provider and sandbox -identities, plan and profile +versioned, identity-addressed transaction records containing the provider and +sandbox identities, plan and profile fingerprints, exact original and replacement IDs, rollback target, and phase. Exact commit and cleanup receipts are durable terminal records, so adapter recreation does not depend on process-local transaction sets or tombstone maps. @@ -97,14 +97,14 @@ namespace, compacts that state to an exact commit receipt, and rejects rollback when a later image-runtime invocation reads that receipt. The provider may retire that receipt only after it proves the external rollback backup is gone, so that this receipt does not block the next bootstrap attempt. -Enumeration reconstructs only unfinished records. The bounded -[3.12b recovery slice](https://github.com/NVIDIA/NemoClaw/issues/7744) owns phase -reconciliation and cross-surface resume or rollback. The adapter reads mutable -OpenShell names only to detect ownership reuse. Unsafe name-only deletion returns -a typed retention error. The dormant adapter assumes the protocol's single -coordinator; multi-process lease/arbitration remains an explicit -production-activation gate. Activation must also inject the selected gateway's -canonical state root. +Direct identity lookup reconstructs one known transaction record. The bounded +[3.12b recovery slice](https://github.com/NVIDIA/NemoClaw/issues/7744) introduces +unfinished-record enumeration together with phase reconciliation and +cross-surface resume or rollback. The adapter reads mutable OpenShell names only +to detect ownership reuse. Unsafe name-only deletion returns a typed retention +error. The dormant adapter assumes the protocol's single coordinator; +multi-process lease/arbitration remains an explicit production-activation gate. +Activation must also inject the selected gateway's canonical state root. ## Architectural disposition From 79fd30710d194b619fe579d3daa201fec10a38e5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 12:48:40 -0700 Subject: [PATCH 111/117] feat(onboard): persist managed bootstrap finalization Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/README.md | 21 +- src/lib/onboard/managed-bootstrap/adapter.ts | 8 +- .../managed-bootstrap/docker-journal.test.ts | 155 ++++++ .../managed-bootstrap/docker-journal.ts | 432 +++++++++++++++- .../managed-bootstrap/docker-shared-state.ts | 3 +- .../managed-bootstrap/docker-test-fixture.ts | 47 +- .../onboard/managed-bootstrap/docker.test.ts | 102 +++- src/lib/onboard/managed-bootstrap/docker.ts | 325 ++++++++---- ...d-startup-shared-state-transaction.test.ts | 251 +++++++++- .../onboard/managed-startup/image-runtime.ts | 53 +- .../shared-state-transaction.ts | 466 +++++++++++++++++- 11 files changed, 1743 insertions(+), 120 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 159ab401e86..b5f8434f79f 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -85,9 +85,24 @@ and sandbox ID and then enter the destructive cutover. Post-cutover rollback publishes `rollback-authorized` before exact replacement deletion; pre-cutover staged cleanup removes only the exact prepared replacement without that journal transition. Commit publishes `shared-state-committed` before exact backup -deletion. Cleanup is bound to full runtime IDs. Mutable OpenShell names are read -only to detect ownership reuse, and unsafe name-only deletion returns a typed -retention error. The dormant adapter assumes the protocol's single coordinator; +deletion. Cleanup is bound to full runtime IDs. Its private state root retains +versioned, identity-addressed transaction records containing the provider and +sandbox identities, plan and profile +fingerprints, exact original and replacement IDs, rollback target, and phase. +Exact commit and cleanup receipts are durable terminal records, so adapter +recreation does not depend on process-local transaction sets or tombstone maps. +The image-owned shared-state transaction uses the same identity-bound model: a +commit atomically moves its pending manifest and backups into a durable receipt +namespace, compacts that state to an exact commit receipt, and rejects rollback +when a later image-runtime invocation reads that receipt. The provider may +retire that receipt only after it proves the external rollback backup is gone, +so that this receipt does not block the next bootstrap attempt. +Direct identity lookup reconstructs one known transaction record. The bounded +[3.12b recovery slice](https://github.com/NVIDIA/NemoClaw/issues/7744) introduces +unfinished-record enumeration together with phase reconciliation and +cross-surface resume or rollback. The adapter reads mutable OpenShell names only +to detect ownership reuse. Unsafe name-only deletion returns a typed retention +error. The dormant adapter assumes the protocol's single coordinator; multi-process lease/arbitration remains an explicit production-activation gate. Activation must also inject the selected gateway's canonical state root. diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index 415fced86cd..ee1a834985f 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -933,13 +933,15 @@ function normalizePreparedReplacement( }); } +export function createManagedBootstrapPlanFingerprint(plan: ManagedBootstrapExpectedPlan): string { + return createHash("sha256").update(canonicalJson(plan), "utf8").digest("hex"); +} + export function createManagedBootstrapPreparedAuthority( transaction: ManagedBootstrapPreparedTransaction, ): ManagedBootstrapPreparedAuthority { const { handle, snapshot, prepared } = transaction; - const planFingerprint = createHash("sha256") - .update(canonicalJson(handle.plan), "utf8") - .digest("hex"); + const planFingerprint = createManagedBootstrapPlanFingerprint(handle.plan); const bound = Object.freeze({ schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, phase: "prepared" as const, diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index 7ed0d8bba6a..25cd9ec0e50 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -9,10 +9,15 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapFinalizationRecord, type DockerManagedBootstrapJournal, + parseDockerManagedBootstrapFinalizationRecord, parseDockerManagedBootstrapJournal, + sameDockerManagedBootstrapReceipt, + serializeDockerManagedBootstrapFinalizationRecord, serializeDockerManagedBootstrapJournal, } from "./docker-journal"; @@ -22,11 +27,13 @@ const journal = Object.freeze({ schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, phase: "staged", bootstrapIdentity: IDENTITY, + providerId: "docker", sandbox: { sandboxName: "alpha", sandboxId: "sandbox-alpha", driverId: "docker", }, + planFingerprint: "9".repeat(64), profileFingerprint: "2".repeat(64), imageReference: `registry.example/image@sha256:${"3".repeat(64)}`, runtimeImageContentId: `sha256:${"4".repeat(64)}`, @@ -37,7 +44,59 @@ const journal = Object.freeze({ backupName: "openshell-alpha-backup", originalSpecHash: "7".repeat(64), replacementSpecHash: "8".repeat(64), + rollbackTargetRuntimeId: "5".repeat(64), + rollbackTargetSpecHash: "7".repeat(64), + preparationReceipt: { + schemaVersion: 1, + sandbox: { + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + driverId: "docker", + }, + bootstrapIdentity: IDENTITY, + authorityFingerprint: "a".repeat(64), + recordId: "prepared-alpha", + recordedAt: "2026-07-31T19:59:59.000Z", + }, + commitReceipt: null, } satisfies DockerManagedBootstrapJournal); +const finalization = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase: "committed", + bootstrapIdentity: IDENTITY, + providerId: "docker", + sandbox: journal.sandbox, + planFingerprint: journal.planFingerprint, + profileFingerprint: journal.profileFingerprint, + imageReference: journal.imageReference, + commitReceipt: { + schemaVersion: 1, + sandbox: journal.sandbox, + runtimeId: journal.replacementRuntimeId, + image: { + repository: "registry.example/image", + manifestDigest: `sha256:${"3".repeat(64)}` as const, + }, + runtimeImageContentId: journal.runtimeImageContentId, + originalSpecHash: journal.originalSpecHash, + replacementSpecHash: journal.replacementSpecHash, + profileFingerprint: journal.profileFingerprint, + bootstrapIdentity: IDENTITY, + transactionPending: false, + completedAt: "2026-07-31T20:00:00.000Z", + }, + cleanupReceipt: { + schemaVersion: 1, + sandbox: journal.sandbox, + bootstrapIdentity: IDENTITY, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: "2026-07-31T20:00:01.000Z", + }, +} satisfies DockerManagedBootstrapFinalizationRecord); function readPinnedPrivateFile(target: string): { readonly mode: number; readonly text: string } { const descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); @@ -78,6 +137,9 @@ describe("Docker managed bootstrap journal", () => { ); expect(store.transition(IDENTITY, "staged", "cutover").phase).toBe("cutover"); + expect(store.recordCompletion(IDENTITY, finalization.commitReceipt).commitReceipt).toEqual( + finalization.commitReceipt, + ); expect(store.transition(IDENTITY, "cutover", "shared-state-committed").phase).toBe( "shared-state-committed", ); @@ -172,4 +234,97 @@ describe("Docker managed bootstrap journal", () => { serializeDockerManagedBootstrapJournal(Object.freeze({ ...journal, phase: "staged" })), ).toBe(`${JSON.stringify(journal)}\n`); }); + + it("compares equivalent receipts independent of property insertion order", () => { + const preparation = journal.preparationReceipt; + const reorderedPreparation = { + recordedAt: preparation.recordedAt, + recordId: preparation.recordId, + authorityFingerprint: preparation.authorityFingerprint, + bootstrapIdentity: preparation.bootstrapIdentity, + sandbox: { + driverId: preparation.sandbox.driverId, + sandboxId: preparation.sandbox.sandboxId, + sandboxName: preparation.sandbox.sandboxName, + }, + schemaVersion: preparation.schemaVersion, + } satisfies typeof preparation; + expect( + sameDockerManagedBootstrapReceipt("preparation", preparation, reorderedPreparation), + ).toBe(true); + + const completion = finalization.commitReceipt; + const reorderedCompletion = { + completedAt: completion.completedAt, + transactionPending: completion.transactionPending, + bootstrapIdentity: completion.bootstrapIdentity, + profileFingerprint: completion.profileFingerprint, + replacementSpecHash: completion.replacementSpecHash, + originalSpecHash: completion.originalSpecHash, + runtimeImageContentId: completion.runtimeImageContentId, + image: { + manifestDigest: completion.image.manifestDigest, + repository: completion.image.repository, + }, + runtimeId: completion.runtimeId, + sandbox: { + driverId: completion.sandbox.driverId, + sandboxId: completion.sandbox.sandboxId, + sandboxName: completion.sandbox.sandboxName, + }, + schemaVersion: completion.schemaVersion, + } satisfies typeof completion; + expect(sameDockerManagedBootstrapReceipt("completion", completion, reorderedCompletion)).toBe( + true, + ); + }); + + it("reloads exact terminal receipts from a new journal store", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const first = createFileDockerManagedBootstrapJournalStore(root); + first.create(journal); + + first.recordFinalization(finalization); + const restarted = createFileDockerManagedBootstrapJournalStore(root); + expect(restarted.loadFinalization(IDENTITY)).toEqual(finalization); + expect( + parseDockerManagedBootstrapFinalizationRecord( + serializeDockerManagedBootstrapFinalizationRecord(finalization), + ), + ).toEqual(finalization); + expect(() => + restarted.recordFinalization({ + ...finalization, + cleanupReceipt: { ...finalization.cleanupReceipt, finalizedAt: "2026-07-31T20:00:02.000Z" }, + }), + ).toThrow("finalization record changed"); + expect(() => + restarted.recordFinalization({ + ...finalization, + phase: "rolled-back", + commitReceipt: null, + cleanupReceipt: { ...finalization.cleanupReceipt, outcome: "rolled-back" }, + }), + ).toThrow("finalization record changed"); + }); + + it("reloads the exact completion receipt from a new journal store", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const first = createFileDockerManagedBootstrapJournalStore(root); + first.create(journal); + first.transition(IDENTITY, "staged", "cutover"); + const completed = first.recordCompletion(IDENTITY, finalization.commitReceipt); + expect(completed.commitReceipt).toEqual(finalization.commitReceipt); + + const restarted = createFileDockerManagedBootstrapJournalStore(root); + expect(restarted.recordCompletion(IDENTITY, finalization.commitReceipt)).toEqual(completed); + expect(() => + restarted.recordCompletion(IDENTITY, { + ...finalization.commitReceipt, + completedAt: "2026-07-31T20:00:02.000Z", + }), + ).toThrow("completion receipt changed"); + }); }); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index c6043409d7d..015bda1d12d 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -4,12 +4,19 @@ import fs from "node:fs"; import path from "node:path"; -import type { ManagedBootstrapSandboxIdentity } from "./adapter"; +import type { + ManagedBootstrapCompletionReceipt, + ManagedBootstrapDurablePreparationReceipt, + ManagedBootstrapFinalizationReceipt, + ManagedBootstrapSandboxIdentity, +} from "./adapter"; -export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 1 as const; +export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 2 as const; export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY = "managed-bootstrap"; +export const DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION = 1 as const; const SHA256_RE = /^[a-f0-9]{64}$/u; +const MANIFEST_DIGEST_RE = /^sha256:[a-f0-9]{64}$/u; const MAX_JOURNAL_BYTES = 32 * 1024; const JOURNAL_DIRECTORY_MODE = 0o700; const JOURNAL_FILE_MODE = 0o600; @@ -28,7 +35,9 @@ export interface DockerManagedBootstrapJournal { readonly schemaVersion: typeof DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION; readonly phase: DockerManagedBootstrapJournalPhase; readonly bootstrapIdentity: string; + readonly providerId: string; readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly planFingerprint: string; readonly profileFingerprint: string; readonly imageReference: string; readonly runtimeImageContentId: string; @@ -39,6 +48,23 @@ export interface DockerManagedBootstrapJournal { readonly backupName: string; readonly originalSpecHash: string; readonly replacementSpecHash: string; + readonly rollbackTargetRuntimeId: string; + readonly rollbackTargetSpecHash: string; + readonly preparationReceipt: ManagedBootstrapDurablePreparationReceipt | null; + readonly commitReceipt: ManagedBootstrapCompletionReceipt | null; +} + +export interface DockerManagedBootstrapFinalizationRecord { + readonly schemaVersion: typeof DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION; + readonly phase: "committed" | "rolled-back"; + readonly bootstrapIdentity: string; + readonly providerId: string; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly planFingerprint: string; + readonly profileFingerprint: string; + readonly imageReference: string; + readonly commitReceipt: ManagedBootstrapCompletionReceipt | null; + readonly cleanupReceipt: ManagedBootstrapFinalizationReceipt; } export interface DockerManagedBootstrapJournalStore { @@ -49,7 +75,13 @@ export interface DockerManagedBootstrapJournalStore { expected: DockerManagedBootstrapJournalPhase, next: DockerManagedBootstrapJournalPhase, ): DockerManagedBootstrapJournal; + recordCompletion( + bootstrapIdentity: string, + receipt: ManagedBootstrapCompletionReceipt, + ): DockerManagedBootstrapJournal; remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]): void; + recordFinalization(record: DockerManagedBootstrapFinalizationRecord): void; + loadFinalization(bootstrapIdentity: string): DockerManagedBootstrapFinalizationRecord | null; } /** @@ -137,15 +169,21 @@ export function normalizeDockerManagedBootstrapJournal( const expectedKeys = [ "backupName", "bootstrapIdentity", + "commitReceipt", "imageReference", "originalName", "originalRuntimeId", "originalSpecHash", "phase", + "planFingerprint", + "preparationReceipt", "profileFingerprint", + "providerId", "replacementRuntimeId", "replacementSpecHash", "replacementStagingName", + "rollbackTargetRuntimeId", + "rollbackTargetSpecHash", "runtimeImageContentId", "sandbox", "schemaVersion", @@ -160,7 +198,9 @@ export function normalizeDockerManagedBootstrapJournal( schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, phase: exactPhase(journal.phase), bootstrapIdentity: exactSha256(journal.bootstrapIdentity, "bootstrap identity"), + providerId: exactString(journal.providerId, "provider ID"), sandbox: exactSandbox(journal.sandbox), + planFingerprint: exactSha256(journal.planFingerprint, "plan fingerprint"), profileFingerprint: exactSha256(journal.profileFingerprint, "profile fingerprint"), imageReference: exactString(journal.imageReference, "image reference"), runtimeImageContentId: exactString(journal.runtimeImageContentId, "runtime image content ID"), @@ -175,6 +215,20 @@ export function normalizeDockerManagedBootstrapJournal( backupName: exactString(journal.backupName, "backup name", 253), originalSpecHash: exactSha256(journal.originalSpecHash, "original spec hash"), replacementSpecHash: exactSha256(journal.replacementSpecHash, "replacement spec hash"), + rollbackTargetRuntimeId: exactSha256( + journal.rollbackTargetRuntimeId, + "rollback target runtime ID", + ), + rollbackTargetSpecHash: exactSha256( + journal.rollbackTargetSpecHash, + "rollback target spec hash", + ), + preparationReceipt: + journal.preparationReceipt === null + ? null + : exactPreparationReceipt(journal.preparationReceipt), + commitReceipt: + journal.commitReceipt === null ? null : exactCompletionReceipt(journal.commitReceipt), } satisfies DockerManagedBootstrapJournal); if (normalized.originalRuntimeId === normalized.replacementRuntimeId) { fail("original and replacement runtime IDs must differ"); @@ -185,6 +239,33 @@ export function normalizeDockerManagedBootstrapJournal( ) { fail("original, staging, and backup names must be distinct"); } + if ( + normalized.providerId !== normalized.sandbox.driverId || + normalized.rollbackTargetRuntimeId !== normalized.originalRuntimeId || + normalized.rollbackTargetSpecHash !== normalized.originalSpecHash + ) { + fail("provider or rollback authority does not match the transaction identity"); + } + if ( + (normalized.preparationReceipt !== null && + (normalized.preparationReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + normalized.preparationReceipt.sandbox.sandboxName !== normalized.sandbox.sandboxName || + normalized.preparationReceipt.sandbox.sandboxId !== normalized.sandbox.sandboxId || + normalized.preparationReceipt.sandbox.driverId !== normalized.sandbox.driverId)) || + (normalized.commitReceipt !== null && + (normalized.commitReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + normalized.commitReceipt.sandbox.sandboxName !== normalized.sandbox.sandboxName || + normalized.commitReceipt.sandbox.sandboxId !== normalized.sandbox.sandboxId || + normalized.commitReceipt.sandbox.driverId !== normalized.sandbox.driverId || + normalized.commitReceipt.runtimeId !== normalized.replacementRuntimeId || + normalized.commitReceipt.profileFingerprint !== normalized.profileFingerprint || + normalized.commitReceipt.originalSpecHash !== normalized.originalSpecHash || + normalized.commitReceipt.replacementSpecHash !== normalized.replacementSpecHash || + `${normalized.commitReceipt.image.repository}@${normalized.commitReceipt.image.manifestDigest}` !== + normalized.imageReference)) + ) { + fail("durable preparation or commit receipt does not match the transaction identity"); + } return normalized; } @@ -220,6 +301,275 @@ export function parseDockerManagedBootstrapJournal(text: string): DockerManagedB return journal; } +function exactTimestamp(value: unknown, label: string): string { + const timestamp = exactString(value, label, 128); + const parsed = new Date(timestamp); + if (!Number.isFinite(parsed.getTime()) || parsed.toISOString() !== timestamp) { + fail(`${label} must be one canonical timestamp`); + } + return timestamp; +} + +function exactBoolean(value: unknown, label: string): boolean { + if (typeof value !== "boolean") fail(`${label} must be boolean`); + return value; +} + +function exactNullableSha256(value: unknown, label: string): string | null { + return value === null ? null : exactSha256(value, label); +} + +function exactImage(value: unknown): ManagedBootstrapCompletionReceipt["image"] { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("completion image identity must be an object"); + } + const image = value as Record; + if (Object.keys(image).sort().join(",") !== "manifestDigest,repository") { + fail("completion image identity schema is invalid"); + } + const manifestDigest = exactString(image.manifestDigest, "completion manifest digest", 128); + if (!MANIFEST_DIGEST_RE.test(manifestDigest)) { + fail("completion manifest digest must be canonical sha256"); + } + return Object.freeze({ + repository: exactString(image.repository, "completion image repository"), + manifestDigest: manifestDigest as `sha256:${string}`, + }); +} + +function exactPreparationReceipt(value: unknown): ManagedBootstrapDurablePreparationReceipt { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("durable preparation receipt must be an object"); + } + const receipt = value as Record; + const expectedKeys = [ + "authorityFingerprint", + "bootstrapIdentity", + "recordId", + "recordedAt", + "sandbox", + "schemaVersion", + ]; + if ( + Object.keys(receipt).sort().join(",") !== expectedKeys.sort().join(",") || + receipt.schemaVersion !== 1 + ) { + fail("durable preparation receipt schema is invalid"); + } + return Object.freeze({ + schemaVersion: 1, + sandbox: exactSandbox(receipt.sandbox), + bootstrapIdentity: exactSha256( + receipt.bootstrapIdentity, + "durable preparation bootstrap identity", + ), + authorityFingerprint: exactSha256( + receipt.authorityFingerprint, + "durable preparation authority fingerprint", + ), + recordId: exactString(receipt.recordId, "durable preparation record ID", 1024), + recordedAt: exactTimestamp(receipt.recordedAt, "durable preparation timestamp"), + }); +} + +function exactCompletionReceipt(value: unknown): ManagedBootstrapCompletionReceipt { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("commit receipt must be an object"); + } + const receipt = value as Record; + const expectedKeys = [ + "bootstrapIdentity", + "completedAt", + "image", + "originalSpecHash", + "profileFingerprint", + "replacementSpecHash", + "runtimeId", + "runtimeImageContentId", + "sandbox", + "schemaVersion", + "transactionPending", + ]; + if ( + Object.keys(receipt).sort().join(",") !== expectedKeys.sort().join(",") || + receipt.schemaVersion !== 1 + ) { + fail("commit receipt schema is invalid"); + } + return Object.freeze({ + schemaVersion: 1, + sandbox: exactSandbox(receipt.sandbox), + runtimeId: exactSha256(receipt.runtimeId, "commit runtime ID"), + image: exactImage(receipt.image), + runtimeImageContentId: exactString( + receipt.runtimeImageContentId, + "commit runtime image content ID", + ), + originalSpecHash: exactSha256(receipt.originalSpecHash, "commit original spec hash"), + replacementSpecHash: exactSha256(receipt.replacementSpecHash, "commit replacement spec hash"), + profileFingerprint: exactSha256(receipt.profileFingerprint, "commit profile fingerprint"), + bootstrapIdentity: exactSha256(receipt.bootstrapIdentity, "commit bootstrap identity"), + transactionPending: exactBoolean(receipt.transactionPending, "commit transaction pending"), + completedAt: exactTimestamp(receipt.completedAt, "commit completion timestamp"), + }); +} + +export function sameDockerManagedBootstrapReceipt( + kind: "preparation", + left: ManagedBootstrapDurablePreparationReceipt, + right: ManagedBootstrapDurablePreparationReceipt, +): boolean; +export function sameDockerManagedBootstrapReceipt( + kind: "completion", + left: ManagedBootstrapCompletionReceipt, + right: ManagedBootstrapCompletionReceipt, +): boolean; +export function sameDockerManagedBootstrapReceipt( + kind: "preparation" | "completion", + left: ManagedBootstrapDurablePreparationReceipt | ManagedBootstrapCompletionReceipt, + right: ManagedBootstrapDurablePreparationReceipt | ManagedBootstrapCompletionReceipt, +): boolean { + if (kind === "preparation") { + return ( + JSON.stringify(exactPreparationReceipt(left)) === + JSON.stringify(exactPreparationReceipt(right)) + ); + } + return ( + JSON.stringify(exactCompletionReceipt(left)) === JSON.stringify(exactCompletionReceipt(right)) + ); +} + +function exactCleanupReceipt(value: unknown): ManagedBootstrapFinalizationReceipt { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("cleanup receipt must be an object"); + } + const receipt = value as Record; + const expectedKeys = [ + "alreadyRolledBack", + "bootstrapIdentity", + "finalizedAt", + "heldWorkloadRemoved", + "outcome", + "restoredRuntimeId", + "restoredSpecHash", + "sandbox", + "schemaVersion", + ]; + if ( + Object.keys(receipt).sort().join(",") !== expectedKeys.sort().join(",") || + receipt.schemaVersion !== 1 || + !["committed", "rolled-back"].includes(String(receipt.outcome)) + ) { + fail("cleanup receipt schema is invalid"); + } + return Object.freeze({ + schemaVersion: 1, + sandbox: exactSandbox(receipt.sandbox), + bootstrapIdentity: exactSha256(receipt.bootstrapIdentity, "cleanup bootstrap identity"), + outcome: receipt.outcome as "committed" | "rolled-back", + restoredRuntimeId: exactNullableSha256(receipt.restoredRuntimeId, "restored runtime ID"), + restoredSpecHash: exactNullableSha256(receipt.restoredSpecHash, "restored spec hash"), + heldWorkloadRemoved: exactBoolean(receipt.heldWorkloadRemoved, "held workload removed"), + alreadyRolledBack: exactBoolean(receipt.alreadyRolledBack, "already rolled back"), + finalizedAt: exactTimestamp(receipt.finalizedAt, "cleanup finalization timestamp"), + }); +} + +export function normalizeDockerManagedBootstrapFinalizationRecord( + value: unknown, +): DockerManagedBootstrapFinalizationRecord { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("finalization record must be an object"); + } + const record = value as Record; + const expectedKeys = [ + "bootstrapIdentity", + "cleanupReceipt", + "commitReceipt", + "imageReference", + "phase", + "planFingerprint", + "profileFingerprint", + "providerId", + "sandbox", + "schemaVersion", + ]; + if ( + Object.keys(record).sort().join(",") !== expectedKeys.sort().join(",") || + record.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION || + !["committed", "rolled-back"].includes(String(record.phase)) + ) { + fail("finalization record schema is invalid"); + } + const phase = record.phase as "committed" | "rolled-back"; + const sandbox = exactSandbox(record.sandbox); + const commitReceipt = + record.commitReceipt === null ? null : exactCompletionReceipt(record.commitReceipt); + const cleanupReceipt = exactCleanupReceipt(record.cleanupReceipt); + const normalized = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase, + bootstrapIdentity: exactSha256(record.bootstrapIdentity, "finalization bootstrap identity"), + providerId: exactString(record.providerId, "finalization provider ID"), + sandbox, + planFingerprint: exactSha256(record.planFingerprint, "finalization plan fingerprint"), + profileFingerprint: exactSha256(record.profileFingerprint, "finalization profile fingerprint"), + imageReference: exactString(record.imageReference, "finalization image reference"), + commitReceipt, + cleanupReceipt, + } satisfies DockerManagedBootstrapFinalizationRecord); + if ( + normalized.providerId !== sandbox.driverId || + normalized.bootstrapIdentity !== cleanupReceipt.bootstrapIdentity || + normalized.phase !== cleanupReceipt.outcome || + JSON.stringify(normalized.sandbox) !== JSON.stringify(cleanupReceipt.sandbox) || + (phase === "committed") !== (commitReceipt !== null) || + (commitReceipt !== null && + (commitReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + commitReceipt.profileFingerprint !== normalized.profileFingerprint || + JSON.stringify(commitReceipt.sandbox) !== JSON.stringify(normalized.sandbox) || + `${commitReceipt.image.repository}@${commitReceipt.image.manifestDigest}` !== + normalized.imageReference)) + ) { + fail("finalization receipts do not match their durable transaction identity"); + } + return normalized; +} + +export function serializeDockerManagedBootstrapFinalizationRecord( + record: DockerManagedBootstrapFinalizationRecord, +): string { + const serialized = `${JSON.stringify(normalizeDockerManagedBootstrapFinalizationRecord(record))}\n`; + if (Buffer.byteLength(serialized, "utf8") > MAX_JOURNAL_BYTES) { + fail("serialized finalization record exceeds its bounded transport"); + } + return serialized; +} + +export function parseDockerManagedBootstrapFinalizationRecord( + text: string, +): DockerManagedBootstrapFinalizationRecord { + if ( + text.length === 0 || + text.includes("\0") || + Buffer.byteLength(text, "utf8") > MAX_JOURNAL_BYTES + ) { + fail("serialized finalization record is empty or too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("serialized finalization record is not valid JSON"); + } + const record = normalizeDockerManagedBootstrapFinalizationRecord(parsed); + if (serializeDockerManagedBootstrapFinalizationRecord(record) !== text) { + fail("serialized finalization record is not canonical"); + } + return record; +} + function assertDirectory(directory: string): void { fs.mkdirSync(directory, { recursive: true, mode: JOURNAL_DIRECTORY_MODE }); const stat = fs.lstatSync(directory); @@ -237,6 +587,10 @@ function decisionPath(target: string): string { return `${target}.decision`; } +function finalizationPath(target: string): string { + return `${target}.finalized`; +} + function sameStableMetadata(left: fs.BigIntStats, right: fs.BigIntStats): boolean { return ( left.dev === right.dev && @@ -361,6 +715,15 @@ function atomicWrite( if (cleanupFailure !== null) throw cleanupFailure.error; } +function sameSerializedJournal( + left: DockerManagedBootstrapJournal, + right: DockerManagedBootstrapJournal, +): boolean { + return ( + serializeDockerManagedBootstrapJournal(left) === serializeDockerManagedBootstrapJournal(right) + ); +} + export function createFileDockerManagedBootstrapJournalStore( stateRoot: string, ): DockerManagedBootstrapJournalStore { @@ -386,9 +749,26 @@ export function createFileDockerManagedBootstrapJournalStore( } return decided; }; + const loadFinalization = ( + bootstrapIdentity: string, + ): DockerManagedBootstrapFinalizationRecord | null => { + assertDirectory(directory); + const contents = readPrivateFile( + finalizationPath(journalPath(directory, bootstrapIdentity)), + "finalization", + ); + return contents === null ? null : parseDockerManagedBootstrapFinalizationRecord(contents); + }; return Object.freeze({ create(journal: DockerManagedBootstrapJournal) { const normalized = normalizeDockerManagedBootstrapJournal(journal); + if ( + normalized.phase !== "staged" || + normalized.preparationReceipt === null || + normalized.commitReceipt !== null + ) { + fail("a new journal requires staged durable preparation authority"); + } assertDirectory(directory); const target = journalPath(directory, normalized.bootstrapIdentity); if (readPrivateFile(decisionPath(target), "decision") !== null) { @@ -429,6 +809,33 @@ export function createFileDockerManagedBootstrapJournalStore( atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(updated), false); return updated; }, + recordCompletion( + bootstrapIdentity: string, + receipt: ManagedBootstrapCompletionReceipt, + ): DockerManagedBootstrapJournal { + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const current = load(bootstrapIdentity); + if (!current || current.phase !== "cutover") { + fail(`completion recording requires phase cutover, found ${current?.phase ?? "absent"}`); + } + const updated = normalizeDockerManagedBootstrapJournal({ + ...current, + commitReceipt: receipt, + }); + if (current.commitReceipt !== null) { + if (!sameSerializedJournal(current, updated)) { + fail("completion receipt changed for this bootstrap identity"); + } + return current; + } + atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(updated), false); + const persisted = load(bootstrapIdentity); + if (!persisted || !sameSerializedJournal(persisted, updated)) { + fail("completion receipt was not durably re-readable"); + } + return persisted; + }, remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]) { assertDirectory(directory); const target = journalPath(directory, bootstrapIdentity); @@ -444,5 +851,26 @@ export function createFileDockerManagedBootstrapJournalStore( fs.unlinkSync(target); fsyncDirectory(directory); }, + recordFinalization(record: DockerManagedBootstrapFinalizationRecord) { + const normalized = normalizeDockerManagedBootstrapFinalizationRecord(record); + assertDirectory(directory); + const target = finalizationPath(journalPath(directory, normalized.bootstrapIdentity)); + const serialized = serializeDockerManagedBootstrapFinalizationRecord(normalized); + const existing = readPrivateFile(target, "finalization"); + if (existing !== null) { + if (existing !== serialized) + fail("finalization record changed for this bootstrap identity"); + return; + } + try { + atomicWrite(directory, target, serialized, true); + } catch (error) { + if (readPrivateFile(target, "finalization") !== serialized) throw error; + } + if (readPrivateFile(target, "finalization") !== serialized) { + fail("finalization record was not durably re-readable"); + } + }, + loadFinalization, }); } diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts index 415fe56fb82..a92d2e335ed 100644 --- a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts @@ -17,6 +17,7 @@ import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "../docker-gpu-pat import { MANAGED_STARTUP_RUNTIME_EXECUTABLE } from "../managed-startup/image-runtime"; import { MANAGED_STARTUP_AGENTS, type ManagedStartupAgent } from "../managed-startup/profile"; import { + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, } from "../managed-startup/shared-state-transaction"; @@ -24,8 +25,6 @@ import { isImmutableDockerImageId } from "../openshell-docker-sandbox-containers import { cleanupTempDir, secureTempFile } from "../temp-files"; const RECEIPT_TEMP_PREFIX = "nemoclaw-managed-startup-receipt"; -const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY = - "/var/lib/nemoclaw/managed-startup-shared-state-commit-v1"; const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; const DURABLE_IDENTITY_RE = /^[a-f0-9]{64}$/u; const DOCKER_MUTATION_OPTIONS = { diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index cb069cb5bca..088a3ed7a05 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -21,13 +21,18 @@ import { } from "./adapter"; import type { DockerManagedBootstrapDeps } from "./docker"; import { + type DockerManagedBootstrapFinalizationRecord, type DockerManagedBootstrapJournal, DockerManagedBootstrapJournalAcknowledgementLostError, type DockerManagedBootstrapJournalPhase, type DockerManagedBootstrapJournalStore, } from "./docker-journal"; import { normalizeDockerManagedBootstrapLaunchSpec } from "./docker-spec"; -import { parseManagedBootstrapEnvelope } from "./envelope"; +import { + MANAGED_BOOTSTRAP_COMPLETION_FILE, + parseManagedBootstrapEnvelope, + serializeManagedBootstrapImageCompletion, +} from "./envelope"; export const IDENTITY = "1".repeat(64); export const OLD_ID = "2".repeat(64); @@ -206,6 +211,7 @@ export function fixture(options: DockerFixtureOptions = {}) { let original: DockerContainerInspect | null = originalInspect(agentInputs(options.agent)); let replacement: DockerContainerInspect | null = null; let journal: DockerManagedBootstrapJournal | null = null; + let finalization: DockerManagedBootstrapFinalizationRecord | null = null; let sharedState: "committed" | "none" | "pending" = options.sharedState ?? "none"; const events: string[] = []; const lostAcknowledgements = new Set(options.lostAcknowledgements ?? []); @@ -240,6 +246,20 @@ export function fixture(options: DockerFixtureOptions = {}) { } return structuredClone(journal); }, + recordCompletion(_identity, receipt) { + if (!journal || journal.phase !== "cutover") { + throw new Error("completion requires cutover journal"); + } + if ( + journal.commitReceipt !== null && + JSON.stringify(journal.commitReceipt) !== JSON.stringify(receipt) + ) { + throw new Error("completion changed"); + } + journal = { ...journal, commitReceipt: structuredClone(receipt) }; + events.push("journal:completion"); + return structuredClone(journal); + }, remove(_identity, expected) { const current = journal; void (current !== null && expected.includes(current.phase) @@ -253,6 +273,14 @@ export function fixture(options: DockerFixtureOptions = {}) { ); } }, + recordFinalization(value) { + if (finalization && JSON.stringify(finalization) !== JSON.stringify(value)) { + throw new Error("finalization changed"); + } + finalization = structuredClone(value); + events.push(`finalization:${value.phase}`); + }, + loadFinalization: () => (finalization ? structuredClone(finalization) : null), }; const inspect = (reference: string): DockerContainerInspect => { const candidates = [original, replacement].filter( @@ -323,6 +351,20 @@ export function fixture(options: DockerFixtureOptions = {}) { return ok(); }; const copyFromContainer = () => { + if (source === `${NEW_ID}:${MANAGED_BOOTSTRAP_COMPLETION_FILE}`) { + fs.writeFileSync( + destination, + serializeManagedBootstrapImageCompletion({ + bootstrapIdentity: IDENTITY, + agent: options.agent ?? "hermes", + profileFingerprint: agentInputs(options.agent).request.profileFingerprint, + transactionPending: sharedState === "pending", + }), + { mode: 0o444 }, + ); + fs.chmodSync(destination, 0o444); + return ok(); + } const receipt = source.split(":")[1]; const expected = receipt?.includes("shared-state-commit") ? "committed" : "pending"; return sharedState === expected @@ -433,6 +475,9 @@ export function fixture(options: DockerFixtureOptions = {}) { get journal() { return journal; }, + get finalization() { + return finalization; + }, get original() { return original; }, diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 7979330b180..8e94b573589 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -3,7 +3,10 @@ import { assert, describe, expect, it, vi } from "vitest"; -import { ManagedBootstrapOwnerCleanupRequiredError } from "./adapter"; +import { + ManagedBootstrapDurableCommitCleanupPendingError, + ManagedBootstrapOwnerCleanupRequiredError, +} from "./adapter"; import { createDockerManagedBootstrapAdapter } from "./docker"; import { normalizeDockerManagedBootstrapLaunchSpec, @@ -49,11 +52,23 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.events).not.toContain(`stop:${OLD_ID}`); fake.events.push("authority:recorded"); const durable = durablePreparation(handle, snapshot, prepared); + const reorderedDurable = { + recordedAt: durable.recordedAt, + recordId: durable.recordId, + authorityFingerprint: durable.authorityFingerprint, + bootstrapIdentity: durable.bootstrapIdentity, + sandbox: { + driverId: durable.sandbox.driverId, + sandboxId: durable.sandbox.sandboxId, + sandboxName: durable.sandbox.sandboxName, + }, + schemaVersion: durable.schemaVersion, + } satisfies typeof durable; const replacement = await adapter.activateBootstrapReplacement({ handle, snapshot, prepared, - durablePreparation: durable, + durablePreparation: reorderedDurable, }); const order = fake.events; expect(order).toContain("authority:recorded"); @@ -68,17 +83,70 @@ describe("Docker managed bootstrap adapter", () => { replacementRuntimeId: NEW_ID, }); + const commitReceipt = await adapter.awaitBootstrap({ + handle, + snapshot, + replacement, + timeoutSecs: 1, + }); + const reorderedCommitReceipt = { + completedAt: commitReceipt.completedAt, + transactionPending: commitReceipt.transactionPending, + bootstrapIdentity: commitReceipt.bootstrapIdentity, + profileFingerprint: commitReceipt.profileFingerprint, + replacementSpecHash: commitReceipt.replacementSpecHash, + originalSpecHash: commitReceipt.originalSpecHash, + runtimeImageContentId: commitReceipt.runtimeImageContentId, + image: { + manifestDigest: commitReceipt.image.manifestDigest, + repository: commitReceipt.image.repository, + }, + runtimeId: commitReceipt.runtimeId, + sandbox: { + driverId: commitReceipt.sandbox.driverId, + sandboxId: commitReceipt.sandbox.sandboxId, + sandboxName: commitReceipt.sandbox.sandboxName, + }, + schemaVersion: commitReceipt.schemaVersion, + } satisfies typeof commitReceipt; + expect(fake.events).toContain("journal:completion"); + expect(fake.events).toContain(`start:${NEW_ID}`); + expect(fake.events.indexOf("journal:completion")).toBeGreaterThan( + fake.events.indexOf(`start:${NEW_ID}`), + ); + const finalized = await adapter.finalizeBootstrap({ + outcome: "commit", + handle, + snapshot, + prepared, + durablePreparation: reorderedDurable, + replacement, + completion: reorderedCommitReceipt, + }); + expect(finalized).toMatchObject({ outcome: "committed" }); + expect(fake.events).toContain("journal:shared-state-committed"); + expect(fake.events).toContain(`rm:${OLD_ID}`); + expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( + fake.events.indexOf(`rm:${OLD_ID}`), + ); + expect(fake.journal).toBeNull(); + expect(fake.finalization).toMatchObject({ phase: "committed", commitReceipt }); + expect(fake.sharedState).toBe("none"); + expect(fake.replacement?.Id).toBe(NEW_ID); + + const eventCount = fake.events.length; await expect( - adapter.finalizeBootstrap({ + createDockerManagedBootstrapAdapter(fake.deps).finalizeBootstrap({ outcome: "commit", handle, snapshot, prepared, - durablePreparation: durable, + durablePreparation: reorderedDurable, replacement, - completion: completion(replacement), + completion: reorderedCommitReceipt, }), - ).resolves.toMatchObject({ outcome: "committed" }); + ).resolves.toEqual(finalized); + expect(fake.events).toHaveLength(eventCount); expect(fake.events).toContain("journal:shared-state-committed"); expect(fake.events).toContain(`rm:${OLD_ID}`); expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( @@ -87,6 +155,20 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.journal).toBeNull(); expect(fake.sharedState).toBe("none"); expect(fake.replacement?.Id).toBe(NEW_ID); + + await expect( + createDockerManagedBootstrapAdapter(fake.deps).finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: reorderedDurable, + replacement, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapDurableCommitCleanupPendingError); + expect(fake.events).toHaveLength(eventCount); + expect(fake.finalization).toMatchObject({ phase: "committed", commitReceipt }); }); it("preserves commit validation failure details when the replacement cannot be quiesced", async () => { @@ -109,6 +191,12 @@ describe("Docker managed bootstrap adapter", () => { prepared, durablePreparation: durable, }); + const commitReceipt = await adapter.awaitBootstrap({ + handle, + snapshot, + replacement, + timeoutSecs: 1, + }); vi.mocked(fake.deps.dockerStop!).mockReturnValue({ status: 1, stderr: "injected quiesce failure", @@ -122,7 +210,7 @@ describe("Docker managed bootstrap adapter", () => { prepared, durablePreparation: durable, replacement, - completion: completion(replacement), + completion: commitReceipt, }), ).rejects.toThrow( /logical commit validation failed: Managed-startup shared-state commit helper failed.*injected commit failure.*new workload could not be quiesced.*injected quiesce failure/u, diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index 03afb2ba280..e633ee9f2ed 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -42,6 +42,7 @@ import { assertManagedBootstrapSafeProcessEnvironmentKey, attachManagedBootstrapRollbackError, createManagedBootstrapIdentity, + createManagedBootstrapPlanFingerprint, createManagedBootstrapPreparedAuthority, MANAGED_BOOTSTRAP_SCHEMA_VERSION, type ManagedBootstrapAdapter, @@ -64,11 +65,15 @@ import { } from "./adapter"; import { createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapFinalizationRecord, type DockerManagedBootstrapJournal, DockerManagedBootstrapJournalAcknowledgementLostError, type DockerManagedBootstrapJournalStore, parseDockerManagedBootstrapJournal, + sameDockerManagedBootstrapReceipt, + serializeDockerManagedBootstrapFinalizationRecord, serializeDockerManagedBootstrapJournal, } from "./docker-journal"; import { @@ -144,12 +149,6 @@ type ResolvedDeps = Required< type DockerBootstrapTransaction = DockerManagedBootstrapJournal; -interface DockerBootstrapRollbackTombstone { - readonly profileFingerprint: string; - readonly imageReference: string; - readonly receipt: ManagedBootstrapFinalizationReceipt; -} - export interface DockerManagedBootstrapAdapter extends ManagedBootstrapAdapter {} function resolveDeps(deps: DockerManagedBootstrapDeps): ResolvedDeps { @@ -1426,6 +1425,26 @@ function sameDockerBootstrapJournal( ); } +function sameDockerBootstrapPreparedAuthority( + left: DockerBootstrapTransaction, + right: DockerBootstrapTransaction, +): boolean { + return sameDockerBootstrapJournal( + Object.freeze({ + ...left, + phase: "staged" as const, + preparationReceipt: null, + commitReceipt: null, + }), + Object.freeze({ + ...right, + phase: "staged" as const, + preparationReceipt: null, + commitReceipt: null, + }), + ); +} + function createDockerBootstrapJournalDurably( journal: DockerBootstrapTransaction, deps: ResolvedDeps, @@ -1467,6 +1486,27 @@ function transitionDockerBootstrapJournalDurably( return persisted; } +function recordDockerBootstrapCompletionDurably( + journal: DockerBootstrapTransaction, + receipt: ManagedBootstrapCompletionReceipt, + deps: ResolvedDeps, +): DockerBootstrapTransaction { + const expected = Object.freeze({ ...journal, commitReceipt: receipt }); + try { + deps.journalStore.recordCompletion(journal.bootstrapIdentity, receipt); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + if (!recovered || !sameDockerBootstrapJournal(recovered, expected)) throw error; + return recovered; + } + const persisted = deps.journalStore.load(journal.bootstrapIdentity); + if (!persisted || !sameDockerBootstrapJournal(persisted, expected)) { + throw new Error("Managed bootstrap Docker completion receipt was not durably re-readable."); + } + return persisted; +} + function removeDockerBootstrapJournalDurably( journal: DockerBootstrapTransaction, deps: ResolvedDeps, @@ -1490,6 +1530,7 @@ function assertDockerBootstrapTransactionAuthority( snapshot: ManagedBootstrapObservedSnapshot, prepared?: ManagedBootstrapPreparedReplacementHandle | null, replacement?: ManagedBootstrapReplacementHandle | null, + durablePreparation?: ManagedBootstrapDurablePreparationReceipt | null, ): void { const originalName = dockerContainerName( parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, @@ -1498,9 +1539,11 @@ function assertDockerBootstrapTransactionAuthority( if ( transaction.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION || transaction.bootstrapIdentity !== handle.bootstrapIdentity || + transaction.providerId !== expectedSandbox.driverId || transaction.sandbox.sandboxName !== expectedSandbox.sandboxName || transaction.sandbox.sandboxId !== expectedSandbox.sandboxId || transaction.sandbox.driverId !== expectedSandbox.driverId || + transaction.planFingerprint !== createManagedBootstrapPlanFingerprint(handle.plan) || transaction.profileFingerprint !== handle.plan.profile.fingerprint || transaction.imageReference !== expectedImageReference(snapshot.image.repository, snapshot.image.manifestDigest) || @@ -1511,6 +1554,22 @@ function assertDockerBootstrapTransactionAuthority( replacementStagingName(originalName, handle.bootstrapIdentity) || transaction.backupName !== backupName(originalName, handle.bootstrapIdentity) || transaction.originalSpecHash !== snapshot.specHash || + transaction.rollbackTargetRuntimeId !== snapshot.runtimeId || + transaction.rollbackTargetSpecHash !== snapshot.specHash || + (transaction.preparationReceipt !== null && + prepared !== undefined && + prepared !== null && + transaction.preparationReceipt.authorityFingerprint !== + createManagedBootstrapPreparedAuthority({ handle, snapshot, prepared }) + .authorityFingerprint) || + (durablePreparation !== undefined && + durablePreparation !== null && + (transaction.preparationReceipt === null || + !sameDockerManagedBootstrapReceipt( + "preparation", + transaction.preparationReceipt, + durablePreparation, + ))) || (prepared !== undefined && prepared !== null && (transaction.originalRuntimeId !== prepared.originalRuntimeId || @@ -1677,8 +1736,64 @@ export function createDockerManagedBootstrapAdapter( dependencies: DockerManagedBootstrapDeps = {}, ): DockerManagedBootstrapAdapter { const deps = resolveDeps(dependencies); - const committedTransactions = new Set(); - const rollbackTombstones = new Map(); + const finalizationRecord = ( + handle: ManagedBootstrapHeldWorkloadHandle, + ): DockerManagedBootstrapFinalizationRecord | null => { + const record = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + if (!record) return null; + if ( + record.providerId !== handle.sandbox.driverId || + record.sandbox.sandboxName !== handle.sandbox.sandboxName || + record.sandbox.sandboxId !== handle.sandbox.sandboxId || + record.sandbox.driverId !== handle.sandbox.driverId || + record.planFingerprint !== createManagedBootstrapPlanFingerprint(handle.plan) || + record.profileFingerprint !== handle.plan.profile.fingerprint || + record.imageReference !== + expectedImageReference(handle.plan.image.repository, handle.plan.image.manifestDigest) + ) { + throw new Error("Managed bootstrap finalization record does not match its durable identity."); + } + return record; + }; + const persistFinalization = ( + handle: ManagedBootstrapHeldWorkloadHandle, + phase: "committed" | "rolled-back", + commitReceipt: ManagedBootstrapCompletionReceipt | null, + cleanupReceipt: ManagedBootstrapFinalizationReceipt, + ): ManagedBootstrapFinalizationReceipt => { + const record = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase, + bootstrapIdentity: handle.bootstrapIdentity, + providerId: handle.sandbox.driverId, + sandbox: handle.sandbox, + planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + handle.plan.image.repository, + handle.plan.image.manifestDigest, + ), + commitReceipt, + cleanupReceipt, + } satisfies DockerManagedBootstrapFinalizationRecord); + const serialized = serializeDockerManagedBootstrapFinalizationRecord(record); + try { + deps.journalStore.recordFinalization(record); + } catch (error) { + const recovered = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + if ( + !recovered || + serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized + ) { + throw error; + } + } + const persisted = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + if (!persisted || serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized) { + throw new Error("Managed bootstrap finalization receipt was not durably re-readable."); + } + return persisted.cleanupReceipt; + }; const completedRollback = ( handle: ManagedBootstrapHeldWorkloadHandle, alreadyRolledBack: boolean, @@ -1694,34 +1809,39 @@ export function createDockerManagedBootstrapAdapter( alreadyRolledBack, finalizedAt: deps.now().toISOString(), } satisfies ManagedBootstrapFinalizationReceipt); - rollbackTombstones.set(handle.bootstrapIdentity, { - profileFingerprint: handle.plan.profile.fingerprint, - imageReference: expectedImageReference( - handle.plan.image.repository, - handle.plan.image.manifestDigest, - ), - receipt, - }); - return receipt; + return persistFinalization(handle, "rolled-back", null, receipt); + }; + const completedCommit = ( + handle: ManagedBootstrapHeldWorkloadHandle, + commitReceipt: ManagedBootstrapCompletionReceipt, + ): ManagedBootstrapFinalizationReceipt => { + const cleanupReceipt = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + } satisfies ManagedBootstrapFinalizationReceipt); + return persistFinalization(handle, "committed", commitReceipt, cleanupReceipt); }; const priorRollback = ( handle: ManagedBootstrapHeldWorkloadHandle, ): ManagedBootstrapFinalizationReceipt | null => { - const tombstone = rollbackTombstones.get(handle.bootstrapIdentity); - if (!tombstone) return null; - const receipt = tombstone.receipt; - if ( - receipt.sandbox.sandboxName !== handle.sandbox.sandboxName || - receipt.sandbox.sandboxId !== handle.sandbox.sandboxId || - receipt.sandbox.driverId !== handle.sandbox.driverId || - tombstone.profileFingerprint !== handle.plan.profile.fingerprint || - tombstone.imageReference !== - expectedImageReference(handle.plan.image.repository, handle.plan.image.manifestDigest) - ) { - throw new Error("Managed bootstrap rollback tombstone does not match its durable identity."); + const finalized = finalizationRecord(handle); + if (!finalized) return null; + if (finalized.phase === "committed") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: handle.bootstrapIdentity, + cleanupRuntimeId: finalized.commitReceipt?.runtimeId ?? "unknown", + detail: "rollback is no longer legal after the durable finalization receipt", + }); } return Object.freeze({ - ...receipt, + ...finalized.cleanupReceipt, alreadyRolledBack: true, }); }; @@ -1743,10 +1863,7 @@ export function createDockerManagedBootstrapAdapter( const finalized = priorRollback(handle); if (finalized) return finalized; const journal = deps.journalStore.load(handle.bootstrapIdentity); - if ( - committedTransactions.has(handle.bootstrapIdentity) || - journal?.phase === "shared-state-committed" - ) { + if (journal?.phase === "shared-state-committed") { throw new ManagedBootstrapDurableCommitCleanupPendingError({ bootstrapIdentity: handle.bootstrapIdentity, cleanupRuntimeId: journal?.originalRuntimeId ?? snapshot?.runtimeId ?? "unknown", @@ -1854,15 +1971,21 @@ export function createDockerManagedBootstrapAdapter( detail: "durable Docker cutover lacks its coordinator-recorded prepared authority", }); } - const stagedJournal = Object.freeze({ ...journal, phase: "staged" as const }); - if (!sameDockerBootstrapJournal(stagedJournal, preparedAuthority)) { + if (!sameDockerBootstrapPreparedAuthority(journal, preparedAuthority)) { throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: handle.bootstrapIdentity, runtimeId: journal.replacementRuntimeId, detail: "durable Docker cutover changed its prepared rollback authority", }); } - assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); + assertDockerBootstrapTransactionAuthority( + journal, + handle, + snapshot, + prepared, + replacement, + durablePreparation, + ); const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); if (!original) { throw new ManagedBootstrapCommitStateIndeterminateError({ @@ -2082,14 +2205,14 @@ export function createDockerManagedBootstrapAdapter( return completedRollback(handle, false); }; const commitBootstrapNow = ( + handle: ManagedBootstrapHeldWorkloadHandle, receipt: ManagedBootstrapCompletionReceipt, transaction: DockerBootstrapTransaction, input: { readonly sharedStateStatus: "committed" | "none"; readonly sharedStateTransaction: ReturnType; }, - ): void => { - if (committedTransactions.has(receipt.bootstrapIdentity)) return; + ): ManagedBootstrapFinalizationReceipt => { if ( transaction.phase !== "shared-state-committed" || transaction.replacementRuntimeId !== receipt.runtimeId || @@ -2181,7 +2304,7 @@ export function createDockerManagedBootstrapAdapter( } } removeDockerBootstrapJournalDurably(transaction, deps); - committedTransactions.add(receipt.bootstrapIdentity); + return completedCommit(handle, receipt); }; const finalizeBootstrap = async ( input: Parameters[0], @@ -2193,6 +2316,21 @@ export function createDockerManagedBootstrapAdapter( if (!completion || !snapshot || !prepared || !durablePreparation || !replacement) { throw new Error("Managed bootstrap commit requires one complete cutover receipt."); } + const finalized = finalizationRecord(handle); + if (finalized) { + if ( + finalized.phase !== "committed" || + !finalized.commitReceipt || + !sameDockerManagedBootstrapReceipt("completion", finalized.commitReceipt, completion) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: "durable finalization cannot change outcome or commit receipt", + }); + } + return finalized.cleanupReceipt; + } const preparedAuthority = transactionFromPreparedAuthority(handle, snapshot, prepared); assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); const sharedTransaction = managedSharedStateTransaction( @@ -2210,19 +2348,6 @@ export function createDockerManagedBootstrapAdapter( let journal = deps.journalStore.load(handle.bootstrapIdentity); if (!journal) { - if (committedTransactions.has(completion.bootstrapIdentity)) { - return Object.freeze({ - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox: handle.sandbox, - bootstrapIdentity: handle.bootstrapIdentity, - outcome: "committed", - restoredRuntimeId: null, - restoredSpecHash: null, - heldWorkloadRemoved: false, - alreadyRolledBack: false, - finalizedAt: deps.now().toISOString(), - }); - } const originalPresence = probeExactDockerContainerAbsence(snapshot.runtimeId, deps); if (originalPresence === "unknown") { throw new ManagedBootstrapCommitStateIndeterminateError({ @@ -2256,33 +2381,34 @@ export function createDockerManagedBootstrapAdapter( detail: "the retired-journal replacement does not match the exact completion receipt", }); } - committedTransactions.add(completion.bootstrapIdentity); - return Object.freeze({ - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox: handle.sandbox, + return completedCommit(handle, completion); + } + + if (!sameDockerBootstrapPreparedAuthority(journal, preparedAuthority)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: handle.bootstrapIdentity, - outcome: "committed", - restoredRuntimeId: null, - restoredSpecHash: null, - heldWorkloadRemoved: false, - alreadyRolledBack: false, - finalizedAt: deps.now().toISOString(), + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker commit changed its prepared rollback authority", }); } - + assertDockerBootstrapTransactionAuthority( + journal, + handle, + snapshot, + prepared, + replacement, + durablePreparation, + ); if ( - !sameDockerBootstrapJournal( - Object.freeze({ ...journal, phase: "staged" as const }), - preparedAuthority, - ) + journal.commitReceipt === null || + !sameDockerManagedBootstrapReceipt("completion", journal.commitReceipt, completion) ) { throw new ManagedBootstrapCommitStateIndeterminateError({ - bootstrapIdentity: handle.bootstrapIdentity, + bootstrapIdentity: journal.bootstrapIdentity, runtimeId: journal.replacementRuntimeId, - detail: "durable Docker commit changed its prepared rollback authority", + detail: "commit requires the exact durable completion receipt", }); } - assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); if (journal.phase === "staged" || journal.phase === "rollback-authorized") { throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: journal.bootstrapIdentity, @@ -2361,21 +2487,10 @@ export function createDockerManagedBootstrapAdapter( }); } - commitBootstrapNow(completion, journal, { + return commitBootstrapNow(handle, completion, journal, { sharedStateStatus: sharedStatus === "committed" ? "committed" : "none", sharedStateTransaction: sharedTransaction, }); - return Object.freeze({ - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox: handle.sandbox, - bootstrapIdentity: handle.bootstrapIdentity, - outcome: "committed", - restoredRuntimeId: null, - restoredSpecHash: null, - heldWorkloadRemoved: false, - alreadyRolledBack: false, - finalizedAt: deps.now().toISOString(), - }); }; return { async createHeldWorkload(input) { @@ -2631,7 +2746,9 @@ export function createDockerManagedBootstrapAdapter( schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, phase: "staged", bootstrapIdentity: handle.bootstrapIdentity, + providerId: handle.sandbox.driverId, sandbox: Object.freeze({ ...handle.sandbox }), + planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), profileFingerprint: handle.plan.profile.fingerprint, imageReference: expectedImageReference( snapshot.image.repository, @@ -2645,6 +2762,10 @@ export function createDockerManagedBootstrapAdapter( backupName: backupContainerName, originalSpecHash: snapshot.specHash, replacementSpecHash: expectedActivatedSpecHash, + rollbackTargetRuntimeId: snapshot.runtimeId, + rollbackTargetSpecHash: snapshot.specHash, + preparationReceipt: null, + commitReceipt: null, }); requestFile = writeProtectedEnvelope(handle.bootstrapIdentity, request); @@ -2724,9 +2845,20 @@ export function createDockerManagedBootstrapAdapter( async activateBootstrapReplacement({ handle, snapshot, prepared, durablePreparation }) { const authority = transactionFromPreparedAuthority(handle, snapshot, prepared); assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); + const durableAuthority = Object.freeze({ + ...authority, + preparationReceipt: durablePreparation, + }); const existingJournal = deps.journalStore.load(handle.bootstrapIdentity); if (existingJournal) { - assertDockerBootstrapTransactionAuthority(existingJournal, handle, snapshot, prepared); + assertDockerBootstrapTransactionAuthority( + existingJournal, + handle, + snapshot, + prepared, + null, + durablePreparation, + ); throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: existingJournal.bootstrapIdentity, runtimeId: existingJournal.replacementRuntimeId, @@ -2756,7 +2888,7 @@ export function createDockerManagedBootstrapAdapter( ); } - let journal = createDockerBootstrapJournalDurably(authority, deps); + let journal = createDockerBootstrapJournalDurably(durableAuthority, deps); const originalAtFence = inspectExact(snapshot.runtimeId, deps); const replacementAtFence = inspectExact(prepared.preparedRuntimeId, deps); assertTransactionOriginal(journal, originalAtFence); @@ -2929,7 +3061,19 @@ export function createDockerManagedBootstrapAdapter( "Managed bootstrap Docker image completion identities do not match the transaction.", ); } - return Object.freeze({ + if (afterWaitJournal.commitReceipt !== null) { + if ( + afterWaitJournal.commitReceipt.transactionPending !== imageCompletion.transactionPending + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: afterWaitJournal.bootstrapIdentity, + runtimeId: afterWaitJournal.replacementRuntimeId, + detail: "durable completion disagrees with the image-owned transaction receipt", + }); + } + return afterWaitJournal.commitReceipt; + } + const completion = Object.freeze({ schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, sandbox: handle.sandbox, runtimeId: replacement.replacementRuntimeId, @@ -2942,6 +3086,15 @@ export function createDockerManagedBootstrapAdapter( transactionPending: imageCompletion.transactionPending, completedAt: deps.now().toISOString(), }); + const completedJournal = recordDockerBootstrapCompletionDurably( + afterWaitJournal, + completion, + deps, + ); + if (completedJournal.commitReceipt === null) { + throw new Error("Managed bootstrap Docker completion receipt disappeared after recording."); + } + return completedJournal.commitReceipt; }, finalizeBootstrap, diff --git a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts index 3022fc71850..6ec1563e73e 100644 --- a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts +++ b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts @@ -10,9 +10,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; import type { SandboxMessagingPlan } from "../messaging/manifest"; import type { ManagedStartupAgent, ManagedStartupProfile } from "./managed-startup/profile"; +import { fingerprintManagedStartupProfile } from "./managed-startup/profile"; import { beginManagedStartupSharedStateTransaction, + clearManagedStartupSharedStateCommitReceipt, commitManagedStartupSharedStateTransaction, + getManagedStartupSharedStateTransactionStatus, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, type ManagedStartupSharedTransactionOptions, rollbackManagedStartupSharedStateTransaction, } from "./managed-startup/shared-state-transaction"; @@ -67,6 +71,13 @@ describe("managed startup shared-state transaction", () => { ); } + function commitReceiptDirectory(): string { + return path.join( + path.dirname(transactionDirectory), + path.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY), + ); + } + it.each([ "openclaw", "hermes", @@ -246,6 +257,192 @@ describe("managed startup shared-state transaction", () => { expect(commitManagedStartupSharedStateTransaction("openclaw", options)).toBe(false); }); + it("fsyncs every transaction namespace before exposing a pending receipt", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "before\n"); + const open = vi.spyOn(fs, "openSync"); + const fsync = vi.spyOn(fs, "fsyncSync"); + + expect( + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options), + ).toBe(true); + + const transactionParent = path.dirname(transactionDirectory); + const backupDirectory = path.join(transactionDirectory, "backups"); + expect(open).toHaveBeenCalledWith(transactionParent, fs.constants.O_RDONLY); + expect(open).toHaveBeenCalledWith(transactionDirectory, fs.constants.O_RDONLY); + expect(open).toHaveBeenCalledWith(backupDirectory, fs.constants.O_RDONLY); + // File contents plus parent, backup, and manifest directory entries all + // reach stable storage before the transaction is returned as pending. + expect(fsync.mock.calls.length).toBeGreaterThanOrEqual(6); + }); + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("persists one exact compact %s bootstrap commit across fresh calls, forbids rollback, and retires it for the next attempt", (agent) => { + const profile = managedStartupE2eProfile(agent); + const bootstrapIdentity = "b".repeat(64); + const nextBootstrapIdentity = "d".repeat(64); + const boundOptions = { ...options, bootstrapIdentity }; + const root = agentRoot(agent); + fs.mkdirSync(root); + const config = path.join( + root, + agent === "openclaw" ? "openclaw.json" : agent === "hermes" ? "config.yaml" : "config.toml", + ); + fs.writeFileSync(config, "before\n"); + + expect(beginManagedStartupSharedStateTransaction(profile, boundOptions)).toBe(true); + fs.writeFileSync(config, "committed\n"); + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("pending"); + expect(() => + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: "e".repeat(64), + bootstrapIdentity, + }, + options, + ), + ).toThrow(/expected agent, profile fingerprint, or bootstrap identity/u); + expect(commitManagedStartupSharedStateTransaction(agent, boundOptions)).toBe(true); + + const receiptDirectory = commitReceiptDirectory(); + const receiptFile = path.join(receiptDirectory, "receipt.json"); + expect(fs.existsSync(transactionDirectory)).toBe(false); + expect(fs.readdirSync(receiptDirectory)).toEqual(["receipt.json"]); + expect(mode(receiptDirectory)).toBe(0o700); + expect(mode(receiptFile)).toBe(0o400); + expect(JSON.parse(fs.readFileSync(receiptFile, "utf8"))).toEqual({ + schemaVersion: 1, + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }); + + // These calls reconstruct state solely from the image-owned receipt. + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("committed"); + expect(commitManagedStartupSharedStateTransaction(agent, boundOptions)).toBe(true); + expect(() => rollbackManagedStartupSharedStateTransaction(agent, boundOptions)).toThrow( + /durably committed and cannot be rolled back/u, + ); + expect(() => + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: "e".repeat(64), + bootstrapIdentity, + }, + options, + ), + ).toThrow(/different bootstrap attempt/u); + expect(fs.readFileSync(config, "utf8")).toBe("committed\n"); + + expect(clearManagedStartupSharedStateCommitReceipt(agent, boundOptions)).toBe(true); + expect(fs.existsSync(receiptDirectory)).toBe(false); + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("none"); + + const nextOptions = { ...options, bootstrapIdentity: nextBootstrapIdentity }; + expect(beginManagedStartupSharedStateTransaction(profile, nextOptions)).toBe(true); + expect(rollbackManagedStartupSharedStateTransaction(agent, nextOptions)).toBe(true); + }); + + it.each([ + "during-compact-receipt-write", + "before-backup-removal", + "during-backup-removal", + "after-backup-removal", + "after-manifest-removal", + ] as const)("recovers an atomically established commit interrupted %s", (interruption) => { + const profile = managedStartupE2eProfile("openclaw"); + const bootstrapIdentity = "b".repeat(64); + const boundOptions = { ...options, bootstrapIdentity }; + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "before\n"); + beginManagedStartupSharedStateTransaction(profile, boundOptions); + fs.writeFileSync(path.join(root, "openclaw.json"), "committed\n"); + + const originalRmSync = fs.rmSync.bind(fs); + const rm = vi.spyOn(fs, "rmSync").mockImplementation((( + target: fs.PathLike, + removeOptions?: fs.RmDirOptions, + ) => + String(target).endsWith(`${path.sep}backups`) + ? (() => { + throw new Error("injected post-rename cleanup interruption"); + })() + : originalRmSync(target, removeOptions)) as typeof fs.rmSync); + expect(() => commitManagedStartupSharedStateTransaction("openclaw", boundOptions)).toThrow( + /injected post-rename cleanup interruption/u, + ); + rm.mockRestore(); + + expect(fs.existsSync(transactionDirectory)).toBe(false); + const committedDirectory = commitReceiptDirectory(); + const backups = path.join(committedDirectory, "backups"); + const manifest = path.join(committedDirectory, "manifest.json"); + const applyInterruption: Record void> = { + "during-compact-receipt-write": () => + fs.renameSync( + path.join(committedDirectory, "receipt.json"), + path.join(committedDirectory, ".receipt.json.1234567890abcdef12345678"), + ), + "before-backup-removal": () => undefined, + "during-backup-removal": () => { + const [firstBackup] = fs.readdirSync(backups); + expect(firstBackup).toBeTruthy(); + fs.unlinkSync(path.join(backups, firstBackup!)); + }, + "after-backup-removal": () => originalRmSync(backups, { force: false, recursive: true }), + "after-manifest-removal": () => fs.unlinkSync(manifest), + }; + applyInterruption[interruption](); + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent: "openclaw", + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("committed"); + expect(commitManagedStartupSharedStateTransaction("openclaw", boundOptions)).toBe(true); + expect(fs.readdirSync(committedDirectory)).toEqual(["receipt.json"]); + expect(clearManagedStartupSharedStateCommitReceipt("openclaw", boundOptions)).toBe(true); + }); + it("resumes the same pending profile idempotently and rejects profile drift", () => { const root = agentRoot("openclaw"); fs.mkdirSync(root); @@ -258,10 +455,62 @@ describe("managed startup shared-state transaction", () => { managedStartupE2eProfile("openclaw", true), options, ), - ).toThrow(/belongs to a different profile/u); + ).toThrow(/belongs to a different agent, profile fingerprint, or bootstrap attempt/u); expect(rollbackManagedStartupSharedStateTransaction("openclaw", options)).toBe(true); }); + it("validates a directly mounted copied receipt under an unchanged 0755 image parent", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "{}\n"); + const profile = managedStartupE2eProfile("openclaw"); + const bootstrapIdentity = "b".repeat(64); + const boundOptions = { ...options, bootstrapIdentity }; + beginManagedStartupSharedStateTransaction(profile, boundOptions); + + const imageParent = path.join(temporaryRoot, "image-var-lib-nemoclaw"); + const copiedReceipt = path.join(imageParent, "managed-startup-shared-state-transaction-v1"); + fs.mkdirSync(imageParent, { mode: 0o755 }); + fs.chmodSync(imageParent, 0o755); + fs.cpSync(transactionDirectory, copiedReceipt, { + recursive: true, + preserveTimestamps: true, + }); + // Node 22.23 normalizes copied directory modes to 0755. Recreate the + // protected modes that the container-copy fixture is intended to model. + fs.chmodSync(copiedReceipt, 0o700); + fs.chmodSync(path.join(copiedReceipt, "backups"), 0o700); + + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent: "openclaw", + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + { + ...boundOptions, + transactionDirectory: copiedReceipt, + }, + ), + ).toBe("pending"); + + fs.chmodSync(imageParent, 0o700); + expect(() => + getManagedStartupSharedStateTransactionStatus( + { + agent: "openclaw", + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + { + ...boundOptions, + transactionDirectory: copiedReceipt, + }, + ), + ).toThrow(/must be .* mode 755/u); + }); + it("rejects planted target and ancestor symlinks before creating a receipt", () => { const outside = path.join(temporaryRoot, "outside"); fs.mkdirSync(outside); diff --git a/src/lib/onboard/managed-startup/image-runtime.ts b/src/lib/onboard/managed-startup/image-runtime.ts index f8e5c7f4edd..898783c7f30 100644 --- a/src/lib/onboard/managed-startup/image-runtime.ts +++ b/src/lib/onboard/managed-startup/image-runtime.ts @@ -35,7 +35,9 @@ import { } from "./root-apply"; import { beginManagedStartupSharedStateTransaction, + clearManagedStartupSharedStateCommitReceipt, commitManagedStartupSharedStateTransaction, + getManagedStartupSharedStateTransactionStatus, MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, rollbackManagedStartupSharedStateTransaction, } from "./shared-state-transaction"; @@ -1540,7 +1542,7 @@ function readCliAgent(argv: readonly string[], expectedLength = 2): string { const index = argv.indexOf("--agent"); if (index < 0 || index + 1 >= argv.length || argv.length !== expectedLength) { fail( - "usage: managed-startup-image-runtime [--apply-root-stdin|--wait-for-completion|--verify-completion|--begin-shared-state-transaction|--commit-shared-state-transaction] --agent ", + "usage: managed-startup-image-runtime [--apply-root-stdin|--wait-for-completion|--verify-completion|--begin-shared-state-transaction|--commit-shared-state-transaction|--clear-shared-state-commit-receipt|--shared-state-transaction-status] --agent ", ); } return argv[index + 1] as string; @@ -1554,6 +1556,14 @@ function readCliFingerprint(argv: readonly string[]): string { return argv[index + 1] as string; } +function readCliBootstrapIdentity(argv: readonly string[]): string { + const index = argv.indexOf("--bootstrap-identity"); + if (index < 0 || index + 1 >= argv.length || !SHA256_RE.test(String(argv[index + 1] ?? ""))) { + fail("managed bootstrap identity argument is missing or invalid"); + } + return argv[index + 1] as string; +} + export async function main(argv: readonly string[] = process.argv.slice(2)): Promise { if (argv.length === 1 && argv[0] === "--internal-write-openclaw-hash") { internalWriteOpenClawHash(); @@ -1600,29 +1610,58 @@ export async function main(argv: readonly string[] = process.argv.slice(2)): Pro return; } if ( - argv.length === 4 && + (argv.length === 4 || argv.length === 6) && argv[0] === "--rollback-shared-state-transaction" && - argv[3] === "--read-only-receipt" + argv[argv.length - 1] === "--read-only-receipt" ) { requireRoot(); - const agent = exactAgent(readCliAgent(argv, 4)); + const agent = exactAgent(readCliAgent(argv, argv.length)); const rolledBack = rollbackManagedStartupSharedStateTransaction(agent, { transactionDirectory: MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, readOnlyReceipt: true, + bootstrapIdentity: argv.length === 6 ? readCliBootstrapIdentity(argv) : null, }); if (!rolledBack) fail("read-only shared-state rollback receipt is missing"); console.log(`[managed-startup] verified and restored ${agent} shared state`); return; } - if (argv.length === 3 && argv[0] === "--commit-shared-state-transaction") { + if ((argv.length === 3 || argv.length === 5) && argv[0] === "--commit-shared-state-transaction") { requireRoot(); - const agent = exactAgent(readCliAgent(argv, 3)); - if (!commitManagedStartupSharedStateTransaction(agent)) { + const agent = exactAgent(readCliAgent(argv, argv.length)); + if ( + !commitManagedStartupSharedStateTransaction(agent, { + bootstrapIdentity: argv.length === 5 ? readCliBootstrapIdentity(argv) : null, + }) + ) { fail("managed startup transaction is missing at commit"); } console.log(`[managed-startup] committed ${agent} shared state`); return; } + if (argv.length === 5 && argv[0] === "--clear-shared-state-commit-receipt") { + requireRoot(); + const agent = exactAgent(readCliAgent(argv, 5)); + const bootstrapIdentity = readCliBootstrapIdentity(argv); + if (!clearManagedStartupSharedStateCommitReceipt(agent, { bootstrapIdentity })) { + fail("managed startup durable commit receipt is missing at cleanup"); + } + console.log(`[managed-startup] cleared ${agent} durable shared-state commit receipt`); + return; + } + if (argv.length === 7 && argv[0] === "--shared-state-transaction-status") { + requireRoot(); + const agent = exactAgent(readCliAgent(argv, 7)); + const profileFingerprint = readCliFingerprint(argv); + const bootstrapIdentity = readCliBootstrapIdentity(argv); + process.stdout.write( + `${getManagedStartupSharedStateTransactionStatus({ + agent, + profileFingerprint, + bootstrapIdentity, + })}\n`, + ); + return; + } const result = await applyManagedStartupImageProfile(readCliAgent(argv)); console.log( result.adapterApplied diff --git a/src/lib/onboard/managed-startup/shared-state-transaction.ts b/src/lib/onboard/managed-startup/shared-state-transaction.ts index d921ec36d89..62894abeb5c 100644 --- a/src/lib/onboard/managed-startup/shared-state-transaction.ts +++ b/src/lib/onboard/managed-startup/shared-state-transaction.ts @@ -20,14 +20,19 @@ const MAX_TRANSACTION_FILES = 128; const MAX_TRANSACTION_FILE_BYTES = 8 * 1024 * 1024; const MAX_TRANSACTION_TOTAL_BYTES = 32 * 1024 * 1024; const MAX_MANIFEST_BYTES = 256 * 1024; +const MAX_COMMIT_RECEIPT_BYTES = 4096; const TRANSACTION_PARENT_DIRECTORY_MODE = 0o755; const TRANSACTION_DIRECTORY_MODE = 0o700; const TRANSACTION_FILE_MODE = 0o400; +const ATOMIC_TEMPORARY_FILE_MODE = 0o600; export const MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY = "/var/lib/nemoclaw/managed-startup-shared-state-transaction-v1"; export const MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY = "/run/nemoclaw/managed-startup-shared-rollback-receipt-v1"; +export const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY = + "/var/lib/nemoclaw/managed-startup-shared-state-commit-v1"; +const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE = "receipt.json"; interface FilePresentReceipt { readonly path: string; @@ -66,13 +71,23 @@ interface TransactionManifest { readonly schemaVersion: typeof TRANSACTION_SCHEMA_VERSION; readonly agent: ManagedStartupAgent; readonly profileFingerprint: string; + readonly bootstrapIdentity: string | null; readonly files: readonly FileReceipt[]; readonly directories: readonly DirectoryReceipt[]; } +interface CommitReceipt { + readonly schemaVersion: typeof TRANSACTION_SCHEMA_VERSION; + readonly agent: ManagedStartupAgent; + readonly profileFingerprint: string; + readonly bootstrapIdentity: string; +} + export interface ManagedStartupSharedTransactionOptions { readonly sandboxRoot?: string; readonly transactionDirectory?: string; + /** Test/helper seam. Production derives the fixed image-owned commit receipt path. */ + readonly commitReceiptDirectory?: string; /** Test seam. Production always retains the root:root defaults. */ readonly trustedUid?: number; /** Test seam. Production always retains the root:root defaults. */ @@ -82,6 +97,8 @@ export interface ManagedStartupSharedTransactionOptions { * so ownership may reflect the Docker CLI user instead of container root. */ readonly readOnlyReceipt?: boolean; + /** One-attempt identity for managed bootstrap; null for legacy root application. */ + readonly bootstrapIdentity?: string | null; } interface ResolvedOptions { @@ -90,9 +107,12 @@ interface ResolvedOptions { readonly transactionDirectory: string; readonly backupDirectory: string; readonly manifestFile: string; + readonly commitReceiptDirectory: string; + readonly commitReceiptFile: string; readonly trustedUid: number; readonly trustedGid: number; readonly readOnlyReceipt: boolean; + readonly bootstrapIdentity: string | null; } interface StableFile { @@ -109,11 +129,28 @@ function resolveOptions(options: ManagedStartupSharedTransactionOptions = {}): R const transactionDirectory = path.resolve( options.transactionDirectory ?? MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, ); + const commitReceiptDirectory = path.resolve( + options.commitReceiptDirectory ?? + (options.transactionDirectory + ? path.join( + path.dirname(transactionDirectory), + path.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY), + ) + : MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY), + ); if ( transactionDirectory === sandboxRoot || - transactionDirectory.startsWith(`${sandboxRoot}${path.sep}`) + transactionDirectory.startsWith(`${sandboxRoot}${path.sep}`) || + commitReceiptDirectory === sandboxRoot || + commitReceiptDirectory.startsWith(`${sandboxRoot}${path.sep}`) || + path.dirname(commitReceiptDirectory) !== path.dirname(transactionDirectory) || + commitReceiptDirectory === transactionDirectory ) { - fail("transaction receipts must not be stored in sandbox-shared state"); + fail("transaction and commit receipts require distinct paths outside sandbox-shared state"); + } + const bootstrapIdentity = options.bootstrapIdentity ?? null; + if (bootstrapIdentity !== null && !/^[a-f0-9]{64}$/u.test(bootstrapIdentity)) { + fail("bootstrap identity must encode 32 lowercase-hex bytes"); } return { sandboxRoot, @@ -121,9 +158,15 @@ function resolveOptions(options: ManagedStartupSharedTransactionOptions = {}): R transactionDirectory, backupDirectory: path.join(transactionDirectory, "backups"), manifestFile: path.join(transactionDirectory, "manifest.json"), + commitReceiptDirectory, + commitReceiptFile: path.join( + commitReceiptDirectory, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE, + ), trustedUid: options.trustedUid ?? 0, trustedGid: options.trustedGid ?? 0, readOnlyReceipt: options.readOnlyReceipt ?? false, + bootstrapIdentity, }; } @@ -482,16 +525,63 @@ function atomicWriteTrustedFile( } } +function fsyncDirectory(directory: string): void { + const descriptor = fs.openSync(directory, fs.constants.O_RDONLY); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + function canonicalManifest(manifest: TransactionManifest): string { return `${JSON.stringify(manifest, null, 2)}\n`; } +function canonicalCommitReceipt(receipt: CommitReceipt): string { + return `${JSON.stringify(receipt, null, 2)}\n`; +} + function requireExactKeys(record: Record, keys: readonly string[]): void { if (Object.keys(record).sort().join(",") !== [...keys].sort().join(",")) { fail("transaction manifest contains unexpected fields"); } } +function parseCommitReceipt(text: string): CommitReceipt { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("commit receipt is not valid JSON"); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + fail("commit receipt must be an object"); + } + const record = parsed as Record; + requireExactKeys(record, ["agent", "bootstrapIdentity", "profileFingerprint", "schemaVersion"]); + if ( + record.schemaVersion !== TRANSACTION_SCHEMA_VERSION || + !["openclaw", "hermes", "langchain-deepagents-code"].includes(String(record.agent)) || + typeof record.profileFingerprint !== "string" || + !/^[a-f0-9]{64}$/u.test(record.profileFingerprint) || + typeof record.bootstrapIdentity !== "string" || + !/^[a-f0-9]{64}$/u.test(record.bootstrapIdentity) + ) { + fail("commit receipt has an invalid envelope"); + } + const receipt: CommitReceipt = { + schemaVersion: TRANSACTION_SCHEMA_VERSION, + agent: record.agent as ManagedStartupAgent, + profileFingerprint: record.profileFingerprint, + bootstrapIdentity: record.bootstrapIdentity, + }; + if (canonicalCommitReceipt(receipt) !== text) { + fail("commit receipt is not canonical"); + } + return receipt; +} + function safeMetadata(value: unknown): value is number { return Number.isSafeInteger(value) && (value as number) >= 0; } @@ -509,6 +599,7 @@ function parseManifest(text: string): TransactionManifest { const record = parsed as Record; requireExactKeys(record, [ "agent", + "bootstrapIdentity", "directories", "files", "profileFingerprint", @@ -519,6 +610,11 @@ function parseManifest(text: string): TransactionManifest { !["openclaw", "hermes", "langchain-deepagents-code"].includes(String(record.agent)) || typeof record.profileFingerprint !== "string" || !/^[a-f0-9]{64}$/u.test(record.profileFingerprint) || + !( + record.bootstrapIdentity === null || + (typeof record.bootstrapIdentity === "string" && + /^[a-f0-9]{64}$/u.test(record.bootstrapIdentity)) + ) || !Array.isArray(record.files) || !Array.isArray(record.directories) || record.files.length > MAX_TRANSACTION_FILES || @@ -613,6 +709,7 @@ function parseManifest(text: string): TransactionManifest { schemaVersion: TRANSACTION_SCHEMA_VERSION, agent: record.agent as ManagedStartupAgent, profileFingerprint: record.profileFingerprint, + bootstrapIdentity: record.bootstrapIdentity as string | null, files, directories, }; @@ -639,9 +736,9 @@ function requireTrustedTransactionPath( } } -function requireReadOnlyReceiptMount(options: ResolvedOptions): void { +function requireReadOnlyReceiptMount(target: string, options: ResolvedOptions): void { if (!options.readOnlyReceipt) return; - const probe = path.join(options.transactionDirectory, ".nemoclaw-write-probe"); + const probe = path.join(target, ".nemoclaw-write-probe"); let descriptor: number | undefined; try { descriptor = fs.openSync( @@ -664,7 +761,7 @@ function loadManifest(options: ResolvedOptions): TransactionManifest | null { requireTransactionBoundaries(options); if (!pathExistsNoFollow(options.transactionDirectory)) return null; requireTrustedTransactionPath(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE, options); - requireReadOnlyReceiptMount(options); + requireReadOnlyReceiptMount(options.transactionDirectory, options); requireTrustedTransactionPath(options.backupDirectory, TRANSACTION_DIRECTORY_MODE, options); requireTrustedTransactionPath(options.manifestFile, TRANSACTION_FILE_MODE, options); const stable = readStableFile(options.manifestFile, MAX_MANIFEST_BYTES); @@ -679,6 +776,59 @@ function loadManifest(options: ResolvedOptions): TransactionManifest | null { return parseManifest(stable.bytes.toString("utf8")); } +function transactionOptionsAt( + options: ResolvedOptions, + transactionDirectory: string, +): ResolvedOptions { + return { + ...options, + transactionDirectory, + backupDirectory: path.join(transactionDirectory, "backups"), + manifestFile: path.join(transactionDirectory, "manifest.json"), + }; +} + +function loadCommitReceipt( + options: ResolvedOptions, +): { readonly receipt: CommitReceipt; readonly compact: boolean } | null { + requireTransactionBoundaries(options); + if (!pathExistsNoFollow(options.commitReceiptDirectory)) return null; + requireTrustedTransactionPath( + options.commitReceiptDirectory, + TRANSACTION_DIRECTORY_MODE, + options, + ); + if (pathExistsNoFollow(options.commitReceiptFile)) { + requireReadOnlyReceiptMount(options.commitReceiptDirectory, options); + requireTrustedTransactionPath(options.commitReceiptFile, TRANSACTION_FILE_MODE, options); + const stable = readStableFile(options.commitReceiptFile, MAX_COMMIT_RECEIPT_BYTES); + if ( + (!options.readOnlyReceipt && + (Number(stable.stat.uid) !== options.trustedUid || + Number(stable.stat.gid) !== options.trustedGid)) || + Number(stable.stat.mode & 0o7777n) !== TRANSACTION_FILE_MODE + ) { + fail("commit receipt ownership changed while it was read"); + } + return { receipt: parseCommitReceipt(stable.bytes.toString("utf8")), compact: true }; + } + const stagedOptions = transactionOptionsAt(options, options.commitReceiptDirectory); + const staged = loadManifest(stagedOptions); + if (!staged || staged.bootstrapIdentity === null) { + fail("durable commit staging receipt is incomplete"); + } + verifyAllBackups(staged.files, stagedOptions); + return { + receipt: { + schemaVersion: TRANSACTION_SCHEMA_VERSION, + agent: staged.agent, + profileFingerprint: staged.profileFingerprint, + bootstrapIdentity: staged.bootstrapIdentity, + }, + compact: false, + }; +} + function verifyBackup(receipt: FilePresentReceipt, options: ResolvedOptions): Buffer { const backupPath = path.join(options.backupDirectory, receipt.backup); requireTrustedTransactionPath(backupPath, TRANSACTION_FILE_MODE, options); @@ -742,11 +892,162 @@ function directoryMatchesReceipt(target: string, receipt: DirectoryPresentReceip function removeTransactionDirectory(options: ResolvedOptions): void { requireTrustedTransactionPath(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE, options); fs.rmSync(options.transactionDirectory, { force: false, recursive: true }); + fsyncDirectory(options.transactionParentDirectory); if (pathExistsNoFollow(options.transactionDirectory)) { fail("transaction directory remained after cleanup"); } } +function assertCommitReceiptMatches( + receipt: CommitReceipt, + expected: { + readonly agent: ManagedStartupAgent; + readonly profileFingerprint?: string; + readonly bootstrapIdentity: string; + }, +): void { + if ( + receipt.agent !== expected.agent || + (expected.profileFingerprint !== undefined && + receipt.profileFingerprint !== expected.profileFingerprint) || + receipt.bootstrapIdentity !== expected.bootstrapIdentity + ) { + fail("durable commit receipt belongs to a different bootstrap attempt"); + } +} + +function loadCommitStagingManifest(options: ResolvedOptions): TransactionManifest | null { + if (!pathExistsNoFollow(options.manifestFile)) return null; + requireTrustedTransactionPath(options.manifestFile, TRANSACTION_FILE_MODE, options); + const stable = readStableFile(options.manifestFile, MAX_MANIFEST_BYTES); + if ( + Number(stable.stat.uid) !== options.trustedUid || + Number(stable.stat.gid) !== options.trustedGid || + Number(stable.stat.mode & 0o7777n) !== TRANSACTION_FILE_MODE + ) { + fail("durable commit staging manifest ownership changed while it was read"); + } + return parseManifest(stable.bytes.toString("utf8")); +} + +function retireInterruptedCommitReceiptWrites( + receipt: CommitReceipt, + options: ResolvedOptions, +): void { + const temporaryPattern = new RegExp( + `^\\.${MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE.replace(".", "\\.")}\\.[a-f0-9]{24}$`, + "u", + ); + for (const entry of fs.readdirSync(options.commitReceiptDirectory)) { + if (!temporaryPattern.test(entry)) continue; + const target = path.join(options.commitReceiptDirectory, entry); + const stat = fs.lstatSync(target); + if ( + stat.isSymbolicLink() || + !stat.isFile() || + stat.nlink !== 1 || + stat.uid !== options.trustedUid || + stat.gid !== options.trustedGid || + ![ATOMIC_TEMPORARY_FILE_MODE, TRANSACTION_FILE_MODE].includes(modeOf(stat)) + ) { + fail("interrupted durable commit receipt write has unsafe metadata"); + } + const stable = readStableFile(target, MAX_COMMIT_RECEIPT_BYTES); + const mode = Number(stable.stat.mode & 0o7777n); + if ( + Number(stable.stat.uid) !== options.trustedUid || + Number(stable.stat.gid) !== options.trustedGid || + ![ATOMIC_TEMPORARY_FILE_MODE, TRANSACTION_FILE_MODE].includes(mode) + ) { + fail("interrupted durable commit receipt write changed during verification"); + } + if (stable.bytes.length > 0) { + let interruptedReceipt: CommitReceipt | null = null; + try { + interruptedReceipt = parseCommitReceipt(stable.bytes.toString("utf8")); + } catch { + // The atomic writer may have crashed after any partial write. The + // trusted 0700 directory and exact random temp-name shape bind this + // artifact to that interrupted write; the established receipt is now + // authoritative. + } + if (interruptedReceipt) assertCommitReceiptMatches(interruptedReceipt, receipt); + } + fs.unlinkSync(target); + fsyncDirectory(options.commitReceiptDirectory); + } +} + +function compactDurableCommitReceipt( + state: { readonly receipt: CommitReceipt; readonly compact: boolean }, + options: ResolvedOptions, +): void { + if (!state.compact) { + atomicWriteTrustedFile( + options.commitReceiptFile, + canonicalCommitReceipt(state.receipt), + TRANSACTION_FILE_MODE, + options.trustedUid, + options.trustedGid, + ); + fsyncDirectory(options.commitReceiptDirectory); + } + retireInterruptedCommitReceiptWrites(state.receipt, options); + const stagedOptions = transactionOptionsAt(options, options.commitReceiptDirectory); + const manifestExists = pathExistsNoFollow(stagedOptions.manifestFile); + const backupsExist = pathExistsNoFollow(stagedOptions.backupDirectory); + const unexpectedBeforeCleanup = fs + .readdirSync(options.commitReceiptDirectory) + .filter( + (entry) => + ![ + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE, + path.basename(stagedOptions.backupDirectory), + path.basename(stagedOptions.manifestFile), + ].includes(entry), + ); + if (unexpectedBeforeCleanup.length !== 0) { + fail("durable commit receipt directory contains unexpected artifacts"); + } + if (manifestExists) { + // The fsynced compact receipt is authoritative after commit. Validate the + // remaining manifest identity without requiring a complete backup tree: + // recursive backup deletion may have been interrupted at any point. + const staged = loadCommitStagingManifest(stagedOptions); + if (!staged || staged.bootstrapIdentity === null) { + fail("durable commit staging receipt disappeared during cleanup"); + } + assertCommitReceiptMatches(state.receipt, { + agent: staged.agent, + profileFingerprint: staged.profileFingerprint, + bootstrapIdentity: staged.bootstrapIdentity, + }); + } + if (backupsExist) { + requireTrustedTransactionPath( + stagedOptions.backupDirectory, + TRANSACTION_DIRECTORY_MODE, + options, + ); + fs.rmSync(stagedOptions.backupDirectory, { force: false, recursive: true }); + fsyncDirectory(options.commitReceiptDirectory); + } + if (manifestExists) { + requireTrustedTransactionPath(stagedOptions.manifestFile, TRANSACTION_FILE_MODE, options); + fs.unlinkSync(stagedOptions.manifestFile); + fsyncDirectory(options.commitReceiptDirectory); + } + const unexpected = fs + .readdirSync(options.commitReceiptDirectory) + .filter((entry) => entry !== MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE); + if (unexpected.length !== 0) { + fail("durable commit receipt directory contains unexpected artifacts"); + } + const verified = loadCommitReceipt(options); + if (!verified?.compact) fail("durable commit receipt did not compact successfully"); + assertCommitReceiptMatches(verified.receipt, state.receipt); +} + export function beginManagedStartupSharedStateTransaction( profile: ManagedStartupProfile, inputOptions: ManagedStartupSharedTransactionOptions = {}, @@ -758,10 +1059,28 @@ export function beginManagedStartupSharedStateTransaction( } requireTransactionBoundaries(options); const profileFingerprint = fingerprintManagedStartupProfile(profile); + const committed = loadCommitReceipt(options); + if (committed) { + if (options.bootstrapIdentity === null) { + fail("a durable managed bootstrap commit receipt already exists"); + } + assertCommitReceiptMatches(committed.receipt, { + agent: profile.agent, + profileFingerprint, + bootstrapIdentity: options.bootstrapIdentity, + }); + fail("this managed bootstrap attempt is already durably committed"); + } const pending = loadManifest(options); if (pending) { - if (pending.agent !== profile.agent || pending.profileFingerprint !== profileFingerprint) { - fail("a pending managed startup transaction belongs to a different profile"); + if ( + pending.agent !== profile.agent || + pending.profileFingerprint !== profileFingerprint || + pending.bootstrapIdentity !== options.bootstrapIdentity + ) { + fail( + "a pending managed startup transaction belongs to a different agent, profile fingerprint, or bootstrap attempt", + ); } verifyAllBackups(pending.files, options); return false; @@ -780,6 +1099,7 @@ export function beginManagedStartupSharedStateTransaction( schemaVersion: TRANSACTION_SCHEMA_VERSION, agent: profile.agent, profileFingerprint, + bootstrapIdentity: options.bootstrapIdentity, files: snapshots.map(({ receipt }) => receipt), directories, }; @@ -801,9 +1121,11 @@ export function beginManagedStartupSharedStateTransaction( }; fs.chownSync(options.transactionDirectory, options.trustedUid, options.trustedGid); fs.chmodSync(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE); + fsyncDirectory(options.transactionParentDirectory); fs.mkdirSync(options.backupDirectory, { mode: TRANSACTION_DIRECTORY_MODE }); fs.chownSync(options.backupDirectory, options.trustedUid, options.trustedGid); fs.chmodSync(options.backupDirectory, TRANSACTION_DIRECTORY_MODE); + fsyncDirectory(options.transactionDirectory); for (const snapshot of snapshots) { if (snapshot.receipt.state !== "file" || snapshot.bytes === null) continue; atomicWriteTrustedFile( @@ -814,6 +1136,7 @@ export function beginManagedStartupSharedStateTransaction( options.trustedGid, ); } + fsyncDirectory(options.backupDirectory); atomicWriteTrustedFile( options.manifestFile, canonicalManifest(manifest), @@ -821,6 +1144,7 @@ export function beginManagedStartupSharedStateTransaction( options.trustedUid, options.trustedGid, ); + fsyncDirectory(options.transactionDirectory); loadManifest(options); } catch (error) { try { @@ -990,11 +1314,25 @@ export function rollbackManagedStartupSharedStateTransaction( ): boolean { const options = resolveOptions(inputOptions); requireTransactionIdentity(options); + const committed = loadCommitReceipt(options); + if (committed) { + if (options.bootstrapIdentity === null) { + fail("shared state is already durably committed"); + } + assertCommitReceiptMatches(committed.receipt, { + agent: expectedAgent, + bootstrapIdentity: options.bootstrapIdentity, + }); + fail("shared state is already durably committed and cannot be rolled back"); + } const manifest = loadManifest(options); if (!manifest) return false; if (manifest.agent !== expectedAgent) { fail(`pending transaction targets ${manifest.agent}, expected ${expectedAgent}`); } + if (manifest.bootstrapIdentity !== options.bootstrapIdentity) { + fail("pending transaction belongs to a different bootstrap attempt"); + } const backups = verifyAllBackups(manifest.files, options); ensureOriginalDirectories(manifest.directories, options); restoreFiles(manifest.files, backups, options); @@ -1015,11 +1353,123 @@ export function commitManagedStartupSharedStateTransaction( if (options.readOnlyReceipt) { fail("cannot commit a read-only rollback receipt"); } + const committed = loadCommitReceipt(options); + if (committed) { + if (options.bootstrapIdentity === null) { + fail("durable commit receipt is missing its expected bootstrap identity"); + } + assertCommitReceiptMatches(committed.receipt, { + agent: expectedAgent, + bootstrapIdentity: options.bootstrapIdentity, + }); + compactDurableCommitReceipt(committed, options); + return true; + } const manifest = loadManifest(options); if (!manifest) return false; if (manifest.agent !== expectedAgent) { fail(`pending transaction targets ${manifest.agent}, expected ${expectedAgent}`); } - removeTransactionDirectory(options); + if (manifest.bootstrapIdentity !== options.bootstrapIdentity) { + fail("pending transaction belongs to a different bootstrap attempt"); + } + if (manifest.bootstrapIdentity === null) { + removeTransactionDirectory(options); + return true; + } + verifyAllBackups(manifest.files, options); + if (pathExistsNoFollow(options.commitReceiptDirectory)) { + fail("durable commit receipt path appeared before transaction commit"); + } + try { + fs.renameSync(options.transactionDirectory, options.commitReceiptDirectory); + fsyncDirectory(options.transactionParentDirectory); + } catch (error) { + fail(`could not atomically establish durable commit state: ${(error as Error).message}`); + } + const renamed = loadCommitReceipt(options); + if (!renamed) fail("durable commit state disappeared after atomic rename"); + assertCommitReceiptMatches(renamed.receipt, { + agent: expectedAgent, + profileFingerprint: manifest.profileFingerprint, + bootstrapIdentity: manifest.bootstrapIdentity, + }); + compactDurableCommitReceipt(renamed, options); return true; } + +/** + * Retire one exact durable bootstrap commit only after the runtime owner has + * proven its external rollback backup is gone. This prevents a completed + * attempt's image-owned receipt from blocking a later bootstrap with a + * different identity in the same persisted workload. + */ +export function clearManagedStartupSharedStateCommitReceipt( + expectedAgent: ManagedStartupAgent, + inputOptions: ManagedStartupSharedTransactionOptions = {}, +): boolean { + const options = resolveOptions(inputOptions); + requireTransactionIdentity(options); + if (options.readOnlyReceipt) { + fail("cannot clear a durable commit from a read-only receipt"); + } + if (options.bootstrapIdentity === null) { + fail("durable commit cleanup requires its bootstrap identity"); + } + const committed = loadCommitReceipt(options); + if (!committed) return false; + assertCommitReceiptMatches(committed.receipt, { + agent: expectedAgent, + bootstrapIdentity: options.bootstrapIdentity, + }); + compactDurableCommitReceipt(committed, options); + requireTrustedTransactionPath( + options.commitReceiptDirectory, + TRANSACTION_DIRECTORY_MODE, + options, + ); + requireTrustedTransactionPath(options.commitReceiptFile, TRANSACTION_FILE_MODE, options); + const entries = fs.readdirSync(options.commitReceiptDirectory); + if (entries.length !== 1 || entries[0] !== MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE) { + fail("durable commit receipt directory contains unexpected artifacts"); + } + fs.rmSync(options.commitReceiptDirectory, { force: false, recursive: true }); + fsyncDirectory(options.transactionParentDirectory); + if (pathExistsNoFollow(options.commitReceiptDirectory)) { + fail("durable commit receipt remained after cleanup"); + } + return true; +} + +export function getManagedStartupSharedStateTransactionStatus( + expected: { + readonly agent: ManagedStartupAgent; + readonly profileFingerprint: string; + readonly bootstrapIdentity: string; + }, + inputOptions: ManagedStartupSharedTransactionOptions = {}, +): "committed" | "none" | "pending" { + const options = resolveOptions({ + ...inputOptions, + bootstrapIdentity: expected.bootstrapIdentity, + }); + requireTransactionIdentity(options); + const manifest = loadManifest(options); + if (manifest) { + if ( + manifest.agent !== expected.agent || + manifest.profileFingerprint !== expected.profileFingerprint || + manifest.bootstrapIdentity !== expected.bootstrapIdentity + ) { + fail( + "pending transaction does not match the expected agent, profile fingerprint, or bootstrap identity", + ); + } + verifyAllBackups(manifest.files, options); + return "pending"; + } + const committed = loadCommitReceipt(options); + if (!committed) return "none"; + assertCommitReceiptMatches(committed.receipt, expected); + return "committed"; +} From 942ce856b0a5324be3319a702a4201adfce87fce Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 12:55:46 -0700 Subject: [PATCH 112/117] feat(onboard): recover durable bootstrap transactions Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/README.md | 32 +- .../onboard/managed-bootstrap/adapter.test.ts | 49 +++ src/lib/onboard/managed-bootstrap/adapter.ts | 102 +++++ .../managed-bootstrap/docker-journal.test.ts | 39 ++ .../managed-bootstrap/docker-journal.ts | 45 ++- .../managed-bootstrap/docker-recovery.test.ts | 201 +++++++++ .../managed-bootstrap/docker-runtime.ts | 4 + .../managed-bootstrap/docker-test-fixture.ts | 28 ++ .../onboard/managed-bootstrap/docker.test.ts | 6 + src/lib/onboard/managed-bootstrap/docker.ts | 381 +++++++++++++++++- src/lib/onboard/managed-bootstrap/index.ts | 2 + .../managed-bootstrap/runtime-create.ts | 2 + .../runtime-provider-contract.test.ts | 1 + .../onboard/sandbox-gpu-create-flow.test.ts | 22 +- .../onboard/sandbox-gpu-create-run-attempt.ts | 1 + 15 files changed, 890 insertions(+), 25 deletions(-) create mode 100644 src/lib/onboard/managed-bootstrap/docker-recovery.test.ts diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index b5f8434f79f..7e7b3ba805d 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -94,17 +94,23 @@ recreation does not depend on process-local transaction sets or tombstone maps. The image-owned shared-state transaction uses the same identity-bound model: a commit atomically moves its pending manifest and backups into a durable receipt namespace, compacts that state to an exact commit receipt, and rejects rollback -when a later image-runtime invocation reads that receipt. The provider may -retire that receipt only after it proves the external rollback backup is gone, -so that this receipt does not block the next bootstrap attempt. -Direct identity lookup reconstructs one known transaction record. The bounded -[3.12b recovery slice](https://github.com/NVIDIA/NemoClaw/issues/7744) introduces -unfinished-record enumeration together with phase reconciliation and -cross-surface resume or rollback. The adapter reads mutable OpenShell names only -to detect ownership reuse. Unsafe name-only deletion returns a typed retention -error. The dormant adapter assumes the protocol's single coordinator; -multi-process lease/arbitration remains an explicit production-activation gate. -Activation must also inject the selected gateway's canonical state root. +after a restart. The provider may retire that receipt only after it proves the +external rollback backup is gone, leaving the next bootstrap attempt unblocked. +Direct identity lookup reconstructs one known transaction record, while managed +create-lifecycle startup uses unfinished-record enumeration to ask the selected +provider to reconcile every identity-addressed record before a new sandbox +create begins. The Docker provider then resumes the durable phase monotonically: +staged work rolls back without entering cutover; cutover work follows a proven +image-owned commit forward or durably authorizes rollback; rollback-authorized +work completes exact restore and cleanup; and shared-state-committed work +completes exact backup cleanup and commit. Recovery persists an identity-bound +finalization receipt before removing the active journal, is idempotent across +another interruption, and returns normalized, provider-owned receipts in stable +identity order. Mutable OpenShell names are read only to detect ownership reuse, +and unsafe name-only deletion returns a typed retention error. The protocol still +assumes a single coordinator; multi-process lease/arbitration remains an explicit +production-activation gate. Activation must also inject the selected gateway's +canonical state root. ## Architectural disposition @@ -113,8 +119,8 @@ candidate Docker surface owns create routing, replacement construction, native-to-compatibility fallback evidence, and deferred commit or rollback. Central onboarding accepts that provider-neutral surface without a Docker or Podman selection branch. Tests register an MXC-style surface through the same -bundle and render held launches for OpenClaw, Hermes, and LangChain Deep Agents -Code. +bundle, render held launches for OpenClaw, Hermes, and LangChain Deep Agents +Code, and exercise recovery phases across all three agents. The coordinator remains the driver-neutral transaction authority: its receipt shapes, normalization, state transitions, and rollback proofs form one cohesive diff --git a/src/lib/onboard/managed-bootstrap/adapter.test.ts b/src/lib/onboard/managed-bootstrap/adapter.test.ts index 9968171289d..3f80e00e61d 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.test.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.test.ts @@ -25,6 +25,7 @@ import { type ManagedBootstrapPreparedReplacementHandle, type ManagedBootstrapReplacementHandle, prepareManagedBootstrapSequence, + recoverManagedBootstrapTransactions, renderManagedBootstrapHeldCommand, } from "./adapter"; @@ -227,6 +228,7 @@ function adapterFor(agent: ManagedStartupAgent): Fixture { const order: string[] = []; const raw: Fixture["raw"] = { handle: null, snapshot: null, prepared: null }; const adapter: ManagedBootstrapAdapter = { + recoverUnfinishedTransactions: vi.fn(async () => []), createHeldWorkload: vi.fn(async (input) => { order.push("create"); const receipt = await input.launch({ @@ -908,6 +910,53 @@ describe("managed bootstrap adapter contract", () => { expect(fixture.adapter.finalizeBootstrap).not.toHaveBeenCalled(); }); + it("normalizes, freezes, and orders provider-owned restart recovery receipts", async () => { + const fixture = adapterFor("openclaw"); + const receipt = cleanupReceipt(); + const candidate = (bootstrapIdentity: string) => ({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: receipt.sandbox.driverId, + sourcePhase: "cutover", + sandbox: receipt.sandbox, + bootstrapIdentity, + outcome: "rolled-back" as const, + finalization: { ...receipt, bootstrapIdentity }, + }); + vi.mocked(fixture.adapter.recoverUnfinishedTransactions).mockResolvedValueOnce([ + candidate("b".repeat(64)), + candidate("a".repeat(64)), + ]); + + const recovered = await recoverManagedBootstrapTransactions(fixture.adapter); + + expect(recovered.map(({ bootstrapIdentity }) => bootstrapIdentity)).toEqual([ + "a".repeat(64), + "b".repeat(64), + ]); + expect(Object.isFrozen(recovered)).toBe(true); + expect(recovered.every((entry) => Object.isFrozen(entry.finalization))).toBe(true); + }); + + it("rejects recovery evidence whose provider does not own the durable sandbox", async () => { + const fixture = adapterFor("openclaw"); + const receipt = cleanupReceipt(); + vi.mocked(fixture.adapter.recoverUnfinishedTransactions).mockResolvedValueOnce([ + { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: "mxc", + sourcePhase: "cutover", + sandbox: receipt.sandbox, + bootstrapIdentity: IDENTITY, + outcome: "rolled-back", + finalization: receipt, + }, + ]); + + await expect(recoverManagedBootstrapTransactions(fixture.adapter)).rejects.toThrow( + "recovery provider does not own", + ); + }); + it.each([ "BASHOPTS=extdebug", "BASH_ENV=/sandbox/attacker", diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index ee1a834985f..9f7e6bf3cf1 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -239,6 +239,21 @@ export interface ManagedBootstrapFinalizationReceipt { readonly finalizedAt: string; } +/** + * Driver-neutral evidence that one durable, process-orphaned transaction was + * reconciled without reconstructing authority from mutable runtime names. + */ +export interface ManagedBootstrapRecoveryReceipt { + readonly schemaVersion: typeof MANAGED_BOOTSTRAP_SCHEMA_VERSION; + readonly providerId: string; + /** Provider-owned phase name retained for diagnostics, never central routing. */ + readonly sourcePhase: string; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly bootstrapIdentity: string; + readonly outcome: "committed" | "rolled-back"; + readonly finalization: ManagedBootstrapFinalizationReceipt; +} + export class ManagedBootstrapDurableCommitCleanupPendingError extends Error { readonly bootstrapIdentity: string; readonly cleanupRuntimeId: string; @@ -310,6 +325,12 @@ export function attachManagedBootstrapRollbackError(failure: Error, rollbackErro } export interface ManagedBootstrapAdapter { + /** + * Enumerate durable unfinished records and reconcile each through the owning + * provider. Implementations must be restart-safe and idempotent. + */ + recoverUnfinishedTransactions(): Promise; + /** Return only after one durable sandbox/driver identity reports Ready. */ createHeldWorkload( input: ManagedBootstrapCreateInput, @@ -376,6 +397,87 @@ export interface ManagedBootstrapAdapter { }): Promise; } +function normalizeRecoveryReceipt( + candidate: ManagedBootstrapRecoveryReceipt, +): ManagedBootstrapRecoveryReceipt { + if ( + typeof candidate !== "object" || + candidate === null || + Array.isArray(candidate) || + candidate.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + !["committed", "rolled-back"].includes(String(candidate.outcome)) + ) { + protocolFail("recovery receipt has an invalid schema or outcome"); + } + assertOpaqueString(candidate.providerId, "recovery provider ID"); + assertOpaqueString(candidate.sourcePhase, "recovery source phase"); + assertSandboxIdentity(candidate.sandbox); + if (candidate.sandbox.driverId !== candidate.providerId) { + protocolFail("recovery provider does not own the recovered sandbox"); + } + if (!SHA256_RE.test(candidate.bootstrapIdentity)) { + protocolFail("recovery bootstrap identity must be lowercase SHA-256"); + } + const finalization = candidate.finalization; + if ( + typeof finalization !== "object" || + finalization === null || + Array.isArray(finalization) || + finalization.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + finalization.outcome !== candidate.outcome || + finalization.bootstrapIdentity !== candidate.bootstrapIdentity || + !isDeepStrictEqual(finalization.sandbox, candidate.sandbox) || + typeof finalization.heldWorkloadRemoved !== "boolean" || + typeof finalization.alreadyRolledBack !== "boolean" + ) { + protocolFail("recovery finalization does not match its durable identity"); + } + if ( + (finalization.restoredRuntimeId !== null && !SHA256_RE.test(finalization.restoredRuntimeId)) || + (finalization.restoredSpecHash !== null && !SHA256_RE.test(finalization.restoredSpecHash)) || + (finalization.restoredRuntimeId === null) !== (finalization.restoredSpecHash === null) || + (candidate.outcome === "committed" && + (finalization.restoredRuntimeId !== null || + finalization.heldWorkloadRemoved || + finalization.alreadyRolledBack)) + ) { + protocolFail("recovery finalization state is inconsistent"); + } + assertTimestamp(finalization.finalizedAt, "recovery finalization timestamp"); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: candidate.providerId, + sourcePhase: candidate.sourcePhase, + sandbox: Object.freeze({ ...candidate.sandbox }), + bootstrapIdentity: candidate.bootstrapIdentity, + outcome: candidate.outcome, + finalization: Object.freeze({ + ...finalization, + sandbox: Object.freeze({ ...candidate.sandbox }), + }), + }); +} + +/** Recover process-orphaned work without relying on coordinator WeakMap state. */ +export async function recoverManagedBootstrapTransactions( + adapter: ManagedBootstrapAdapter, +): Promise { + const candidates = await adapter.recoverUnfinishedTransactions(); + if (!Array.isArray(candidates)) { + protocolFail("provider recovery must return a receipt array"); + } + const receipts = candidates.map(normalizeRecoveryReceipt); + const identities = receipts.map(({ bootstrapIdentity }) => bootstrapIdentity); + if (new Set(identities).size !== identities.length) { + protocolFail("provider recovery returned duplicate bootstrap identities"); + } + return Object.freeze( + [...receipts].sort((left, right) => + left.bootstrapIdentity.localeCompare(right.bootstrapIdentity), + ), + ); +} + export interface ManagedBootstrapPreparationInput { readonly create: ManagedBootstrapCreateInput; readonly request: ManagedStartupRootApplyRequest; diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index 25cd9ec0e50..b1ed163b954 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -28,6 +28,7 @@ const journal = Object.freeze({ phase: "staged", bootstrapIdentity: IDENTITY, providerId: "docker", + agent: "hermes", sandbox: { sandboxName: "alpha", sandboxId: "sandbox-alpha", @@ -65,6 +66,7 @@ const finalization = Object.freeze({ phase: "committed", bootstrapIdentity: IDENTITY, providerId: "docker", + agent: journal.agent, sandbox: journal.sandbox, planFingerprint: journal.planFingerprint, profileFingerprint: journal.profileFingerprint, @@ -286,6 +288,9 @@ describe("Docker managed bootstrap journal", () => { first.create(journal); first.recordFinalization(finalization); + expect(first.listUnfinished()).toEqual([journal]); + first.remove(IDENTITY, ["staged"]); + expect(first.listUnfinished()).toEqual([]); const restarted = createFileDockerManagedBootstrapJournalStore(root); expect(restarted.loadFinalization(IDENTITY)).toEqual(finalization); expect( @@ -309,6 +314,40 @@ describe("Docker managed bootstrap journal", () => { ).toThrow("finalization record changed"); }); + it.each([ + { label: "journal", suffix: "" }, + { label: "decision", suffix: ".decision" }, + { label: "finalization", suffix: ".finalized" }, + ])("ignores an atomic $label write left by a crash during enumeration", ({ suffix }) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + const directory = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); + fs.writeFileSync( + path.join(directory, `.${IDENTITY}.json${suffix}.1234.deadbeef.tmp`), + "partial", + { + mode: 0o600, + }, + ); + + expect(store.listUnfinished()).toEqual([journal]); + }); + + it("rejects an unsupported journal-directory entry during enumeration", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + const directory = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); + fs.writeFileSync(path.join(directory, `${IDENTITY}.json.unknown`), "unexpected", { + mode: 0o600, + }); + + expect(() => store.listUnfinished()).toThrow("journal directory contains an unsupported entry"); + }); + it("reloads the exact completion receipt from a new journal store", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); roots.push(root); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index 015bda1d12d..db37c5ce213 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -3,7 +3,7 @@ import fs from "node:fs"; import path from "node:path"; - +import type { ManagedStartupAgent } from "../managed-startup/profile"; import type { ManagedBootstrapCompletionReceipt, ManagedBootstrapDurablePreparationReceipt, @@ -11,9 +11,9 @@ import type { ManagedBootstrapSandboxIdentity, } from "./adapter"; -export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 2 as const; +export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 3 as const; export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY = "managed-bootstrap"; -export const DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION = 1 as const; +export const DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION = 2 as const; const SHA256_RE = /^[a-f0-9]{64}$/u; const MANIFEST_DIGEST_RE = /^sha256:[a-f0-9]{64}$/u; @@ -36,6 +36,7 @@ export interface DockerManagedBootstrapJournal { readonly phase: DockerManagedBootstrapJournalPhase; readonly bootstrapIdentity: string; readonly providerId: string; + readonly agent: ManagedStartupAgent; readonly sandbox: ManagedBootstrapSandboxIdentity; readonly planFingerprint: string; readonly profileFingerprint: string; @@ -59,6 +60,7 @@ export interface DockerManagedBootstrapFinalizationRecord { readonly phase: "committed" | "rolled-back"; readonly bootstrapIdentity: string; readonly providerId: string; + readonly agent: ManagedStartupAgent; readonly sandbox: ManagedBootstrapSandboxIdentity; readonly planFingerprint: string; readonly profileFingerprint: string; @@ -70,6 +72,7 @@ export interface DockerManagedBootstrapFinalizationRecord { export interface DockerManagedBootstrapJournalStore { create(journal: DockerManagedBootstrapJournal): void; load(bootstrapIdentity: string): DockerManagedBootstrapJournal | null; + listUnfinished(): readonly DockerManagedBootstrapJournal[]; transition( bootstrapIdentity: string, expected: DockerManagedBootstrapJournalPhase, @@ -144,6 +147,13 @@ function exactPhase(value: unknown): DockerManagedBootstrapJournalPhase { return value as DockerManagedBootstrapJournalPhase; } +function exactAgent(value: unknown): ManagedStartupAgent { + if (!["openclaw", "hermes", "langchain-deepagents-code"].includes(String(value))) { + fail("agent is unsupported"); + } + return value as ManagedStartupAgent; +} + function exactSandbox(value: unknown): ManagedBootstrapSandboxIdentity { if (typeof value !== "object" || value === null || Array.isArray(value)) { fail("sandbox identity must be an object"); @@ -167,6 +177,7 @@ export function normalizeDockerManagedBootstrapJournal( } const journal = value as Record; const expectedKeys = [ + "agent", "backupName", "bootstrapIdentity", "commitReceipt", @@ -199,6 +210,7 @@ export function normalizeDockerManagedBootstrapJournal( phase: exactPhase(journal.phase), bootstrapIdentity: exactSha256(journal.bootstrapIdentity, "bootstrap identity"), providerId: exactString(journal.providerId, "provider ID"), + agent: exactAgent(journal.agent), sandbox: exactSandbox(journal.sandbox), planFingerprint: exactSha256(journal.planFingerprint, "plan fingerprint"), profileFingerprint: exactSha256(journal.profileFingerprint, "profile fingerprint"), @@ -484,6 +496,7 @@ export function normalizeDockerManagedBootstrapFinalizationRecord( } const record = value as Record; const expectedKeys = [ + "agent", "bootstrapIdentity", "cleanupReceipt", "commitReceipt", @@ -512,6 +525,7 @@ export function normalizeDockerManagedBootstrapFinalizationRecord( phase, bootstrapIdentity: exactSha256(record.bootstrapIdentity, "finalization bootstrap identity"), providerId: exactString(record.providerId, "finalization provider ID"), + agent: exactAgent(record.agent), sandbox, planFingerprint: exactSha256(record.planFingerprint, "finalization plan fingerprint"), profileFingerprint: exactSha256(record.profileFingerprint, "finalization profile fingerprint"), @@ -777,6 +791,31 @@ export function createFileDockerManagedBootstrapJournalStore( atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(normalized), true); }, load, + listUnfinished() { + assertDirectory(directory); + const identities: string[] = []; + for (const name of fs.readdirSync(directory)) { + const match = name.match(/^([a-f0-9]{64})\.json$/u); + if (match) { + identities.push(match[1]); + continue; + } + if ( + /^\.[a-f0-9]{64}\.json(?:\.decision|\.finalized)?\.[0-9]+\.[a-f0-9]+\.tmp$/u.test(name) || + /^[a-f0-9]{64}\.json\.(?:decision|finalized)$/u.test(name) + ) { + continue; + } + fail(`journal directory contains an unsupported entry: ${name}`); + } + return Object.freeze( + identities.sort().map((identity) => { + const journal = load(identity); + if (!journal) fail(`enumerated journal ${identity} disappeared`); + return journal; + }), + ); + }, transition( bootstrapIdentity: string, expected: DockerManagedBootstrapJournalPhase, diff --git a/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts b/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts new file mode 100644 index 00000000000..fc9f22b8d1a --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts @@ -0,0 +1,201 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { createDockerManagedBootstrapAdapter } from "./docker"; +import type { DockerManagedBootstrapJournalStore } from "./docker-journal"; +import { + authority, + type DockerFixtureOptions, + durablePreparation, + fixture, +} from "./docker-test-fixture"; + +async function prepareTransaction( + fake: ReturnType, + agent: Parameters[0] = "hermes", +) { + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority(agent); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request, + replacementOptions: { values: {} }, + }); + return { + adapter, + handle, + prepared, + snapshot, + durable: durablePreparation(handle, snapshot, prepared), + }; +} + +describe("Docker managed bootstrap restart recovery", () => { + it.each([ + { + label: "staged", + options: { + journalCreateFailures: [new Error("injected crash after durable staged fence")], + }, + phase: "staged", + }, + { + label: "cutover", + options: { + journalTransitionFailures: { + cutover: new Error("injected crash after durable cutover fence"), + }, + }, + phase: "cutover", + }, + ] satisfies readonly { + readonly label: string; + readonly options: DockerFixtureOptions; + readonly phase: "cutover" | "staged"; + }[])("reconciles a process restart from the durable $label phase", async ({ options, phase }) => { + const fake = fixture(options); + const transaction = await prepareTransaction(fake); + + await expect( + transaction.adapter.activateBootstrapReplacement({ + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + }), + ).rejects.toThrow(`crash after durable ${phase} fence`); + expect(fake.journal?.phase).toBe(phase); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject([ + { sourcePhase: phase, outcome: "rolled-back" }, + ]); + expect(fake.journal).toBeNull(); + expect(fake.finalization?.phase).toBe("rolled-back"); + expect(fake.replacement).toBeNull(); + expect(fake.original?.State?.Running).toBe(true); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toEqual([]); + }); + + it("finishes rollback-authorized recovery after shared-state rollback is interrupted", async () => { + const fake = fixture({ + agent: "openclaw", + journalTransitionFailures: { + "rollback-authorized": new Error("injected crash after durable rollback fence"), + }, + sharedState: "pending", + }); + const transaction = await prepareTransaction(fake, "openclaw"); + const replacement = await transaction.adapter.activateBootstrapReplacement({ + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + }); + + await expect( + transaction.adapter.finalizeBootstrap({ + outcome: "rollback", + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + replacement, + completion: null, + }), + ).rejects.toThrow("crash after durable rollback fence"); + expect(fake.journal?.phase).toBe("rollback-authorized"); + expect(fake.sharedState).toBe("pending"); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject([ + { sourcePhase: "rollback-authorized", outcome: "rolled-back" }, + ]); + expect(fake.sharedState).toBe("none"); + expect(fake.replacement).toBeNull(); + expect(fake.original?.State?.Running).toBe(true); + }); + + it("compacts a terminal commit journal after another restart interruption", async () => { + const fake = fixture({ + agent: "langchain-deepagents-code", + dockerRemoveFailures: [new Error("injected crash before exact Docker removal")], + journalRemoveFailures: [new Error("injected crash before terminal journal removal")], + sharedState: "pending", + }); + const transaction = await prepareTransaction(fake, "langchain-deepagents-code"); + const replacement = await transaction.adapter.activateBootstrapReplacement({ + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + }); + const completion = await transaction.adapter.awaitBootstrap({ + handle: transaction.handle, + snapshot: transaction.snapshot, + replacement, + timeoutSecs: 1, + }); + + await expect( + transaction.adapter.finalizeBootstrap({ + outcome: "commit", + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + replacement, + completion, + }), + ).rejects.toThrow("crash before exact Docker removal"); + expect(fake.journal?.phase).toBe("shared-state-committed"); + expect(fake.sharedState).toBe("committed"); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect(restarted.recoverUnfinishedTransactions()).rejects.toThrow( + "crash before terminal journal removal", + ); + expect(fake.journal?.phase).toBe("shared-state-committed"); + expect(fake.finalization?.phase).toBe("committed"); + + const journalStore = fake.deps.journalStore; + if (!journalStore) throw new Error("fixture journal store is missing"); + const reorderedFinalizationStore = { + ...journalStore, + loadFinalization(bootstrapIdentity: string) { + const record = journalStore.loadFinalization(bootstrapIdentity); + const receipt = record?.commitReceipt; + if (!record || !receipt) return record; + return { + ...record, + commitReceipt: { + completedAt: receipt.completedAt, + transactionPending: receipt.transactionPending, + bootstrapIdentity: receipt.bootstrapIdentity, + profileFingerprint: receipt.profileFingerprint, + replacementSpecHash: receipt.replacementSpecHash, + originalSpecHash: receipt.originalSpecHash, + runtimeImageContentId: receipt.runtimeImageContentId, + image: receipt.image, + runtimeId: receipt.runtimeId, + sandbox: receipt.sandbox, + schemaVersion: receipt.schemaVersion, + } satisfies typeof receipt, + }; + }, + } satisfies DockerManagedBootstrapJournalStore; + const resumed = createDockerManagedBootstrapAdapter({ + ...fake.deps, + journalStore: reorderedFinalizationStore, + }); + await expect(resumed.recoverUnfinishedTransactions()).resolves.toMatchObject([ + { sourcePhase: "shared-state-committed", outcome: "committed" }, + ]); + expect(fake.journal).toBeNull(); + expect(fake.sharedState).toBe("none"); + expect(fake.replacement?.State?.Running).toBe(true); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.ts index 6da0f754f1b..dba07739d18 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.ts @@ -21,6 +21,7 @@ import { finalizeManagedBootstrapSequence, MANAGED_BOOTSTRAP_SCHEMA_VERSION, prepareManagedBootstrapSequence, + recoverManagedBootstrapTransactions, } from "./adapter"; import { createDockerManagedBootstrapAdapter } from "./docker"; import type { @@ -137,6 +138,9 @@ function createDockerLifecycle( return { launchArgv: input.launchArgv, patch, + async recoverUnfinished() { + return recoverManagedBootstrapTransactions(adapter); + }, async prepareNetwork() { if (input.route !== "compatibility") return; const { enforceDockerGpuPatchPreserveNetwork } = await import( diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index 088a3ed7a05..42f0fc3ae1d 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -65,7 +65,10 @@ export type DockerFixtureAcknowledgement = export type DockerFixtureOptions = { readonly agent?: ManagedStartupAgent; + readonly dockerRemoveFailures?: readonly Error[]; readonly dockerStartResults?: Readonly>; + readonly journalCreateFailures?: readonly Error[]; + readonly journalRemoveFailures?: readonly Error[]; readonly journalTransitionFailures?: Partial< Readonly> >; @@ -214,6 +217,9 @@ export function fixture(options: DockerFixtureOptions = {}) { let finalization: DockerManagedBootstrapFinalizationRecord | null = null; let sharedState: "committed" | "none" | "pending" = options.sharedState ?? "none"; const events: string[] = []; + const dockerRemoveFailures = [...(options.dockerRemoveFailures ?? [])]; + const journalCreateFailures = [...(options.journalCreateFailures ?? [])]; + const journalRemoveFailures = [...(options.journalRemoveFailures ?? [])]; const lostAcknowledgements = new Set(options.lostAcknowledgements ?? []); const losesAcknowledgement = (operation: DockerFixtureAcknowledgement) => lostAcknowledgements.has(operation); @@ -223,6 +229,13 @@ export function fixture(options: DockerFixtureOptions = {}) { create(value) { journal = structuredClone(value); events.push("journal:staged"); + const injectedFailure = journalCreateFailures.shift(); + switch (injectedFailure) { + case undefined: + break; + default: + throw injectedFailure; + } if (losesAcknowledgement("journal:create")) { throw new DockerManagedBootstrapJournalAcknowledgementLostError( "lost journal create acknowledgement", @@ -230,6 +243,7 @@ export function fixture(options: DockerFixtureOptions = {}) { } }, load: () => copyJournal(), + listUnfinished: () => (journal ? [structuredClone(journal)] : []), transition(_identity, expected, next) { const current = journal !== null && journal.phase === expected @@ -265,6 +279,13 @@ export function fixture(options: DockerFixtureOptions = {}) { void (current !== null && expected.includes(current.phase) ? current : failFixture("stale journal remove")); + const injectedFailure = journalRemoveFailures.shift(); + switch (injectedFailure) { + case undefined: + break; + default: + throw injectedFailure; + } journal = null; events.push("journal:removed"); if (losesAcknowledgement("journal:remove")) { @@ -453,6 +474,13 @@ export function fixture(options: DockerFixtureOptions = {}) { }), dockerRm: vi.fn((id) => { events.push(`rm:${id}`); + const injectedFailure = dockerRemoveFailures.shift(); + switch (injectedFailure) { + case undefined: + break; + default: + throw injectedFailure; + } switch (id) { case OLD_ID: original = null; diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 8e94b573589..842c14f654e 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -129,6 +129,9 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( fake.events.indexOf(`rm:${OLD_ID}`), ); + expect(fake.events.indexOf("finalization:committed")).toBeLessThan( + fake.events.indexOf("journal:removed"), + ); expect(fake.journal).toBeNull(); expect(fake.finalization).toMatchObject({ phase: "committed", commitReceipt }); expect(fake.sharedState).toBe("none"); @@ -260,6 +263,9 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( fake.events.indexOf(`rm:${NEW_ID}`), ); + expect(fake.events.indexOf("finalization:rolled-back")).toBeLessThan( + fake.events.indexOf("journal:removed"), + ); expect(fake.journal).toBeNull(); expect(fake.replacement).toBeNull(); expect(fake.original).not.toBeNull(); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index e633ee9f2ed..89c3c83b76a 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -58,6 +58,7 @@ import { type ManagedBootstrapObservedSnapshot, ManagedBootstrapOwnerCleanupRequiredError, type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapRecoveryReceipt, type ManagedBootstrapReplacementHandle, type ManagedBootstrapReplacementOptions, type ManagedBootstrapSandboxIdentity, @@ -1416,6 +1417,16 @@ function managedSharedStateTransaction( } as const; } +function recoveredManagedSharedStateTransaction(journal: DockerBootstrapTransaction) { + return { + agent: journal.agent, + bootstrapIdentity: journal.bootstrapIdentity, + containerId: journal.replacementRuntimeId, + image: journal.runtimeImageContentId, + profileFingerprint: journal.profileFingerprint, + } as const; +} + function sameDockerBootstrapJournal( left: DockerBootstrapTransaction, right: DockerBootstrapTransaction, @@ -1766,6 +1777,7 @@ export function createDockerManagedBootstrapAdapter( phase, bootstrapIdentity: handle.bootstrapIdentity, providerId: handle.sandbox.driverId, + agent: handle.plan.profile.agent, sandbox: handle.sandbox, planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), profileFingerprint: handle.plan.profile.fingerprint, @@ -1811,6 +1823,21 @@ export function createDockerManagedBootstrapAdapter( } satisfies ManagedBootstrapFinalizationReceipt); return persistFinalization(handle, "rolled-back", null, receipt); }; + const completeRollbackTransaction = ( + handle: ManagedBootstrapHeldWorkloadHandle, + journal: DockerBootstrapTransaction, + ): ManagedBootstrapFinalizationReceipt => { + let ownerCleanupFailure: { readonly error: unknown } | null = null; + try { + retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, journal.originalRuntimeId); + } catch (error) { + ownerCleanupFailure = { error }; + } + const finalization = completedRollback(handle, false); + removeDockerBootstrapJournalDurably(journal, deps); + if (ownerCleanupFailure) throw ownerCleanupFailure.error; + return finalization; + }; const completedCommit = ( handle: ManagedBootstrapHeldWorkloadHandle, commitReceipt: ManagedBootstrapCompletionReceipt, @@ -1845,6 +1872,332 @@ export function createDockerManagedBootstrapAdapter( alreadyRolledBack: true, }); }; + const persistRecoveredFinalization = ( + journal: DockerBootstrapTransaction, + phase: "committed" | "rolled-back", + commitReceipt: ManagedBootstrapCompletionReceipt | null, + cleanupReceipt: ManagedBootstrapFinalizationReceipt, + ): ManagedBootstrapFinalizationReceipt => { + const record = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase, + bootstrapIdentity: journal.bootstrapIdentity, + providerId: journal.providerId, + agent: journal.agent, + sandbox: journal.sandbox, + planFingerprint: journal.planFingerprint, + profileFingerprint: journal.profileFingerprint, + imageReference: journal.imageReference, + commitReceipt, + cleanupReceipt, + } satisfies DockerManagedBootstrapFinalizationRecord); + const serialized = serializeDockerManagedBootstrapFinalizationRecord(record); + try { + deps.journalStore.recordFinalization(record); + } catch (error) { + const recovered = deps.journalStore.loadFinalization(journal.bootstrapIdentity); + if ( + !recovered || + serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized + ) { + throw error; + } + } + const persisted = deps.journalStore.loadFinalization(journal.bootstrapIdentity); + if (!persisted || serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized) { + throw new Error("Managed bootstrap recovered finalization was not durably re-readable."); + } + return persisted.cleanupReceipt; + }; + const recoveredReceipt = ( + journal: DockerBootstrapTransaction, + sourcePhase: DockerBootstrapTransaction["phase"], + finalization: ManagedBootstrapFinalizationReceipt, + ): ManagedBootstrapRecoveryReceipt => + Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: journal.providerId, + sourcePhase, + sandbox: journal.sandbox, + bootstrapIdentity: journal.bootstrapIdentity, + outcome: finalization.outcome, + finalization, + }); + const compactRecoveredFinalization = ( + journal: DockerBootstrapTransaction, + sourcePhase: DockerBootstrapTransaction["phase"], + ): ManagedBootstrapRecoveryReceipt | null => { + const finalization = deps.journalStore.loadFinalization(journal.bootstrapIdentity); + if (!finalization) return null; + const phaseMatches = + (finalization.phase === "committed" && + journal.phase === "shared-state-committed" && + finalization.commitReceipt !== null && + journal.commitReceipt !== null && + sameDockerManagedBootstrapReceipt( + "completion", + finalization.commitReceipt, + journal.commitReceipt, + )) || + (finalization.phase === "rolled-back" && + (journal.phase === "staged" || journal.phase === "rollback-authorized") && + finalization.commitReceipt === null); + if ( + !phaseMatches || + finalization.bootstrapIdentity !== journal.bootstrapIdentity || + finalization.providerId !== journal.providerId || + finalization.agent !== journal.agent || + finalization.sandbox.sandboxName !== journal.sandbox.sandboxName || + finalization.sandbox.sandboxId !== journal.sandbox.sandboxId || + finalization.sandbox.driverId !== journal.sandbox.driverId || + finalization.planFingerprint !== journal.planFingerprint || + finalization.profileFingerprint !== journal.profileFingerprint || + finalization.imageReference !== journal.imageReference + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: + journal.phase === "shared-state-committed" + ? journal.replacementRuntimeId + : journal.originalRuntimeId, + detail: "terminal finalization does not match its retained durable journal", + }); + } + removeDockerBootstrapJournalDurably(journal, deps); + return recoveredReceipt(journal, sourcePhase, finalization.cleanupReceipt); + }; + const finishRecoveredRollback = ( + journal: DockerBootstrapTransaction, + sourcePhase: DockerBootstrapTransaction["phase"], + ): ManagedBootstrapRecoveryReceipt => { + const cleanupReceipt = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: journal.sandbox, + bootstrapIdentity: journal.bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: journal.originalRuntimeId, + restoredSpecHash: journal.originalSpecHash, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + } satisfies ManagedBootstrapFinalizationReceipt); + const finalization = persistRecoveredFinalization(journal, "rolled-back", null, cleanupReceipt); + removeDockerBootstrapJournalDurably(journal, deps); + return recoveredReceipt(journal, sourcePhase, finalization); + }; + const finishRecoveredCommit = ( + journal: DockerBootstrapTransaction, + sourcePhase: DockerBootstrapTransaction["phase"], + ): ManagedBootstrapRecoveryReceipt => { + if (journal.phase !== "shared-state-committed" || journal.commitReceipt === null) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable commit recovery requires its exact completion receipt and commit fence", + }); + } + const replacement = inspectTransactionRuntime(journal, journal.replacementRuntimeId, deps); + if (!replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "the exact committed replacement is absent during restart recovery", + }); + } + assertTransactionReplacement(journal, replacement); + if ( + dockerContainerName(replacement) !== journal.originalName || + !isStableRunning(replacement) || + normalizeDockerManagedBootstrapLaunchSpec(replacement).hash !== journal.replacementSpecHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "the committed replacement does not match its durable runtime authority", + }); + } + const sharedTransaction = recoveredManagedSharedStateTransaction(journal); + const sharedStatus = probeDockerManagedStartupSharedState( + { transaction: sharedTransaction, profileFingerprint: journal.profileFingerprint }, + deps, + ); + if (sharedStatus === "pending") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "shared state is pending after the durable commit fence", + }); + } + const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); + if (original) { + assertTransactionOriginal(journal, original); + if ( + dockerContainerName(original) !== journal.backupName || + !isExplicitlyStopped(original) || + normalizeDockerManagedBootstrapLaunchSpec({ + ...original, + Name: `/${journal.originalName}`, + }).hash !== journal.originalSpecHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "the exact rollback backup changed before recovered commit cleanup", + }); + } + if (sharedStatus === "none") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "the shared commit receipt was retired before exact backup absence was proven", + }); + } + const removed = deps.dockerRm(journal.originalRuntimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(journal.originalRuntimeId, deps) !== "absent" + ) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: journal.bootstrapIdentity, + cleanupRuntimeId: journal.originalRuntimeId, + detail: `${commandDetail(removed) || "Docker removal failed"}; exact backup absence was not proven`, + }); + } + } + if (sharedStatus === "committed") { + clearDockerManagedStartupSharedStateCommitReceipt(sharedTransaction, deps); + } + if (probeExactDockerContainerAbsence(journal.originalRuntimeId, deps) !== "absent") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: journal.bootstrapIdentity, + cleanupRuntimeId: journal.originalRuntimeId, + detail: "exact rollback-backup absence was not durable after restart recovery", + }); + } + const cleanupReceipt = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: journal.sandbox, + bootstrapIdentity: journal.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + } satisfies ManagedBootstrapFinalizationReceipt); + const finalization = persistRecoveredFinalization( + journal, + "committed", + journal.commitReceipt, + cleanupReceipt, + ); + removeDockerBootstrapJournalDurably(journal, deps); + return recoveredReceipt(journal, sourcePhase, finalization); + }; + const finishRecoveredRollbackPhase = ( + journal: DockerBootstrapTransaction, + sourcePhase: DockerBootstrapTransaction["phase"], + ): ManagedBootstrapRecoveryReceipt => { + const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); + if (!original) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "the exact rollback original is absent during restart recovery", + }); + } + assertTransactionOriginal(journal, original); + const replacement = inspectTransactionRuntime(journal, journal.replacementRuntimeId, deps); + if (replacement) assertTransactionReplacement(journal, replacement); + if (journal.phase === "staged") { + if ( + dockerContainerName(original) !== journal.originalName || + !isStableRunning(original) || + normalizeDockerManagedBootstrapLaunchSpec(original).hash !== journal.originalSpecHash || + (replacement !== null && + (dockerContainerName(replacement) !== journal.replacementStagingName || + !isExplicitlyStopped(replacement))) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "staged restart recovery does not match its pre-cutover fence", + }); + } + if (replacement) removeExactReplacement(journal, replacement, deps); + return finishRecoveredRollback(journal, sourcePhase); + } + if (journal.phase !== "cutover" && journal.phase !== "rollback-authorized") { + return finishRecoveredCommit(journal, sourcePhase); + } + let activeJournal = journal; + if (!replacement && dockerContainerName(original) !== journal.originalName) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "the replacement disappeared before exact rollback restoration was proven", + }); + } + if (replacement) { + const sharedTransaction = recoveredManagedSharedStateTransaction(journal); + const sharedStatus = probeDockerManagedStartupSharedState( + { transaction: sharedTransaction, profileFingerprint: journal.profileFingerprint }, + deps, + ); + if (sharedStatus === "committed") { + if (journal.phase === "rollback-authorized") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "shared state committed after durable rollback authorization", + }); + } + activeJournal = transitionDockerBootstrapJournalDurably( + journal, + "shared-state-committed", + deps, + ); + return finishRecoveredCommit(activeJournal, sourcePhase); + } + if (journal.phase === "cutover") { + activeJournal = transitionDockerBootstrapJournalDurably( + journal, + "rollback-authorized", + deps, + ); + } + if (sharedStatus === "pending") { + finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: false, + retainContainerAfterRollback: true, + }, + deps, + ); + } + } else if (journal.phase === "cutover") { + activeJournal = transitionDockerBootstrapJournalDurably(journal, "rollback-authorized", deps); + } + restoreOriginal(activeJournal, deps); + const restored = inspectExact(activeJournal.originalRuntimeId, deps); + if ( + !isStableRunning(restored) || + dockerContainerName(restored) !== activeJournal.originalName || + normalizeDockerManagedBootstrapLaunchSpec(restored).hash !== activeJournal.originalSpecHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: activeJournal.bootstrapIdentity, + runtimeId: activeJournal.originalRuntimeId, + detail: "restart recovery did not restore the exact original runtime and launch spec", + }); + } + return finishRecoveredRollback(activeJournal, sourcePhase); + }; const rollbackBootstrapNow = ({ handle, snapshot, @@ -2002,6 +2355,7 @@ export function createDockerManagedBootstrapAdapter( ); if (journal.phase === "staged") { + const stagedJournal: DockerBootstrapTransaction = journal; assertStableRunning(original, "staged original"); if (observedReplacement) { assertExplicitlyStopped(observedReplacement, "staged replacement"); @@ -2020,9 +2374,7 @@ export function createDockerManagedBootstrapAdapter( if (observedReplacement) { removeExactReplacement(journal, observedReplacement, deps); } - removeDockerBootstrapJournalDurably(journal, deps); - retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, journal.originalRuntimeId); - return completedRollback(handle, false); + return completeRollbackTransaction(handle, stagedJournal); } if (journal.phase !== "cutover" && journal.phase !== "rollback-authorized") { @@ -2200,9 +2552,7 @@ export function createDockerManagedBootstrapAdapter( ) { throw new Error("Managed bootstrap Docker rollback did not restore its exact original."); } - removeDockerBootstrapJournalDurably(activeJournal, deps); - retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, activeJournal.originalRuntimeId); - return completedRollback(handle, false); + return completeRollbackTransaction(handle, activeJournal); }; const commitBootstrapNow = ( handle: ManagedBootstrapHeldWorkloadHandle, @@ -2303,8 +2653,9 @@ export function createDockerManagedBootstrapAdapter( }); } } + const finalization = completedCommit(handle, receipt); removeDockerBootstrapJournalDurably(transaction, deps); - return completedCommit(handle, receipt); + return finalization; }; const finalizeBootstrap = async ( input: Parameters[0], @@ -2493,6 +2844,21 @@ export function createDockerManagedBootstrapAdapter( }); }; return { + async recoverUnfinishedTransactions() { + const receipts: ManagedBootstrapRecoveryReceipt[] = []; + for (const journal of deps.journalStore.listUnfinished()) { + const sourcePhase = journal.phase; + const finalized = compactRecoveredFinalization(journal, sourcePhase); + receipts.push( + finalized ?? + (journal.phase === "shared-state-committed" + ? finishRecoveredCommit(journal, sourcePhase) + : finishRecoveredRollbackPhase(journal, sourcePhase)), + ); + } + return Object.freeze(receipts); + }, + async createHeldWorkload(input) { if ( input.plan.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || @@ -2747,6 +3113,7 @@ export function createDockerManagedBootstrapAdapter( phase: "staged", bootstrapIdentity: handle.bootstrapIdentity, providerId: handle.sandbox.driverId, + agent: handle.plan.profile.agent, sandbox: Object.freeze({ ...handle.sandbox }), planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), profileFingerprint: handle.plan.profile.fingerprint, diff --git a/src/lib/onboard/managed-bootstrap/index.ts b/src/lib/onboard/managed-bootstrap/index.ts index c55768afe02..abeddd0a06d 100644 --- a/src/lib/onboard/managed-bootstrap/index.ts +++ b/src/lib/onboard/managed-bootstrap/index.ts @@ -10,7 +10,9 @@ export { type ManagedBootstrapAuthorityStore, type ManagedBootstrapExpectedPlan, type ManagedBootstrapPreparedTransaction, + type ManagedBootstrapRecoveryReceipt, prepareManagedBootstrapSequence, + recoverManagedBootstrapTransactions, } from "./adapter"; export { MANAGED_BOOTSTRAP_COMPLETION_FILE, diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.ts b/src/lib/onboard/managed-bootstrap/runtime-create.ts index 198d91710f0..639e4cfa6e4 100644 --- a/src/lib/onboard/managed-bootstrap/runtime-create.ts +++ b/src/lib/onboard/managed-bootstrap/runtime-create.ts @@ -10,6 +10,7 @@ import type { ManagedBootstrapAuthorityStore, ManagedBootstrapCreateReceipt, ManagedBootstrapImageIdentity, + ManagedBootstrapRecoveryReceipt, } from "./adapter"; export interface ManagedBootstrapRuntimeCommandResult { @@ -92,6 +93,7 @@ export interface ManagedBootstrapRuntimeCreateLaunchResult { export interface ManagedBootstrapRuntimeCreateLifecycle { readonly launchArgv: readonly string[]; readonly patch: ManagedBootstrapRuntimePatch; + recoverUnfinished(): Promise; prepareNetwork(): Promise; runCreate( launch: (input: { diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index b5628f99b78..6628831a980 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -209,6 +209,7 @@ describe("RuntimeProviderBundle registry contract", () => { printReadinessFailureIfEnabled: vi.fn(), verifyGpuOrExit: vi.fn(async (verify) => verify("alpha")), }, + recoverUnfinished: vi.fn(async () => []), prepareNetwork: vi.fn(async () => undefined), runCreate: vi.fn(), })); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index 8602f2e51f5..a7362d037ce 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -166,7 +166,7 @@ beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); describe("runSandboxGpuCreateFlow provider-owned managed create", () => { - it("runs an MXC-style bundle without a Docker branch in central orchestration", async () => { + it("recovers before an MXC-style create without a Docker branch in central orchestration", async () => { const input = createInput(); input.sandboxGpuConfig = { mode: "0", @@ -201,11 +201,14 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { input.sandboxEnv = launch.sandboxEnv; input.sandboxStartupCommand = launch.sandboxStartupCommand; const patch = createPatch() as unknown as ManagedBootstrapRuntimePatch; + const recoverUnfinished = vi.fn(async () => []); + const prepareNetwork = vi.fn(async () => undefined); const createLifecycle = vi.fn( (lifecycleInput: ManagedBootstrapRuntimeCreateLifecycleInput) => ({ launchArgv: ["mxc-launch", ...lifecycleInput.launchArgv.slice(1)], patch, - prepareNetwork: vi.fn(async () => undefined), + recoverUnfinished, + prepareNetwork, runCreate: async ( start: (held: { readonly heldWorkloadArgv: readonly string[]; @@ -284,6 +287,15 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { vi.mocked(deps.runCaptureOpenshell).mockImplementation((args) => args[1] === "get" ? "ID: mxc-alpha\n" : "alpha Ready", ); + recoverUnfinished.mockRejectedValueOnce(new Error("unfinished recovery failed")); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "unfinished recovery failed", + ); + expect(prepareNetwork).not.toHaveBeenCalled(); + expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); + recoverUnfinished.mockClear(); + createLifecycle.mockClear(); const result = await runSandboxGpuCreateFlow(input, deps); @@ -297,6 +309,12 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { input.sandboxEnv, expect.anything(), ); + expect(recoverUnfinished.mock.invocationCallOrder[0]).toBeLessThan( + prepareNetwork.mock.invocationCallOrder[0], + ); + expect(prepareNetwork.mock.invocationCallOrder[0]).toBeLessThan( + mocks.streamSandboxCreate.mock.invocationCallOrder[0], + ); expect(mocks.createDockerGpuSandboxCreatePatch).not.toHaveBeenCalled(); expect(mocks.queryOpenShellDockerSandboxContainers).not.toHaveBeenCalled(); expect(mocks.queryOpenShellDockerSandboxRuntimeSnapshot).not.toHaveBeenCalled(); diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 8eeb4a8aac0..5f1d98fa198 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -143,6 +143,7 @@ export function createSandboxGpuCreateAttemptRunner( backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", deps, }); + await managedLifecycle?.recoverUnfinished(); await managedLifecycle?.prepareNetwork(); const [createExecutable, ...createExecutableArgs] = managedLifecycle?.launchArgv ?? attemptArgv; if (!createExecutable) throw new Error("Sandbox create executable is missing."); From f1b50a457bfb0181abe4ff48a819ec8d93afc282 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 13:19:50 -0700 Subject: [PATCH 113/117] feat(onboard): recover durable bootstrap transactions Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/README.md | 32 +- .../onboard/managed-bootstrap/adapter.test.ts | 49 +++ src/lib/onboard/managed-bootstrap/adapter.ts | 102 +++++ .../managed-bootstrap/docker-journal.test.ts | 39 ++ .../managed-bootstrap/docker-journal.ts | 45 ++- .../managed-bootstrap/docker-recovery.test.ts | 201 +++++++++ .../managed-bootstrap/docker-runtime.ts | 4 + .../managed-bootstrap/docker-test-fixture.ts | 28 ++ .../onboard/managed-bootstrap/docker.test.ts | 6 + src/lib/onboard/managed-bootstrap/docker.ts | 381 +++++++++++++++++- src/lib/onboard/managed-bootstrap/index.ts | 2 + .../managed-bootstrap/runtime-create.ts | 2 + .../runtime-provider-contract.test.ts | 1 + .../onboard/sandbox-gpu-create-flow.test.ts | 22 +- .../onboard/sandbox-gpu-create-run-attempt.ts | 1 + 15 files changed, 890 insertions(+), 25 deletions(-) create mode 100644 src/lib/onboard/managed-bootstrap/docker-recovery.test.ts diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index b5f8434f79f..7e7b3ba805d 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -94,17 +94,23 @@ recreation does not depend on process-local transaction sets or tombstone maps. The image-owned shared-state transaction uses the same identity-bound model: a commit atomically moves its pending manifest and backups into a durable receipt namespace, compacts that state to an exact commit receipt, and rejects rollback -when a later image-runtime invocation reads that receipt. The provider may -retire that receipt only after it proves the external rollback backup is gone, -so that this receipt does not block the next bootstrap attempt. -Direct identity lookup reconstructs one known transaction record. The bounded -[3.12b recovery slice](https://github.com/NVIDIA/NemoClaw/issues/7744) introduces -unfinished-record enumeration together with phase reconciliation and -cross-surface resume or rollback. The adapter reads mutable OpenShell names only -to detect ownership reuse. Unsafe name-only deletion returns a typed retention -error. The dormant adapter assumes the protocol's single coordinator; -multi-process lease/arbitration remains an explicit production-activation gate. -Activation must also inject the selected gateway's canonical state root. +after a restart. The provider may retire that receipt only after it proves the +external rollback backup is gone, leaving the next bootstrap attempt unblocked. +Direct identity lookup reconstructs one known transaction record, while managed +create-lifecycle startup uses unfinished-record enumeration to ask the selected +provider to reconcile every identity-addressed record before a new sandbox +create begins. The Docker provider then resumes the durable phase monotonically: +staged work rolls back without entering cutover; cutover work follows a proven +image-owned commit forward or durably authorizes rollback; rollback-authorized +work completes exact restore and cleanup; and shared-state-committed work +completes exact backup cleanup and commit. Recovery persists an identity-bound +finalization receipt before removing the active journal, is idempotent across +another interruption, and returns normalized, provider-owned receipts in stable +identity order. Mutable OpenShell names are read only to detect ownership reuse, +and unsafe name-only deletion returns a typed retention error. The protocol still +assumes a single coordinator; multi-process lease/arbitration remains an explicit +production-activation gate. Activation must also inject the selected gateway's +canonical state root. ## Architectural disposition @@ -113,8 +119,8 @@ candidate Docker surface owns create routing, replacement construction, native-to-compatibility fallback evidence, and deferred commit or rollback. Central onboarding accepts that provider-neutral surface without a Docker or Podman selection branch. Tests register an MXC-style surface through the same -bundle and render held launches for OpenClaw, Hermes, and LangChain Deep Agents -Code. +bundle, render held launches for OpenClaw, Hermes, and LangChain Deep Agents +Code, and exercise recovery phases across all three agents. The coordinator remains the driver-neutral transaction authority: its receipt shapes, normalization, state transitions, and rollback proofs form one cohesive diff --git a/src/lib/onboard/managed-bootstrap/adapter.test.ts b/src/lib/onboard/managed-bootstrap/adapter.test.ts index 9968171289d..3f80e00e61d 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.test.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.test.ts @@ -25,6 +25,7 @@ import { type ManagedBootstrapPreparedReplacementHandle, type ManagedBootstrapReplacementHandle, prepareManagedBootstrapSequence, + recoverManagedBootstrapTransactions, renderManagedBootstrapHeldCommand, } from "./adapter"; @@ -227,6 +228,7 @@ function adapterFor(agent: ManagedStartupAgent): Fixture { const order: string[] = []; const raw: Fixture["raw"] = { handle: null, snapshot: null, prepared: null }; const adapter: ManagedBootstrapAdapter = { + recoverUnfinishedTransactions: vi.fn(async () => []), createHeldWorkload: vi.fn(async (input) => { order.push("create"); const receipt = await input.launch({ @@ -908,6 +910,53 @@ describe("managed bootstrap adapter contract", () => { expect(fixture.adapter.finalizeBootstrap).not.toHaveBeenCalled(); }); + it("normalizes, freezes, and orders provider-owned restart recovery receipts", async () => { + const fixture = adapterFor("openclaw"); + const receipt = cleanupReceipt(); + const candidate = (bootstrapIdentity: string) => ({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: receipt.sandbox.driverId, + sourcePhase: "cutover", + sandbox: receipt.sandbox, + bootstrapIdentity, + outcome: "rolled-back" as const, + finalization: { ...receipt, bootstrapIdentity }, + }); + vi.mocked(fixture.adapter.recoverUnfinishedTransactions).mockResolvedValueOnce([ + candidate("b".repeat(64)), + candidate("a".repeat(64)), + ]); + + const recovered = await recoverManagedBootstrapTransactions(fixture.adapter); + + expect(recovered.map(({ bootstrapIdentity }) => bootstrapIdentity)).toEqual([ + "a".repeat(64), + "b".repeat(64), + ]); + expect(Object.isFrozen(recovered)).toBe(true); + expect(recovered.every((entry) => Object.isFrozen(entry.finalization))).toBe(true); + }); + + it("rejects recovery evidence whose provider does not own the durable sandbox", async () => { + const fixture = adapterFor("openclaw"); + const receipt = cleanupReceipt(); + vi.mocked(fixture.adapter.recoverUnfinishedTransactions).mockResolvedValueOnce([ + { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: "mxc", + sourcePhase: "cutover", + sandbox: receipt.sandbox, + bootstrapIdentity: IDENTITY, + outcome: "rolled-back", + finalization: receipt, + }, + ]); + + await expect(recoverManagedBootstrapTransactions(fixture.adapter)).rejects.toThrow( + "recovery provider does not own", + ); + }); + it.each([ "BASHOPTS=extdebug", "BASH_ENV=/sandbox/attacker", diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index ee1a834985f..9f7e6bf3cf1 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -239,6 +239,21 @@ export interface ManagedBootstrapFinalizationReceipt { readonly finalizedAt: string; } +/** + * Driver-neutral evidence that one durable, process-orphaned transaction was + * reconciled without reconstructing authority from mutable runtime names. + */ +export interface ManagedBootstrapRecoveryReceipt { + readonly schemaVersion: typeof MANAGED_BOOTSTRAP_SCHEMA_VERSION; + readonly providerId: string; + /** Provider-owned phase name retained for diagnostics, never central routing. */ + readonly sourcePhase: string; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly bootstrapIdentity: string; + readonly outcome: "committed" | "rolled-back"; + readonly finalization: ManagedBootstrapFinalizationReceipt; +} + export class ManagedBootstrapDurableCommitCleanupPendingError extends Error { readonly bootstrapIdentity: string; readonly cleanupRuntimeId: string; @@ -310,6 +325,12 @@ export function attachManagedBootstrapRollbackError(failure: Error, rollbackErro } export interface ManagedBootstrapAdapter { + /** + * Enumerate durable unfinished records and reconcile each through the owning + * provider. Implementations must be restart-safe and idempotent. + */ + recoverUnfinishedTransactions(): Promise; + /** Return only after one durable sandbox/driver identity reports Ready. */ createHeldWorkload( input: ManagedBootstrapCreateInput, @@ -376,6 +397,87 @@ export interface ManagedBootstrapAdapter { }): Promise; } +function normalizeRecoveryReceipt( + candidate: ManagedBootstrapRecoveryReceipt, +): ManagedBootstrapRecoveryReceipt { + if ( + typeof candidate !== "object" || + candidate === null || + Array.isArray(candidate) || + candidate.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + !["committed", "rolled-back"].includes(String(candidate.outcome)) + ) { + protocolFail("recovery receipt has an invalid schema or outcome"); + } + assertOpaqueString(candidate.providerId, "recovery provider ID"); + assertOpaqueString(candidate.sourcePhase, "recovery source phase"); + assertSandboxIdentity(candidate.sandbox); + if (candidate.sandbox.driverId !== candidate.providerId) { + protocolFail("recovery provider does not own the recovered sandbox"); + } + if (!SHA256_RE.test(candidate.bootstrapIdentity)) { + protocolFail("recovery bootstrap identity must be lowercase SHA-256"); + } + const finalization = candidate.finalization; + if ( + typeof finalization !== "object" || + finalization === null || + Array.isArray(finalization) || + finalization.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + finalization.outcome !== candidate.outcome || + finalization.bootstrapIdentity !== candidate.bootstrapIdentity || + !isDeepStrictEqual(finalization.sandbox, candidate.sandbox) || + typeof finalization.heldWorkloadRemoved !== "boolean" || + typeof finalization.alreadyRolledBack !== "boolean" + ) { + protocolFail("recovery finalization does not match its durable identity"); + } + if ( + (finalization.restoredRuntimeId !== null && !SHA256_RE.test(finalization.restoredRuntimeId)) || + (finalization.restoredSpecHash !== null && !SHA256_RE.test(finalization.restoredSpecHash)) || + (finalization.restoredRuntimeId === null) !== (finalization.restoredSpecHash === null) || + (candidate.outcome === "committed" && + (finalization.restoredRuntimeId !== null || + finalization.heldWorkloadRemoved || + finalization.alreadyRolledBack)) + ) { + protocolFail("recovery finalization state is inconsistent"); + } + assertTimestamp(finalization.finalizedAt, "recovery finalization timestamp"); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: candidate.providerId, + sourcePhase: candidate.sourcePhase, + sandbox: Object.freeze({ ...candidate.sandbox }), + bootstrapIdentity: candidate.bootstrapIdentity, + outcome: candidate.outcome, + finalization: Object.freeze({ + ...finalization, + sandbox: Object.freeze({ ...candidate.sandbox }), + }), + }); +} + +/** Recover process-orphaned work without relying on coordinator WeakMap state. */ +export async function recoverManagedBootstrapTransactions( + adapter: ManagedBootstrapAdapter, +): Promise { + const candidates = await adapter.recoverUnfinishedTransactions(); + if (!Array.isArray(candidates)) { + protocolFail("provider recovery must return a receipt array"); + } + const receipts = candidates.map(normalizeRecoveryReceipt); + const identities = receipts.map(({ bootstrapIdentity }) => bootstrapIdentity); + if (new Set(identities).size !== identities.length) { + protocolFail("provider recovery returned duplicate bootstrap identities"); + } + return Object.freeze( + [...receipts].sort((left, right) => + left.bootstrapIdentity.localeCompare(right.bootstrapIdentity), + ), + ); +} + export interface ManagedBootstrapPreparationInput { readonly create: ManagedBootstrapCreateInput; readonly request: ManagedStartupRootApplyRequest; diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index 25cd9ec0e50..b1ed163b954 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -28,6 +28,7 @@ const journal = Object.freeze({ phase: "staged", bootstrapIdentity: IDENTITY, providerId: "docker", + agent: "hermes", sandbox: { sandboxName: "alpha", sandboxId: "sandbox-alpha", @@ -65,6 +66,7 @@ const finalization = Object.freeze({ phase: "committed", bootstrapIdentity: IDENTITY, providerId: "docker", + agent: journal.agent, sandbox: journal.sandbox, planFingerprint: journal.planFingerprint, profileFingerprint: journal.profileFingerprint, @@ -286,6 +288,9 @@ describe("Docker managed bootstrap journal", () => { first.create(journal); first.recordFinalization(finalization); + expect(first.listUnfinished()).toEqual([journal]); + first.remove(IDENTITY, ["staged"]); + expect(first.listUnfinished()).toEqual([]); const restarted = createFileDockerManagedBootstrapJournalStore(root); expect(restarted.loadFinalization(IDENTITY)).toEqual(finalization); expect( @@ -309,6 +314,40 @@ describe("Docker managed bootstrap journal", () => { ).toThrow("finalization record changed"); }); + it.each([ + { label: "journal", suffix: "" }, + { label: "decision", suffix: ".decision" }, + { label: "finalization", suffix: ".finalized" }, + ])("ignores an atomic $label write left by a crash during enumeration", ({ suffix }) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + const directory = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); + fs.writeFileSync( + path.join(directory, `.${IDENTITY}.json${suffix}.1234.deadbeef.tmp`), + "partial", + { + mode: 0o600, + }, + ); + + expect(store.listUnfinished()).toEqual([journal]); + }); + + it("rejects an unsupported journal-directory entry during enumeration", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + const directory = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); + fs.writeFileSync(path.join(directory, `${IDENTITY}.json.unknown`), "unexpected", { + mode: 0o600, + }); + + expect(() => store.listUnfinished()).toThrow("journal directory contains an unsupported entry"); + }); + it("reloads the exact completion receipt from a new journal store", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); roots.push(root); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index 015bda1d12d..db37c5ce213 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -3,7 +3,7 @@ import fs from "node:fs"; import path from "node:path"; - +import type { ManagedStartupAgent } from "../managed-startup/profile"; import type { ManagedBootstrapCompletionReceipt, ManagedBootstrapDurablePreparationReceipt, @@ -11,9 +11,9 @@ import type { ManagedBootstrapSandboxIdentity, } from "./adapter"; -export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 2 as const; +export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 3 as const; export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY = "managed-bootstrap"; -export const DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION = 1 as const; +export const DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION = 2 as const; const SHA256_RE = /^[a-f0-9]{64}$/u; const MANIFEST_DIGEST_RE = /^sha256:[a-f0-9]{64}$/u; @@ -36,6 +36,7 @@ export interface DockerManagedBootstrapJournal { readonly phase: DockerManagedBootstrapJournalPhase; readonly bootstrapIdentity: string; readonly providerId: string; + readonly agent: ManagedStartupAgent; readonly sandbox: ManagedBootstrapSandboxIdentity; readonly planFingerprint: string; readonly profileFingerprint: string; @@ -59,6 +60,7 @@ export interface DockerManagedBootstrapFinalizationRecord { readonly phase: "committed" | "rolled-back"; readonly bootstrapIdentity: string; readonly providerId: string; + readonly agent: ManagedStartupAgent; readonly sandbox: ManagedBootstrapSandboxIdentity; readonly planFingerprint: string; readonly profileFingerprint: string; @@ -70,6 +72,7 @@ export interface DockerManagedBootstrapFinalizationRecord { export interface DockerManagedBootstrapJournalStore { create(journal: DockerManagedBootstrapJournal): void; load(bootstrapIdentity: string): DockerManagedBootstrapJournal | null; + listUnfinished(): readonly DockerManagedBootstrapJournal[]; transition( bootstrapIdentity: string, expected: DockerManagedBootstrapJournalPhase, @@ -144,6 +147,13 @@ function exactPhase(value: unknown): DockerManagedBootstrapJournalPhase { return value as DockerManagedBootstrapJournalPhase; } +function exactAgent(value: unknown): ManagedStartupAgent { + if (!["openclaw", "hermes", "langchain-deepagents-code"].includes(String(value))) { + fail("agent is unsupported"); + } + return value as ManagedStartupAgent; +} + function exactSandbox(value: unknown): ManagedBootstrapSandboxIdentity { if (typeof value !== "object" || value === null || Array.isArray(value)) { fail("sandbox identity must be an object"); @@ -167,6 +177,7 @@ export function normalizeDockerManagedBootstrapJournal( } const journal = value as Record; const expectedKeys = [ + "agent", "backupName", "bootstrapIdentity", "commitReceipt", @@ -199,6 +210,7 @@ export function normalizeDockerManagedBootstrapJournal( phase: exactPhase(journal.phase), bootstrapIdentity: exactSha256(journal.bootstrapIdentity, "bootstrap identity"), providerId: exactString(journal.providerId, "provider ID"), + agent: exactAgent(journal.agent), sandbox: exactSandbox(journal.sandbox), planFingerprint: exactSha256(journal.planFingerprint, "plan fingerprint"), profileFingerprint: exactSha256(journal.profileFingerprint, "profile fingerprint"), @@ -484,6 +496,7 @@ export function normalizeDockerManagedBootstrapFinalizationRecord( } const record = value as Record; const expectedKeys = [ + "agent", "bootstrapIdentity", "cleanupReceipt", "commitReceipt", @@ -512,6 +525,7 @@ export function normalizeDockerManagedBootstrapFinalizationRecord( phase, bootstrapIdentity: exactSha256(record.bootstrapIdentity, "finalization bootstrap identity"), providerId: exactString(record.providerId, "finalization provider ID"), + agent: exactAgent(record.agent), sandbox, planFingerprint: exactSha256(record.planFingerprint, "finalization plan fingerprint"), profileFingerprint: exactSha256(record.profileFingerprint, "finalization profile fingerprint"), @@ -777,6 +791,31 @@ export function createFileDockerManagedBootstrapJournalStore( atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(normalized), true); }, load, + listUnfinished() { + assertDirectory(directory); + const identities: string[] = []; + for (const name of fs.readdirSync(directory)) { + const match = name.match(/^([a-f0-9]{64})\.json$/u); + if (match) { + identities.push(match[1]); + continue; + } + if ( + /^\.[a-f0-9]{64}\.json(?:\.decision|\.finalized)?\.[0-9]+\.[a-f0-9]+\.tmp$/u.test(name) || + /^[a-f0-9]{64}\.json\.(?:decision|finalized)$/u.test(name) + ) { + continue; + } + fail(`journal directory contains an unsupported entry: ${name}`); + } + return Object.freeze( + identities.sort().map((identity) => { + const journal = load(identity); + if (!journal) fail(`enumerated journal ${identity} disappeared`); + return journal; + }), + ); + }, transition( bootstrapIdentity: string, expected: DockerManagedBootstrapJournalPhase, diff --git a/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts b/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts new file mode 100644 index 00000000000..fc9f22b8d1a --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts @@ -0,0 +1,201 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { createDockerManagedBootstrapAdapter } from "./docker"; +import type { DockerManagedBootstrapJournalStore } from "./docker-journal"; +import { + authority, + type DockerFixtureOptions, + durablePreparation, + fixture, +} from "./docker-test-fixture"; + +async function prepareTransaction( + fake: ReturnType, + agent: Parameters[0] = "hermes", +) { + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority(agent); + const prepared = await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request, + replacementOptions: { values: {} }, + }); + return { + adapter, + handle, + prepared, + snapshot, + durable: durablePreparation(handle, snapshot, prepared), + }; +} + +describe("Docker managed bootstrap restart recovery", () => { + it.each([ + { + label: "staged", + options: { + journalCreateFailures: [new Error("injected crash after durable staged fence")], + }, + phase: "staged", + }, + { + label: "cutover", + options: { + journalTransitionFailures: { + cutover: new Error("injected crash after durable cutover fence"), + }, + }, + phase: "cutover", + }, + ] satisfies readonly { + readonly label: string; + readonly options: DockerFixtureOptions; + readonly phase: "cutover" | "staged"; + }[])("reconciles a process restart from the durable $label phase", async ({ options, phase }) => { + const fake = fixture(options); + const transaction = await prepareTransaction(fake); + + await expect( + transaction.adapter.activateBootstrapReplacement({ + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + }), + ).rejects.toThrow(`crash after durable ${phase} fence`); + expect(fake.journal?.phase).toBe(phase); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject([ + { sourcePhase: phase, outcome: "rolled-back" }, + ]); + expect(fake.journal).toBeNull(); + expect(fake.finalization?.phase).toBe("rolled-back"); + expect(fake.replacement).toBeNull(); + expect(fake.original?.State?.Running).toBe(true); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toEqual([]); + }); + + it("finishes rollback-authorized recovery after shared-state rollback is interrupted", async () => { + const fake = fixture({ + agent: "openclaw", + journalTransitionFailures: { + "rollback-authorized": new Error("injected crash after durable rollback fence"), + }, + sharedState: "pending", + }); + const transaction = await prepareTransaction(fake, "openclaw"); + const replacement = await transaction.adapter.activateBootstrapReplacement({ + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + }); + + await expect( + transaction.adapter.finalizeBootstrap({ + outcome: "rollback", + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + replacement, + completion: null, + }), + ).rejects.toThrow("crash after durable rollback fence"); + expect(fake.journal?.phase).toBe("rollback-authorized"); + expect(fake.sharedState).toBe("pending"); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject([ + { sourcePhase: "rollback-authorized", outcome: "rolled-back" }, + ]); + expect(fake.sharedState).toBe("none"); + expect(fake.replacement).toBeNull(); + expect(fake.original?.State?.Running).toBe(true); + }); + + it("compacts a terminal commit journal after another restart interruption", async () => { + const fake = fixture({ + agent: "langchain-deepagents-code", + dockerRemoveFailures: [new Error("injected crash before exact Docker removal")], + journalRemoveFailures: [new Error("injected crash before terminal journal removal")], + sharedState: "pending", + }); + const transaction = await prepareTransaction(fake, "langchain-deepagents-code"); + const replacement = await transaction.adapter.activateBootstrapReplacement({ + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + }); + const completion = await transaction.adapter.awaitBootstrap({ + handle: transaction.handle, + snapshot: transaction.snapshot, + replacement, + timeoutSecs: 1, + }); + + await expect( + transaction.adapter.finalizeBootstrap({ + outcome: "commit", + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + replacement, + completion, + }), + ).rejects.toThrow("crash before exact Docker removal"); + expect(fake.journal?.phase).toBe("shared-state-committed"); + expect(fake.sharedState).toBe("committed"); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect(restarted.recoverUnfinishedTransactions()).rejects.toThrow( + "crash before terminal journal removal", + ); + expect(fake.journal?.phase).toBe("shared-state-committed"); + expect(fake.finalization?.phase).toBe("committed"); + + const journalStore = fake.deps.journalStore; + if (!journalStore) throw new Error("fixture journal store is missing"); + const reorderedFinalizationStore = { + ...journalStore, + loadFinalization(bootstrapIdentity: string) { + const record = journalStore.loadFinalization(bootstrapIdentity); + const receipt = record?.commitReceipt; + if (!record || !receipt) return record; + return { + ...record, + commitReceipt: { + completedAt: receipt.completedAt, + transactionPending: receipt.transactionPending, + bootstrapIdentity: receipt.bootstrapIdentity, + profileFingerprint: receipt.profileFingerprint, + replacementSpecHash: receipt.replacementSpecHash, + originalSpecHash: receipt.originalSpecHash, + runtimeImageContentId: receipt.runtimeImageContentId, + image: receipt.image, + runtimeId: receipt.runtimeId, + sandbox: receipt.sandbox, + schemaVersion: receipt.schemaVersion, + } satisfies typeof receipt, + }; + }, + } satisfies DockerManagedBootstrapJournalStore; + const resumed = createDockerManagedBootstrapAdapter({ + ...fake.deps, + journalStore: reorderedFinalizationStore, + }); + await expect(resumed.recoverUnfinishedTransactions()).resolves.toMatchObject([ + { sourcePhase: "shared-state-committed", outcome: "committed" }, + ]); + expect(fake.journal).toBeNull(); + expect(fake.sharedState).toBe("none"); + expect(fake.replacement?.State?.Running).toBe(true); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.ts index 6da0f754f1b..dba07739d18 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.ts @@ -21,6 +21,7 @@ import { finalizeManagedBootstrapSequence, MANAGED_BOOTSTRAP_SCHEMA_VERSION, prepareManagedBootstrapSequence, + recoverManagedBootstrapTransactions, } from "./adapter"; import { createDockerManagedBootstrapAdapter } from "./docker"; import type { @@ -137,6 +138,9 @@ function createDockerLifecycle( return { launchArgv: input.launchArgv, patch, + async recoverUnfinished() { + return recoverManagedBootstrapTransactions(adapter); + }, async prepareNetwork() { if (input.route !== "compatibility") return; const { enforceDockerGpuPatchPreserveNetwork } = await import( diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index 088a3ed7a05..42f0fc3ae1d 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -65,7 +65,10 @@ export type DockerFixtureAcknowledgement = export type DockerFixtureOptions = { readonly agent?: ManagedStartupAgent; + readonly dockerRemoveFailures?: readonly Error[]; readonly dockerStartResults?: Readonly>; + readonly journalCreateFailures?: readonly Error[]; + readonly journalRemoveFailures?: readonly Error[]; readonly journalTransitionFailures?: Partial< Readonly> >; @@ -214,6 +217,9 @@ export function fixture(options: DockerFixtureOptions = {}) { let finalization: DockerManagedBootstrapFinalizationRecord | null = null; let sharedState: "committed" | "none" | "pending" = options.sharedState ?? "none"; const events: string[] = []; + const dockerRemoveFailures = [...(options.dockerRemoveFailures ?? [])]; + const journalCreateFailures = [...(options.journalCreateFailures ?? [])]; + const journalRemoveFailures = [...(options.journalRemoveFailures ?? [])]; const lostAcknowledgements = new Set(options.lostAcknowledgements ?? []); const losesAcknowledgement = (operation: DockerFixtureAcknowledgement) => lostAcknowledgements.has(operation); @@ -223,6 +229,13 @@ export function fixture(options: DockerFixtureOptions = {}) { create(value) { journal = structuredClone(value); events.push("journal:staged"); + const injectedFailure = journalCreateFailures.shift(); + switch (injectedFailure) { + case undefined: + break; + default: + throw injectedFailure; + } if (losesAcknowledgement("journal:create")) { throw new DockerManagedBootstrapJournalAcknowledgementLostError( "lost journal create acknowledgement", @@ -230,6 +243,7 @@ export function fixture(options: DockerFixtureOptions = {}) { } }, load: () => copyJournal(), + listUnfinished: () => (journal ? [structuredClone(journal)] : []), transition(_identity, expected, next) { const current = journal !== null && journal.phase === expected @@ -265,6 +279,13 @@ export function fixture(options: DockerFixtureOptions = {}) { void (current !== null && expected.includes(current.phase) ? current : failFixture("stale journal remove")); + const injectedFailure = journalRemoveFailures.shift(); + switch (injectedFailure) { + case undefined: + break; + default: + throw injectedFailure; + } journal = null; events.push("journal:removed"); if (losesAcknowledgement("journal:remove")) { @@ -453,6 +474,13 @@ export function fixture(options: DockerFixtureOptions = {}) { }), dockerRm: vi.fn((id) => { events.push(`rm:${id}`); + const injectedFailure = dockerRemoveFailures.shift(); + switch (injectedFailure) { + case undefined: + break; + default: + throw injectedFailure; + } switch (id) { case OLD_ID: original = null; diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 8e94b573589..842c14f654e 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -129,6 +129,9 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( fake.events.indexOf(`rm:${OLD_ID}`), ); + expect(fake.events.indexOf("finalization:committed")).toBeLessThan( + fake.events.indexOf("journal:removed"), + ); expect(fake.journal).toBeNull(); expect(fake.finalization).toMatchObject({ phase: "committed", commitReceipt }); expect(fake.sharedState).toBe("none"); @@ -260,6 +263,9 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( fake.events.indexOf(`rm:${NEW_ID}`), ); + expect(fake.events.indexOf("finalization:rolled-back")).toBeLessThan( + fake.events.indexOf("journal:removed"), + ); expect(fake.journal).toBeNull(); expect(fake.replacement).toBeNull(); expect(fake.original).not.toBeNull(); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index e633ee9f2ed..89c3c83b76a 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -58,6 +58,7 @@ import { type ManagedBootstrapObservedSnapshot, ManagedBootstrapOwnerCleanupRequiredError, type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapRecoveryReceipt, type ManagedBootstrapReplacementHandle, type ManagedBootstrapReplacementOptions, type ManagedBootstrapSandboxIdentity, @@ -1416,6 +1417,16 @@ function managedSharedStateTransaction( } as const; } +function recoveredManagedSharedStateTransaction(journal: DockerBootstrapTransaction) { + return { + agent: journal.agent, + bootstrapIdentity: journal.bootstrapIdentity, + containerId: journal.replacementRuntimeId, + image: journal.runtimeImageContentId, + profileFingerprint: journal.profileFingerprint, + } as const; +} + function sameDockerBootstrapJournal( left: DockerBootstrapTransaction, right: DockerBootstrapTransaction, @@ -1766,6 +1777,7 @@ export function createDockerManagedBootstrapAdapter( phase, bootstrapIdentity: handle.bootstrapIdentity, providerId: handle.sandbox.driverId, + agent: handle.plan.profile.agent, sandbox: handle.sandbox, planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), profileFingerprint: handle.plan.profile.fingerprint, @@ -1811,6 +1823,21 @@ export function createDockerManagedBootstrapAdapter( } satisfies ManagedBootstrapFinalizationReceipt); return persistFinalization(handle, "rolled-back", null, receipt); }; + const completeRollbackTransaction = ( + handle: ManagedBootstrapHeldWorkloadHandle, + journal: DockerBootstrapTransaction, + ): ManagedBootstrapFinalizationReceipt => { + let ownerCleanupFailure: { readonly error: unknown } | null = null; + try { + retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, journal.originalRuntimeId); + } catch (error) { + ownerCleanupFailure = { error }; + } + const finalization = completedRollback(handle, false); + removeDockerBootstrapJournalDurably(journal, deps); + if (ownerCleanupFailure) throw ownerCleanupFailure.error; + return finalization; + }; const completedCommit = ( handle: ManagedBootstrapHeldWorkloadHandle, commitReceipt: ManagedBootstrapCompletionReceipt, @@ -1845,6 +1872,332 @@ export function createDockerManagedBootstrapAdapter( alreadyRolledBack: true, }); }; + const persistRecoveredFinalization = ( + journal: DockerBootstrapTransaction, + phase: "committed" | "rolled-back", + commitReceipt: ManagedBootstrapCompletionReceipt | null, + cleanupReceipt: ManagedBootstrapFinalizationReceipt, + ): ManagedBootstrapFinalizationReceipt => { + const record = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase, + bootstrapIdentity: journal.bootstrapIdentity, + providerId: journal.providerId, + agent: journal.agent, + sandbox: journal.sandbox, + planFingerprint: journal.planFingerprint, + profileFingerprint: journal.profileFingerprint, + imageReference: journal.imageReference, + commitReceipt, + cleanupReceipt, + } satisfies DockerManagedBootstrapFinalizationRecord); + const serialized = serializeDockerManagedBootstrapFinalizationRecord(record); + try { + deps.journalStore.recordFinalization(record); + } catch (error) { + const recovered = deps.journalStore.loadFinalization(journal.bootstrapIdentity); + if ( + !recovered || + serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized + ) { + throw error; + } + } + const persisted = deps.journalStore.loadFinalization(journal.bootstrapIdentity); + if (!persisted || serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized) { + throw new Error("Managed bootstrap recovered finalization was not durably re-readable."); + } + return persisted.cleanupReceipt; + }; + const recoveredReceipt = ( + journal: DockerBootstrapTransaction, + sourcePhase: DockerBootstrapTransaction["phase"], + finalization: ManagedBootstrapFinalizationReceipt, + ): ManagedBootstrapRecoveryReceipt => + Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: journal.providerId, + sourcePhase, + sandbox: journal.sandbox, + bootstrapIdentity: journal.bootstrapIdentity, + outcome: finalization.outcome, + finalization, + }); + const compactRecoveredFinalization = ( + journal: DockerBootstrapTransaction, + sourcePhase: DockerBootstrapTransaction["phase"], + ): ManagedBootstrapRecoveryReceipt | null => { + const finalization = deps.journalStore.loadFinalization(journal.bootstrapIdentity); + if (!finalization) return null; + const phaseMatches = + (finalization.phase === "committed" && + journal.phase === "shared-state-committed" && + finalization.commitReceipt !== null && + journal.commitReceipt !== null && + sameDockerManagedBootstrapReceipt( + "completion", + finalization.commitReceipt, + journal.commitReceipt, + )) || + (finalization.phase === "rolled-back" && + (journal.phase === "staged" || journal.phase === "rollback-authorized") && + finalization.commitReceipt === null); + if ( + !phaseMatches || + finalization.bootstrapIdentity !== journal.bootstrapIdentity || + finalization.providerId !== journal.providerId || + finalization.agent !== journal.agent || + finalization.sandbox.sandboxName !== journal.sandbox.sandboxName || + finalization.sandbox.sandboxId !== journal.sandbox.sandboxId || + finalization.sandbox.driverId !== journal.sandbox.driverId || + finalization.planFingerprint !== journal.planFingerprint || + finalization.profileFingerprint !== journal.profileFingerprint || + finalization.imageReference !== journal.imageReference + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: + journal.phase === "shared-state-committed" + ? journal.replacementRuntimeId + : journal.originalRuntimeId, + detail: "terminal finalization does not match its retained durable journal", + }); + } + removeDockerBootstrapJournalDurably(journal, deps); + return recoveredReceipt(journal, sourcePhase, finalization.cleanupReceipt); + }; + const finishRecoveredRollback = ( + journal: DockerBootstrapTransaction, + sourcePhase: DockerBootstrapTransaction["phase"], + ): ManagedBootstrapRecoveryReceipt => { + const cleanupReceipt = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: journal.sandbox, + bootstrapIdentity: journal.bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: journal.originalRuntimeId, + restoredSpecHash: journal.originalSpecHash, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + } satisfies ManagedBootstrapFinalizationReceipt); + const finalization = persistRecoveredFinalization(journal, "rolled-back", null, cleanupReceipt); + removeDockerBootstrapJournalDurably(journal, deps); + return recoveredReceipt(journal, sourcePhase, finalization); + }; + const finishRecoveredCommit = ( + journal: DockerBootstrapTransaction, + sourcePhase: DockerBootstrapTransaction["phase"], + ): ManagedBootstrapRecoveryReceipt => { + if (journal.phase !== "shared-state-committed" || journal.commitReceipt === null) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "durable commit recovery requires its exact completion receipt and commit fence", + }); + } + const replacement = inspectTransactionRuntime(journal, journal.replacementRuntimeId, deps); + if (!replacement) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "the exact committed replacement is absent during restart recovery", + }); + } + assertTransactionReplacement(journal, replacement); + if ( + dockerContainerName(replacement) !== journal.originalName || + !isStableRunning(replacement) || + normalizeDockerManagedBootstrapLaunchSpec(replacement).hash !== journal.replacementSpecHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "the committed replacement does not match its durable runtime authority", + }); + } + const sharedTransaction = recoveredManagedSharedStateTransaction(journal); + const sharedStatus = probeDockerManagedStartupSharedState( + { transaction: sharedTransaction, profileFingerprint: journal.profileFingerprint }, + deps, + ); + if (sharedStatus === "pending") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "shared state is pending after the durable commit fence", + }); + } + const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); + if (original) { + assertTransactionOriginal(journal, original); + if ( + dockerContainerName(original) !== journal.backupName || + !isExplicitlyStopped(original) || + normalizeDockerManagedBootstrapLaunchSpec({ + ...original, + Name: `/${journal.originalName}`, + }).hash !== journal.originalSpecHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "the exact rollback backup changed before recovered commit cleanup", + }); + } + if (sharedStatus === "none") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "the shared commit receipt was retired before exact backup absence was proven", + }); + } + const removed = deps.dockerRm(journal.originalRuntimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if ( + !hasZeroDockerExitStatus(removed) && + probeExactDockerContainerAbsence(journal.originalRuntimeId, deps) !== "absent" + ) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: journal.bootstrapIdentity, + cleanupRuntimeId: journal.originalRuntimeId, + detail: `${commandDetail(removed) || "Docker removal failed"}; exact backup absence was not proven`, + }); + } + } + if (sharedStatus === "committed") { + clearDockerManagedStartupSharedStateCommitReceipt(sharedTransaction, deps); + } + if (probeExactDockerContainerAbsence(journal.originalRuntimeId, deps) !== "absent") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: journal.bootstrapIdentity, + cleanupRuntimeId: journal.originalRuntimeId, + detail: "exact rollback-backup absence was not durable after restart recovery", + }); + } + const cleanupReceipt = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: journal.sandbox, + bootstrapIdentity: journal.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + } satisfies ManagedBootstrapFinalizationReceipt); + const finalization = persistRecoveredFinalization( + journal, + "committed", + journal.commitReceipt, + cleanupReceipt, + ); + removeDockerBootstrapJournalDurably(journal, deps); + return recoveredReceipt(journal, sourcePhase, finalization); + }; + const finishRecoveredRollbackPhase = ( + journal: DockerBootstrapTransaction, + sourcePhase: DockerBootstrapTransaction["phase"], + ): ManagedBootstrapRecoveryReceipt => { + const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); + if (!original) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "the exact rollback original is absent during restart recovery", + }); + } + assertTransactionOriginal(journal, original); + const replacement = inspectTransactionRuntime(journal, journal.replacementRuntimeId, deps); + if (replacement) assertTransactionReplacement(journal, replacement); + if (journal.phase === "staged") { + if ( + dockerContainerName(original) !== journal.originalName || + !isStableRunning(original) || + normalizeDockerManagedBootstrapLaunchSpec(original).hash !== journal.originalSpecHash || + (replacement !== null && + (dockerContainerName(replacement) !== journal.replacementStagingName || + !isExplicitlyStopped(replacement))) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "staged restart recovery does not match its pre-cutover fence", + }); + } + if (replacement) removeExactReplacement(journal, replacement, deps); + return finishRecoveredRollback(journal, sourcePhase); + } + if (journal.phase !== "cutover" && journal.phase !== "rollback-authorized") { + return finishRecoveredCommit(journal, sourcePhase); + } + let activeJournal = journal; + if (!replacement && dockerContainerName(original) !== journal.originalName) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "the replacement disappeared before exact rollback restoration was proven", + }); + } + if (replacement) { + const sharedTransaction = recoveredManagedSharedStateTransaction(journal); + const sharedStatus = probeDockerManagedStartupSharedState( + { transaction: sharedTransaction, profileFingerprint: journal.profileFingerprint }, + deps, + ); + if (sharedStatus === "committed") { + if (journal.phase === "rollback-authorized") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.replacementRuntimeId, + detail: "shared state committed after durable rollback authorization", + }); + } + activeJournal = transitionDockerBootstrapJournalDurably( + journal, + "shared-state-committed", + deps, + ); + return finishRecoveredCommit(activeJournal, sourcePhase); + } + if (journal.phase === "cutover") { + activeJournal = transitionDockerBootstrapJournalDurably( + journal, + "rollback-authorized", + deps, + ); + } + if (sharedStatus === "pending") { + finalizeDockerManagedStartupSharedState( + { + transaction: sharedTransaction, + supervisorReady: false, + retainContainerAfterRollback: true, + }, + deps, + ); + } + } else if (journal.phase === "cutover") { + activeJournal = transitionDockerBootstrapJournalDurably(journal, "rollback-authorized", deps); + } + restoreOriginal(activeJournal, deps); + const restored = inspectExact(activeJournal.originalRuntimeId, deps); + if ( + !isStableRunning(restored) || + dockerContainerName(restored) !== activeJournal.originalName || + normalizeDockerManagedBootstrapLaunchSpec(restored).hash !== activeJournal.originalSpecHash + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: activeJournal.bootstrapIdentity, + runtimeId: activeJournal.originalRuntimeId, + detail: "restart recovery did not restore the exact original runtime and launch spec", + }); + } + return finishRecoveredRollback(activeJournal, sourcePhase); + }; const rollbackBootstrapNow = ({ handle, snapshot, @@ -2002,6 +2355,7 @@ export function createDockerManagedBootstrapAdapter( ); if (journal.phase === "staged") { + const stagedJournal: DockerBootstrapTransaction = journal; assertStableRunning(original, "staged original"); if (observedReplacement) { assertExplicitlyStopped(observedReplacement, "staged replacement"); @@ -2020,9 +2374,7 @@ export function createDockerManagedBootstrapAdapter( if (observedReplacement) { removeExactReplacement(journal, observedReplacement, deps); } - removeDockerBootstrapJournalDurably(journal, deps); - retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, journal.originalRuntimeId); - return completedRollback(handle, false); + return completeRollbackTransaction(handle, stagedJournal); } if (journal.phase !== "cutover" && journal.phase !== "rollback-authorized") { @@ -2200,9 +2552,7 @@ export function createDockerManagedBootstrapAdapter( ) { throw new Error("Managed bootstrap Docker rollback did not restore its exact original."); } - removeDockerBootstrapJournalDurably(activeJournal, deps); - retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, activeJournal.originalRuntimeId); - return completedRollback(handle, false); + return completeRollbackTransaction(handle, activeJournal); }; const commitBootstrapNow = ( handle: ManagedBootstrapHeldWorkloadHandle, @@ -2303,8 +2653,9 @@ export function createDockerManagedBootstrapAdapter( }); } } + const finalization = completedCommit(handle, receipt); removeDockerBootstrapJournalDurably(transaction, deps); - return completedCommit(handle, receipt); + return finalization; }; const finalizeBootstrap = async ( input: Parameters[0], @@ -2493,6 +2844,21 @@ export function createDockerManagedBootstrapAdapter( }); }; return { + async recoverUnfinishedTransactions() { + const receipts: ManagedBootstrapRecoveryReceipt[] = []; + for (const journal of deps.journalStore.listUnfinished()) { + const sourcePhase = journal.phase; + const finalized = compactRecoveredFinalization(journal, sourcePhase); + receipts.push( + finalized ?? + (journal.phase === "shared-state-committed" + ? finishRecoveredCommit(journal, sourcePhase) + : finishRecoveredRollbackPhase(journal, sourcePhase)), + ); + } + return Object.freeze(receipts); + }, + async createHeldWorkload(input) { if ( input.plan.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || @@ -2747,6 +3113,7 @@ export function createDockerManagedBootstrapAdapter( phase: "staged", bootstrapIdentity: handle.bootstrapIdentity, providerId: handle.sandbox.driverId, + agent: handle.plan.profile.agent, sandbox: Object.freeze({ ...handle.sandbox }), planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), profileFingerprint: handle.plan.profile.fingerprint, diff --git a/src/lib/onboard/managed-bootstrap/index.ts b/src/lib/onboard/managed-bootstrap/index.ts index c55768afe02..abeddd0a06d 100644 --- a/src/lib/onboard/managed-bootstrap/index.ts +++ b/src/lib/onboard/managed-bootstrap/index.ts @@ -10,7 +10,9 @@ export { type ManagedBootstrapAuthorityStore, type ManagedBootstrapExpectedPlan, type ManagedBootstrapPreparedTransaction, + type ManagedBootstrapRecoveryReceipt, prepareManagedBootstrapSequence, + recoverManagedBootstrapTransactions, } from "./adapter"; export { MANAGED_BOOTSTRAP_COMPLETION_FILE, diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.ts b/src/lib/onboard/managed-bootstrap/runtime-create.ts index 198d91710f0..639e4cfa6e4 100644 --- a/src/lib/onboard/managed-bootstrap/runtime-create.ts +++ b/src/lib/onboard/managed-bootstrap/runtime-create.ts @@ -10,6 +10,7 @@ import type { ManagedBootstrapAuthorityStore, ManagedBootstrapCreateReceipt, ManagedBootstrapImageIdentity, + ManagedBootstrapRecoveryReceipt, } from "./adapter"; export interface ManagedBootstrapRuntimeCommandResult { @@ -92,6 +93,7 @@ export interface ManagedBootstrapRuntimeCreateLaunchResult { export interface ManagedBootstrapRuntimeCreateLifecycle { readonly launchArgv: readonly string[]; readonly patch: ManagedBootstrapRuntimePatch; + recoverUnfinished(): Promise; prepareNetwork(): Promise; runCreate( launch: (input: { diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index b5628f99b78..6628831a980 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -209,6 +209,7 @@ describe("RuntimeProviderBundle registry contract", () => { printReadinessFailureIfEnabled: vi.fn(), verifyGpuOrExit: vi.fn(async (verify) => verify("alpha")), }, + recoverUnfinished: vi.fn(async () => []), prepareNetwork: vi.fn(async () => undefined), runCreate: vi.fn(), })); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index 8602f2e51f5..a7362d037ce 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -166,7 +166,7 @@ beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); describe("runSandboxGpuCreateFlow provider-owned managed create", () => { - it("runs an MXC-style bundle without a Docker branch in central orchestration", async () => { + it("recovers before an MXC-style create without a Docker branch in central orchestration", async () => { const input = createInput(); input.sandboxGpuConfig = { mode: "0", @@ -201,11 +201,14 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { input.sandboxEnv = launch.sandboxEnv; input.sandboxStartupCommand = launch.sandboxStartupCommand; const patch = createPatch() as unknown as ManagedBootstrapRuntimePatch; + const recoverUnfinished = vi.fn(async () => []); + const prepareNetwork = vi.fn(async () => undefined); const createLifecycle = vi.fn( (lifecycleInput: ManagedBootstrapRuntimeCreateLifecycleInput) => ({ launchArgv: ["mxc-launch", ...lifecycleInput.launchArgv.slice(1)], patch, - prepareNetwork: vi.fn(async () => undefined), + recoverUnfinished, + prepareNetwork, runCreate: async ( start: (held: { readonly heldWorkloadArgv: readonly string[]; @@ -284,6 +287,15 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { vi.mocked(deps.runCaptureOpenshell).mockImplementation((args) => args[1] === "get" ? "ID: mxc-alpha\n" : "alpha Ready", ); + recoverUnfinished.mockRejectedValueOnce(new Error("unfinished recovery failed")); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "unfinished recovery failed", + ); + expect(prepareNetwork).not.toHaveBeenCalled(); + expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); + recoverUnfinished.mockClear(); + createLifecycle.mockClear(); const result = await runSandboxGpuCreateFlow(input, deps); @@ -297,6 +309,12 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { input.sandboxEnv, expect.anything(), ); + expect(recoverUnfinished.mock.invocationCallOrder[0]).toBeLessThan( + prepareNetwork.mock.invocationCallOrder[0], + ); + expect(prepareNetwork.mock.invocationCallOrder[0]).toBeLessThan( + mocks.streamSandboxCreate.mock.invocationCallOrder[0], + ); expect(mocks.createDockerGpuSandboxCreatePatch).not.toHaveBeenCalled(); expect(mocks.queryOpenShellDockerSandboxContainers).not.toHaveBeenCalled(); expect(mocks.queryOpenShellDockerSandboxRuntimeSnapshot).not.toHaveBeenCalled(); diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 8eeb4a8aac0..5f1d98fa198 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -143,6 +143,7 @@ export function createSandboxGpuCreateAttemptRunner( backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", deps, }); + await managedLifecycle?.recoverUnfinished(); await managedLifecycle?.prepareNetwork(); const [createExecutable, ...createExecutableArgs] = managedLifecycle?.launchArgv ?? attemptArgv; if (!createExecutable) throw new Error("Sandbox create executable is missing."); From 2d1c54276ac5bba78fc950eddb86106edef35d99 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 14:07:37 -0700 Subject: [PATCH 114/117] fix(onboard): close durable recovery feedback Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/README.md | 25 +- .../onboard/managed-bootstrap/adapter.test.ts | 169 ++++- src/lib/onboard/managed-bootstrap/adapter.ts | 154 ++++- .../managed-bootstrap/docker-journal.test.ts | 314 +++++++++- .../managed-bootstrap/docker-journal.ts | 582 ++++++++++++++++-- .../managed-bootstrap/docker-recovery.test.ts | 325 ++++++++-- .../managed-bootstrap/docker-test-fixture.ts | 38 +- .../onboard/managed-bootstrap/docker.test.ts | 54 +- src/lib/onboard/managed-bootstrap/docker.ts | 282 +++++++-- src/lib/onboard/managed-bootstrap/index.ts | 4 + .../managed-bootstrap/runtime-create.ts | 4 +- .../runtime-provider-contract.test.ts | 2 +- .../onboard/sandbox-gpu-create-flow.test.ts | 42 +- .../onboard/sandbox-gpu-create-run-attempt.ts | 8 +- 14 files changed, 1749 insertions(+), 254 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 7e7b3ba805d..77cc960dc94 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -91,6 +91,12 @@ sandbox identities, plan and profile fingerprints, exact original and replacement IDs, rollback target, and phase. Exact commit and cleanup receipts are durable terminal records, so adapter recreation does not depend on process-local transaction sets or tombstone maps. +Rollback retains an `owner-cleanup-required` phase only after image-owned shared +state is restored and the exact replacement is absent. That phase keeps the +restored original quiescent and preserves the journal without a terminal +receipt until the owning sandbox service removes the exact runtime and the +provider proves its absence. Unknown runtime presence is a retryable durable +cleanup failure, never evidence of absence. The image-owned shared-state transaction uses the same identity-bound model: a commit atomically moves its pending manifest and backups into a durable receipt namespace, compacts that state to an exact commit receipt, and rejects rollback @@ -105,12 +111,19 @@ image-owned commit forward or durably authorizes rollback; rollback-authorized work completes exact restore and cleanup; and shared-state-committed work completes exact backup cleanup and commit. Recovery persists an identity-bound finalization receipt before removing the active journal, is idempotent across -another interruption, and returns normalized, provider-owned receipts in stable -identity order. Mutable OpenShell names are read only to detect ownership reuse, -and unsafe name-only deletion returns a typed retention error. The protocol still -assumes a single coordinator; multi-process lease/arbitration remains an explicit -production-activation gate. Activation must also inject the selected gateway's -canonical state root. +another interruption, and enumerates durable identities before loading each +record so one unreadable transaction does not hide other results. The provider +returns bounded `{ receipts, failures }` evidence; the coordinator validates, +copies, freezes, and orders both arrays without routing on provider phases or +failure codes. A failure for the requested sandbox name, or one whose sandbox +identity cannot be proven, blocks create. An exact failure for another sandbox +is warned and retained without blocking the requested create. The code reads +mutable OpenShell names only to detect ownership reuse, and unsafe name-only +deletion returns a typed retention error. Docker mutations use the previously +journaled full container ID, whose identity cannot be rebound, then re-inspect +that same ID after quiescence. Multi-process lease/arbitration remains an +explicit production-activation gate. Activation must also inject the selected +gateway's canonical state root. ## Architectural disposition diff --git a/src/lib/onboard/managed-bootstrap/adapter.test.ts b/src/lib/onboard/managed-bootstrap/adapter.test.ts index 3f80e00e61d..5d86254de2b 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.test.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.test.ts @@ -13,6 +13,7 @@ import { import { createManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; import { activateManagedBootstrapSequence, + enforceManagedBootstrapRecoveryForSandbox, finalizeManagedBootstrapSequence, MANAGED_BOOTSTRAP_SCHEMA_VERSION, type ManagedBootstrapAdapter, @@ -23,6 +24,7 @@ import { type ManagedBootstrapHeldWorkloadHandle, type ManagedBootstrapObservedSnapshot, type ManagedBootstrapPreparedReplacementHandle, + ManagedBootstrapRecoveryBlockedError, type ManagedBootstrapReplacementHandle, prepareManagedBootstrapSequence, recoverManagedBootstrapTransactions, @@ -228,7 +230,7 @@ function adapterFor(agent: ManagedStartupAgent): Fixture { const order: string[] = []; const raw: Fixture["raw"] = { handle: null, snapshot: null, prepared: null }; const adapter: ManagedBootstrapAdapter = { - recoverUnfinishedTransactions: vi.fn(async () => []), + recoverUnfinishedTransactions: vi.fn(async () => ({ receipts: [], failures: [] })), createHeldWorkload: vi.fn(async (input) => { order.push("create"); const receipt = await input.launch({ @@ -922,41 +924,172 @@ describe("managed bootstrap adapter contract", () => { outcome: "rolled-back" as const, finalization: { ...receipt, bootstrapIdentity }, }); - vi.mocked(fixture.adapter.recoverUnfinishedTransactions).mockResolvedValueOnce([ - candidate("b".repeat(64)), - candidate("a".repeat(64)), - ]); + vi.mocked(fixture.adapter.recoverUnfinishedTransactions).mockResolvedValueOnce({ + receipts: [candidate("b".repeat(64)), candidate("a".repeat(64))], + failures: [], + }); const recovered = await recoverManagedBootstrapTransactions(fixture.adapter); - expect(recovered.map(({ bootstrapIdentity }) => bootstrapIdentity)).toEqual([ + expect(recovered.receipts.map(({ bootstrapIdentity }) => bootstrapIdentity)).toEqual([ "a".repeat(64), "b".repeat(64), ]); expect(Object.isFrozen(recovered)).toBe(true); - expect(recovered.every((entry) => Object.isFrozen(entry.finalization))).toBe(true); + expect(Object.isFrozen(recovered.receipts)).toBe(true); + expect(recovered.receipts.every((entry) => Object.isFrozen(entry.finalization))).toBe(true); }); it("rejects recovery evidence whose provider does not own the durable sandbox", async () => { const fixture = adapterFor("openclaw"); const receipt = cleanupReceipt(); - vi.mocked(fixture.adapter.recoverUnfinishedTransactions).mockResolvedValueOnce([ - { - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - providerId: "mxc", - sourcePhase: "cutover", - sandbox: receipt.sandbox, - bootstrapIdentity: IDENTITY, - outcome: "rolled-back", - finalization: receipt, - }, - ]); + vi.mocked(fixture.adapter.recoverUnfinishedTransactions).mockResolvedValueOnce({ + receipts: [ + { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: "mxc", + sourcePhase: "cutover", + sandbox: receipt.sandbox, + bootstrapIdentity: IDENTITY, + outcome: "rolled-back", + finalization: receipt, + }, + ], + failures: [], + }); await expect(recoverManagedBootstrapTransactions(fixture.adapter)).rejects.toThrow( "recovery provider does not own", ); }); + it("normalizes provider-neutral failures and preserves bounded MXC-style diagnostics", async () => { + const fixture = adapterFor("hermes"); + const failure = (bootstrapIdentity: string, sandboxName: string | null) => ({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: "mxc", + sourcePhase: "provider-owned-cleanup", + sandbox: + sandboxName === null + ? null + : { sandboxName, sandboxId: `mxc-${sandboxName}`, driverId: "mxc" }, + bootstrapIdentity, + code: "provider-owned-retry", + retryable: true, + detail: "opaque MXC recovery evidence", + }); + vi.mocked(fixture.adapter.recoverUnfinishedTransactions).mockResolvedValueOnce({ + receipts: [], + failures: [failure("b".repeat(64), "bravo"), failure("a".repeat(64), null)], + }); + + const recovered = await recoverManagedBootstrapTransactions(fixture.adapter); + + expect(recovered.failures.map(({ bootstrapIdentity }) => bootstrapIdentity)).toEqual([ + "a".repeat(64), + "b".repeat(64), + ]); + expect(recovered.failures[0]).toMatchObject({ sandbox: null, providerId: "mxc" }); + expect(Object.isFrozen(recovered.failures)).toBe(true); + expect(recovered.failures.every(Object.isFrozen)).toBe(true); + }); + + it("rejects duplicate identities across recovered receipts and failures", async () => { + const fixture = adapterFor("openclaw"); + const receipt = cleanupReceipt(); + vi.mocked(fixture.adapter.recoverUnfinishedTransactions).mockResolvedValueOnce({ + receipts: [ + { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: receipt.sandbox.driverId, + sourcePhase: "cutover", + sandbox: receipt.sandbox, + bootstrapIdentity: IDENTITY, + outcome: "rolled-back", + finalization: receipt, + }, + ], + failures: [ + { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: receipt.sandbox.driverId, + sourcePhase: "cleanup", + sandbox: receipt.sandbox, + bootstrapIdentity: IDENTITY, + code: "retry", + retryable: true, + detail: "retained", + }, + ], + }); + + await expect(recoverManagedBootstrapTransactions(fixture.adapter)).rejects.toThrow( + "duplicate bootstrap identities", + ); + }); + + it("rejects an unbounded provider recovery result before normalizing records", async () => { + const fixture = adapterFor("hermes"); + const candidate = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: "mxc", + sourcePhase: "provider-owned-cleanup", + sandbox: null, + bootstrapIdentity: IDENTITY, + code: "provider-owned-retry", + retryable: true, + detail: "opaque MXC recovery evidence", + } as const; + vi.mocked(fixture.adapter.recoverUnfinishedTransactions).mockResolvedValueOnce({ + receipts: [], + failures: Array.from({ length: 4097 }, () => candidate), + }); + + await expect(recoverManagedBootstrapTransactions(fixture.adapter)).rejects.toThrow( + "provider recovery returned too many records", + ); + }); + + it("blocks same-name and identity-unknown failures while warning for unrelated sandboxes", () => { + const failure = (bootstrapIdentity: string, sandboxName: string | null) => + Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: "mxc", + sourcePhase: "cleanup", + sandbox: + sandboxName === null + ? null + : Object.freeze({ sandboxName, sandboxId: `mxc-${sandboxName}`, driverId: "mxc" }), + bootstrapIdentity, + code: "provider-owned-retry", + retryable: true, + detail: "opaque provider detail", + }); + const warn = vi.fn(); + const unrelated = failure("a".repeat(64), "bravo"); + const sameName = failure("b".repeat(64), "alpha"); + const identityUnknown = failure("c".repeat(64), null); + + expect( + enforceManagedBootstrapRecoveryForSandbox( + Object.freeze({ receipts: Object.freeze([]), failures: Object.freeze([unrelated]) }), + "alpha", + warn, + ), + ).toMatchObject({ failures: [unrelated] }); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("unrelated sandbox 'bravo'")); + + for (const blocking of [sameName, identityUnknown]) { + expect(() => + enforceManagedBootstrapRecoveryForSandbox( + Object.freeze({ receipts: Object.freeze([]), failures: Object.freeze([blocking]) }), + "alpha", + warn, + ), + ).toThrow(ManagedBootstrapRecoveryBlockedError); + } + }); + it.each([ "BASHOPTS=extdebug", "BASH_ENV=/sandbox/attacker", diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index 9f7e6bf3cf1..bbf8df337e0 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -18,6 +18,7 @@ export const MANAGED_BOOTSTRAP_IDENTITY_BYTES = 32; const SHA256_RE = /^[a-f0-9]{64}$/u; const MANIFEST_DIGEST_RE = /^sha256:[a-f0-9]{64}$/u; const ENV_ASSIGNMENT_RE = /^[A-Za-z_][A-Za-z0-9_]*=/u; +const MAX_MANAGED_BOOTSTRAP_RECOVERY_RECORDS = 4096; const PROCESS_INJECTION_ENV_KEYS = new Set([ "BASHOPTS", "BASH_ENV", @@ -254,6 +255,27 @@ export interface ManagedBootstrapRecoveryReceipt { readonly finalization: ManagedBootstrapFinalizationReceipt; } +/** Bounded provider-owned evidence that one durable transaction still needs attention. */ +export interface ManagedBootstrapRecoveryFailure { + readonly schemaVersion: typeof MANAGED_BOOTSTRAP_SCHEMA_VERSION; + readonly providerId: string; + /** Null when the durable record could not prove its provider-owned phase. */ + readonly sourcePhase: string | null; + /** Null when the durable record could not prove its sandbox identity. */ + readonly sandbox: ManagedBootstrapSandboxIdentity | null; + readonly bootstrapIdentity: string; + /** Provider-owned diagnostic code. Central orchestration must not branch on this value. */ + readonly code: string; + readonly retryable: boolean; + readonly detail: string; +} + +/** Lossless provider-neutral recovery output for one bounded enumeration pass. */ +export interface ManagedBootstrapRecoveryReport { + readonly receipts: readonly ManagedBootstrapRecoveryReceipt[]; + readonly failures: readonly ManagedBootstrapRecoveryFailure[]; +} + export class ManagedBootstrapDurableCommitCleanupPendingError extends Error { readonly bootstrapIdentity: string; readonly cleanupRuntimeId: string; @@ -312,6 +334,25 @@ export class ManagedBootstrapOwnerCleanupRequiredError extends Error { } } +export class ManagedBootstrapRecoveryBlockedError extends Error { + readonly sandboxName: string; + readonly failures: readonly ManagedBootstrapRecoveryFailure[]; + + constructor(sandboxName: string, failures: readonly ManagedBootstrapRecoveryFailure[]) { + const first = failures[0]; + super( + `Managed bootstrap recovery blocks sandbox '${sandboxName}' because ${String( + failures.length, + )} durable transaction${failures.length === 1 ? "" : "s"} still need attention.${ + first ? ` First failure ${first.bootstrapIdentity} (${first.code}): ${first.detail}` : "" + }`, + ); + this.name = "ManagedBootstrapRecoveryBlockedError"; + this.sandboxName = sandboxName; + this.failures = Object.freeze([...failures]); + } +} + export function attachManagedBootstrapRollbackError(failure: Error, rollbackError: unknown): void { ( failure as Error & { @@ -329,7 +370,7 @@ export interface ManagedBootstrapAdapter { * Enumerate durable unfinished records and reconcile each through the owning * provider. Implementations must be restart-safe and idempotent. */ - recoverUnfinishedTransactions(): Promise; + recoverUnfinishedTransactions(): Promise; /** Return only after one durable sandbox/driver identity reports Ready. */ createHeldWorkload( @@ -458,24 +499,109 @@ function normalizeRecoveryReceipt( }); } +function normalizeRecoveryFailure( + candidate: ManagedBootstrapRecoveryFailure, +): ManagedBootstrapRecoveryFailure { + if ( + typeof candidate !== "object" || + candidate === null || + Array.isArray(candidate) || + candidate.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + typeof candidate.retryable !== "boolean" + ) { + protocolFail("recovery failure has an invalid schema"); + } + assertOpaqueString(candidate.providerId, "recovery failure provider ID", 256); + if (candidate.sourcePhase !== null) { + assertOpaqueString(candidate.sourcePhase, "recovery failure source phase", 256); + } + if (candidate.sandbox !== null) { + assertSandboxIdentity(candidate.sandbox); + if (candidate.sandbox.driverId !== candidate.providerId) { + protocolFail("recovery failure provider does not own the durable sandbox"); + } + } + if (!SHA256_RE.test(candidate.bootstrapIdentity)) { + protocolFail("recovery failure bootstrap identity must be lowercase SHA-256"); + } + assertOpaqueString(candidate.code, "recovery failure code", 256); + assertOpaqueString(candidate.detail, "recovery failure detail", 8 * 1024); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: candidate.providerId, + sourcePhase: candidate.sourcePhase, + sandbox: candidate.sandbox === null ? null : Object.freeze({ ...candidate.sandbox }), + bootstrapIdentity: candidate.bootstrapIdentity, + code: candidate.code, + retryable: candidate.retryable, + detail: candidate.detail, + }); +} + /** Recover process-orphaned work without relying on coordinator WeakMap state. */ export async function recoverManagedBootstrapTransactions( adapter: ManagedBootstrapAdapter, -): Promise { +): Promise { const candidates = await adapter.recoverUnfinishedTransactions(); - if (!Array.isArray(candidates)) { - protocolFail("provider recovery must return a receipt array"); + if ( + typeof candidates !== "object" || + candidates === null || + Array.isArray(candidates) || + !Array.isArray(candidates.receipts) || + !Array.isArray(candidates.failures) + ) { + protocolFail("provider recovery must return bounded receipt and failure arrays"); } - const receipts = candidates.map(normalizeRecoveryReceipt); - const identities = receipts.map(({ bootstrapIdentity }) => bootstrapIdentity); + if ( + candidates.receipts.length + candidates.failures.length > + MAX_MANAGED_BOOTSTRAP_RECOVERY_RECORDS + ) { + protocolFail("provider recovery returned too many records"); + } + const receipts = candidates.receipts.map(normalizeRecoveryReceipt); + const failures = candidates.failures.map(normalizeRecoveryFailure); + const identities = [...receipts, ...failures].map(({ bootstrapIdentity }) => bootstrapIdentity); if (new Set(identities).size !== identities.length) { protocolFail("provider recovery returned duplicate bootstrap identities"); } - return Object.freeze( - [...receipts].sort((left, right) => - left.bootstrapIdentity.localeCompare(right.bootstrapIdentity), - ), + const byBootstrapIdentity = ( + left: ManagedBootstrapRecoveryReceipt | ManagedBootstrapRecoveryFailure, + right: ManagedBootstrapRecoveryReceipt | ManagedBootstrapRecoveryFailure, + ) => left.bootstrapIdentity.localeCompare(right.bootstrapIdentity); + return Object.freeze({ + receipts: Object.freeze([...receipts].sort(byBootstrapIdentity)), + failures: Object.freeze([...failures].sort(byBootstrapIdentity)), + }); +} + +/** Block only failures that can own the requested name; warn for exact unrelated sandboxes. */ +export function enforceManagedBootstrapRecoveryForSandbox( + report: ManagedBootstrapRecoveryReport, + sandboxName: string, + warn: (message: string) => void, +): ManagedBootstrapRecoveryReport { + assertOpaqueString(sandboxName, "recovery target sandbox name"); + const blocking = report.failures.filter( + (failure) => failure.sandbox === null || failure.sandbox.sandboxName === sandboxName, ); + for (const failure of report.failures) { + if (failure.sandbox === null || failure.sandbox.sandboxName === sandboxName) continue; + warn( + `Managed bootstrap recovery retained unrelated sandbox '${failure.sandbox.sandboxName}' ` + + `(${failure.bootstrapIdentity}, ${failure.code}).`, + ); + } + if (blocking.length > 0) { + throw new ManagedBootstrapRecoveryBlockedError( + sandboxName, + Object.freeze( + [...blocking].sort((left, right) => + left.bootstrapIdentity.localeCompare(right.bootstrapIdentity), + ), + ), + ); + } + return report; } export interface ManagedBootstrapPreparationInput { @@ -506,12 +632,16 @@ function protocolFail(message: string): never { throw new Error(`Managed bootstrap protocol violation: ${message}`); } -function assertOpaqueString(value: unknown, label: string): asserts value is string { +function assertOpaqueString( + value: unknown, + label: string, + maxBytes = 64 * 1024, +): asserts value is string { if ( typeof value !== "string" || value.length === 0 || value.includes("\0") || - Buffer.byteLength(value, "utf8") > 64 * 1024 + Buffer.byteLength(value, "utf8") > maxBytes ) { protocolFail(`${label} must be one bounded non-empty string`); } diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index b1ed163b954..69d2a336d3c 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -12,8 +12,12 @@ import { DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapFinalizationContext, type DockerManagedBootstrapFinalizationRecord, type DockerManagedBootstrapJournal, + type DockerManagedBootstrapJournalStore, + DockerManagedBootstrapLegacyRecordRequiresAgentError, + normalizeDockerManagedBootstrapJournal, parseDockerManagedBootstrapFinalizationRecord, parseDockerManagedBootstrapJournal, sameDockerManagedBootstrapReceipt, @@ -23,6 +27,17 @@ import { const roots: string[] = []; const IDENTITY = "1".repeat(64); +const OTHER_IDENTITY = "0".repeat(64); + +function loadUnfinished( + store: DockerManagedBootstrapJournalStore, +): readonly DockerManagedBootstrapJournal[] { + return store.listUnfinishedIdentities().map((identity) => { + const record = store.load(identity); + expect(record, `enumerated journal ${identity} must remain loadable`).not.toBeNull(); + return record as DockerManagedBootstrapJournal; + }); +} const journal = Object.freeze({ schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, phase: "staged", @@ -100,6 +115,75 @@ const finalization = Object.freeze({ }, } satisfies DockerManagedBootstrapFinalizationRecord); +const finalizationContext = Object.freeze({ + bootstrapIdentity: finalization.bootstrapIdentity, + providerId: finalization.providerId, + agent: finalization.agent, + sandbox: finalization.sandbox, + planFingerprint: finalization.planFingerprint, + profileFingerprint: finalization.profileFingerprint, + imageReference: finalization.imageReference, +} satisfies DockerManagedBootstrapFinalizationContext); + +function legacyJournalV1() { + return Object.freeze({ + schemaVersion: 1 as const, + phase: journal.phase, + bootstrapIdentity: journal.bootstrapIdentity, + sandbox: journal.sandbox, + profileFingerprint: journal.profileFingerprint, + imageReference: journal.imageReference, + runtimeImageContentId: journal.runtimeImageContentId, + originalRuntimeId: journal.originalRuntimeId, + replacementRuntimeId: journal.replacementRuntimeId, + originalName: journal.originalName, + replacementStagingName: journal.replacementStagingName, + backupName: journal.backupName, + originalSpecHash: journal.originalSpecHash, + replacementSpecHash: journal.replacementSpecHash, + }); +} + +function legacyJournalV2() { + return Object.freeze({ + schemaVersion: 2 as const, + phase: journal.phase, + bootstrapIdentity: journal.bootstrapIdentity, + providerId: journal.providerId, + sandbox: journal.sandbox, + planFingerprint: journal.planFingerprint, + profileFingerprint: journal.profileFingerprint, + imageReference: journal.imageReference, + runtimeImageContentId: journal.runtimeImageContentId, + originalRuntimeId: journal.originalRuntimeId, + replacementRuntimeId: journal.replacementRuntimeId, + originalName: journal.originalName, + replacementStagingName: journal.replacementStagingName, + backupName: journal.backupName, + originalSpecHash: journal.originalSpecHash, + replacementSpecHash: journal.replacementSpecHash, + rollbackTargetRuntimeId: journal.rollbackTargetRuntimeId, + rollbackTargetSpecHash: journal.rollbackTargetSpecHash, + preparationReceipt: journal.preparationReceipt, + commitReceipt: journal.commitReceipt, + }); +} + +function legacyFinalizationV1() { + return Object.freeze({ + schemaVersion: 1 as const, + phase: finalization.phase, + bootstrapIdentity: finalization.bootstrapIdentity, + providerId: finalization.providerId, + sandbox: finalization.sandbox, + planFingerprint: finalization.planFingerprint, + profileFingerprint: finalization.profileFingerprint, + imageReference: finalization.imageReference, + commitReceipt: finalization.commitReceipt, + cleanupReceipt: finalization.cleanupReceipt, + }); +} + function readPinnedPrivateFile(target: string): { readonly mode: number; readonly text: string } { const descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); try { @@ -122,6 +206,17 @@ afterEach(() => { }); describe("Docker managed bootstrap journal", () => { + it("rejects comma-joined keys as a different schema", () => { + const { agent: _agent, backupName: _backupName, ...withoutSeparateKeys } = journal; + + expect(() => + normalizeDockerManagedBootstrapJournal({ + ...withoutSeparateKeys, + "agent,backupName": "hermes", + }), + ).toThrow("journal schema is invalid"); + }); + it("publishes private canonical state through only monotonic phases", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); roots.push(root); @@ -149,6 +244,26 @@ describe("Docker managed bootstrap journal", () => { expect(store.load(IDENTITY)).toBeNull(); }); + it("persists owner cleanup as a restart-safe non-terminal phase", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const first = createFileDockerManagedBootstrapJournalStore(root); + first.create(journal); + + const retained = first.transition(IDENTITY, "staged", "owner-cleanup-required"); + expect(retained.phase).toBe("owner-cleanup-required"); + expect(first.listUnfinishedIdentities()).toEqual([IDENTITY]); + + const restarted = createFileDockerManagedBootstrapJournalStore(root); + expect(restarted.load(IDENTITY)).toEqual(retained); + expect(restarted.listUnfinishedIdentities()).toEqual([IDENTITY]); + expect(() => + restarted.transition(IDENTITY, "owner-cleanup-required", "shared-state-committed"), + ).toThrow("unsupported"); + restarted.remove(IDENTITY, ["owner-cleanup-required"]); + expect(restarted.load(IDENTITY)).toBeNull(); + }); + it("recovers one durable cutover decision before journal replacement", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); roots.push(root); @@ -286,11 +401,12 @@ describe("Docker managed bootstrap journal", () => { roots.push(root); const first = createFileDockerManagedBootstrapJournalStore(root); first.create(journal); + expect(loadUnfinished(first)).toEqual([journal]); first.recordFinalization(finalization); - expect(first.listUnfinished()).toEqual([journal]); + expect(loadUnfinished(first)).toEqual([journal]); first.remove(IDENTITY, ["staged"]); - expect(first.listUnfinished()).toEqual([]); + expect(loadUnfinished(first)).toEqual([]); const restarted = createFileDockerManagedBootstrapJournalStore(root); expect(restarted.loadFinalization(IDENTITY)).toEqual(finalization); expect( @@ -332,7 +448,7 @@ describe("Docker managed bootstrap journal", () => { }, ); - expect(store.listUnfinished()).toEqual([journal]); + expect(loadUnfinished(store)).toEqual([journal]); }); it("rejects an unsupported journal-directory entry during enumeration", () => { @@ -345,7 +461,9 @@ describe("Docker managed bootstrap journal", () => { mode: 0o600, }); - expect(() => store.listUnfinished()).toThrow("journal directory contains an unsupported entry"); + expect(() => store.listUnfinishedIdentities()).toThrow( + "journal directory contains an unsupported entry", + ); }); it("reloads the exact completion receipt from a new journal store", () => { @@ -359,6 +477,7 @@ describe("Docker managed bootstrap journal", () => { const restarted = createFileDockerManagedBootstrapJournalStore(root); expect(restarted.recordCompletion(IDENTITY, finalization.commitReceipt)).toEqual(completed); + expect(loadUnfinished(restarted)).toEqual([completed]); expect(() => restarted.recordCompletion(IDENTITY, { ...finalization.commitReceipt, @@ -366,4 +485,191 @@ describe("Docker managed bootstrap journal", () => { }), ).toThrow("completion receipt changed"); }); + + it.each([ + [1, legacyJournalV1], + [2, legacyJournalV2], + ] as const)("fails typed and closed for exact legacy journal schema %i", (schemaVersion, legacy) => { + const serialized = `${JSON.stringify(legacy())}\n`; + let failure: unknown; + try { + parseDockerManagedBootstrapJournal(serialized); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(DockerManagedBootstrapLegacyRecordRequiresAgentError); + expect(failure).toMatchObject({ + bootstrapIdentity: IDENTITY, + recordKind: "journal", + schemaVersion, + }); + + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + expect(loadUnfinished(store)).toEqual([]); + const target = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, `${IDENTITY}.json`); + fs.writeFileSync(target, serialized, { mode: 0o600 }); + expect(store.listUnfinishedIdentities()).toEqual([IDENTITY]); + expect(() => store.load(IDENTITY)).toThrowError( + DockerManagedBootstrapLegacyRecordRequiresAgentError, + ); + expect(readPinnedPrivateFile(target).text).toBe(serialized); + }); + + it("does not classify a malformed legacy journal as upgradeable authority", () => { + const malformed = { ...legacyJournalV2(), agent: "hermes" }; + expect(() => parseDockerManagedBootstrapJournal(`${JSON.stringify(malformed)}\n`)).toThrow( + "legacy journal schema is invalid", + ); + try { + parseDockerManagedBootstrapJournal(`${JSON.stringify(malformed)}\n`); + } catch (error) { + expect(error).not.toBeInstanceOf(DockerManagedBootstrapLegacyRecordRequiresAgentError); + } + }); + + it("rejects owner cleanup as authority invented by a legacy journal", () => { + expect(() => + parseDockerManagedBootstrapJournal( + `${JSON.stringify({ ...legacyJournalV2(), phase: "owner-cleanup-required" })}\n`, + ), + ).toThrow("legacy phase is unsupported"); + }); + + it("upgrades legacy finalization only with exact immutable transaction context", () => { + const serialized = `${JSON.stringify(legacyFinalizationV1())}\n`; + let missingContextFailure: unknown; + try { + parseDockerManagedBootstrapFinalizationRecord(serialized); + } catch (error) { + missingContextFailure = error; + } + expect(missingContextFailure).toBeInstanceOf( + DockerManagedBootstrapLegacyRecordRequiresAgentError, + ); + expect(missingContextFailure).toMatchObject({ reason: "missing-context" }); + + let contextMismatchFailure: unknown; + try { + parseDockerManagedBootstrapFinalizationRecord(serialized, { + ...finalizationContext, + planFingerprint: "0".repeat(64), + }); + } catch (error) { + contextMismatchFailure = error; + } + expect(contextMismatchFailure).toBeInstanceOf( + DockerManagedBootstrapLegacyRecordRequiresAgentError, + ); + expect(contextMismatchFailure).toMatchObject({ + message: expect.stringContaining("supplied durable context does not match this record"), + reason: "context-mismatch", + }); + expect(parseDockerManagedBootstrapFinalizationRecord(serialized, finalizationContext)).toEqual( + finalization, + ); + + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + expect(loadUnfinished(store)).toEqual([]); + const target = path.join( + root, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, + `${IDENTITY}.json.finalized`, + ); + fs.writeFileSync(target, serialized, { mode: 0o600 }); + expect(() => store.loadFinalization(IDENTITY)).toThrowError( + DockerManagedBootstrapLegacyRecordRequiresAgentError, + ); + expect(() => store.recordFinalization(finalization)).toThrowError( + DockerManagedBootstrapLegacyRecordRequiresAgentError, + ); + expect(readPinnedPrivateFile(target).text).toBe(serialized); + + expect(() => + store.recordFinalization(finalization, { + ...finalizationContext, + agent: "openclaw", + }), + ).toThrow("does not match supplied durable context"); + expect(readPinnedPrivateFile(target).text).toBe(serialized); + + store.recordFinalization(finalization, finalizationContext); + expect(store.loadFinalization(IDENTITY)).toEqual(finalization); + expect(readPinnedPrivateFile(target).text).toBe( + serializeDockerManagedBootstrapFinalizationRecord(finalization), + ); + }); + + it("rejects a current finalization that contradicts supplied durable context", () => { + const wrongAgent = Object.freeze({ ...finalization, agent: "openclaw" as const }); + expect(() => + parseDockerManagedBootstrapFinalizationRecord( + serializeDockerManagedBootstrapFinalizationRecord(wrongAgent), + finalizationContext, + ), + ).toThrow("does not match supplied durable context"); + }); + + it("does not create a finalization before validating durable context", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + expect(loadUnfinished(store)).toEqual([]); + const target = path.join( + root, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, + `${IDENTITY}.json.finalized`, + ); + + expect(() => + store.recordFinalization(finalization, { + ...finalizationContext, + agent: "openclaw", + }), + ).toThrow("does not match supplied durable context"); + expect(fs.existsSync(target)).toBe(false); + }); + + it("rejects current journal and finalization records stored under another identity", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + expect(loadUnfinished(store)).toEqual([]); + const directory = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); + const misplacedJournal = path.join(directory, `${OTHER_IDENTITY}.json`); + const misplacedFinalization = `${misplacedJournal}.finalized`; + fs.writeFileSync(misplacedJournal, serializeDockerManagedBootstrapJournal(journal), { + mode: 0o600, + }); + fs.writeFileSync( + misplacedFinalization, + serializeDockerManagedBootstrapFinalizationRecord(finalization), + { mode: 0o600 }, + ); + + expect(() => store.load(OTHER_IDENTITY)).toThrow( + "journal bootstrap identity does not match its file name", + ); + expect(() => store.loadFinalization(OTHER_IDENTITY)).toThrow( + "finalization bootstrap identity does not match its file name", + ); + expect(fs.existsSync(misplacedJournal)).toBe(true); + expect(fs.existsSync(misplacedFinalization)).toBe(true); + }); + + it("fails closed when enumeration encounters an unsupported state entry", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + fs.writeFileSync( + path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, "unexpected.json"), + "{}\n", + { mode: 0o600 }, + ); + expect(() => store.listUnfinishedIdentities()).toThrow("unsupported entry"); + }); }); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index db37c5ce213..03e745acb34 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -29,6 +29,7 @@ export type DockerManagedBootstrapJournalPhase = | "staged" | "cutover" | "rollback-authorized" + | "owner-cleanup-required" | "shared-state-committed"; export interface DockerManagedBootstrapJournal { @@ -69,10 +70,21 @@ export interface DockerManagedBootstrapFinalizationRecord { readonly cleanupReceipt: ManagedBootstrapFinalizationReceipt; } +export type DockerManagedBootstrapFinalizationContext = Pick< + DockerManagedBootstrapFinalizationRecord, + | "agent" + | "bootstrapIdentity" + | "imageReference" + | "planFingerprint" + | "profileFingerprint" + | "providerId" + | "sandbox" +>; + export interface DockerManagedBootstrapJournalStore { create(journal: DockerManagedBootstrapJournal): void; load(bootstrapIdentity: string): DockerManagedBootstrapJournal | null; - listUnfinished(): readonly DockerManagedBootstrapJournal[]; + listUnfinishedIdentities(): readonly string[]; transition( bootstrapIdentity: string, expected: DockerManagedBootstrapJournalPhase, @@ -83,8 +95,14 @@ export interface DockerManagedBootstrapJournalStore { receipt: ManagedBootstrapCompletionReceipt, ): DockerManagedBootstrapJournal; remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]): void; - recordFinalization(record: DockerManagedBootstrapFinalizationRecord): void; - loadFinalization(bootstrapIdentity: string): DockerManagedBootstrapFinalizationRecord | null; + recordFinalization( + record: DockerManagedBootstrapFinalizationRecord, + context?: DockerManagedBootstrapFinalizationContext, + ): void; + loadFinalization( + bootstrapIdentity: string, + context?: DockerManagedBootstrapFinalizationContext, + ): DockerManagedBootstrapFinalizationRecord | null; } /** @@ -99,6 +117,34 @@ export class DockerManagedBootstrapJournalAcknowledgementLostError extends Error } } +export class DockerManagedBootstrapLegacyRecordRequiresAgentError extends Error { + readonly bootstrapIdentity: string; + readonly recordKind: "finalization" | "journal"; + readonly reason: "context-mismatch" | "missing-context" | undefined; + readonly schemaVersion: number; + + constructor(input: { + readonly bootstrapIdentity: string; + readonly recordKind: "finalization" | "journal"; + readonly reason?: "context-mismatch" | "missing-context"; + readonly schemaVersion: number; + }) { + const reason = input.reason; + super( + `Managed bootstrap Docker ${input.recordKind} schema ${input.schemaVersion} for ` + + `${input.bootstrapIdentity} lacks durable agent identity` + + (reason === "context-mismatch" + ? "; supplied durable context does not match this record" + : ""), + ); + this.name = "DockerManagedBootstrapLegacyRecordRequiresAgentError"; + this.bootstrapIdentity = input.bootstrapIdentity; + this.recordKind = input.recordKind; + this.reason = reason; + this.schemaVersion = input.schemaVersion; + } +} + class DockerManagedBootstrapJournalExistsError extends Error { constructor() { super( @@ -110,14 +156,25 @@ class DockerManagedBootstrapJournalExistsError extends Error { const ALLOWED_TRANSITIONS = new Set([ "staged->cutover", + "staged->owner-cleanup-required", "cutover->rollback-authorized", "cutover->shared-state-committed", + "rollback-authorized->owner-cleanup-required", ]); function fail(message: string): never { throw new Error(`Managed bootstrap Docker journal is invalid: ${message}`); } +function hasExactKeys(record: Readonly>, expected: readonly string[]) { + const actualKeys = Object.keys(record).sort(); + const expectedKeys = [...expected].sort(); + return ( + actualKeys.length === expectedKeys.length && + actualKeys.every((key, index) => key === expectedKeys[index]) + ); +} + function exactString(value: unknown, label: string, maxBytes = 4096): string { if ( typeof value !== "string" || @@ -140,13 +197,30 @@ function exactSha256(value: unknown, label: string): string { function exactPhase(value: unknown): DockerManagedBootstrapJournalPhase { if ( - !["staged", "cutover", "rollback-authorized", "shared-state-committed"].includes(String(value)) + ![ + "staged", + "cutover", + "rollback-authorized", + "owner-cleanup-required", + "shared-state-committed", + ].includes(String(value)) ) { fail("phase is unsupported"); } return value as DockerManagedBootstrapJournalPhase; } +function exactLegacyPhase( + value: unknown, +): Exclude { + if ( + !["staged", "cutover", "rollback-authorized", "shared-state-committed"].includes(String(value)) + ) { + fail("legacy phase is unsupported"); + } + return value as Exclude; +} + function exactAgent(value: unknown): ManagedStartupAgent { if (!["openclaw", "hermes", "langchain-deepagents-code"].includes(String(value))) { fail("agent is unsupported"); @@ -169,6 +243,174 @@ function exactSandbox(value: unknown): ManagedBootstrapSandboxIdentity { }); } +function sameSandboxIdentity( + left: ManagedBootstrapSandboxIdentity, + right: ManagedBootstrapSandboxIdentity, +): boolean { + return ( + left.sandboxName === right.sandboxName && + left.sandboxId === right.sandboxId && + left.driverId === right.driverId + ); +} + +// Frozen historical schemas: these branches must reproduce the exact canonical +// bytes written by schema 1 and schema 2. Do not share their implementation with +// the current normalizer or update them when the current schema changes. +function normalizeLegacyDockerManagedBootstrapJournal( + journal: Readonly>, + schemaVersion: 1 | 2, +): { readonly bootstrapIdentity: string; readonly canonical: string } { + if (schemaVersion === 1) { + const expectedKeys = [ + "backupName", + "bootstrapIdentity", + "imageReference", + "originalName", + "originalRuntimeId", + "originalSpecHash", + "phase", + "profileFingerprint", + "replacementRuntimeId", + "replacementSpecHash", + "replacementStagingName", + "runtimeImageContentId", + "sandbox", + "schemaVersion", + ]; + if (!hasExactKeys(journal, expectedKeys)) fail("legacy journal schema is invalid"); + const normalized = Object.freeze({ + schemaVersion: 1 as const, + phase: exactLegacyPhase(journal.phase), + bootstrapIdentity: exactSha256(journal.bootstrapIdentity, "bootstrap identity"), + sandbox: exactSandbox(journal.sandbox), + profileFingerprint: exactSha256(journal.profileFingerprint, "profile fingerprint"), + imageReference: exactString(journal.imageReference, "image reference"), + runtimeImageContentId: exactString(journal.runtimeImageContentId, "runtime image content ID"), + originalRuntimeId: exactSha256(journal.originalRuntimeId, "original runtime ID"), + replacementRuntimeId: exactSha256(journal.replacementRuntimeId, "replacement runtime ID"), + originalName: exactString(journal.originalName, "original name", 253), + replacementStagingName: exactString( + journal.replacementStagingName, + "replacement staging name", + 253, + ), + backupName: exactString(journal.backupName, "backup name", 253), + originalSpecHash: exactSha256(journal.originalSpecHash, "original spec hash"), + replacementSpecHash: exactSha256(journal.replacementSpecHash, "replacement spec hash"), + }); + if (normalized.originalRuntimeId === normalized.replacementRuntimeId) { + fail("original and replacement runtime IDs must differ"); + } + if ( + new Set([normalized.originalName, normalized.replacementStagingName, normalized.backupName]) + .size !== 3 + ) { + fail("original, staging, and backup names must be distinct"); + } + return { + bootstrapIdentity: normalized.bootstrapIdentity, + canonical: `${JSON.stringify(normalized)}\n`, + }; + } + + const expectedKeys = [ + "backupName", + "bootstrapIdentity", + "commitReceipt", + "imageReference", + "originalName", + "originalRuntimeId", + "originalSpecHash", + "phase", + "planFingerprint", + "preparationReceipt", + "profileFingerprint", + "providerId", + "replacementRuntimeId", + "replacementSpecHash", + "replacementStagingName", + "rollbackTargetRuntimeId", + "rollbackTargetSpecHash", + "runtimeImageContentId", + "sandbox", + "schemaVersion", + ]; + if (!hasExactKeys(journal, expectedKeys)) fail("legacy journal schema is invalid"); + const normalized = Object.freeze({ + schemaVersion: 2 as const, + phase: exactLegacyPhase(journal.phase), + bootstrapIdentity: exactSha256(journal.bootstrapIdentity, "bootstrap identity"), + providerId: exactString(journal.providerId, "provider ID"), + sandbox: exactSandbox(journal.sandbox), + planFingerprint: exactSha256(journal.planFingerprint, "plan fingerprint"), + profileFingerprint: exactSha256(journal.profileFingerprint, "profile fingerprint"), + imageReference: exactString(journal.imageReference, "image reference"), + runtimeImageContentId: exactString(journal.runtimeImageContentId, "runtime image content ID"), + originalRuntimeId: exactSha256(journal.originalRuntimeId, "original runtime ID"), + replacementRuntimeId: exactSha256(journal.replacementRuntimeId, "replacement runtime ID"), + originalName: exactString(journal.originalName, "original name", 253), + replacementStagingName: exactString( + journal.replacementStagingName, + "replacement staging name", + 253, + ), + backupName: exactString(journal.backupName, "backup name", 253), + originalSpecHash: exactSha256(journal.originalSpecHash, "original spec hash"), + replacementSpecHash: exactSha256(journal.replacementSpecHash, "replacement spec hash"), + rollbackTargetRuntimeId: exactSha256( + journal.rollbackTargetRuntimeId, + "rollback target runtime ID", + ), + rollbackTargetSpecHash: exactSha256( + journal.rollbackTargetSpecHash, + "rollback target spec hash", + ), + preparationReceipt: + journal.preparationReceipt === null + ? null + : exactPreparationReceipt(journal.preparationReceipt), + commitReceipt: + journal.commitReceipt === null ? null : exactCompletionReceipt(journal.commitReceipt), + }); + if (normalized.originalRuntimeId === normalized.replacementRuntimeId) { + fail("original and replacement runtime IDs must differ"); + } + if ( + new Set([normalized.originalName, normalized.replacementStagingName, normalized.backupName]) + .size !== 3 + ) { + fail("original, staging, and backup names must be distinct"); + } + if ( + normalized.providerId !== normalized.sandbox.driverId || + normalized.rollbackTargetRuntimeId !== normalized.originalRuntimeId || + normalized.rollbackTargetSpecHash !== normalized.originalSpecHash + ) { + fail("provider or rollback authority does not match the transaction identity"); + } + if ( + (normalized.preparationReceipt !== null && + (normalized.preparationReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + !sameSandboxIdentity(normalized.preparationReceipt.sandbox, normalized.sandbox))) || + (normalized.commitReceipt !== null && + (normalized.commitReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + !sameSandboxIdentity(normalized.commitReceipt.sandbox, normalized.sandbox) || + normalized.commitReceipt.runtimeId !== normalized.replacementRuntimeId || + normalized.commitReceipt.profileFingerprint !== normalized.profileFingerprint || + normalized.commitReceipt.originalSpecHash !== normalized.originalSpecHash || + normalized.commitReceipt.replacementSpecHash !== normalized.replacementSpecHash || + `${normalized.commitReceipt.image.repository}@${normalized.commitReceipt.image.manifestDigest}` !== + normalized.imageReference)) + ) { + fail("durable preparation or commit receipt does not match the transaction identity"); + } + return { + bootstrapIdentity: normalized.bootstrapIdentity, + canonical: `${JSON.stringify(normalized)}\n`, + }; +} + export function normalizeDockerManagedBootstrapJournal( value: unknown, ): DockerManagedBootstrapJournal { @@ -176,6 +418,14 @@ export function normalizeDockerManagedBootstrapJournal( fail("journal must be an object"); } const journal = value as Record; + if (journal.schemaVersion === 1 || journal.schemaVersion === 2) { + const legacy = normalizeLegacyDockerManagedBootstrapJournal(journal, journal.schemaVersion); + throw new DockerManagedBootstrapLegacyRecordRequiresAgentError({ + bootstrapIdentity: legacy.bootstrapIdentity, + recordKind: "journal", + schemaVersion: journal.schemaVersion, + }); + } const expectedKeys = [ "agent", "backupName", @@ -200,7 +450,7 @@ export function normalizeDockerManagedBootstrapJournal( "schemaVersion", ]; if ( - Object.keys(journal).sort().join(",") !== expectedKeys.sort().join(",") || + !hasExactKeys(journal, expectedKeys) || journal.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION ) { fail("journal schema is invalid"); @@ -261,14 +511,10 @@ export function normalizeDockerManagedBootstrapJournal( if ( (normalized.preparationReceipt !== null && (normalized.preparationReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || - normalized.preparationReceipt.sandbox.sandboxName !== normalized.sandbox.sandboxName || - normalized.preparationReceipt.sandbox.sandboxId !== normalized.sandbox.sandboxId || - normalized.preparationReceipt.sandbox.driverId !== normalized.sandbox.driverId)) || + !sameSandboxIdentity(normalized.preparationReceipt.sandbox, normalized.sandbox))) || (normalized.commitReceipt !== null && (normalized.commitReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || - normalized.commitReceipt.sandbox.sandboxName !== normalized.sandbox.sandboxName || - normalized.commitReceipt.sandbox.sandboxId !== normalized.sandbox.sandboxId || - normalized.commitReceipt.sandbox.driverId !== normalized.sandbox.driverId || + !sameSandboxIdentity(normalized.commitReceipt.sandbox, normalized.sandbox) || normalized.commitReceipt.runtimeId !== normalized.replacementRuntimeId || normalized.commitReceipt.profileFingerprint !== normalized.profileFingerprint || normalized.commitReceipt.originalSpecHash !== normalized.originalSpecHash || @@ -306,6 +552,22 @@ export function parseDockerManagedBootstrapJournal(text: string): DockerManagedB } catch { fail("serialized journal is not valid JSON"); } + if ( + typeof parsed === "object" && + parsed !== null && + !Array.isArray(parsed) && + ((parsed as Record).schemaVersion === 1 || + (parsed as Record).schemaVersion === 2) + ) { + const record = parsed as Record & { readonly schemaVersion: 1 | 2 }; + const legacy = normalizeLegacyDockerManagedBootstrapJournal(record, record.schemaVersion); + if (legacy.canonical !== text) fail("serialized legacy journal is not canonical"); + throw new DockerManagedBootstrapLegacyRecordRequiresAgentError({ + bootstrapIdentity: legacy.bootstrapIdentity, + recordKind: "journal", + schemaVersion: record.schemaVersion, + }); + } const journal = normalizeDockerManagedBootstrapJournal(parsed); if (serializeDockerManagedBootstrapJournal(journal) !== text) { fail("serialized journal is not canonical"); @@ -488,32 +750,16 @@ function exactCleanupReceipt(value: unknown): ManagedBootstrapFinalizationReceip }); } -export function normalizeDockerManagedBootstrapFinalizationRecord( - value: unknown, -): DockerManagedBootstrapFinalizationRecord { - if (typeof value !== "object" || value === null || Array.isArray(value)) { - fail("finalization record must be an object"); - } - const record = value as Record; - const expectedKeys = [ - "agent", - "bootstrapIdentity", - "cleanupReceipt", - "commitReceipt", - "imageReference", - "phase", - "planFingerprint", - "profileFingerprint", - "providerId", - "sandbox", - "schemaVersion", - ]; - if ( - Object.keys(record).sort().join(",") !== expectedKeys.sort().join(",") || - record.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION || - !["committed", "rolled-back"].includes(String(record.phase)) - ) { - fail("finalization record schema is invalid"); +type DockerManagedBootstrapFinalizationWithoutAgent = Omit< + DockerManagedBootstrapFinalizationRecord, + "agent" | "schemaVersion" +>; + +function normalizeFinalizationWithoutAgent( + record: Readonly>, +): DockerManagedBootstrapFinalizationWithoutAgent { + if (!["committed", "rolled-back"].includes(String(record.phase))) { + fail("finalization phase is invalid"); } const phase = record.phase as "committed" | "rolled-back"; const sandbox = exactSandbox(record.sandbox); @@ -521,28 +767,26 @@ export function normalizeDockerManagedBootstrapFinalizationRecord( record.commitReceipt === null ? null : exactCompletionReceipt(record.commitReceipt); const cleanupReceipt = exactCleanupReceipt(record.cleanupReceipt); const normalized = Object.freeze({ - schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, phase, bootstrapIdentity: exactSha256(record.bootstrapIdentity, "finalization bootstrap identity"), providerId: exactString(record.providerId, "finalization provider ID"), - agent: exactAgent(record.agent), sandbox, planFingerprint: exactSha256(record.planFingerprint, "finalization plan fingerprint"), profileFingerprint: exactSha256(record.profileFingerprint, "finalization profile fingerprint"), imageReference: exactString(record.imageReference, "finalization image reference"), commitReceipt, cleanupReceipt, - } satisfies DockerManagedBootstrapFinalizationRecord); + } satisfies DockerManagedBootstrapFinalizationWithoutAgent); if ( normalized.providerId !== sandbox.driverId || normalized.bootstrapIdentity !== cleanupReceipt.bootstrapIdentity || normalized.phase !== cleanupReceipt.outcome || - JSON.stringify(normalized.sandbox) !== JSON.stringify(cleanupReceipt.sandbox) || + !sameSandboxIdentity(normalized.sandbox, cleanupReceipt.sandbox) || (phase === "committed") !== (commitReceipt !== null) || (commitReceipt !== null && (commitReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || commitReceipt.profileFingerprint !== normalized.profileFingerprint || - JSON.stringify(commitReceipt.sandbox) !== JSON.stringify(normalized.sandbox) || + !sameSandboxIdentity(commitReceipt.sandbox, normalized.sandbox) || `${commitReceipt.image.repository}@${commitReceipt.image.manifestDigest}` !== normalized.imageReference)) ) { @@ -551,6 +795,176 @@ export function normalizeDockerManagedBootstrapFinalizationRecord( return normalized; } +function normalizeLegacyFinalizationShape(value: unknown): { + readonly canonical: string; + readonly record: DockerManagedBootstrapFinalizationWithoutAgent; +} { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("finalization record must be an object"); + } + const record = value as Record; + const expectedKeys = [ + "bootstrapIdentity", + "cleanupReceipt", + "commitReceipt", + "imageReference", + "phase", + "planFingerprint", + "profileFingerprint", + "providerId", + "sandbox", + "schemaVersion", + ]; + if (!hasExactKeys(record, expectedKeys) || record.schemaVersion !== 1) { + fail("legacy finalization record schema is invalid"); + } + const normalized = normalizeFinalizationWithoutAgent(record); + const legacy = Object.freeze({ + schemaVersion: 1 as const, + phase: normalized.phase, + bootstrapIdentity: normalized.bootstrapIdentity, + providerId: normalized.providerId, + sandbox: normalized.sandbox, + planFingerprint: normalized.planFingerprint, + profileFingerprint: normalized.profileFingerprint, + imageReference: normalized.imageReference, + commitReceipt: normalized.commitReceipt, + cleanupReceipt: normalized.cleanupReceipt, + }); + return { canonical: `${JSON.stringify(legacy)}\n`, record: normalized }; +} + +function normalizeFinalizationContext( + context: DockerManagedBootstrapFinalizationContext, +): DockerManagedBootstrapFinalizationContext { + return Object.freeze({ + bootstrapIdentity: exactSha256( + context.bootstrapIdentity, + "finalization context bootstrap identity", + ), + providerId: exactString(context.providerId, "finalization context provider ID"), + agent: exactAgent(context.agent), + sandbox: exactSandbox(context.sandbox), + planFingerprint: exactSha256(context.planFingerprint, "finalization context plan fingerprint"), + profileFingerprint: exactSha256( + context.profileFingerprint, + "finalization context profile fingerprint", + ), + imageReference: exactString(context.imageReference, "finalization context image reference"), + }); +} + +function matchesFinalizationContext( + record: DockerManagedBootstrapFinalizationWithoutAgent, + context: DockerManagedBootstrapFinalizationContext, +): boolean { + return ( + context.bootstrapIdentity === record.bootstrapIdentity && + context.providerId === record.providerId && + sameSandboxIdentity(context.sandbox, record.sandbox) && + context.planFingerprint === record.planFingerprint && + context.profileFingerprint === record.profileFingerprint && + context.imageReference === record.imageReference + ); +} + +function assertFinalizationMatchesContext( + record: DockerManagedBootstrapFinalizationRecord, + context: DockerManagedBootstrapFinalizationContext, +): void { + const normalizedContext = normalizeFinalizationContext(context); + if ( + record.agent !== normalizedContext.agent || + !matchesFinalizationContext(record, normalizedContext) + ) { + fail("finalization record does not match supplied durable context"); + } +} + +function upgradeLegacyFinalization( + legacy: DockerManagedBootstrapFinalizationWithoutAgent, + context: DockerManagedBootstrapFinalizationContext | undefined, +): DockerManagedBootstrapFinalizationRecord { + const missingAgent = (reason: "context-mismatch" | "missing-context" = "missing-context") => + new DockerManagedBootstrapLegacyRecordRequiresAgentError({ + bootstrapIdentity: legacy.bootstrapIdentity, + recordKind: "finalization", + reason, + schemaVersion: 1, + }); + // Runtime names and image repositories are mutable descriptions, never + // agent authority. Only an exact live handle or current journal may supply + // the field omitted by schema v1. + if (!context) throw missingAgent(); + const normalizedContext = normalizeFinalizationContext(context); + if (!matchesFinalizationContext(legacy, normalizedContext)) { + throw missingAgent("context-mismatch"); + } + return Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase: legacy.phase, + bootstrapIdentity: legacy.bootstrapIdentity, + providerId: legacy.providerId, + agent: normalizedContext.agent, + sandbox: legacy.sandbox, + planFingerprint: legacy.planFingerprint, + profileFingerprint: legacy.profileFingerprint, + imageReference: legacy.imageReference, + commitReceipt: legacy.commitReceipt, + cleanupReceipt: legacy.cleanupReceipt, + }); +} + +export function normalizeDockerManagedBootstrapFinalizationRecord( + value: unknown, +): DockerManagedBootstrapFinalizationRecord { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("finalization record must be an object"); + } + const record = value as Record; + if (record.schemaVersion === 1) { + const legacy = normalizeLegacyFinalizationShape(record).record; + throw new DockerManagedBootstrapLegacyRecordRequiresAgentError({ + bootstrapIdentity: legacy.bootstrapIdentity, + recordKind: "finalization", + schemaVersion: 1, + }); + } + const expectedKeys = [ + "agent", + "bootstrapIdentity", + "cleanupReceipt", + "commitReceipt", + "imageReference", + "phase", + "planFingerprint", + "profileFingerprint", + "providerId", + "sandbox", + "schemaVersion", + ]; + if ( + !hasExactKeys(record, expectedKeys) || + record.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION + ) { + fail("finalization record schema is invalid"); + } + const normalized = normalizeFinalizationWithoutAgent(record); + return Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase: normalized.phase, + bootstrapIdentity: normalized.bootstrapIdentity, + providerId: normalized.providerId, + agent: exactAgent(record.agent), + sandbox: normalized.sandbox, + planFingerprint: normalized.planFingerprint, + profileFingerprint: normalized.profileFingerprint, + imageReference: normalized.imageReference, + commitReceipt: normalized.commitReceipt, + cleanupReceipt: normalized.cleanupReceipt, + }); +} + export function serializeDockerManagedBootstrapFinalizationRecord( record: DockerManagedBootstrapFinalizationRecord, ): string { @@ -561,9 +975,10 @@ export function serializeDockerManagedBootstrapFinalizationRecord( return serialized; } -export function parseDockerManagedBootstrapFinalizationRecord( +function parseDockerManagedBootstrapFinalizationRecordWithContext( text: string, -): DockerManagedBootstrapFinalizationRecord { + context?: DockerManagedBootstrapFinalizationContext, +): { readonly record: DockerManagedBootstrapFinalizationRecord; readonly upgradedLegacy: boolean } { if ( text.length === 0 || text.includes("\0") || @@ -577,11 +992,29 @@ export function parseDockerManagedBootstrapFinalizationRecord( } catch { fail("serialized finalization record is not valid JSON"); } + if ( + typeof parsed === "object" && + parsed !== null && + !Array.isArray(parsed) && + (parsed as Record).schemaVersion === 1 + ) { + const legacy = normalizeLegacyFinalizationShape(parsed); + if (legacy.canonical !== text) fail("serialized legacy finalization record is not canonical"); + return { record: upgradeLegacyFinalization(legacy.record, context), upgradedLegacy: true }; + } const record = normalizeDockerManagedBootstrapFinalizationRecord(parsed); if (serializeDockerManagedBootstrapFinalizationRecord(record) !== text) { fail("serialized finalization record is not canonical"); } - return record; + if (context) assertFinalizationMatchesContext(record, context); + return { record, upgradedLegacy: false }; +} + +export function parseDockerManagedBootstrapFinalizationRecord( + text: string, + context?: DockerManagedBootstrapFinalizationContext, +): DockerManagedBootstrapFinalizationRecord { + return parseDockerManagedBootstrapFinalizationRecordWithContext(text, context).record; } function assertDirectory(directory: string): void { @@ -748,6 +1181,9 @@ export function createFileDockerManagedBootstrapJournalStore( const contents = readPrivateFile(target, "journal"); if (contents === null) return null; const journal = parseDockerManagedBootstrapJournal(contents); + if (journal.bootstrapIdentity !== bootstrapIdentity) { + fail("journal bootstrap identity does not match its file name"); + } const decision = readPrivateFile(decisionPath(target), "decision"); if (decision === null) return journal; const phase = decision.endsWith("\n") ? decision.slice(0, -1) : ""; @@ -765,13 +1201,24 @@ export function createFileDockerManagedBootstrapJournalStore( }; const loadFinalization = ( bootstrapIdentity: string, + context?: DockerManagedBootstrapFinalizationContext, ): DockerManagedBootstrapFinalizationRecord | null => { assertDirectory(directory); - const contents = readPrivateFile( - finalizationPath(journalPath(directory, bootstrapIdentity)), - "finalization", - ); - return contents === null ? null : parseDockerManagedBootstrapFinalizationRecord(contents); + const target = finalizationPath(journalPath(directory, bootstrapIdentity)); + const contents = readPrivateFile(target, "finalization"); + if (contents === null) return null; + const parsed = parseDockerManagedBootstrapFinalizationRecordWithContext(contents, context); + if (parsed.record.bootstrapIdentity !== bootstrapIdentity) { + fail("finalization bootstrap identity does not match its file name"); + } + if (parsed.upgradedLegacy) { + const serialized = serializeDockerManagedBootstrapFinalizationRecord(parsed.record); + atomicWrite(directory, target, serialized, false); + if (readPrivateFile(target, "finalization") !== serialized) { + fail("upgraded finalization record was not durably re-readable"); + } + } + return parsed.record; }; return Object.freeze({ create(journal: DockerManagedBootstrapJournal) { @@ -791,7 +1238,7 @@ export function createFileDockerManagedBootstrapJournalStore( atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(normalized), true); }, load, - listUnfinished() { + listUnfinishedIdentities() { assertDirectory(directory); const identities: string[] = []; for (const name of fs.readdirSync(directory)) { @@ -808,13 +1255,7 @@ export function createFileDockerManagedBootstrapJournalStore( } fail(`journal directory contains an unsupported entry: ${name}`); } - return Object.freeze( - identities.sort().map((identity) => { - const journal = load(identity); - if (!journal) fail(`enumerated journal ${identity} disappeared`); - return journal; - }), - ); + return Object.freeze(identities.sort()); }, transition( bootstrapIdentity: string, @@ -890,23 +1331,38 @@ export function createFileDockerManagedBootstrapJournalStore( fs.unlinkSync(target); fsyncDirectory(directory); }, - recordFinalization(record: DockerManagedBootstrapFinalizationRecord) { + recordFinalization( + record: DockerManagedBootstrapFinalizationRecord, + context?: DockerManagedBootstrapFinalizationContext, + ) { const normalized = normalizeDockerManagedBootstrapFinalizationRecord(record); + if (context) assertFinalizationMatchesContext(normalized, context); assertDirectory(directory); const target = finalizationPath(journalPath(directory, normalized.bootstrapIdentity)); const serialized = serializeDockerManagedBootstrapFinalizationRecord(normalized); - const existing = readPrivateFile(target, "finalization"); + const existing = loadFinalization(normalized.bootstrapIdentity, context); if (existing !== null) { - if (existing !== serialized) + if (serializeDockerManagedBootstrapFinalizationRecord(existing) !== serialized) { fail("finalization record changed for this bootstrap identity"); + } return; } try { atomicWrite(directory, target, serialized, true); } catch (error) { - if (readPrivateFile(target, "finalization") !== serialized) throw error; + const recovered = loadFinalization(normalized.bootstrapIdentity, context); + if ( + !recovered || + serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized + ) { + throw error; + } } - if (readPrivateFile(target, "finalization") !== serialized) { + const persisted = loadFinalization(normalized.bootstrapIdentity, context); + if ( + !persisted || + serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized + ) { fail("finalization record was not durably re-readable"); } }, diff --git a/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts b/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts index fc9f22b8d1a..0bd01ccc9ec 100644 --- a/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; +import { recoverManagedBootstrapTransactions } from "./adapter"; import { createDockerManagedBootstrapAdapter } from "./docker"; import type { DockerManagedBootstrapJournalStore } from "./docker-journal"; import { @@ -10,6 +11,9 @@ import { type DockerFixtureOptions, durablePreparation, fixture, + IDENTITY, + NEW_ID, + OLD_ID, } from "./docker-test-fixture"; async function prepareTransaction( @@ -33,6 +37,16 @@ async function prepareTransaction( }; } +function expectEventBefore(events: readonly string[], before: string, after: string): void { + expect(events).toContain(before); + expect(events).toContain(after); + expect(events.indexOf(before)).toBeLessThan(events.indexOf(after)); +} + +function dockerMutationEvents(events: readonly string[]): readonly string[] { + return events.filter((event) => /^(?:create:|rename:|rm:|start:|stop:)/u.test(event)); +} + describe("Docker managed bootstrap restart recovery", () => { it.each([ { @@ -55,7 +69,10 @@ describe("Docker managed bootstrap restart recovery", () => { readonly label: string; readonly options: DockerFixtureOptions; readonly phase: "cutover" | "staged"; - }[])("reconciles a process restart from the durable $label phase", async ({ options, phase }) => { + }[])("retains owner cleanup after a process restart from the durable $label phase", async ({ + options, + phase, + }) => { const fake = fixture(options); const transaction = await prepareTransaction(fake); @@ -70,17 +87,59 @@ describe("Docker managed bootstrap restart recovery", () => { expect(fake.journal?.phase).toBe(phase); const restarted = createDockerManagedBootstrapAdapter(fake.deps); - await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject([ - { sourcePhase: phase, outcome: "rolled-back" }, - ]); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject({ + receipts: [], + failures: [ + { + bootstrapIdentity: IDENTITY, + sourcePhase: "owner-cleanup-required", + code: "owner-cleanup-required", + retryable: true, + }, + ], + }); + expect(fake.journal?.phase).toBe("owner-cleanup-required"); + expect(fake.finalization).toBeNull(); + expect(fake.replacement).toBeNull(); + expect(fake.original?.State?.Running).toBe(false); + expectEventBefore(fake.events, `rm:${NEW_ID}`, "journal:owner-cleanup-required"); + + const ownerTransitions = fake.events.filter( + (event) => event === "journal:owner-cleanup-required", + ).length; + const mutationsAfterFirstRecovery = dockerMutationEvents(fake.events); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject({ + receipts: [], + failures: [{ code: "owner-cleanup-required" }], + }); + expect(fake.events.filter((event) => event === "journal:owner-cleanup-required")).toHaveLength( + ownerTransitions, + ); + expect(fake.journal?.phase).toBe("owner-cleanup-required"); + expect(fake.finalization).toBeNull(); + expect(dockerMutationEvents(fake.events)).toEqual(mutationsAfterFirstRecovery); + + fake.removeOriginalExternally(); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject({ + receipts: [ + { + bootstrapIdentity: IDENTITY, + sourcePhase: "owner-cleanup-required", + outcome: "rolled-back", + }, + ], + failures: [], + }); expect(fake.journal).toBeNull(); expect(fake.finalization?.phase).toBe("rolled-back"); expect(fake.replacement).toBeNull(); - expect(fake.original?.State?.Running).toBe(true); - await expect(restarted.recoverUnfinishedTransactions()).resolves.toEqual([]); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toEqual({ + receipts: [], + failures: [], + }); }); - it("finishes rollback-authorized recovery after shared-state rollback is interrupted", async () => { + it("publishes owner cleanup only after shared rollback, replacement cleanup, and restoration", async () => { const fake = fixture({ agent: "openclaw", journalTransitionFailures: { @@ -111,19 +170,77 @@ describe("Docker managed bootstrap restart recovery", () => { expect(fake.sharedState).toBe("pending"); const restarted = createDockerManagedBootstrapAdapter(fake.deps); - await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject([ - { sourcePhase: "rollback-authorized", outcome: "rolled-back" }, - ]); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject({ + receipts: [], + failures: [{ sourcePhase: "owner-cleanup-required", code: "owner-cleanup-required" }], + }); expect(fake.sharedState).toBe("none"); expect(fake.replacement).toBeNull(); - expect(fake.original?.State?.Running).toBe(true); + expect(fake.original?.State?.Running).toBe(false); + expect(fake.journal?.phase).toBe("owner-cleanup-required"); + expect(fake.finalization).toBeNull(); + expectEventBefore(fake.events, "shared:rollback", `rm:${NEW_ID}`); + expectEventBefore(fake.events, `rm:${NEW_ID}`, "journal:owner-cleanup-required"); + + const mutationsAfterFirstRecovery = dockerMutationEvents(fake.events); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject({ + receipts: [], + failures: [{ sourcePhase: "owner-cleanup-required", code: "owner-cleanup-required" }], + }); + expect(dockerMutationEvents(fake.events)).toEqual(mutationsAfterFirstRecovery); + + fake.removeOriginalExternally(); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject({ + receipts: [{ outcome: "rolled-back", sourcePhase: "owner-cleanup-required" }], + failures: [], + }); }); - it("compacts a terminal commit journal after another restart interruption", async () => { + it("retains owner authority while exact runtime presence is unknown", async () => { + const fake = fixture({ + journalCreateFailures: [new Error("injected crash after durable staged fence")], + }); + const transaction = await prepareTransaction(fake); + await expect( + transaction.adapter.activateBootstrapReplacement({ + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + }), + ).rejects.toThrow("crash after durable staged fence"); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await restarted.recoverUnfinishedTransactions(); + expect(fake.journal?.phase).toBe("owner-cleanup-required"); + fake.setDockerInspectUnknown(OLD_ID, true); + + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject({ + receipts: [], + failures: [ + { + sourcePhase: "owner-cleanup-required", + code: "commit-state-indeterminate", + retryable: true, + }, + ], + }); + expect(fake.journal?.phase).toBe("owner-cleanup-required"); + expect(fake.finalization).toBeNull(); + + fake.setDockerInspectUnknown(OLD_ID, false); + fake.removeOriginalExternally(); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject({ + receipts: [{ outcome: "rolled-back" }], + failures: [], + }); + }); + + it("retains durable commit authority when image receipt retirement fails", async () => { const fake = fixture({ agent: "langchain-deepagents-code", dockerRemoveFailures: [new Error("injected crash before exact Docker removal")], - journalRemoveFailures: [new Error("injected crash before terminal journal removal")], + sharedReceiptClearFailures: [new Error("injected image receipt cleanup failure")], sharedState: "pending", }); const transaction = await prepareTransaction(fake, "langchain-deepagents-code"); @@ -155,47 +272,159 @@ describe("Docker managed bootstrap restart recovery", () => { expect(fake.sharedState).toBe("committed"); const restarted = createDockerManagedBootstrapAdapter(fake.deps); - await expect(restarted.recoverUnfinishedTransactions()).rejects.toThrow( - "crash before terminal journal removal", - ); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject({ + receipts: [], + failures: [ + { + sourcePhase: "shared-state-committed", + code: "durable-cleanup-pending", + retryable: true, + }, + ], + }); expect(fake.journal?.phase).toBe("shared-state-committed"); + expect(fake.finalization).toBeNull(); + expect(fake.sharedState).toBe("committed"); + + const mutationsAfterFailedRetirement = dockerMutationEvents(fake.events); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject({ + receipts: [{ outcome: "committed", sourcePhase: "shared-state-committed" }], + failures: [], + }); + expect(fake.journal).toBeNull(); expect(fake.finalization?.phase).toBe("committed"); + expect(fake.sharedState).toBe("none"); + expect(dockerMutationEvents(fake.events)).toEqual(mutationsAfterFailedRetirement); + }); - const journalStore = fake.deps.journalStore; - if (!journalStore) throw new Error("fixture journal store is missing"); - const reorderedFinalizationStore = { - ...journalStore, - loadFinalization(bootstrapIdentity: string) { - const record = journalStore.loadFinalization(bootstrapIdentity); - const receipt = record?.commitReceipt; - if (!record || !receipt) return record; - return { - ...record, - commitReceipt: { - completedAt: receipt.completedAt, - transactionPending: receipt.transactionPending, - bootstrapIdentity: receipt.bootstrapIdentity, - profileFingerprint: receipt.profileFingerprint, - replacementSpecHash: receipt.replacementSpecHash, - originalSpecHash: receipt.originalSpecHash, - runtimeImageContentId: receipt.runtimeImageContentId, - image: receipt.image, - runtimeId: receipt.runtimeId, - sandbox: receipt.sandbox, - schemaVersion: receipt.schemaVersion, - } satisfies typeof receipt, - }; - }, - } satisfies DockerManagedBootstrapJournalStore; - const resumed = createDockerManagedBootstrapAdapter({ - ...fake.deps, - journalStore: reorderedFinalizationStore, + it("compacts a terminal commit journal after another restart interruption", async () => { + const fake = fixture({ + agent: "langchain-deepagents-code", + dockerRemoveFailures: [new Error("injected crash before exact Docker removal")], + journalRemoveFailures: [new Error("injected crash before terminal journal removal")], + sharedState: "pending", + }); + const transaction = await prepareTransaction(fake, "langchain-deepagents-code"); + const replacement = await transaction.adapter.activateBootstrapReplacement({ + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + }); + const completion = await transaction.adapter.awaitBootstrap({ + handle: transaction.handle, + snapshot: transaction.snapshot, + replacement, + timeoutSecs: 1, + }); + + await expect( + transaction.adapter.finalizeBootstrap({ + outcome: "commit", + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + replacement, + completion, + }), + ).rejects.toThrow("crash before exact Docker removal"); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject({ + receipts: [], + failures: [ + { + sourcePhase: "shared-state-committed", + code: "provider-recovery-failed", + detail: "injected crash before terminal journal removal", + }, + ], + }); + expect(fake.journal?.phase).toBe("shared-state-committed"); + expect(fake.finalization?.phase).toBe("committed"); + + const resumed = createDockerManagedBootstrapAdapter(fake.deps); + await expect(resumed.recoverUnfinishedTransactions()).resolves.toMatchObject({ + receipts: [{ sourcePhase: "shared-state-committed", outcome: "committed" }], + failures: [], }); - await expect(resumed.recoverUnfinishedTransactions()).resolves.toMatchObject([ - { sourcePhase: "shared-state-committed", outcome: "committed" }, - ]); expect(fake.journal).toBeNull(); expect(fake.sharedState).toBe("none"); expect(fake.replacement?.State?.Running).toBe(true); }); + + it("isolates identity-first failures and returns bounded lossless mixed evidence", async () => { + const fake = fixture({ + dockerRemoveFailures: [new Error("injected crash before exact Docker removal")], + journalRemoveFailures: [new Error("injected crash before terminal journal removal")], + sharedState: "pending", + }); + const transaction = await prepareTransaction(fake); + const replacement = await transaction.adapter.activateBootstrapReplacement({ + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + }); + const completion = await transaction.adapter.awaitBootstrap({ + handle: transaction.handle, + snapshot: transaction.snapshot, + replacement, + timeoutSecs: 1, + }); + await expect( + transaction.adapter.finalizeBootstrap({ + outcome: "commit", + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + replacement, + completion, + }), + ).rejects.toThrow("crash before exact Docker removal"); + await createDockerManagedBootstrapAdapter(fake.deps).recoverUnfinishedTransactions(); + expect(fake.finalization?.phase).toBe("committed"); + expect(fake.journal).not.toBeNull(); + + const badIdentity = "0".repeat(64); + const delegate = fake.deps.journalStore as DockerManagedBootstrapJournalStore; + const failUnreadableRecord = (): never => { + throw new Error(`${"💥".repeat(3000)}\0tail`); + }; + const mixedStore: DockerManagedBootstrapJournalStore = { + ...delegate, + listUnfinishedIdentities: () => [badIdentity, IDENTITY], + load(bootstrapIdentity) { + return bootstrapIdentity === badIdentity + ? failUnreadableRecord() + : delegate.load(bootstrapIdentity); + }, + }; + const adapter = createDockerManagedBootstrapAdapter({ ...fake.deps, journalStore: mixedStore }); + + const report = await recoverManagedBootstrapTransactions(adapter); + + expect(report.receipts).toMatchObject([ + { bootstrapIdentity: IDENTITY, sourcePhase: "shared-state-committed", outcome: "committed" }, + ]); + expect(report.failures).toMatchObject([ + { + bootstrapIdentity: badIdentity, + providerId: "docker", + sourcePhase: null, + sandbox: null, + code: "provider-recovery-failed", + }, + ]); + expect(report.failures[0]?.detail).not.toContain("\0"); + expect(Buffer.byteLength(report.failures[0]?.detail ?? "", "utf8")).toBeLessThanOrEqual(8192); + expect(Object.isFrozen(report)).toBe(true); + expect(Object.isFrozen(report.receipts)).toBe(true); + expect(Object.isFrozen(report.failures)).toBe(true); + expect(report.receipts.every(Object.isFrozen)).toBe(true); + expect(report.failures.every(Object.isFrozen)).toBe(true); + expect(fake.journal).toBeNull(); + }); }); diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index 42f0fc3ae1d..5464e6ad06c 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -26,6 +26,8 @@ import { DockerManagedBootstrapJournalAcknowledgementLostError, type DockerManagedBootstrapJournalPhase, type DockerManagedBootstrapJournalStore, + sameDockerManagedBootstrapReceipt, + serializeDockerManagedBootstrapFinalizationRecord, } from "./docker-journal"; import { normalizeDockerManagedBootstrapLaunchSpec } from "./docker-spec"; import { @@ -58,6 +60,8 @@ export type DockerFixtureAcknowledgement = | "container:stop" | "journal:create" | "journal:cutover" + | "journal:completion" + | "journal:owner-cleanup-required" | "journal:remove" | "journal:rollback-authorized" | "journal:staged" @@ -66,6 +70,7 @@ export type DockerFixtureAcknowledgement = export type DockerFixtureOptions = { readonly agent?: ManagedStartupAgent; readonly dockerRemoveFailures?: readonly Error[]; + readonly dockerInspectUnknownIds?: readonly string[]; readonly dockerStartResults?: Readonly>; readonly journalCreateFailures?: readonly Error[]; readonly journalRemoveFailures?: readonly Error[]; @@ -76,6 +81,7 @@ export type DockerFixtureOptions = { readonly ownerId?: string; readonly sharedState?: "committed" | "none" | "pending"; readonly sharedStateCommitResult?: FixtureCommandResult; + readonly sharedReceiptClearFailures?: readonly Error[]; }; function agentInputs(agent: ManagedStartupAgent = "hermes") { @@ -220,6 +226,8 @@ export function fixture(options: DockerFixtureOptions = {}) { const dockerRemoveFailures = [...(options.dockerRemoveFailures ?? [])]; const journalCreateFailures = [...(options.journalCreateFailures ?? [])]; const journalRemoveFailures = [...(options.journalRemoveFailures ?? [])]; + const sharedReceiptClearFailures = [...(options.sharedReceiptClearFailures ?? [])]; + const dockerInspectUnknownIds = new Set(options.dockerInspectUnknownIds ?? []); const lostAcknowledgements = new Set(options.lostAcknowledgements ?? []); const losesAcknowledgement = (operation: DockerFixtureAcknowledgement) => lostAcknowledgements.has(operation); @@ -243,7 +251,7 @@ export function fixture(options: DockerFixtureOptions = {}) { } }, load: () => copyJournal(), - listUnfinished: () => (journal ? [structuredClone(journal)] : []), + listUnfinishedIdentities: () => (journal ? [journal.bootstrapIdentity] : []), transition(_identity, expected, next) { const current = journal !== null && journal.phase === expected @@ -266,12 +274,17 @@ export function fixture(options: DockerFixtureOptions = {}) { } if ( journal.commitReceipt !== null && - JSON.stringify(journal.commitReceipt) !== JSON.stringify(receipt) + !sameDockerManagedBootstrapReceipt("completion", journal.commitReceipt, receipt) ) { throw new Error("completion changed"); } journal = { ...journal, commitReceipt: structuredClone(receipt) }; events.push("journal:completion"); + if (losesAcknowledgement("journal:completion")) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal completion acknowledgement", + ); + } return structuredClone(journal); }, remove(_identity, expected) { @@ -295,7 +308,11 @@ export function fixture(options: DockerFixtureOptions = {}) { } }, recordFinalization(value) { - if (finalization && JSON.stringify(finalization) !== JSON.stringify(value)) { + if ( + finalization && + serializeDockerManagedBootstrapFinalizationRecord(finalization) !== + serializeDockerManagedBootstrapFinalizationRecord(value) + ) { throw new Error("finalization changed"); } finalization = structuredClone(value); @@ -355,6 +372,9 @@ export function fixture(options: DockerFixtureOptions = {}) { return ok(original ? OLD_ID : ""); case "inspect": { const id = String(args[3] ?? ""); + if (dockerInspectUnknownIds.has(id)) { + return { status: 1, stderr: `injected unknown inspect state for ${id}` }; + } try { inspect(id); return ok(`[{"Id":"${id}"}]`); @@ -419,6 +439,10 @@ export function fixture(options: DockerFixtureOptions = {}) { return result; } case args.includes("--clear-shared-state-commit-receipt"): + { + const injectedFailure = sharedReceiptClearFailures.shift(); + if (injectedFailure) throw injectedFailure; + } sharedState = "none"; events.push("shared:clear"); return ok(); @@ -515,6 +539,14 @@ export function fixture(options: DockerFixtureOptions = {}) { get sharedState() { return sharedState; }, + removeOriginalExternally() { + original = null as unknown as DockerContainerInspect; + events.push(`external-rm:${OLD_ID}`); + }, + setDockerInspectUnknown(runtimeId: string, indeterminate: boolean) { + if (indeterminate) dockerInspectUnknownIds.add(runtimeId); + else dockerInspectUnknownIds.delete(runtimeId); + }, }; } diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 842c14f654e..cf1e1e63fdd 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -24,6 +24,12 @@ import { SUPPORTED_AGENTS, } from "./docker-test-fixture"; +function expectEventBefore(events: readonly string[], before: string, after: string): void { + expect(events).toContain(before); + expect(events).toContain(after); + expect(events.indexOf(before)).toBeLessThan(events.indexOf(after)); +} + describe("Docker managed bootstrap adapter", () => { it("publishes durable commit authority before deleting the rollback backup after lost acknowledgements", async () => { const fake = fixture({ @@ -35,6 +41,7 @@ describe("Docker managed bootstrap adapter", () => { "container:stop", "journal:create", "journal:cutover", + "journal:completion", "journal:remove", "journal:shared-state-committed", ], @@ -71,12 +78,8 @@ describe("Docker managed bootstrap adapter", () => { durablePreparation: reorderedDurable, }); const order = fake.events; - expect(order).toContain("authority:recorded"); - expect(order).toContain("journal:staged"); - expect(order.indexOf("journal:staged")).toBeGreaterThan(order.indexOf("authority:recorded")); - expect(order).toContain("journal:cutover"); - expect(order).toContain(`stop:${OLD_ID}`); - expect(order.indexOf("journal:cutover")).toBeLessThan(order.indexOf(`stop:${OLD_ID}`)); + expectEventBefore(order, "authority:recorded", "journal:staged"); + expectEventBefore(order, "journal:cutover", `stop:${OLD_ID}`); expect(fake.journal).toMatchObject({ phase: "cutover", originalRuntimeId: OLD_ID, @@ -124,14 +127,8 @@ describe("Docker managed bootstrap adapter", () => { completion: reorderedCommitReceipt, }); expect(finalized).toMatchObject({ outcome: "committed" }); - expect(fake.events).toContain("journal:shared-state-committed"); - expect(fake.events).toContain(`rm:${OLD_ID}`); - expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( - fake.events.indexOf(`rm:${OLD_ID}`), - ); - expect(fake.events.indexOf("finalization:committed")).toBeLessThan( - fake.events.indexOf("journal:removed"), - ); + expectEventBefore(fake.events, "journal:shared-state-committed", `rm:${OLD_ID}`); + expectEventBefore(fake.events, "finalization:committed", "journal:removed"); expect(fake.journal).toBeNull(); expect(fake.finalization).toMatchObject({ phase: "committed", commitReceipt }); expect(fake.sharedState).toBe("none"); @@ -258,15 +255,10 @@ describe("Docker managed bootstrap adapter", () => { completion: null, }), ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); - expect(fake.events).toContain("journal:rollback-authorized"); - expect(fake.events).toContain(`rm:${NEW_ID}`); - expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( - fake.events.indexOf(`rm:${NEW_ID}`), - ); - expect(fake.events.indexOf("finalization:rolled-back")).toBeLessThan( - fake.events.indexOf("journal:removed"), - ); - expect(fake.journal).toBeNull(); + expectEventBefore(fake.events, "journal:rollback-authorized", `rm:${NEW_ID}`); + expectEventBefore(fake.events, `rm:${NEW_ID}`, "journal:owner-cleanup-required"); + expect(fake.finalization).toBeNull(); + expect(fake.journal?.phase).toBe("owner-cleanup-required"); expect(fake.replacement).toBeNull(); expect(fake.original).not.toBeNull(); expect(fake.original?.Name).toBe("/openshell-alpha"); @@ -308,12 +300,10 @@ describe("Docker managed bootstrap adapter", () => { completion: null, }), ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); - expect(fake.events).toContain("journal:rollback-authorized"); - expect(fake.events).toContain(`rm:${NEW_ID}`); - expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( - fake.events.indexOf(`rm:${NEW_ID}`), - ); - expect(fake.journal).toBeNull(); + expectEventBefore(fake.events, "journal:rollback-authorized", `rm:${NEW_ID}`); + expectEventBefore(fake.events, `rm:${NEW_ID}`, "journal:owner-cleanup-required"); + expect(fake.finalization).toBeNull(); + expect(fake.journal?.phase).toBe("owner-cleanup-required"); }); it("fences rollback when image-owned shared state is already committed", async () => { @@ -417,8 +407,12 @@ describe("Docker managed bootstrap adapter", () => { completion: null, }), ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); - expect(fake.journal).toBeNull(); + expect(fake.journal?.phase).toBe("owner-cleanup-required"); + expect(fake.finalization).toBeNull(); expect(fake.replacement).toBeNull(); + expect(fake.original?.State?.Running).toBe(false); + expectEventBefore(fake.events, "shared:rollback", `rm:${NEW_ID}`); + expectEventBefore(fake.events, `rm:${NEW_ID}`, "journal:owner-cleanup-required"); expect( vi.mocked(fake.deps.dockerRun!).mock.calls.some(([args]) => { const agentIndex = args.indexOf("--agent"); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index 89c3c83b76a..86cb093d92b 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -58,7 +58,9 @@ import { type ManagedBootstrapObservedSnapshot, ManagedBootstrapOwnerCleanupRequiredError, type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapRecoveryFailure, type ManagedBootstrapRecoveryReceipt, + type ManagedBootstrapRecoveryReport, type ManagedBootstrapReplacementHandle, type ManagedBootstrapReplacementOptions, type ManagedBootstrapSandboxIdentity, @@ -72,6 +74,7 @@ import { type DockerManagedBootstrapJournal, DockerManagedBootstrapJournalAcknowledgementLostError, type DockerManagedBootstrapJournalStore, + DockerManagedBootstrapLegacyRecordRequiresAgentError, parseDockerManagedBootstrapJournal, sameDockerManagedBootstrapReceipt, serializeDockerManagedBootstrapFinalizationRecord, @@ -103,9 +106,49 @@ const REQUEST_TEMP_PREFIX = "nemoclaw-managed-bootstrap-request"; const COMPLETION_TEMP_PREFIX = "nemoclaw-managed-bootstrap-completion"; const COMPLETION_MAX_BYTES = 4096; const DOCKER_DRIVER_ID = "docker"; +const MAX_RECOVERY_FAILURE_DETAIL_BYTES = 8 * 1024; export const MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE = "/usr/local/bin/nemoclaw-managed-bootstrap"; +function boundedRecoveryFailureDetail(error: unknown): string { + const raw = (error instanceof Error ? error.message : String(error)).replaceAll("\0", "�"); + const detail = raw.length > 0 ? raw : "Docker recovery failed without diagnostic detail"; + const bytes = Buffer.from(detail, "utf8"); + if (bytes.length <= MAX_RECOVERY_FAILURE_DETAIL_BYTES) return detail; + let bounded = bytes.subarray(0, MAX_RECOVERY_FAILURE_DETAIL_BYTES).toString("utf8"); + while (Buffer.byteLength(bounded, "utf8") > MAX_RECOVERY_FAILURE_DETAIL_BYTES) { + bounded = [...bounded].slice(0, -1).join(""); + } + return bounded; +} + +function dockerManagedBootstrapRecoveryFailure( + bootstrapIdentity: string, + journal: DockerBootstrapTransaction | null, + error: unknown, +): ManagedBootstrapRecoveryFailure { + const classified = + error instanceof ManagedBootstrapOwnerCleanupRequiredError + ? { code: "owner-cleanup-required", retryable: true } + : error instanceof ManagedBootstrapDurableCommitCleanupPendingError + ? { code: "durable-cleanup-pending", retryable: true } + : error instanceof ManagedBootstrapCommitStateIndeterminateError + ? { code: "commit-state-indeterminate", retryable: true } + : error instanceof DockerManagedBootstrapLegacyRecordRequiresAgentError + ? { code: "legacy-agent-required", retryable: false } + : { code: "provider-recovery-failed", retryable: true }; + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: journal?.providerId ?? DOCKER_DRIVER_ID, + sourcePhase: journal?.phase ?? null, + sandbox: journal?.sandbox ?? null, + bootstrapIdentity, + code: classified.code, + retryable: classified.retryable, + detail: boundedRecoveryFailureDetail(error), + }); +} + type DockerCommandResult = { readonly status?: number | null; readonly stdout?: string | Buffer | null; @@ -1272,24 +1315,26 @@ function retainOwnedWorkloadForOwnerCleanup( `Managed bootstrap owner cleanup refused retention after exact runtime ${runtimeId} ownership changed for sandbox ${sandbox.sandboxId}.`, ); } - let stopped: DockerCommandResult; - try { - stopped = deps.dockerStop(runtimeId, { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, - }); - } catch (error) { - throw new Error( - `Managed bootstrap could not quiesce retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ - error instanceof Error ? error.message : String(error) - }`, + if (!isExplicitlyStopped(inspect)) { + let stopped: DockerCommandResult; + try { + stopped = deps.dockerStop(runtimeId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + } catch (error) { + throw new Error( + `Managed bootstrap could not quiesce retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + assertZero( + stopped, + `Managed bootstrap could not quiesce retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}`, ); } - assertZero( - stopped, - `Managed bootstrap could not quiesce retained sandbox ${sandbox.sandboxId} exact runtime ${runtimeId}`, - ); let retained: DockerContainerInspect; try { retained = inspectExact(runtimeId, deps); @@ -1477,7 +1522,7 @@ function createDockerBootstrapJournalDurably( function transitionDockerBootstrapJournalDurably( journal: DockerBootstrapTransaction, - next: "cutover" | "rollback-authorized" | "shared-state-committed", + next: "cutover" | "rollback-authorized" | "owner-cleanup-required" | "shared-state-committed", deps: ResolvedDeps, ): DockerBootstrapTransaction { try { @@ -1747,20 +1792,35 @@ export function createDockerManagedBootstrapAdapter( dependencies: DockerManagedBootstrapDeps = {}, ): DockerManagedBootstrapAdapter { const deps = resolveDeps(dependencies); + const finalizationContext = (handle: ManagedBootstrapHeldWorkloadHandle) => + Object.freeze({ + bootstrapIdentity: handle.bootstrapIdentity, + providerId: handle.sandbox.driverId, + agent: handle.plan.profile.agent, + sandbox: handle.sandbox, + planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + handle.plan.image.repository, + handle.plan.image.manifestDigest, + ), + }); const finalizationRecord = ( handle: ManagedBootstrapHeldWorkloadHandle, ): DockerManagedBootstrapFinalizationRecord | null => { - const record = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + const context = finalizationContext(handle); + const record = deps.journalStore.loadFinalization(handle.bootstrapIdentity, context); if (!record) return null; if ( - record.providerId !== handle.sandbox.driverId || - record.sandbox.sandboxName !== handle.sandbox.sandboxName || - record.sandbox.sandboxId !== handle.sandbox.sandboxId || - record.sandbox.driverId !== handle.sandbox.driverId || - record.planFingerprint !== createManagedBootstrapPlanFingerprint(handle.plan) || - record.profileFingerprint !== handle.plan.profile.fingerprint || - record.imageReference !== - expectedImageReference(handle.plan.image.repository, handle.plan.image.manifestDigest) + record.bootstrapIdentity !== context.bootstrapIdentity || + record.providerId !== context.providerId || + record.agent !== context.agent || + record.sandbox.sandboxName !== context.sandbox.sandboxName || + record.sandbox.sandboxId !== context.sandbox.sandboxId || + record.sandbox.driverId !== context.sandbox.driverId || + record.planFingerprint !== context.planFingerprint || + record.profileFingerprint !== context.profileFingerprint || + record.imageReference !== context.imageReference ) { throw new Error("Managed bootstrap finalization record does not match its durable identity."); } @@ -1772,27 +1832,25 @@ export function createDockerManagedBootstrapAdapter( commitReceipt: ManagedBootstrapCompletionReceipt | null, cleanupReceipt: ManagedBootstrapFinalizationReceipt, ): ManagedBootstrapFinalizationReceipt => { + const context = finalizationContext(handle); const record = Object.freeze({ schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, phase, - bootstrapIdentity: handle.bootstrapIdentity, - providerId: handle.sandbox.driverId, - agent: handle.plan.profile.agent, - sandbox: handle.sandbox, - planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), - profileFingerprint: handle.plan.profile.fingerprint, - imageReference: expectedImageReference( - handle.plan.image.repository, - handle.plan.image.manifestDigest, - ), + bootstrapIdentity: context.bootstrapIdentity, + providerId: context.providerId, + agent: context.agent, + sandbox: context.sandbox, + planFingerprint: context.planFingerprint, + profileFingerprint: context.profileFingerprint, + imageReference: context.imageReference, commitReceipt, cleanupReceipt, } satisfies DockerManagedBootstrapFinalizationRecord); const serialized = serializeDockerManagedBootstrapFinalizationRecord(record); try { - deps.journalStore.recordFinalization(record); + deps.journalStore.recordFinalization(record, context); } catch (error) { - const recovered = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + const recovered = deps.journalStore.loadFinalization(handle.bootstrapIdentity, context); if ( !recovered || serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized @@ -1800,7 +1858,7 @@ export function createDockerManagedBootstrapAdapter( throw error; } } - const persisted = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + const persisted = deps.journalStore.loadFinalization(handle.bootstrapIdentity, context); if (!persisted || serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized) { throw new Error("Managed bootstrap finalization receipt was not durably re-readable."); } @@ -1823,19 +1881,71 @@ export function createDockerManagedBootstrapAdapter( } satisfies ManagedBootstrapFinalizationReceipt); return persistFinalization(handle, "rolled-back", null, receipt); }; + const requireExactOwnerCleanup = (journal: DockerBootstrapTransaction): void => { + const presence = probeExactDockerContainerAbsence(journal.originalRuntimeId, deps); + if (presence === "unknown") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "exact owner-cleanup runtime presence is unknown", + }); + } + if (presence === "absent") return; + const original = inspectExact(journal.originalRuntimeId, deps); + assertTransactionOriginal(journal, original); + if ( + dockerContainerName(original) !== journal.originalName || + normalizeDockerManagedBootstrapLaunchSpec(original).hash !== journal.originalSpecHash || + (journal.phase === "owner-cleanup-required" && !isExplicitlyStopped(original)) || + (journal.phase !== "owner-cleanup-required" && + !isExplicitlyStopped(original) && + !isStableRunning(original)) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "owner-cleanup runtime does not match its exact restored durable authority", + }); + } + try { + retainOwnedWorkloadForOwnerCleanup(journal.sandbox, deps, journal.originalRuntimeId); + } catch (error) { + if (!(error instanceof ManagedBootstrapOwnerCleanupRequiredError)) throw error; + if (error.runtimeId !== journal.originalRuntimeId) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "owner cleanup retained a runtime other than the durable original", + }); + } + if (journal.phase !== "owner-cleanup-required") { + if (journal.phase !== "staged" && journal.phase !== "rollback-authorized") { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: `owner cleanup cannot be retained from durable phase ${journal.phase}`, + }); + } + const current = deps.journalStore.load(journal.bootstrapIdentity); + if (!current || !sameDockerBootstrapJournal(current, journal)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: journal.bootstrapIdentity, + runtimeId: journal.originalRuntimeId, + detail: "durable authority changed before owner cleanup was retained", + }); + } + transitionDockerBootstrapJournalDurably(journal, "owner-cleanup-required", deps); + } + throw error; + } + }; const completeRollbackTransaction = ( handle: ManagedBootstrapHeldWorkloadHandle, journal: DockerBootstrapTransaction, ): ManagedBootstrapFinalizationReceipt => { - let ownerCleanupFailure: { readonly error: unknown } | null = null; - try { - retainOwnedWorkloadForOwnerCleanup(handle.sandbox, deps, journal.originalRuntimeId); - } catch (error) { - ownerCleanupFailure = { error }; - } + requireExactOwnerCleanup(journal); const finalization = completedRollback(handle, false); removeDockerBootstrapJournalDurably(journal, deps); - if (ownerCleanupFailure) throw ownerCleanupFailure.error; return finalization; }; const completedCommit = ( @@ -1893,9 +2003,9 @@ export function createDockerManagedBootstrapAdapter( } satisfies DockerManagedBootstrapFinalizationRecord); const serialized = serializeDockerManagedBootstrapFinalizationRecord(record); try { - deps.journalStore.recordFinalization(record); + deps.journalStore.recordFinalization(record, journal); } catch (error) { - const recovered = deps.journalStore.loadFinalization(journal.bootstrapIdentity); + const recovered = deps.journalStore.loadFinalization(journal.bootstrapIdentity, journal); if ( !recovered || serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized @@ -1903,7 +2013,7 @@ export function createDockerManagedBootstrapAdapter( throw error; } } - const persisted = deps.journalStore.loadFinalization(journal.bootstrapIdentity); + const persisted = deps.journalStore.loadFinalization(journal.bootstrapIdentity, journal); if (!persisted || serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized) { throw new Error("Managed bootstrap recovered finalization was not durably re-readable."); } @@ -1927,7 +2037,7 @@ export function createDockerManagedBootstrapAdapter( journal: DockerBootstrapTransaction, sourcePhase: DockerBootstrapTransaction["phase"], ): ManagedBootstrapRecoveryReceipt | null => { - const finalization = deps.journalStore.loadFinalization(journal.bootstrapIdentity); + const finalization = deps.journalStore.loadFinalization(journal.bootstrapIdentity, journal); if (!finalization) return null; const phaseMatches = (finalization.phase === "committed" && @@ -1940,7 +2050,9 @@ export function createDockerManagedBootstrapAdapter( journal.commitReceipt, )) || (finalization.phase === "rolled-back" && - (journal.phase === "staged" || journal.phase === "rollback-authorized") && + (journal.phase === "staged" || + journal.phase === "rollback-authorized" || + journal.phase === "owner-cleanup-required") && finalization.commitReceipt === null); if ( !phaseMatches || @@ -1963,6 +2075,7 @@ export function createDockerManagedBootstrapAdapter( detail: "terminal finalization does not match its retained durable journal", }); } + if (finalization.phase === "rolled-back") requireExactOwnerCleanup(journal); removeDockerBootstrapJournalDurably(journal, deps); return recoveredReceipt(journal, sourcePhase, finalization.cleanupReceipt); }; @@ -1970,14 +2083,15 @@ export function createDockerManagedBootstrapAdapter( journal: DockerBootstrapTransaction, sourcePhase: DockerBootstrapTransaction["phase"], ): ManagedBootstrapRecoveryReceipt => { + requireExactOwnerCleanup(journal); const cleanupReceipt = Object.freeze({ schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, sandbox: journal.sandbox, bootstrapIdentity: journal.bootstrapIdentity, outcome: "rolled-back", - restoredRuntimeId: journal.originalRuntimeId, - restoredSpecHash: journal.originalSpecHash, - heldWorkloadRemoved: false, + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: true, alreadyRolledBack: false, finalizedAt: deps.now().toISOString(), } satisfies ManagedBootstrapFinalizationReceipt); @@ -2069,7 +2183,17 @@ export function createDockerManagedBootstrapAdapter( } } if (sharedStatus === "committed") { - clearDockerManagedStartupSharedStateCommitReceipt(sharedTransaction, deps); + try { + clearDockerManagedStartupSharedStateCommitReceipt(sharedTransaction, deps); + } catch (error) { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: journal.bootstrapIdentity, + cleanupRuntimeId: journal.replacementRuntimeId, + detail: `the image-owned commit receipt could not be retired during restart recovery: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + } } if (probeExactDockerContainerAbsence(journal.originalRuntimeId, deps) !== "absent") { throw new ManagedBootstrapDurableCommitCleanupPendingError({ @@ -2102,6 +2226,9 @@ export function createDockerManagedBootstrapAdapter( journal: DockerBootstrapTransaction, sourcePhase: DockerBootstrapTransaction["phase"], ): ManagedBootstrapRecoveryReceipt => { + if (journal.phase === "owner-cleanup-required") { + return finishRecoveredRollback(journal, sourcePhase); + } const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); if (!original) { throw new ManagedBootstrapCommitStateIndeterminateError({ @@ -2339,6 +2466,9 @@ export function createDockerManagedBootstrapAdapter( replacement, durablePreparation, ); + if (journal.phase === "owner-cleanup-required") { + return completeRollbackTransaction(handle, journal); + } const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); if (!original) { throw new ManagedBootstrapCommitStateIndeterminateError({ @@ -2846,17 +2976,39 @@ export function createDockerManagedBootstrapAdapter( return { async recoverUnfinishedTransactions() { const receipts: ManagedBootstrapRecoveryReceipt[] = []; - for (const journal of deps.journalStore.listUnfinished()) { - const sourcePhase = journal.phase; - const finalized = compactRecoveredFinalization(journal, sourcePhase); - receipts.push( - finalized ?? - (journal.phase === "shared-state-committed" - ? finishRecoveredCommit(journal, sourcePhase) - : finishRecoveredRollbackPhase(journal, sourcePhase)), - ); + const failures: ManagedBootstrapRecoveryFailure[] = []; + for (const bootstrapIdentity of deps.journalStore.listUnfinishedIdentities()) { + let journal: DockerBootstrapTransaction | null = null; + try { + journal = deps.journalStore.load(bootstrapIdentity); + if (!journal) { + throw new Error("durable journal disappeared after identity enumeration"); + } + const sourcePhase = journal.phase; + const finalized = compactRecoveredFinalization(journal, sourcePhase); + receipts.push( + finalized ?? + (journal.phase === "shared-state-committed" + ? finishRecoveredCommit(journal, sourcePhase) + : finishRecoveredRollbackPhase(journal, sourcePhase)), + ); + } catch (error) { + try { + journal = deps.journalStore.load(bootstrapIdentity) ?? journal; + } catch { + // Preserve the first per-record failure when the durable re-read also fails. + } + failures.push(dockerManagedBootstrapRecoveryFailure(bootstrapIdentity, journal, error)); + } } - return Object.freeze(receipts); + const byBootstrapIdentity = ( + left: ManagedBootstrapRecoveryReceipt | ManagedBootstrapRecoveryFailure, + right: ManagedBootstrapRecoveryReceipt | ManagedBootstrapRecoveryFailure, + ) => left.bootstrapIdentity.localeCompare(right.bootstrapIdentity); + return Object.freeze({ + receipts: Object.freeze(receipts.sort(byBootstrapIdentity)), + failures: Object.freeze(failures.sort(byBootstrapIdentity)), + } satisfies ManagedBootstrapRecoveryReport); }, async createHeldWorkload(input) { diff --git a/src/lib/onboard/managed-bootstrap/index.ts b/src/lib/onboard/managed-bootstrap/index.ts index abeddd0a06d..78873418467 100644 --- a/src/lib/onboard/managed-bootstrap/index.ts +++ b/src/lib/onboard/managed-bootstrap/index.ts @@ -3,6 +3,7 @@ export { activateManagedBootstrapSequence, + enforceManagedBootstrapRecoveryForSandbox, finalizeManagedBootstrapSequence, MANAGED_BOOTSTRAP_SCHEMA_VERSION, type ManagedBootstrapActivatedTransaction, @@ -10,7 +11,10 @@ export { type ManagedBootstrapAuthorityStore, type ManagedBootstrapExpectedPlan, type ManagedBootstrapPreparedTransaction, + ManagedBootstrapRecoveryBlockedError, + type ManagedBootstrapRecoveryFailure, type ManagedBootstrapRecoveryReceipt, + type ManagedBootstrapRecoveryReport, prepareManagedBootstrapSequence, recoverManagedBootstrapTransactions, } from "./adapter"; diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.ts b/src/lib/onboard/managed-bootstrap/runtime-create.ts index 639e4cfa6e4..6ffcc028966 100644 --- a/src/lib/onboard/managed-bootstrap/runtime-create.ts +++ b/src/lib/onboard/managed-bootstrap/runtime-create.ts @@ -10,7 +10,7 @@ import type { ManagedBootstrapAuthorityStore, ManagedBootstrapCreateReceipt, ManagedBootstrapImageIdentity, - ManagedBootstrapRecoveryReceipt, + ManagedBootstrapRecoveryReport, } from "./adapter"; export interface ManagedBootstrapRuntimeCommandResult { @@ -93,7 +93,7 @@ export interface ManagedBootstrapRuntimeCreateLaunchResult { export interface ManagedBootstrapRuntimeCreateLifecycle { readonly launchArgv: readonly string[]; readonly patch: ManagedBootstrapRuntimePatch; - recoverUnfinished(): Promise; + recoverUnfinished(): Promise; prepareNetwork(): Promise; runCreate( launch: (input: { diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index 6628831a980..818ae30eb30 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -209,7 +209,7 @@ describe("RuntimeProviderBundle registry contract", () => { printReadinessFailureIfEnabled: vi.fn(), verifyGpuOrExit: vi.fn(async (verify) => verify("alpha")), }, - recoverUnfinished: vi.fn(async () => []), + recoverUnfinished: vi.fn(async () => ({ receipts: [], failures: [] })), prepareNetwork: vi.fn(async () => undefined), runCreate: vi.fn(), })); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index a7362d037ce..e2be8548e33 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -62,6 +62,10 @@ import { setupGpuFlowMocks, VERIFIED_GPU_PROOF as VERIFIED_PROOF, } from "./__test-helpers__/sandbox-gpu-create-flow"; +import { + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapRecoveryReport, +} from "./managed-bootstrap/adapter"; import type { ManagedBootstrapRuntimeCreateLifecycleInput, ManagedBootstrapRuntimePatch, @@ -201,7 +205,30 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { input.sandboxEnv = launch.sandboxEnv; input.sandboxStartupCommand = launch.sandboxStartupCommand; const patch = createPatch() as unknown as ManagedBootstrapRuntimePatch; - const recoverUnfinished = vi.fn(async () => []); + const recoveryReport = (sandboxName: string | null): ManagedBootstrapRecoveryReport => + Object.freeze({ + receipts: Object.freeze([]), + failures: Object.freeze([ + Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: "mxc", + sourcePhase: "provider-owned-cleanup", + sandbox: + sandboxName === null + ? null + : Object.freeze({ + sandboxName, + sandboxId: `mxc-${sandboxName}`, + driverId: "mxc", + }), + bootstrapIdentity: "e".repeat(64), + code: "mxc-recovery-retry", + retryable: true, + detail: "opaque MXC recovery detail", + }), + ]), + }); + const recoverUnfinished = vi.fn(async () => recoveryReport("bravo")); const prepareNetwork = vi.fn(async () => undefined); const createLifecycle = vi.fn( (lifecycleInput: ManagedBootstrapRuntimeCreateLifecycleInput) => ({ @@ -319,6 +346,19 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { expect(mocks.queryOpenShellDockerSandboxContainers).not.toHaveBeenCalled(); expect(mocks.queryOpenShellDockerSandboxRuntimeSnapshot).not.toHaveBeenCalled(); expect(mocks.enforceDockerGpuPatchPreserveNetwork).not.toHaveBeenCalled(); + + expect(vi.mocked(console.warn).mock.calls.flat().join("\n")).toContain( + "unrelated sandbox 'bravo'", + ); + recoverUnfinished.mockResolvedValueOnce(recoveryReport("alpha")); + prepareNetwork.mockClear(); + mocks.streamSandboxCreate.mockClear(); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "recovery blocks sandbox 'alpha'", + ); + expect(prepareNetwork).not.toHaveBeenCalled(); + expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 5f1d98fa198..49c3271ddd6 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -14,6 +14,7 @@ import { reportSandboxCreateFailure } from "./created-sandbox-failure"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +import { enforceManagedBootstrapRecoveryForSandbox } from "./managed-bootstrap/adapter"; import type { ManagedBootstrapRuntimeSnapshot } from "./managed-bootstrap/runtime-create"; import { queryOpenShellDockerSandboxContainers, @@ -143,7 +144,12 @@ export function createSandboxGpuCreateAttemptRunner( backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", deps, }); - await managedLifecycle?.recoverUnfinished(); + const recovery = await managedLifecycle?.recoverUnfinished(); + if (recovery) { + enforceManagedBootstrapRecoveryForSandbox(recovery, input.sandboxName, (message) => + console.warn(` ⚠ ${message}`), + ); + } await managedLifecycle?.prepareNetwork(); const [createExecutable, ...createExecutableArgs] = managedLifecycle?.launchArgv ?? attemptArgv; if (!createExecutable) throw new Error("Sandbox create executable is missing."); From fd260f2e0fd7da6aa667f149e9c9edfd4d6283a3 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 14:25:42 -0700 Subject: [PATCH 115/117] fix(onboard): close remaining recovery review notes Signed-off-by: Aaron Erickson --- .../managed-bootstrap/docker-journal.ts | 4 +- .../managed-bootstrap/docker-recovery.test.ts | 55 +++++++++++++++++ .../managed-bootstrap/docker-test-fixture.ts | 22 ++++--- src/lib/onboard/managed-bootstrap/docker.ts | 59 ++++++++----------- 4 files changed, 96 insertions(+), 44 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index 03e745acb34..8743dd92075 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -3,7 +3,7 @@ import fs from "node:fs"; import path from "node:path"; -import type { ManagedStartupAgent } from "../managed-startup/profile"; +import { MANAGED_STARTUP_AGENTS, type ManagedStartupAgent } from "../managed-startup/profile"; import type { ManagedBootstrapCompletionReceipt, ManagedBootstrapDurablePreparationReceipt, @@ -222,7 +222,7 @@ function exactLegacyPhase( } function exactAgent(value: unknown): ManagedStartupAgent { - if (!["openclaw", "hermes", "langchain-deepagents-code"].includes(String(value))) { + if (!MANAGED_STARTUP_AGENTS.includes(value as ManagedStartupAgent)) { fail("agent is unsupported"); } return value as ManagedStartupAgent; diff --git a/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts b/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts index 0bd01ccc9ec..12307cc93ac 100644 --- a/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts @@ -297,6 +297,61 @@ describe("Docker managed bootstrap restart recovery", () => { expect(dockerMutationEvents(fake.events)).toEqual(mutationsAfterFailedRetirement); }); + it("retains durable commit authority after a non-zero Docker removal result", async () => { + const fake = fixture({ + dockerRemoveFailures: [new Error("injected crash before exact Docker removal")], + dockerRemoveResults: [{ status: 1, stderr: "injected non-zero Docker removal" }], + sharedState: "pending", + }); + const transaction = await prepareTransaction(fake); + const replacement = await transaction.adapter.activateBootstrapReplacement({ + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + }); + const completion = await transaction.adapter.awaitBootstrap({ + handle: transaction.handle, + snapshot: transaction.snapshot, + replacement, + timeoutSecs: 1, + }); + + await expect( + transaction.adapter.finalizeBootstrap({ + outcome: "commit", + handle: transaction.handle, + snapshot: transaction.snapshot, + prepared: transaction.prepared, + durablePreparation: transaction.durable, + replacement, + completion, + }), + ).rejects.toThrow("crash before exact Docker removal"); + + const restarted = createDockerManagedBootstrapAdapter(fake.deps); + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject({ + receipts: [], + failures: [ + { + sourcePhase: "shared-state-committed", + code: "durable-cleanup-pending", + detail: expect.stringContaining("injected non-zero Docker removal"), + }, + ], + }); + expect(fake.journal?.phase).toBe("shared-state-committed"); + expect(fake.finalization).toBeNull(); + expect(fake.original).not.toBeNull(); + + await expect(restarted.recoverUnfinishedTransactions()).resolves.toMatchObject({ + receipts: [{ sourcePhase: "shared-state-committed", outcome: "committed" }], + failures: [], + }); + expect(fake.journal).toBeNull(); + expect(fake.finalization?.phase).toBe("committed"); + }); + it("compacts a terminal commit journal after another restart interruption", async () => { const fake = fixture({ agent: "langchain-deepagents-code", diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index 5464e6ad06c..dd1a56a0f9c 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -70,6 +70,7 @@ export type DockerFixtureAcknowledgement = export type DockerFixtureOptions = { readonly agent?: ManagedStartupAgent; readonly dockerRemoveFailures?: readonly Error[]; + readonly dockerRemoveResults?: readonly FixtureCommandResult[]; readonly dockerInspectUnknownIds?: readonly string[]; readonly dockerStartResults?: Readonly>; readonly journalCreateFailures?: readonly Error[]; @@ -224,6 +225,7 @@ export function fixture(options: DockerFixtureOptions = {}) { let sharedState: "committed" | "none" | "pending" = options.sharedState ?? "none"; const events: string[] = []; const dockerRemoveFailures = [...(options.dockerRemoveFailures ?? [])]; + const dockerRemoveResults = [...(options.dockerRemoveResults ?? [])]; const journalCreateFailures = [...(options.journalCreateFailures ?? [])]; const journalRemoveFailures = [...(options.journalRemoveFailures ?? [])]; const sharedReceiptClearFailures = [...(options.sharedReceiptClearFailures ?? [])]; @@ -505,17 +507,21 @@ export function fixture(options: DockerFixtureOptions = {}) { default: throw injectedFailure; } - switch (id) { - case OLD_ID: - original = null; - break; - case NEW_ID: - replacement = null; - break; + const result = dockerRemoveResults.shift() ?? ok(); + switch (result.status) { + case 0: + switch (id) { + case OLD_ID: + original = null; + break; + case NEW_ID: + replacement = null; + break; + } } return losesAcknowledgement("container:remove") ? { status: 1, stderr: "lost rm acknowledgement" } - : ok(); + : result; }), runCaptureOpenshell: vi.fn(() => `Name: alpha\nID: ${options.ownerId ?? "sandbox-alpha"}\n`), runOpenshell: vi.fn(() => ok()), diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index 86cb093d92b..abb04816432 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -70,6 +70,7 @@ import { createFileDockerManagedBootstrapJournalStore, DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapFinalizationContext, type DockerManagedBootstrapFinalizationRecord, type DockerManagedBootstrapJournal, DockerManagedBootstrapJournalAcknowledgementLostError, @@ -1826,6 +1827,28 @@ export function createDockerManagedBootstrapAdapter( } return record; }; + const persistFinalizationRecord = ( + record: DockerManagedBootstrapFinalizationRecord, + context: DockerManagedBootstrapFinalizationContext, + ): ManagedBootstrapFinalizationReceipt => { + const serialized = serializeDockerManagedBootstrapFinalizationRecord(record); + try { + deps.journalStore.recordFinalization(record, context); + } catch (error) { + const recovered = deps.journalStore.loadFinalization(record.bootstrapIdentity, context); + if ( + !recovered || + serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized + ) { + throw error; + } + } + const persisted = deps.journalStore.loadFinalization(record.bootstrapIdentity, context); + if (!persisted || serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized) { + throw new Error("Managed bootstrap finalization receipt was not durably re-readable."); + } + return persisted.cleanupReceipt; + }; const persistFinalization = ( handle: ManagedBootstrapHeldWorkloadHandle, phase: "committed" | "rolled-back", @@ -1846,23 +1869,7 @@ export function createDockerManagedBootstrapAdapter( commitReceipt, cleanupReceipt, } satisfies DockerManagedBootstrapFinalizationRecord); - const serialized = serializeDockerManagedBootstrapFinalizationRecord(record); - try { - deps.journalStore.recordFinalization(record, context); - } catch (error) { - const recovered = deps.journalStore.loadFinalization(handle.bootstrapIdentity, context); - if ( - !recovered || - serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized - ) { - throw error; - } - } - const persisted = deps.journalStore.loadFinalization(handle.bootstrapIdentity, context); - if (!persisted || serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized) { - throw new Error("Managed bootstrap finalization receipt was not durably re-readable."); - } - return persisted.cleanupReceipt; + return persistFinalizationRecord(record, context); }; const completedRollback = ( handle: ManagedBootstrapHeldWorkloadHandle, @@ -2001,23 +2008,7 @@ export function createDockerManagedBootstrapAdapter( commitReceipt, cleanupReceipt, } satisfies DockerManagedBootstrapFinalizationRecord); - const serialized = serializeDockerManagedBootstrapFinalizationRecord(record); - try { - deps.journalStore.recordFinalization(record, journal); - } catch (error) { - const recovered = deps.journalStore.loadFinalization(journal.bootstrapIdentity, journal); - if ( - !recovered || - serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized - ) { - throw error; - } - } - const persisted = deps.journalStore.loadFinalization(journal.bootstrapIdentity, journal); - if (!persisted || serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized) { - throw new Error("Managed bootstrap recovered finalization was not durably re-readable."); - } - return persisted.cleanupReceipt; + return persistFinalizationRecord(record, journal); }; const recoveredReceipt = ( journal: DockerBootstrapTransaction, From 456fd86e3f45c030651a67ca0afe14be53ee4d7e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 15:05:48 -0700 Subject: [PATCH 116/117] fix(onboard): scope legacy bootstrap recovery Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/README.md | 34 +++++++++ .../managed-bootstrap/docker-journal.test.ts | 21 +++++- .../managed-bootstrap/docker-journal.ts | 70 ++++++++++++++++++- .../managed-bootstrap/docker-recovery.test.ts | 66 ++++++++++++++++- src/lib/onboard/managed-bootstrap/docker.ts | 13 +++- .../onboard/sandbox-gpu-create-flow.test.ts | 10 ++- src/lib/onboard/sandbox-gpu-create-flow.ts | 55 ++++++++++++--- 7 files changed, 246 insertions(+), 23 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 77cc960dc94..fafd40611d3 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -125,6 +125,40 @@ that same ID after quiescence. Multi-process lease/arbitration remains an explicit production-activation gate. Activation must also inject the selected gateway's canonical state root. +## Legacy journal drain (schema 1 and 2) + +Schema 1 and schema 2 journal bodies predate durable agent identity. They cannot +be upgraded by guessing from a mutable sandbox name, image repository, or the +agent selected by a later command. Recovery therefore preserves the canonical +record and any decision sidecar, reports its exact bootstrap, provider, sandbox, +original-runtime, and replacement-runtime identities, and fences only that +sandbox name. A create for another sandbox may continue after warning about the +retained record. + +When recovery reports one of these records: + +1. Stop onboarding the named sandbox. Save the complete diagnostic and back up + the canonical state root's + `managed-bootstrap/.json` file and any adjacent decision + sidecar without editing either record. +2. Inspect the reported full runtime IDs through the owning provider. Treat + sandbox and container names as diagnostic text only. Never delete, rename, + or adopt a runtime by name, and never copy agent identity from the current + invocation into the old record. +3. If either exact runtime is present, or its presence cannot be proven, leave + the journal in place and recover the provider-owned transaction using those + immutable IDs. A legacy cutover decision may be newer than the journal-body + phase, so the body alone never authorizes commit or rollback. +4. If both exact runtimes are proven absent, still preserve the journal and its + image-owned shared-state evidence. Record the exact absence proof on + [epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) for the + identity-checked retirement path. Until that path ships, use a different + sandbox name rather than deleting durable authority. + +Production activation must include the identity-checked retirement path and +protected recovery qualification. This candidate remains inert, so it does not +expose a runtime that could create these legacy records without that support. + ## Architectural disposition The runtime-provider bundle is the only bootstrap registration boundary. The diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index 69d2a336d3c..457e9b4e154 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -490,7 +490,8 @@ describe("Docker managed bootstrap journal", () => { [1, legacyJournalV1], [2, legacyJournalV2], ] as const)("fails typed and closed for exact legacy journal schema %i", (schemaVersion, legacy) => { - const serialized = `${JSON.stringify(legacy())}\n`; + const record = legacy(); + const serialized = `${JSON.stringify(record)}\n`; let failure: unknown; try { parseDockerManagedBootstrapJournal(serialized); @@ -500,9 +501,21 @@ describe("Docker managed bootstrap journal", () => { expect(failure).toBeInstanceOf(DockerManagedBootstrapLegacyRecordRequiresAgentError); expect(failure).toMatchObject({ bootstrapIdentity: IDENTITY, + journalContext: { + schemaVersion, + phase: record.phase, + bootstrapIdentity: IDENTITY, + providerId: record.sandbox.driverId, + sandbox: record.sandbox, + originalRuntimeId: record.originalRuntimeId, + replacementRuntimeId: record.replacementRuntimeId, + }, recordKind: "journal", schemaVersion, }); + const legacyFailure = failure as DockerManagedBootstrapLegacyRecordRequiresAgentError; + expect(Object.isFrozen(legacyFailure.journalContext)).toBe(true); + expect(Object.isFrozen(legacyFailure.journalContext?.sandbox)).toBe(true); const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); roots.push(root); @@ -656,6 +669,12 @@ describe("Docker managed bootstrap journal", () => { expect(() => store.loadFinalization(OTHER_IDENTITY)).toThrow( "finalization bootstrap identity does not match its file name", ); + fs.writeFileSync(misplacedJournal, `${JSON.stringify(legacyJournalV2())}\n`, { + mode: 0o600, + }); + expect(() => store.load(OTHER_IDENTITY)).toThrow( + "journal bootstrap identity does not match its file name", + ); expect(fs.existsSync(misplacedJournal)).toBe(true); expect(fs.existsSync(misplacedFinalization)).toBe(true); }); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index 8743dd92075..47f1022e884 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -105,6 +105,16 @@ export interface DockerManagedBootstrapJournalStore { ): DockerManagedBootstrapFinalizationRecord | null; } +export interface DockerManagedBootstrapLegacyJournalContext { + readonly schemaVersion: 1 | 2; + readonly phase: Exclude; + readonly bootstrapIdentity: string; + readonly providerId: string; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly originalRuntimeId: string; + readonly replacementRuntimeId: string; +} + /** * Alternate stores may use this only when the durable mutation completed and * the caller lost its acknowledgement. Ordinary I/O and fsync failures must @@ -119,26 +129,44 @@ export class DockerManagedBootstrapJournalAcknowledgementLostError extends Error export class DockerManagedBootstrapLegacyRecordRequiresAgentError extends Error { readonly bootstrapIdentity: string; + readonly journalContext: DockerManagedBootstrapLegacyJournalContext | null; readonly recordKind: "finalization" | "journal"; readonly reason: "context-mismatch" | "missing-context" | undefined; readonly schemaVersion: number; constructor(input: { readonly bootstrapIdentity: string; + readonly journalContext?: DockerManagedBootstrapLegacyJournalContext; readonly recordKind: "finalization" | "journal"; readonly reason?: "context-mismatch" | "missing-context"; readonly schemaVersion: number; }) { const reason = input.reason; + const journalContext = input.journalContext + ? Object.freeze({ + ...input.journalContext, + sandbox: Object.freeze({ ...input.journalContext.sandbox }), + }) + : undefined; + const journalGuidance = journalContext + ? `; recovery is fenced to sandbox '${journalContext.sandbox.sandboxName}' ` + + `(ID ${journalContext.sandbox.sandboxId}, provider ${journalContext.providerId}, ` + + `journal-body phase ${journalContext.phase}) with exact original runtime ` + + `${journalContext.originalRuntimeId} and replacement runtime ` + + `${journalContext.replacementRuntimeId}; preserve the journal and follow ` + + "https://github.com/NVIDIA/NemoClaw/blob/main/src/lib/onboard/managed-bootstrap/README.md#legacy-journal-drain-schema-1-and-2" + : ""; super( `Managed bootstrap Docker ${input.recordKind} schema ${input.schemaVersion} for ` + `${input.bootstrapIdentity} lacks durable agent identity` + (reason === "context-mismatch" ? "; supplied durable context does not match this record" - : ""), + : "") + + journalGuidance, ); this.name = "DockerManagedBootstrapLegacyRecordRequiresAgentError"; this.bootstrapIdentity = input.bootstrapIdentity; + this.journalContext = journalContext ?? null; this.recordKind = input.recordKind; this.reason = reason; this.schemaVersion = input.schemaVersion; @@ -260,7 +288,11 @@ function sameSandboxIdentity( function normalizeLegacyDockerManagedBootstrapJournal( journal: Readonly>, schemaVersion: 1 | 2, -): { readonly bootstrapIdentity: string; readonly canonical: string } { +): { + readonly bootstrapIdentity: string; + readonly canonical: string; + readonly journalContext: DockerManagedBootstrapLegacyJournalContext; +} { if (schemaVersion === 1) { const expectedKeys = [ "backupName", @@ -311,6 +343,15 @@ function normalizeLegacyDockerManagedBootstrapJournal( return { bootstrapIdentity: normalized.bootstrapIdentity, canonical: `${JSON.stringify(normalized)}\n`, + journalContext: Object.freeze({ + schemaVersion, + phase: normalized.phase, + bootstrapIdentity: normalized.bootstrapIdentity, + providerId: normalized.sandbox.driverId, + sandbox: normalized.sandbox, + originalRuntimeId: normalized.originalRuntimeId, + replacementRuntimeId: normalized.replacementRuntimeId, + }), }; } @@ -408,6 +449,15 @@ function normalizeLegacyDockerManagedBootstrapJournal( return { bootstrapIdentity: normalized.bootstrapIdentity, canonical: `${JSON.stringify(normalized)}\n`, + journalContext: Object.freeze({ + schemaVersion, + phase: normalized.phase, + bootstrapIdentity: normalized.bootstrapIdentity, + providerId: normalized.providerId, + sandbox: normalized.sandbox, + originalRuntimeId: normalized.originalRuntimeId, + replacementRuntimeId: normalized.replacementRuntimeId, + }), }; } @@ -422,6 +472,7 @@ export function normalizeDockerManagedBootstrapJournal( const legacy = normalizeLegacyDockerManagedBootstrapJournal(journal, journal.schemaVersion); throw new DockerManagedBootstrapLegacyRecordRequiresAgentError({ bootstrapIdentity: legacy.bootstrapIdentity, + journalContext: legacy.journalContext, recordKind: "journal", schemaVersion: journal.schemaVersion, }); @@ -564,6 +615,7 @@ export function parseDockerManagedBootstrapJournal(text: string): DockerManagedB if (legacy.canonical !== text) fail("serialized legacy journal is not canonical"); throw new DockerManagedBootstrapLegacyRecordRequiresAgentError({ bootstrapIdentity: legacy.bootstrapIdentity, + journalContext: legacy.journalContext, recordKind: "journal", schemaVersion: record.schemaVersion, }); @@ -1180,7 +1232,19 @@ export function createFileDockerManagedBootstrapJournalStore( const target = journalPath(directory, bootstrapIdentity); const contents = readPrivateFile(target, "journal"); if (contents === null) return null; - const journal = parseDockerManagedBootstrapJournal(contents); + let journal: DockerManagedBootstrapJournal; + try { + journal = parseDockerManagedBootstrapJournal(contents); + } catch (error) { + if ( + error instanceof DockerManagedBootstrapLegacyRecordRequiresAgentError && + error.recordKind === "journal" && + error.bootstrapIdentity !== bootstrapIdentity + ) { + fail("journal bootstrap identity does not match its file name"); + } + throw error; + } if (journal.bootstrapIdentity !== bootstrapIdentity) { fail("journal bootstrap identity does not match its file name"); } diff --git a/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts b/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts index 12307cc93ac..d041388f488 100644 --- a/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts @@ -1,11 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; -import { recoverManagedBootstrapTransactions } from "./adapter"; +import { + enforceManagedBootstrapRecoveryForSandbox, + ManagedBootstrapRecoveryBlockedError, + recoverManagedBootstrapTransactions, +} from "./adapter"; import { createDockerManagedBootstrapAdapter } from "./docker"; -import type { DockerManagedBootstrapJournalStore } from "./docker-journal"; +import { + type DockerManagedBootstrapJournalStore, + DockerManagedBootstrapLegacyRecordRequiresAgentError, +} from "./docker-journal"; import { authority, type DockerFixtureOptions, @@ -48,6 +55,59 @@ function dockerMutationEvents(events: readonly string[]): readonly string[] { } describe("Docker managed bootstrap restart recovery", () => { + it("scopes an exact legacy journal to its durable sandbox without inventing agent authority", async () => { + const fake = fixture(); + const delegate = fake.deps.journalStore as DockerManagedBootstrapJournalStore; + const legacyStore: DockerManagedBootstrapJournalStore = { + ...delegate, + listUnfinishedIdentities: () => [IDENTITY], + load() { + throw new DockerManagedBootstrapLegacyRecordRequiresAgentError({ + bootstrapIdentity: IDENTITY, + journalContext: { + schemaVersion: 2, + phase: "cutover", + bootstrapIdentity: IDENTITY, + providerId: "docker", + sandbox: authority().handle.sandbox, + originalRuntimeId: OLD_ID, + replacementRuntimeId: NEW_ID, + }, + recordKind: "journal", + schemaVersion: 2, + }); + }, + }; + const adapter = createDockerManagedBootstrapAdapter({ + ...fake.deps, + journalStore: legacyStore, + }); + + const report = await recoverManagedBootstrapTransactions(adapter); + expect(report).toMatchObject({ + receipts: [], + failures: [ + { + bootstrapIdentity: IDENTITY, + providerId: "docker", + sourcePhase: null, + sandbox: authority().handle.sandbox, + code: "legacy-agent-required", + retryable: true, + detail: expect.stringContaining(OLD_ID), + }, + ], + }); + const warn = vi.fn(); + expect(enforceManagedBootstrapRecoveryForSandbox(report, "bravo", warn)).toBe(report); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("unrelated sandbox 'alpha'")); + expect(() => enforceManagedBootstrapRecoveryForSandbox(report, "alpha", warn)).toThrow( + ManagedBootstrapRecoveryBlockedError, + ); + expect(dockerMutationEvents(fake.events)).toEqual([]); + expect(fake.journal).toBeNull(); + }); + it.each([ { label: "staged", diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index abb04816432..8dc146f1df7 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -128,6 +128,10 @@ function dockerManagedBootstrapRecoveryFailure( journal: DockerBootstrapTransaction | null, error: unknown, ): ManagedBootstrapRecoveryFailure { + const legacyJournalContext = + error instanceof DockerManagedBootstrapLegacyRecordRequiresAgentError + ? error.journalContext + : null; const classified = error instanceof ManagedBootstrapOwnerCleanupRequiredError ? { code: "owner-cleanup-required", retryable: true } @@ -136,13 +140,16 @@ function dockerManagedBootstrapRecoveryFailure( : error instanceof ManagedBootstrapCommitStateIndeterminateError ? { code: "commit-state-indeterminate", retryable: true } : error instanceof DockerManagedBootstrapLegacyRecordRequiresAgentError - ? { code: "legacy-agent-required", retryable: false } + ? { code: "legacy-agent-required", retryable: true } : { code: "provider-recovery-failed", retryable: true }; return Object.freeze({ schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - providerId: journal?.providerId ?? DOCKER_DRIVER_ID, + providerId: journal?.providerId ?? legacyJournalContext?.providerId ?? DOCKER_DRIVER_ID, + // A legacy cutover decision sidecar may have advanced beyond the phase in + // the journal body. Keep the provider phase unknown until agent-bound + // recovery can validate both records, while retaining its exact sandbox. sourcePhase: journal?.phase ?? null, - sandbox: journal?.sandbox ?? null, + sandbox: journal?.sandbox ?? legacyJournalContext?.sandbox ?? null, bootstrapIdentity, code: classified.code, retryable: classified.retryable, diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index e2be8548e33..d152f615e5a 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -353,12 +353,16 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { recoverUnfinished.mockResolvedValueOnce(recoveryReport("alpha")); prepareNetwork.mockClear(); mocks.streamSandboxCreate.mockClear(); + mockExit(); - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "recovery blocks sandbox 'alpha'", - ); + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1"); expect(prepareNetwork).not.toHaveBeenCalled(); expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); + expect(errorOutput()).toContain("recovery stopped before sandbox 'alpha' was created"); + expect(errorOutput()).toContain("Transaction"); + expect(errorOutput()).toContain("durable sandbox ID mxc-alpha"); + expect(errorOutput()).toContain("OpenShell's sandbox get command"); + expect(errorOutput()).toContain("never delete a runtime by mutable sandbox name"); }); }); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 48acddb658d..46f34359a2e 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -10,11 +10,12 @@ import type { DockerGpuPatchDeps, DockerUlimit } from "./docker-gpu-patch-types" import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { renderCompatibilityFallbackCreateArgs } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; -import type { - ManagedBootstrapAdapter, - ManagedBootstrapAgentIdentity, - ManagedBootstrapAuthorityStore, - ManagedBootstrapImageIdentity, +import { + type ManagedBootstrapAdapter, + type ManagedBootstrapAgentIdentity, + type ManagedBootstrapAuthorityStore, + type ManagedBootstrapImageIdentity, + ManagedBootstrapRecoveryBlockedError, } from "./managed-bootstrap/adapter"; import type { ManagedBootstrapRuntimePatch } from "./managed-bootstrap/runtime-create"; import type { ManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; @@ -31,6 +32,36 @@ import { addTraceEvent } from "./tracing"; export { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; +/* + * Keep recovery rendering at this public command boundary. Providers own the + * detail and remediation; central orchestration only renders their bounded, + * identity-bound evidence and never branches on provider IDs or error codes. + */ +function exitForManagedBootstrapRecovery(error: ManagedBootstrapRecoveryBlockedError): never { + console.error(""); + console.error( + ` Managed bootstrap recovery stopped before sandbox '${error.sandboxName}' was created.`, + ); + for (const failure of error.failures) { + const scope = failure.sandbox + ? `sandbox '${failure.sandbox.sandboxName}' (durable sandbox ID ${failure.sandbox.sandboxId}, provider ${failure.providerId})` + : "a sandbox whose durable identity could not be recovered"; + console.error( + ` Transaction ${failure.bootstrapIdentity} requires ${failure.retryable ? "a provider retry after its recovery condition is resolved" : "operator recovery"} for ${scope}.`, + ); + console.error(` ${redactFull(failure.detail)}`); + if (failure.sandbox) { + console.error( + ` Before any provider action, query that exact name with OpenShell's sandbox get command and verify it returns durable sandbox ID ${failure.sandbox.sandboxId}.`, + ); + } + } + console.error( + " Preserve every durable recovery record. Act only on exact provider, sandbox, and runtime IDs from the provider guidance; never delete a runtime by mutable sandbox name.", + ); + process.exit(1); +} + type RunOpenshell = NonNullable; type RunCaptureOpenshell = NonNullable; type Sleep = NonNullable; @@ -104,9 +135,8 @@ export async function runSandboxGpuCreateFlow( ): Promise { let registryImageRef: string | null = input.prebuild.imageRef; const attemptRunner = createSandboxGpuCreateAttemptRunner(input, deps); - const gpuCreateOutcome = await sandboxGpuCreateAttempt.executeSandboxGpuCreatePlan( - input.gpuRoutePlan, - { + const gpuCreateOutcome = await sandboxGpuCreateAttempt + .executeSandboxGpuCreatePlan(input.gpuRoutePlan, { runAttempt: attemptRunner.runAttempt, captureNativeFailure: (failure) => { const routeAdapter = adaptDockerGpuRouteForPatch(failure.route); @@ -192,8 +222,13 @@ export async function runSandboxGpuCreateFlow( input.sandboxGpuConfig.sandboxGpuProof = null; }, traceEvent: addTraceEvent, - }, - ); + }) + .catch((error: unknown) => { + if (error instanceof ManagedBootstrapRecoveryBlockedError) { + exitForManagedBootstrapRecovery(error); + } + throw error; + }); if (!gpuCreateOutcome.ok) { console.error(""); console.error(" Operator-authorized GPU fallback stopped before compatibility retry."); From aca6e3d1787b4b9d5952c86e290300fb48fea683 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 3 Aug 2026 15:16:11 -0700 Subject: [PATCH 117/117] test(onboard): harden recovery evidence Signed-off-by: Aaron Erickson --- .../managed-bootstrap/docker-recovery.test.ts | 1 + src/lib/onboard/sandbox-gpu-create-flow.test.ts | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts b/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts index d041388f488..2d6fb3c1110 100644 --- a/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts @@ -105,6 +105,7 @@ describe("Docker managed bootstrap restart recovery", () => { ManagedBootstrapRecoveryBlockedError, ); expect(dockerMutationEvents(fake.events)).toEqual([]); + expect(fake.events).toEqual([]); expect(fake.journal).toBeNull(); }); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index d152f615e5a..f60f0a59abe 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -205,7 +205,10 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { input.sandboxEnv = launch.sandboxEnv; input.sandboxStartupCommand = launch.sandboxStartupCommand; const patch = createPatch() as unknown as ManagedBootstrapRuntimePatch; - const recoveryReport = (sandboxName: string | null): ManagedBootstrapRecoveryReport => + const recoveryReport = ( + sandboxName: string | null, + detail = "opaque MXC recovery detail", + ): ManagedBootstrapRecoveryReport => Object.freeze({ receipts: Object.freeze([]), failures: Object.freeze([ @@ -224,7 +227,7 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { bootstrapIdentity: "e".repeat(64), code: "mxc-recovery-retry", retryable: true, - detail: "opaque MXC recovery detail", + detail, }), ]), }); @@ -350,7 +353,10 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { expect(vi.mocked(console.warn).mock.calls.flat().join("\n")).toContain( "unrelated sandbox 'bravo'", ); - recoverUnfinished.mockResolvedValueOnce(recoveryReport("alpha")); + const recoverySecret = "opaque-recovery-token"; + recoverUnfinished.mockResolvedValueOnce( + recoveryReport("alpha", `Authorization: Bearer ${recoverySecret}`), + ); prepareNetwork.mockClear(); mocks.streamSandboxCreate.mockClear(); mockExit(); @@ -363,6 +369,8 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { expect(errorOutput()).toContain("durable sandbox ID mxc-alpha"); expect(errorOutput()).toContain("OpenShell's sandbox get command"); expect(errorOutput()).toContain("never delete a runtime by mutable sandbox name"); + expect(errorOutput()).toContain("Authorization: Bearer "); + expect(errorOutput()).not.toContain(recoverySecret); }); });