diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index ceb977e9b7b..efd32b52324 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -2771,6 +2771,40 @@ A rebuild preserves the recorded Deep Agents Code auto-approval capability unles A sandbox onboarded with an explicit GPU opt-out (stored as `sandboxGpuMode: "0"`, plus legacy registry entries that only record `gpuEnabled: false`) is recreated with the same opt-out, so the inner `onboard --resume` skips the Docker CDI GPU preflight on hosts without an NVIDIA GPU. Auto-mode sandboxes remain auto. + + +Before backup or deletion, `rebuild` builds and validates the replacement image with Docker's BuildKit path. +For a NemoClaw-generated OpenClaw image on a local Docker-driver gateway, NemoClaw can use one compatibility retry when Docker reports `BuildKit is enabled but the buildx component is missing or broken`. +NemoClaw runs `docker buildx version` independently and uses the compatibility retry only when that command also fails. +The retry rebuilds the same verified build context once with Docker's legacy builder. +NemoClaw then binds the retained image to the verified Docker engine and its immutable image ID. +It verifies the build context, Docker engine, mutable tag, and immutable image ID again at the final delete boundary. +Any failed build or pre-delete verification stops the rebuild without deleting the existing sandbox. +After deletion, Docker operations for replacement creation remain bound to the retained engine and immutable image ID. +If either identity changes during replacement creation, NemoClaw stops instead of operating on a different Docker target. + +The legacy-builder retry does not apply in the following cases: + +- The sandbox was recorded with a custom Dockerfile. +- The OpenShell gateway is not local. +- The host-side local prebuild is disabled. +- The build has an unrelated failure. +- The independent `docker buildx version` command succeeds. + +When the retry does not apply and Buildx is missing or broken, NemoClaw stops and leaves the existing sandbox intact. +Install or repair Docker Buildx, then verify it with `docker buildx version`. +Rerun the rebuild after that command succeeds. + + + + +Before backup or deletion, `rebuild` builds and validates the replacement image with Docker Buildx. +If Buildx is missing or broken, NemoClaw stops and leaves the existing sandbox intact. +Install or repair Docker Buildx, then verify it with `docker buildx version`. +Rerun the rebuild after that command succeeds. + + + ```bash $$nemoclaw my-assistant rebuild [--yes|-y|--force] [--verbose|-v] [--tool-disclosure ] [--dcode-auto-approval ] [--observability|--no-observability] ``` diff --git a/src/lib/actions/sandbox/rebuild-buildx-mutation-boundary.test.ts b/src/lib/actions/sandbox/rebuild-buildx-mutation-boundary.test.ts new file mode 100644 index 00000000000..ddd39293888 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-buildx-mutation-boundary.test.ts @@ -0,0 +1,147 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { expectNoSandboxDelete } from "../../../../test/helpers/rebuild-delete-assertions"; +import { + createRebuildFlowHarness, + resetRebuildFlowTestEnvironment, + restoreRebuildFlowTestEnvironment, +} from "../../../../test/helpers/rebuild-flow-harness"; +import { fingerprintBuildContext } from "../../adapters/fs/build-context-fingerprint"; +import type { + PreparedOpenClawLegacyImage, + PreparedSandboxBuildContext, +} from "../../onboard/build-context-stage"; + +type RetainedImageFixture = { + buildContext: PreparedSandboxBuildContext & { + contextFingerprint: string; + verifyBuildCtx(): boolean; + }; + lease: PreparedOpenClawLegacyImage; + verifyImage: ReturnType; + retainForRecreate: ReturnType; +}; + +function createRetainedImageFixture(imageVerificationResults: boolean[]): RetainedImageFixture { + const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-boundary-")); + const stagedDockerfile = path.join(buildCtx, "Dockerfile"); + fs.writeFileSync(stagedDockerfile, "FROM scratch\n"); + const contextFingerprint = fingerprintBuildContext(buildCtx); + const verifyImage = vi.fn(() => imageVerificationResults.shift() ?? true); + const retainForRecreate = vi.fn(() => true); + const lease = Object.freeze({ + dockerEnv: Object.freeze({ DOCKER_CONTEXT: "verified-builder" }), + engineId: "verified-engine", + imageRef: "nemoclaw-sandbox-local:alpha-rebuild-preflight", + imageId: `sha256:${"d".repeat(64)}`, + verify: verifyImage, + retainForRecreate, + verifyForCreate: vi.fn(() => true), + finalizeAfterCreate: vi.fn(() => ({ + mutableTagVerified: true, + registryImageRef: null, + })), + abort: vi.fn(() => true), + dispose: vi.fn(() => true), + }); + let buildContext: RetainedImageFixture["buildContext"]; + buildContext = { + buildCtx, + stagedDockerfile, + cleanupBuildCtx: vi.fn(() => { + fs.rmSync(buildCtx, { recursive: true, force: true }); + return true; + }), + origin: "generated", + buildId: "retained-build", + contextFingerprint, + preparedOpenClawLegacyImage: lease, + verifyBuildCtx(this: RetainedImageFixture["buildContext"]) { + return ( + this === buildContext && + this.preparedOpenClawLegacyImage === lease && + fingerprintBuildContext(buildCtx) === contextFingerprint + ); + }, + rebuildTarget: { + agentName: null, + fromDockerfile: null, + }, + }; + return { buildContext, lease, verifyImage, retainForRecreate }; +} + +function sandboxDeleteCallOrder(runOpenshellSpy: ReturnType): number { + const deleteCallIndex = runOpenshellSpy.mock.calls.findIndex( + ([args]) => Array.isArray(args) && args.join(" ") === "sandbox delete -g nemoclaw alpha", + ); + expect(deleteCallIndex).toBeGreaterThanOrEqual(0); + return runOpenshellSpy.mock.invocationCallOrder[deleteCallIndex] ?? Number.POSITIVE_INFINITY; +} + +describe("rebuildSandbox retained OpenClaw image mutation boundary", () => { + beforeEach(resetRebuildFlowTestEnvironment); + afterEach(restoreRebuildFlowTestEnvironment); + + it("keeps the sandbox intact when the retained image drifts at the final delete edge (#7253)", async () => { + const fixture = createRetainedImageFixture([true, true, false]); + const harness = createRebuildFlowHarness({ + rebuildImagePreflightResult: { + ok: true, + imageTag: fixture.lease.imageRef, + prepared: fixture.buildContext, + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow( + "The retained replacement image inputs changed before sandbox deletion. Retry the rebuild.", + ); + + expect(harness.preflightRebuildImageSpy).toHaveBeenCalledOnce(); + expect(harness.backupSandboxStateSpy).toHaveBeenCalledOnce(); + expect(fixture.verifyImage).toHaveBeenCalledTimes(3); + expect(fixture.retainForRecreate).not.toHaveBeenCalled(); + expect(harness.reattachMcpProvidersAfterRebuildAbortSpy).toHaveBeenCalledOnce(); + expect(harness.relockSpy).toHaveBeenCalled(); + expectNoSandboxDelete(harness.runOpenshellSpy); + expect(harness.removeSandboxRegistryEntrySpy).not.toHaveBeenCalled(); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + }); + + it("commits the exact retained image immediately before delete and never rebuilds it afterward (#7253)", async () => { + const fixture = createRetainedImageFixture([true, true, true]); + const harness = createRebuildFlowHarness({ + rebuildImagePreflightResult: { + ok: true, + imageTag: fixture.lease.imageRef, + prepared: fixture.buildContext, + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.preflightRebuildImageSpy).toHaveBeenCalledOnce(); + expect(harness.backupSandboxStateSpy).toHaveBeenCalledOnce(); + expect(fixture.verifyImage).toHaveBeenCalledTimes(3); + expect(fixture.retainForRecreate).toHaveBeenCalledOnce(); + expect(fixture.verifyImage.mock.invocationCallOrder.at(-1)).toBeLessThan( + fixture.retainForRecreate.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + const deleteOrder = sandboxDeleteCallOrder(harness.runOpenshellSpy as never); + expect(fixture.retainForRecreate.mock.invocationCallOrder[0]).toBeLessThan(deleteOrder); + expect(deleteOrder).toBeLessThan( + harness.onboardSpy.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + expect(harness.removeSandboxRegistryEntrySpy).toHaveBeenCalledOnce(); + expect(harness.onboardSpy).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts index f1c635744fc..62b3ca1b445 100644 --- a/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts +++ b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts @@ -7,8 +7,10 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { fingerprintBuildContext } from "../../adapters/fs/build-context-fingerprint"; +import type { PreparedOpenClawLegacyImage } from "../../onboard/build-context-stage"; import { ROOT } from "../../runner"; import type { SandboxBaseImageResolutionMetadata } from "../../sandbox-base-image"; +import type { OpenClawLegacyDockerBinding } from "./rebuild/openclaw-legacy-image"; import { finalizePreparedRebuildImageMessagingPlan, type PreparedRebuildImage, @@ -22,6 +24,7 @@ import { } from "./rebuild-prepared-image-context"; type SuccessfulPreflight = Extract; +const LEGACY_IMAGE_ID = `sha256:${"d".repeat(64)}`; function successful(result: RebuildImagePreflightResult): SuccessfulPreflight { expect(result.ok).toBe(true); @@ -48,11 +51,44 @@ function input(fromDockerfile: string | null) { sandboxGpuDevice: null, errors: [], }, + sandboxName: "alpha", + localPrebuildEnabled: false, gatewayPort: 8080, chatUiUrl: "http://127.0.0.1:18789", }; } +function createLegacyBinding(): OpenClawLegacyDockerBinding { + return Object.freeze({ + dockerEnv: Object.freeze({ + DOCKER_CONFIG: "/home/test/.docker", + DOCKER_CONTEXT: "verified-builder", + }), + engineId: "verified-engine", + }); +} + +function createLegacyLease( + binding: OpenClawLegacyDockerBinding, + imageRef: string, +): PreparedOpenClawLegacyImage { + return Object.freeze({ + dockerEnv: binding.dockerEnv, + engineId: binding.engineId, + imageRef, + imageId: LEGACY_IMAGE_ID, + verify: vi.fn(() => true), + retainForRecreate: vi.fn(() => true), + verifyForCreate: vi.fn(() => true), + finalizeAfterCreate: vi.fn(() => ({ + mutableTagVerified: true, + registryImageRef: null, + })), + abort: vi.fn(() => true), + dispose: vi.fn(() => true), + }); +} + function hermesMessagingPlan() { return { schemaVersion: 1 as const, @@ -180,6 +216,52 @@ describe("preflightRebuildImage", () => { } }); + it("removes a successful BuildKit candidate only with its captured same-engine image ID", async () => { + const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-buildkit-")); + const stagedDockerfile = path.join(buildCtx, "Dockerfile"); + fs.writeFileSync(stagedDockerfile, "FROM scratch\n"); + const binding = createLegacyBinding(); + const buildImage = vi.fn(() => ({ status: 0 }) as never); + const disposeLegacyImage = vi.fn(() => true); + + try { + const result = successful( + await preflightRebuildImage( + { ...input(null), localPrebuildEnabled: true }, + { + stageBuildContext: vi.fn(() => ({ + buildCtx, + stagedDockerfile, + cleanupBuildCtx: vi.fn(() => true), + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "buildkit-success", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + captureLegacyDockerBinding: vi.fn(() => binding), + inspectLegacyImageId: vi.fn(() => LEGACY_IMAGE_ID), + disposeLegacyImage, + }, + ), + ); + + expect(buildImage).toHaveBeenCalledWith( + stagedDockerfile, + result.imageTag, + buildCtx, + expect.objectContaining({ env: binding.dockerEnv }), + ); + expect(result.prepared.preparedOpenClawLegacyImage).toBeUndefined(); + expect(disposeLegacyImage).toHaveBeenCalledWith(binding, result.imageTag, LEGACY_IMAGE_ID); + expect(disposePreparedBuildContext(result.prepared)).toBe(true); + } finally { + fs.rmSync(buildCtx, { recursive: true, force: true }); + } + }); + it.runIf(process.platform !== "win32")( "rejects a symlinked build-context root before the preflight build", async () => { @@ -286,6 +368,390 @@ describe("preflightRebuildImage", () => { } }); + it.each([ + ["error string", "error", false], + ["error buffer", "error", true], + ["stderr string", "stderr", false], + ["stderr buffer", "stderr", true], + ["stdout string", "stdout", false], + ["stdout buffer", "stdout", true], + ] as const)("retains one exact generated OpenClaw legacy image when %s contains the diagnostic (#7111)", async (_case, stream, buffered) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-fallback-")); + const dockerfile = path.join(dir, "Dockerfile"); + fs.writeFileSync(dockerfile, "FROM scratch\n"); + const diagnostic = "ERROR: BuildKit is enabled but the buildx component is missing or broken."; + const buildImage = vi + .fn() + .mockReturnValueOnce({ + status: 1, + [stream]: buffered ? Buffer.from(diagnostic) : diagnostic, + } as never) + .mockReturnValueOnce({ status: 0 } as never); + const cleanupBuildCtx = vi.fn(() => true); + const binding = createLegacyBinding(); + let lease: PreparedOpenClawLegacyImage | null = null; + const createLegacyImage = vi.fn((_binding: OpenClawLegacyDockerBinding, imageRef: string) => { + lease = createLegacyLease(binding, imageRef); + return lease; + }); + const removeImage = vi.fn(() => ({ status: 0 }) as never); + const disposeLegacyImage = vi.fn(() => true); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + try { + const result = successful( + await preflightRebuildImage( + { ...input(null), localPrebuildEnabled: true }, + { + stageBuildContext: vi.fn(() => ({ + buildCtx: dir, + stagedDockerfile: dockerfile, + cleanupBuildCtx, + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "buildx-fallback", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + removeImage, + buildxAvailable: vi.fn(() => false), + captureLegacyDockerBinding: vi.fn(() => binding), + inspectLegacyImageId: vi.fn(() => LEGACY_IMAGE_ID), + createLegacyImage, + disposeLegacyImage, + }, + ), + ); + + expect(buildImage).toHaveBeenCalledTimes(2); + expect(buildImage.mock.calls[0]?.slice(0, 3)).toEqual(buildImage.mock.calls[1]?.slice(0, 3)); + expect(buildImage.mock.calls[0]?.[3]).toEqual( + expect.objectContaining({ cwd: ROOT, env: binding.dockerEnv }), + ); + expect(buildImage.mock.calls[1]?.[3]).toEqual( + expect.objectContaining({ + cwd: ROOT, + env: { ...binding.dockerEnv, DOCKER_BUILDKIT: "0" }, + }), + ); + expect(result.imageTag).toMatch(/^nemoclaw-sandbox-local:/); + expect(result.prepared.preparedOpenClawLegacyImage).toBe(lease); + expect(createLegacyImage).toHaveBeenCalledWith(binding, result.imageTag, LEGACY_IMAGE_ID); + expect(removeImage).not.toHaveBeenCalled(); + expect(disposeLegacyImage).not.toHaveBeenCalled(); + expect(cleanupBuildCtx).not.toHaveBeenCalled(); + expect(verifyPreparedBuildContext(result.prepared)).toBe(true); + expect(disposePreparedBuildContext(result.prepared)).toBe(true); + } finally { + warn.mockRestore(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it.each([ + { + label: "the independent Buildx probe succeeds", + agent: null, + fromDockerfile: null, + origin: "generated" as const, + localPrebuildEnabled: true, + buildxAvailable: true, + capturesBinding: true, + }, + { + label: "the generated target is Hermes", + agent: { name: "hermes" }, + fromDockerfile: null, + origin: "generated" as const, + localPrebuildEnabled: true, + buildxAvailable: false, + capturesBinding: false, + }, + { + label: "local image consumption is unavailable", + agent: null, + fromDockerfile: null, + origin: "generated" as const, + localPrebuildEnabled: false, + buildxAvailable: false, + capturesBinding: false, + }, + { + label: "the Dockerfile is user supplied", + agent: null, + fromDockerfile: "/tmp/Dockerfile.custom", + origin: "custom" as const, + localPrebuildEnabled: true, + buildxAvailable: false, + capturesBinding: false, + }, + ])("does not retry when $label (#7111)", async (testCase) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-no-fallback-")); + const dockerfile = path.join(dir, "Dockerfile"); + fs.writeFileSync(dockerfile, "FROM scratch\n"); + const diagnostic = "ERROR: BuildKit is enabled but the buildx component is missing or broken."; + const buildImage = vi.fn(() => ({ status: 1, stderr: diagnostic }) as never); + const cleanupBuildCtx = vi.fn(() => true); + const binding = createLegacyBinding(); + const captureLegacyDockerBinding = vi.fn(() => binding); + const buildxAvailable = vi.fn(() => testCase.buildxAvailable); + + try { + const result = await preflightRebuildImage( + { + ...input(testCase.fromDockerfile), + agent: testCase.agent as never, + localPrebuildEnabled: testCase.localPrebuildEnabled, + }, + { + stageBuildContext: vi.fn(() => ({ + buildCtx: dir, + stagedDockerfile: dockerfile, + cleanupBuildCtx, + origin: testCase.origin, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "no-fallback", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + removeImage: vi.fn(() => ({ status: 0 }) as never), + buildxAvailable, + captureLegacyDockerBinding, + disposeLegacyImage: vi.fn(() => true), + }, + ); + + expect(result).toEqual({ ok: false, detail: diagnostic }); + expect(buildImage).toHaveBeenCalledOnce(); + expect(captureLegacyDockerBinding.mock.calls).toEqual( + testCase.capturesBinding + ? [ + [ + { + buildDockerEnv: expect.any(Function), + cwd: ROOT, + }, + ], + ] + : [], + ); + expect(buildxAvailable).toHaveBeenCalledTimes(testCase.capturesBinding ? 1 : 0); + expect(cleanupBuildCtx).toHaveBeenCalledOnce(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects generated build-context drift before the legacy retry (#7111)", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-drift-")); + const dockerfile = path.join(dir, "Dockerfile"); + fs.writeFileSync(dockerfile, "FROM scratch\n"); + const buildImage = vi.fn(() => { + fs.writeFileSync(path.join(dir, "mutated"), "changed\n"); + return { + status: 1, + stderr: "ERROR: BuildKit is enabled but the buildx component is missing or broken.", + } as never; + }); + const cleanupBuildCtx = vi.fn(() => true); + + try { + await expect( + preflightRebuildImage( + { ...input(null), localPrebuildEnabled: true }, + { + stageBuildContext: vi.fn(() => ({ + buildCtx: dir, + stagedDockerfile: dockerfile, + cleanupBuildCtx, + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "buildx-drift", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + buildxAvailable: vi.fn(() => false), + captureLegacyDockerBinding: vi.fn(() => createLegacyBinding()), + disposeLegacyImage: vi.fn(() => true), + }, + ), + ).resolves.toEqual({ + ok: false, + detail: "replacement build context changed during preflight", + }); + expect(buildImage).toHaveBeenCalledOnce(); + expect(cleanupBuildCtx).toHaveBeenCalledOnce(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("leaves a unique GC-visible tag when legacy image identity cannot be established (#7253)", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-identity-failure-")); + const dockerfile = path.join(dir, "Dockerfile"); + fs.writeFileSync(dockerfile, "FROM scratch\n"); + const binding = createLegacyBinding(); + const buildImage = vi + .fn() + .mockReturnValueOnce({ + status: 1, + stderr: "ERROR: BuildKit is enabled but the buildx component is missing or broken.", + } as never) + .mockReturnValueOnce({ status: 0 } as never); + const disposeLegacyImage = vi.fn(() => true); + const removeImage = vi.fn(() => ({ status: 0 }) as never); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + try { + const result = await preflightRebuildImage( + { ...input(null), localPrebuildEnabled: true }, + { + stageBuildContext: vi.fn(() => ({ + buildCtx: dir, + stagedDockerfile: dockerfile, + cleanupBuildCtx: vi.fn(() => true), + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "identity-failure", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + removeImage, + buildxAvailable: vi.fn(() => false), + captureLegacyDockerBinding: vi.fn(() => binding), + inspectLegacyImageId: vi.fn(() => { + throw new Error("OpenClaw legacy-image identity could not be verified."); + }), + disposeLegacyImage, + }, + ); + + expect(result).toEqual({ + ok: false, + detail: "OpenClaw legacy-image identity could not be verified.", + }); + expect(disposeLegacyImage).not.toHaveBeenCalled(); + expect(removeImage).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + expect.stringMatching( + /nemoclaw-sandbox-local:.*no verified immutable cleanup identity.*maintenance cleanup/, + ), + ); + } finally { + warn.mockRestore(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("does not retry an unrelated generated-image build failure (#7111)", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-build-no-retry-")); + const dockerfile = path.join(dir, "Dockerfile"); + fs.writeFileSync(dockerfile, "FROM scratch\n"); + const buildImage = vi.fn(() => ({ status: 1, stderr: "registry timeout" }) as never); + const buildxAvailable = vi.fn(() => false); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + try { + await expect( + preflightRebuildImage( + { ...input(null), localPrebuildEnabled: true }, + { + stageBuildContext: vi.fn(() => ({ + buildCtx: dir, + stagedDockerfile: dockerfile, + cleanupBuildCtx: vi.fn(() => true), + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "unrelated-build-failure", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + buildxAvailable, + captureLegacyDockerBinding: vi.fn(() => createLegacyBinding()), + disposeLegacyImage: vi.fn(() => true), + }, + ), + ).resolves.toEqual({ ok: false, detail: "registry timeout" }); + expect(buildImage).toHaveBeenCalledOnce(); + expect(buildxAvailable).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + expect.stringMatching( + /nemoclaw-sandbox-local:.*no verified immutable cleanup identity.*maintenance cleanup/, + ), + ); + } finally { + warn.mockRestore(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("reports the redacted legacy retry failure before the BuildKit diagnostic (#7111)", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-double-fail-")); + const dockerfile = path.join(dir, "Dockerfile"); + const credential = ["legacy", "retry", "credential"].join("-"); + fs.writeFileSync(dockerfile, "FROM scratch\n"); + const buildImage = vi + .fn() + .mockReturnValueOnce({ + status: 1, + stderr: + "ERROR: BuildKit is enabled but the buildx component is missing or broken.\n" + + "x".repeat(9_000), + } as never) + .mockReturnValueOnce({ + status: 1, + stderr: + `legacy build could not read ${os.homedir()}/private-context\n` + + `Authorization: Bearer ${credential}`, + } as never); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + try { + const result = await preflightRebuildImage( + { ...input(null), localPrebuildEnabled: true }, + { + stageBuildContext: vi.fn(() => ({ + buildCtx: dir, + stagedDockerfile: dockerfile, + cleanupBuildCtx: vi.fn(() => true), + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "double-failure", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + buildxAvailable: vi.fn(() => false), + captureLegacyDockerBinding: vi.fn(() => createLegacyBinding()), + disposeLegacyImage: vi.fn(() => true), + }, + ); + + expect(result.ok).toBe(false); + const failure = result as Extract; + expect(failure.detail).toContain("Legacy-builder retry failed"); + expect(failure.detail).toContain("legacy build could not read ~/private-context"); + expect(failure.detail).toContain("Authorization: Bearer "); + expect(failure.detail).not.toContain(credential); + expect(failure.detail.length).toBeLessThan(8_100); + expect(buildImage).toHaveBeenCalledTimes(2); + } finally { + warn.mockRestore(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + it("builds and removes the exact staged custom context on success", async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-custom-preflight-")); const dockerfile = path.join(dir, "Dockerfile.custom"); @@ -306,7 +772,7 @@ describe("preflightRebuildImage", () => { ); expect(buildImage).toHaveBeenCalledWith( expect.stringContaining("Dockerfile"), - expect.stringMatching(/^nemoclaw-rebuild-preflight:/), + expect.stringMatching(/^nemoclaw-sandbox-local:/), expect.any(String), expect.objectContaining({ ignoreError: true }), ); diff --git a/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts b/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts index 1ba0549930d..d2c2abcf83a 100644 --- a/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts @@ -3,7 +3,14 @@ import path from "node:path"; -import { dockerBuild, dockerRmi } from "../../adapters/docker"; +import { + dockerBuild, + dockerRmi, + type DockerBuildOptions, + type DockerRunOptions, + type DockerRunResult, +} from "../../adapters/docker"; +import { dockerSpawnSync } from "../../adapters/docker/exec"; import { fingerprintBuildContext } from "../../adapters/fs/build-context-fingerprint"; import type { AgentDefinition } from "../../agent/defs"; import { createAgentSandbox } from "../../agent/onboard"; @@ -18,6 +25,7 @@ import { } from "../../onboard/reasoning-mode"; import { prepareSandboxDockerfilePatch } from "../../onboard/sandbox-dockerfile-patch-flow"; import type { SandboxGpuConfig } from "../../onboard/sandbox-gpu-mode"; +import { dockerBuildSubprocessEnv, sandboxLocalImageRef } from "../../onboard/sandbox-prebuild"; import { ROOT } from "../../runner"; import { formatBuildFailureDiagnostics, @@ -27,6 +35,13 @@ import { } from "../../sandbox-base-image"; import type { PreservedEnvFile } from "../../state/preserved-env"; import type { ToolDisclosure } from "../../tool-disclosure"; +import { + captureOpenClawLegacyDockerBinding, + createPreparedOpenClawLegacyImage, + disposeOpenClawLegacyDockerImage, + inspectOpenClawLegacyImageId, + type OpenClawLegacyDockerBinding, +} from "./rebuild/openclaw-legacy-image"; import { createBuildContextVerifier, createIdempotentBuildContextCleanup, @@ -46,6 +61,9 @@ type PreflightInput = { toolDisclosure: ToolDisclosure; hermesToolGateways: string[]; sandboxGpuConfig: SandboxGpuConfig; + sandboxName: string; + /** Whether recreation can consume an image built by the host Docker engine. */ + localPrebuildEnabled: boolean; gatewayPort: number; chatUiUrl: string; preResolvedBaseImageMetadata?: SandboxBaseImageResolutionMetadata | null; @@ -54,11 +72,31 @@ type PreflightInput = { type PreflightDeps = { stageBuildContext?: typeof stageCreateSandboxBuildContext; prepareDockerfilePatch?: typeof prepareSandboxDockerfilePatch; - buildImage?: typeof dockerBuild; - removeImage?: typeof dockerRmi; + buildImage?: BuildImage; + removeImage?: RemoveImage; + buildxAvailable?: (process: DockerProofProcess) => boolean; + buildDockerEnv?: () => Record; + captureLegacyDockerBinding?: typeof captureOpenClawLegacyDockerBinding; + inspectLegacyImageId?: typeof inspectOpenClawLegacyImageId; + createLegacyImage?: typeof createPreparedOpenClawLegacyImage; + disposeLegacyImage?: typeof disposeOpenClawLegacyDockerImage; registerExitHandler?: (listener: () => void) => void; }; +type BuildImage = ( + dockerfilePath: string, + tag: string, + contextDir: string, + options: DockerBuildOptions, +) => DockerRunResult; + +type RemoveImage = (imageRef: string, options: NonNullable) => DockerRunResult; + +type DockerProofProcess = { + cwd: string; + env: NodeJS.ProcessEnv; +}; + export type PreparedRebuildImage = FingerprintedPreparedBuildContext & { rebuildTarget: { agentName: string | null; @@ -89,10 +127,97 @@ function resultDetail(result: { ); } +const BUILDX_UNAVAILABLE_DIAGNOSTIC = + "BuildKit is enabled but the buildx component is missing or broken"; + +function hasBuildxUnavailableDiagnostic(result: { + error?: unknown; + stderr?: unknown; + stdout?: unknown; +}): boolean { + return [result.error, result.stderr, result.stdout].some((stream) => { + if (stream == null) return false; + const text = Buffer.isBuffer(stream) ? stream.toString("utf8") : String(stream); + return text.includes(BUILDX_UNAVAILABLE_DIAGNOSTIC); + }); +} + +function legacyRetryFailureDetail( + buildKitResult: Parameters[0], + legacyResult: Parameters[0], +): string { + return formatBuildFailureDiagnostics({ + stderr: + `Legacy-builder retry failed:\n${resultDetail(legacyResult)}\n` + + `Initial BuildKit attempt failed:\n${resultDetail(buildKitResult)}`, + }); +} + +function exactDockerBuild( + dockerfilePath: string, + tag: string, + contextDir: string, + options: DockerBuildOptions, +): DockerRunResult { + const { + env, + ignoreError: _ignoreError, + quiet, + stdio, + suppressOutput: _suppressOutput, + ...spawnOptions + } = options; + return dockerSpawnSync( + ["build", ...(quiet ? ["--quiet"] : []), "-f", dockerfilePath, "-t", tag, contextDir], + { + ...spawnOptions, + cwd: ROOT, + env: { ...env, DOCKER_BUILDKIT: env?.DOCKER_BUILDKIT ?? "1" }, + shell: false, + stdio: stdio ?? ["ignore", "pipe", "pipe"], + }, + ); +} + +function exactDockerRemoveImage( + imageRef: string, + options: NonNullable, +): DockerRunResult { + const { + env, + ignoreError: _ignoreError, + stdio, + suppressOutput: _suppressOutput, + ...spawnOptions + } = options; + return dockerSpawnSync(["rmi", imageRef], { + ...spawnOptions, + cwd: ROOT, + env, + shell: false, + stdio: stdio ?? ["ignore", "pipe", "pipe"], + }); +} + +function defaultBuildxAvailable(process: DockerProofProcess): boolean { + try { + return ( + dockerSpawnSync(["buildx", "version"], { + cwd: process.cwd, + env: process.env, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }).status === 0 + ); + } catch { + return false; + } +} + function removeTemporaryRebuildImage( imageTag: string | null, imageBuilt: boolean, - label: "preflight" | "finalization", + label: "finalization", removeImage: typeof dockerRmi, registerExitHandler: (listener: () => void) => void, ): void { @@ -124,14 +249,25 @@ export async function preflightRebuildImage( ): Promise { const stage = deps.stageBuildContext ?? stageCreateSandboxBuildContext; const preparePatch = deps.prepareDockerfilePatch ?? prepareSandboxDockerfilePatch; - const buildImage = deps.buildImage ?? dockerBuild; - const removeImage = deps.removeImage ?? dockerRmi; + const buildImage = deps.buildImage ?? exactDockerBuild; + const removeImage = deps.removeImage ?? exactDockerRemoveImage; + const buildxAvailable = deps.buildxAvailable ?? defaultBuildxAvailable; + const buildDockerEnv = deps.buildDockerEnv ?? dockerBuildSubprocessEnv; + const captureLegacyDockerBinding = + deps.captureLegacyDockerBinding ?? captureOpenClawLegacyDockerBinding; + const inspectLegacyImageId = deps.inspectLegacyImageId ?? inspectOpenClawLegacyImageId; + const createLegacyImage = deps.createLegacyImage ?? createPreparedOpenClawLegacyImage; + const disposeLegacyImage = deps.disposeLegacyImage ?? disposeOpenClawLegacyDockerImage; const registerExitHandler = deps.registerExitHandler ?? ((listener: () => void) => process.once("exit", listener)); let cleanup: (() => boolean) | null = null; let imageTag: string | null = null; let imageBuilt = false; let retainBuildContext = false; + let dockerEnv: Readonly> | null = null; + let legacyBinding: OpenClawLegacyDockerBinding | null = null; + let legacyImageId: string | null = null; + let preparedLegacyImage: ReturnType | null = null; const previousReasoning = process.env.NEMOCLAW_REASONING; const previousReasoningEffort = process.env[REASONING_EFFORT_ENV]; try { @@ -175,44 +311,163 @@ export async function preflightRebuildImage( warn: () => {}, }); const contextFingerprint = fingerprintBuildContext(staged.buildCtx); - imageTag = `nemoclaw-rebuild-preflight:${String(process.pid)}-${String(Date.now())}`; - const result = buildImage(staged.stagedDockerfile, imageTag, staged.buildCtx, { + const legacyFallbackEligible = + staged.origin === "generated" && + input.agent === null && + input.fromDockerfile === null && + input.localPrebuildEnabled; + if (legacyFallbackEligible) { + legacyBinding = captureLegacyDockerBinding({ buildDockerEnv, cwd: ROOT }); + dockerEnv = legacyBinding.dockerEnv; + } else { + dockerEnv = Object.freeze({ ...buildDockerEnv() }); + } + imageTag = sandboxLocalImageRef( + input.sandboxName, + `rebuild-preflight-${buildId}-${String(process.pid)}-${String(Date.now())}`, + ); + const buildOptions: DockerBuildOptions = { + cwd: ROOT, + env: dockerEnv, ignoreError: true, suppressOutput: true, stdio: ["ignore", "pipe", "pipe"], - }); + }; + const buildKitResult = buildImage( + staged.stagedDockerfile, + imageTag, + staged.buildCtx, + buildOptions, + ); + let result = buildKitResult; + let usedLegacyFallback = false; + if ( + result.status !== 0 && + legacyFallbackEligible && + legacyBinding !== null && + hasBuildxUnavailableDiagnostic(result) && + !buildxAvailable({ cwd: ROOT, env: legacyBinding.dockerEnv }) + ) { + // SOURCE_OF_TRUTH_REVIEW (#7111): the generated OpenClaw final-image + // Dockerfile does not use BuildKit-only instructions. Retry its exact + // fingerprinted bytes once with Docker's compatibility builder only + // after an independent Buildx probe confirms the host CLI lacks it. + // Dockerfile.base, other agents, and custom --from contexts never enter + // this fallback. Remove it when the supported Docker floor no longer + // provides the legacy builder. + if (fingerprintBuildContext(staged.buildCtx) !== contextFingerprint) { + return { ok: false, detail: "replacement build context changed during preflight" }; + } + console.warn( + " Warning: Docker Buildx is unavailable; retrying the generated OpenClaw rebuild image with Docker's legacy builder.", + ); + usedLegacyFallback = true; + result = buildImage(staged.stagedDockerfile, imageTag, staged.buildCtx, { + ...buildOptions, + env: { ...legacyBinding.dockerEnv, DOCKER_BUILDKIT: "0" }, + }); + } + if (result.status !== 0 && usedLegacyFallback) { + return { ok: false, detail: legacyRetryFailureDetail(buildKitResult, result) }; + } if (result.status !== 0) return { ok: false, detail: resultDetail(result) }; imageBuilt = true; if (fingerprintBuildContext(staged.buildCtx) !== contextFingerprint) { return { ok: false, detail: "replacement build context changed during preflight" }; } + if (legacyBinding) { + legacyImageId = inspectLegacyImageId(legacyBinding, imageTag); + } + if (usedLegacyFallback && legacyBinding && legacyImageId) { + preparedLegacyImage = createLegacyImage(legacyBinding, imageTag, legacyImageId); + } retainBuildContext = true; + const verifyFingerprint = createBuildContextVerifier(staged.buildCtx, contextFingerprint); + const prepared: PreparedRebuildImage = { + ...staged, + cleanupBuildCtx: cleanup, + buildId, + dashboardRemoteBindPrepared, + preparedOpenClawLegacyImage: preparedLegacyImage ?? undefined, + contextFingerprint, + verifyBuildCtx(this: PreparedRebuildImage) { + return ( + this === prepared && + this.preparedOpenClawLegacyImage === (preparedLegacyImage ?? undefined) && + verifyFingerprint() + ); + }, + rebuildTarget: { + agentName: input.agent?.name ?? null, + fromDockerfile: input.fromDockerfile ? path.resolve(input.fromDockerfile) : null, + }, + }; return { ok: true, imageTag, - prepared: { - ...staged, - cleanupBuildCtx: cleanup, - buildId, - dashboardRemoteBindPrepared, - contextFingerprint, - verifyBuildCtx: createBuildContextVerifier(staged.buildCtx, contextFingerprint), - rebuildTarget: { - agentName: input.agent?.name ?? null, - fromDockerfile: input.fromDockerfile ? path.resolve(input.fromDockerfile) : null, - }, - }, + prepared, }; } catch (err) { return { ok: false, detail: err instanceof Error ? err.message : String(err) }; } finally { - removeTemporaryRebuildImage( - imageTag, - imageBuilt, - "preflight", - removeImage, - registerExitHandler, - ); + let imageRemoved = false; + if (preparedLegacyImage === null && imageTag !== null) { + try { + imageRemoved = legacyBinding + ? legacyImageId !== null && disposeLegacyImage(legacyBinding, imageTag, legacyImageId) + : removeImage(imageTag, { + cwd: ROOT, + env: dockerEnv ?? undefined, + ignoreError: true, + suppressOutput: true, + }).status === 0; + } catch { + // Best effort; retained-context ownership and environment restoration must continue. + } + } + if ( + preparedLegacyImage === null && + imageTag !== null && + legacyBinding !== null && + legacyImageId === null + ) { + console.warn( + ` Warning: temporary rebuild preflight image '${imageTag}' has no verified immutable cleanup identity; leaving its unique tag for maintenance cleanup.`, + ); + } + if ( + imageBuilt && + preparedLegacyImage === null && + imageTag && + !imageRemoved && + (legacyBinding === null || legacyImageId !== null) + ) { + const retainedImageTag = imageTag; + const retainedDockerEnv = dockerEnv; + const retainedLegacyBinding = legacyBinding; + const retainedLegacyImageId = legacyImageId; + console.warn( + ` Warning: failed to remove temporary rebuild preflight image '${retainedImageTag}'.`, + ); + registerExitHandler(() => { + try { + if (retainedLegacyBinding) { + if (retainedLegacyImageId !== null) { + disposeLegacyImage(retainedLegacyBinding, retainedImageTag, retainedLegacyImageId); + } + } else { + removeImage(retainedImageTag, { + cwd: ROOT, + env: retainedDockerEnv ?? undefined, + ignoreError: true, + suppressOutput: true, + }); + } + } catch { + // Best effort process-exit retry. + } + }); + } if (!retainBuildContext) { try { cleanup?.(); diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index 411b9b9b17f..36c088070ab 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -356,6 +356,133 @@ describe("rebuild destroy phase", () => { expect(relockShieldsIfNeeded).toHaveBeenCalledWith(true); }); + it("reattaches MCP state and keeps the sandbox when the synchronous delete-edge gate fails (#7253)", async () => { + const revalidateBeforeDelete = vi.fn().mockResolvedValue(undefined); + const assertDeleteEdgeUnchanged = vi.fn(); + const validateDeleteEdge = vi.fn(() => ({ + ok: false as const, + message: "Replacement image changed before sandbox deletion.", + code: 7, + })); + const detachedProviderEntries = [{ server: "github" }]; + const scrubbedAdapterEntries = [{ server: "filesystem" }]; + mocks.prepareMcpForRebuild.mockResolvedValue({ + entries: detachedProviderEntries, + detachedProviderEntries, + scrubbedAdapterEntries, + revalidateBeforeDelete, + assertDeleteEdgeUnchanged, + }); + const onDeleted = vi.fn(); + const relockShieldsIfNeeded = vi.fn(() => true); + const bail = vi.fn((message: string): never => { + throw new Error(message); + }); + + await expect( + runRebuildDestroyPhase({ + sandboxName: "alpha", + sandboxEntry: { name: "alpha", agent: "openclaw" }, + staleRecovery: false, + recreateJournal: stubRecreateJournal(), + backupManifest: null, + force: true, + log: vi.fn(), + bail, + relockShieldsIfNeeded, + validateDeleteEdge, + onDeleted, + }), + ).rejects.toThrow("Replacement image changed before sandbox deletion."); + + expect(revalidateBeforeDelete).toHaveBeenCalledOnce(); + expect(assertDeleteEdgeUnchanged).toHaveBeenCalledOnce(); + expect(validateDeleteEdge).toHaveBeenCalledOnce(); + expect(revalidateBeforeDelete.mock.invocationCallOrder[0]).toBeLessThan( + assertDeleteEdgeUnchanged.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + expect(assertDeleteEdgeUnchanged.mock.invocationCallOrder[0]).toBeLessThan( + mocks.getSandbox.mock.invocationCallOrder[1] ?? Number.POSITIVE_INFINITY, + ); + expect(mocks.getSandbox.mock.invocationCallOrder[1]).toBeLessThan( + validateDeleteEdge.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + expect(mocks.runOpenshell).not.toHaveBeenCalled(); + expect(mocks.reattachMcpAfterDeleteFailure).toHaveBeenCalledWith( + "alpha", + detachedProviderEntries, + scrubbedAdapterEntries, + ); + expect(relockShieldsIfNeeded).toHaveBeenCalledWith(true); + expect(bail).toHaveBeenCalledWith("Replacement image changed before sandbox deletion.", 7); + expect(mocks.removeSandboxRegistryEntryWithReceipt).not.toHaveBeenCalled(); + expect(mocks.stopNimContainer).not.toHaveBeenCalled(); + expect(mocks.stopNimContainerByName).not.toHaveBeenCalled(); + expect(onDeleted).not.toHaveBeenCalled(); + }); + + it("runs the synchronous delete-edge gate after registry validation and before delete (#7253)", async () => { + const deleteEdgeEvents: string[] = []; + const revalidateBeforeDelete = vi.fn().mockResolvedValue(undefined); + const assertDeleteEdgeUnchanged = vi.fn(); + const validateDeleteEdge = vi.fn(() => { + deleteEdgeEvents.push("validate"); + return { ok: true as const }; + }); + const log = vi.fn((message: string) => { + deleteEdgeEvents.push( + ...(message.startsWith("Running: openshell sandbox delete") ? ["log"] : []), + ); + }); + mocks.runOpenshell.mockImplementation((args: string[]) => { + deleteEdgeEvents.push(args[1] ?? "unknown"); + return { status: 0, stdout: "", stderr: "" }; + }); + mocks.prepareMcpForRebuild.mockResolvedValue({ + entries: [], + detachedProviderEntries: [], + scrubbedAdapterEntries: [], + revalidateBeforeDelete, + assertDeleteEdgeUnchanged, + }); + + await runRebuildDestroyPhase({ + sandboxName: "alpha", + sandboxEntry: { name: "alpha", agent: "openclaw" }, + staleRecovery: false, + recreateJournal: stubRecreateJournal(), + backupManifest: null, + force: true, + log, + bail: vi.fn((message: string): never => { + throw new Error(message); + }), + relockShieldsIfNeeded: vi.fn(() => true), + validateDeleteEdge, + onDeleted: vi.fn(), + }); + + expect(revalidateBeforeDelete).toHaveBeenCalledOnce(); + expect(assertDeleteEdgeUnchanged).toHaveBeenCalledOnce(); + expect(validateDeleteEdge).toHaveBeenCalledOnce(); + expect(assertDeleteEdgeUnchanged.mock.invocationCallOrder[0]).toBeLessThan( + mocks.getSandbox.mock.invocationCallOrder[1] ?? Number.POSITIVE_INFINITY, + ); + expect(mocks.getSandbox.mock.invocationCallOrder[1]).toBeLessThan( + validateDeleteEdge.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + expect(validateDeleteEdge.mock.invocationCallOrder[0]).toBeLessThan( + mocks.runOpenshell.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + expect(mocks.runOpenshell).toHaveBeenNthCalledWith( + 1, + ["sandbox", "delete", "-g", "nemoclaw", "alpha"], + expect.any(Object), + ); + expect(mocks.reattachMcpAfterDeleteFailure).not.toHaveBeenCalled(); + expect(deleteEdgeEvents).toEqual(["log", "validate", "delete"]); + }); + it("refuses sandbox deletion when read-only MCP state drifts at the delete edge (#7062)", async () => { const revalidateBeforeDelete = vi.fn().mockRejectedValue(new Error("live policy drifted")); mocks.prepareMcpForRebuild.mockResolvedValue({ @@ -408,6 +535,7 @@ describe("rebuild destroy phase", () => { .mockReturnValueOnce({ status: 9, stdout: "", stderr: "delete failed" }) .mockReturnValueOnce({ status: 0, stdout: "Phase: Ready\n", stderr: "" }); const onDeleted = vi.fn(); + const abortPreparedImageRecreate = vi.fn(() => true); const relockShieldsIfNeeded = vi.fn(() => true); const bail = vi.fn((message: string): never => { throw new Error(message); @@ -424,6 +552,8 @@ describe("rebuild destroy phase", () => { log: vi.fn(), bail, relockShieldsIfNeeded, + validateDeleteEdge: vi.fn(() => ({ ok: true as const })), + abortPreparedImageRecreate, onDeleted, }), ).rejects.toThrow("Failed to delete sandbox."); @@ -438,6 +568,7 @@ describe("rebuild destroy phase", () => { expect(mocks.stopNimContainer).not.toHaveBeenCalled(); expect(mocks.stopNimContainerByName).not.toHaveBeenCalled(); expect(relockShieldsIfNeeded).toHaveBeenCalledWith(true); + expect(abortPreparedImageRecreate).toHaveBeenCalledOnce(); expect(mocks.runOpenshell).toHaveBeenNthCalledWith( 2, ["sandbox", "get", "-g", "nemoclaw", "alpha"], @@ -627,6 +758,7 @@ describe("rebuild destroy phase", () => { .mockReturnValueOnce({ status: 0, stdout: "Phase: Terminating\n", stderr: "" }); const onDeleted = vi.fn(); const onDeleteStateAmbiguous = vi.fn(); + const abortPreparedImageRecreate = vi.fn(() => true); const relockShieldsIfNeeded = vi.fn(() => true); await expect( @@ -642,6 +774,8 @@ describe("rebuild destroy phase", () => { throw new Error(message); }), relockShieldsIfNeeded, + validateDeleteEdge: vi.fn(() => ({ ok: true as const })), + abortPreparedImageRecreate, onDeleted, onDeleteStateAmbiguous, }), @@ -649,6 +783,7 @@ describe("rebuild destroy phase", () => { expect(onDeleted).not.toHaveBeenCalled(); expect(onDeleteStateAmbiguous).toHaveBeenCalledOnce(); + expect(abortPreparedImageRecreate).toHaveBeenCalledOnce(); expect(mocks.stopNimContainer).not.toHaveBeenCalled(); expect(mocks.stopNimContainerByName).not.toHaveBeenCalled(); expect(mocks.reattachMcpAfterDeleteFailure).not.toHaveBeenCalled(); diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.ts index 9688c4cfa82..cea8c5dcd64 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.ts @@ -44,6 +44,8 @@ export interface RebuildDestroyPhaseInput { force?: boolean; validateAfterMcpPreparation?: () => Promise; validateAtDeleteEdge?: () => RebuildDeleteValidationResult; + validateDeleteEdge?: () => RebuildDeleteValidationResult; + abortPreparedImageRecreate?: () => boolean; onDeleted: () => void; onDeleteStateAmbiguous?: () => void; } @@ -236,6 +238,7 @@ export async function runRebuildDestroyPhase( relockShieldsIfNeeded, validateAfterMcpPreparation, validateAtDeleteEdge, + validateDeleteEdge, onDeleted, } = input; const deleteTarget = resolveRebuildDeleteTarget(sandboxName, input.sandboxEntry); @@ -321,6 +324,15 @@ export async function runRebuildDestroyPhase( if (!mcpPreparation) return null; const rebuildDetachedMcpProviderEntries = mcpPreparation.detachedProviderEntries; const rebuildScrubbedMcpAdapterEntries = mcpPreparation.scrubbedAdapterEntries; + const abortPreparedImageRecreate = (): void => { + try { + if (input.abortPreparedImageRecreate?.() === false) { + log("The unused retained replacement image could not be released safely."); + } + } catch { + log("The unused retained replacement image could not be released safely."); + } + }; // Exec-unavailable recovery deliberately made no MCP mutation during // preparation. Re-prove target, policy, provider, and registry state while @@ -416,6 +428,38 @@ export async function runRebuildDestroyPhase( log(`Skipping delete: gateway ${gatewayName} reports '${sandboxName}' already absent`); } else { log(`Running: openshell sandbox delete -g ${gatewayName} ${sandboxName}`); + + // Keep this synchronous and at the final delete edge. A successful check + // must be the last executed code before OpenShell receives the delete + // command. + if (validateDeleteEdge) { + let validation: RebuildDeleteValidationResult; + try { + validation = validateDeleteEdge(); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + log(`Unexpected rebuild delete-edge validation failure: ${redactFull(detail)}`); + validation = { + ok: false, + message: "Replacement image validation failed before sandbox deletion.", + }; + } + if (!validation.ok) { + const mcpRecoveryFailure = await reattachMcpAfterDeleteFailure( + sandboxName, + rebuildDetachedMcpProviderEntries, + rebuildScrubbedMcpAdapterEntries, + ); + relockShieldsIfNeeded(true); + bail( + mcpRecoveryFailure + ? `${validation.message} MCP provider recovery also failed: ${mcpRecoveryFailure}` + : validation.message, + validation.code, + ); + return null; + } + } } const deleteResult = sourcePresence === "missing" @@ -451,6 +495,7 @@ export async function runRebuildDestroyPhase( console.error(" State backup is preserved at: " + backupManifest.backupPath); } relockShieldsIfNeeded(true); + abortPreparedImageRecreate(); bail( mcpRecoveryFailure ? `Failed to delete sandbox; MCP provider recovery also failed: ${mcpRecoveryFailure}` @@ -469,6 +514,7 @@ export async function runRebuildDestroyPhase( console.error(" State backup is preserved at: " + backupManifest.backupPath); } input.onDeleteStateAmbiguous?.(); + abortPreparedImageRecreate(); bail( "Sandbox delete failed and exact post-delete state is ambiguous; recovery state was preserved.", deleteResult.status || 1, @@ -486,6 +532,7 @@ export async function runRebuildDestroyPhase( console.error(" State backup is preserved at: " + backupManifest.backupPath); } input.onDeleteStateAmbiguous?.(); + abortPreparedImageRecreate(); bail("Sandbox deletion could not be confirmed."); return null; } diff --git a/src/lib/actions/sandbox/rebuild-openclaw-legacy-image.test.ts b/src/lib/actions/sandbox/rebuild-openclaw-legacy-image.test.ts new file mode 100644 index 00000000000..f70b49b5299 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-openclaw-legacy-image.test.ts @@ -0,0 +1,505 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SpawnSyncOptions } from "node:child_process"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { dockerSpawnSync } from "../../adapters/docker/exec"; +import { + captureOpenClawLegacyDockerBinding, + createPreparedOpenClawLegacyImage, + disposeOpenClawLegacyDockerImage, + inspectOpenClawLegacyImageId, + type OpenClawLegacyDockerBindingDeps, +} from "./rebuild/openclaw-legacy-image"; + +const IMAGE_ID = `sha256:${"a".repeat(64)}`; +const OTHER_IMAGE_ID = `sha256:${"b".repeat(64)}`; +const IMAGE_REF = "nemoclaw-sandbox-local:alpha-rebuild"; +const CANONICAL_CWD = "/canonical/nemoclaw"; + +type DockerState = { + context: string; + contextStatus: number; + engineId: string; + infoStatus: number; + tagImageId: string; + directImageId: string; + rmiStatus: number; +}; + +function dockerResult(stdout: string, status = 0): ReturnType { + return { + error: undefined, + output: [null, stdout, ""], + pid: 123, + signal: null, + status, + stderr: "", + stdout, + }; +} + +function createDockerHarness(overrides: Partial = {}) { + const state: DockerState = { + context: "desktop-linux", + contextStatus: 0, + engineId: "engine-a", + infoStatus: 0, + tagImageId: IMAGE_ID, + directImageId: IMAGE_ID, + rmiStatus: 0, + ...overrides, + }; + const exitListeners = new Set<() => void>(); + const runDocker = vi.fn((args: readonly string[], _options: SpawnSyncOptions = {}) => { + switch (args.join(" ")) { + case "context show": + return dockerResult(state.context, state.contextStatus); + case "info --format {{.ID}}": + return dockerResult(state.engineId, state.infoStatus); + case `image inspect --format {{.Id}} ${IMAGE_REF}`: + return dockerResult(state.tagImageId); + case `image inspect --format {{.Id}} ${IMAGE_ID}`: + return dockerResult(state.directImageId); + case `image inspect --format {{.Id}} ${OTHER_IMAGE_ID}`: + return dockerResult(OTHER_IMAGE_ID); + case `rmi ${IMAGE_ID}`: + return dockerResult("", state.rmiStatus); + default: + throw new Error(`Unexpected Docker command: ${args.join(" ")}`); + } + }); + const deps: OpenClawLegacyDockerBindingDeps = { + cwd: CANONICAL_CWD, + buildDockerEnv: () => ({ PATH: "/usr/bin" }), + runDocker: runDocker as typeof dockerSpawnSync, + addExitListener: (listener) => { + exitListeners.add(listener); + }, + removeExitListener: (listener) => { + exitListeners.delete(listener); + }, + }; + return { deps, exitListeners, runDocker, state }; +} + +function commandCalls(harness: ReturnType): string[][] { + return harness.runDocker.mock.calls.map(([args]) => [...args]); +} + +afterEach(() => vi.unstubAllEnvs()); + +describe("OpenClaw legacy-image Docker binding", () => { + it("pins the current Docker context when no selector is explicit", () => { + const harness = createDockerHarness(); + + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + + expect(binding).toEqual({ + dockerEnv: { DOCKER_CONTEXT: "desktop-linux", PATH: "/usr/bin" }, + engineId: "engine-a", + }); + expect(Object.isFrozen(binding)).toBe(true); + expect(Object.isFrozen(binding.dockerEnv)).toBe(true); + expect(commandCalls(harness)).toEqual([ + ["context", "show"], + ["info", "--format", "{{.ID}}"], + ]); + const contextOptions = harness.runDocker.mock.calls[0]?.[1] as SpawnSyncOptions; + const infoOptions = harness.runDocker.mock.calls[1]?.[1] as SpawnSyncOptions; + expect(Object.isFrozen(contextOptions.env)).toBe(true); + expect(infoOptions.env).toBe(binding.dockerEnv); + expect(infoOptions).toMatchObject({ + encoding: "utf-8", + shell: false, + stdio: ["ignore", "pipe", "pipe"], + timeout: 30_000, + }); + }); + + it("preserves an explicit Docker host without consulting ambient context", () => { + const harness = createDockerHarness(); + harness.deps.buildDockerEnv = () => ({ + DOCKER_HOST: "unix:///var/run/docker.sock", + PATH: "/usr/bin", + }); + + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + + expect(binding.dockerEnv).toEqual({ + DOCKER_HOST: "unix:///var/run/docker.sock", + PATH: "/usr/bin", + }); + expect(commandCalls(harness)).toEqual([["info", "--format", "{{.ID}}"]]); + }); + + it("uses the Docker subprocess allowlist before freezing the selector", () => { + vi.stubEnv("DOCKER_HOST", "unix:///var/run/docker.sock"); + vi.stubEnv("KUBECONFIG", "/secret/kubeconfig"); + vi.stubEnv("OPENSHELL_SECRET", "secret"); + vi.stubEnv("GITHUB_TOKEN", "secret"); + const harness = createDockerHarness(); + delete harness.deps.buildDockerEnv; + + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + + expect(binding.dockerEnv.DOCKER_HOST).toBe("unix:///var/run/docker.sock"); + expect(binding.dockerEnv).not.toHaveProperty("KUBECONFIG"); + expect(binding.dockerEnv).not.toHaveProperty("OPENSHELL_SECRET"); + expect(binding.dockerEnv).not.toHaveProperty("GITHUB_TOKEN"); + }); + + it("keeps a relative Docker config on canonical engine A for identity and cleanup", () => { + const harness = createDockerHarness(); + const runEngineA = harness.runDocker.getMockImplementation()!; + const engineBCalls: string[][] = []; + harness.deps.buildDockerEnv = () => ({ + DOCKER_CONFIG: "relative/docker-config", + PATH: "/usr/bin", + }); + harness.runDocker.mockImplementation( + (args: readonly string[], options: SpawnSyncOptions = {}) => { + const runEngineB = () => { + engineBCalls.push([...args]); + switch (args.join(" ")) { + case "context show": + return dockerResult("ambient-context"); + case "info --format {{.ID}}": + return dockerResult("engine-b"); + case `image inspect --format {{.Id}} ${IMAGE_REF}`: + case `image inspect --format {{.Id}} ${IMAGE_ID}`: + return dockerResult(OTHER_IMAGE_ID); + case `rmi ${IMAGE_ID}`: + return dockerResult(""); + default: + throw new Error(`Unexpected engine B Docker command: ${args.join(" ")}`); + } + }; + return options.cwd === CANONICAL_CWD ? runEngineA(args, options) : runEngineB(); + }, + ); + + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const imageId = inspectOpenClawLegacyImageId(binding, IMAGE_REF); + const removed = disposeOpenClawLegacyDockerImage(binding, IMAGE_REF, imageId); + + expect(binding).toEqual({ + dockerEnv: { + DOCKER_CONFIG: "relative/docker-config", + DOCKER_CONTEXT: "desktop-linux", + PATH: "/usr/bin", + }, + engineId: "engine-a", + }); + expect(imageId).toBe(IMAGE_ID); + expect(removed).toBe(true); + expect(engineBCalls).toEqual([]); + expect(commandCalls(harness)).toEqual([ + ["context", "show"], + ["info", "--format", "{{.ID}}"], + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ["rmi", IMAGE_ID], + ]); + for (const [, options] of harness.runDocker.mock.calls) { + expect(options?.cwd).toBe(CANONICAL_CWD); + } + }); + + it.each([ + ["a malformed explicit selector", { DOCKER_CONTEXT: "bad\ncontext" }, {}, /malformed/], + ["a failed context query", {}, { contextStatus: 1 }, /context could not be captured/], + ["an empty context", {}, { context: "" }, /context could not be captured/], + ["a failed engine query", {}, { infoStatus: 1 }, /engine identity/], + ["a malformed engine identity", {}, { engineId: "engine a" }, /engine identity/], + ] as const)("fails closed for %s", (_scenario, env, overrides, expected) => { + const harness = createDockerHarness(overrides); + harness.deps.buildDockerEnv = () => ({ ...env }); + + expect(() => captureOpenClawLegacyDockerBinding(harness.deps)).toThrow(expected); + }); +}); + +describe("OpenClaw legacy-image identity and lifecycle", () => { + it("inspects a mutable tag and its direct immutable ID using the exact bound environment", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + harness.runDocker.mockClear(); + + expect(inspectOpenClawLegacyImageId(binding, IMAGE_REF)).toBe(IMAGE_ID); + + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ]); + for (const [, options] of harness.runDocker.mock.calls) { + expect(options?.env).toBe(binding.dockerEnv); + } + }); + + it("constructs only a verified frozen lease from an authentic binding", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + harness.runDocker.mockClear(); + + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID.toUpperCase()); + + expect(lease).toMatchObject({ + dockerEnv: binding.dockerEnv, + engineId: "engine-a", + imageId: IMAGE_ID, + imageRef: IMAGE_REF, + }); + expect(Object.isFrozen(lease)).toBe(true); + expect(harness.exitListeners.size).toBe(1); + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ]); + }); + + it("rejects counterfeit bindings, non-tag references, and mismatched image identity", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + + expect(() => + createPreparedOpenClawLegacyImage( + Object.freeze({ dockerEnv: Object.freeze({}), engineId: "engine-a" }), + IMAGE_REF, + IMAGE_ID, + ), + ).toThrow(/not authentic/); + expect(() => createPreparedOpenClawLegacyImage(binding, IMAGE_ID, IMAGE_ID)).toThrow( + /mutable Docker tag/, + ); + harness.state.tagImageId = OTHER_IMAGE_ID; + expect(() => createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID)).toThrow( + /identity changed/, + ); + expect(harness.exitListeners.size).toBe(0); + }); + + it("guards method receivers and finalizes only after create verification", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + harness.runDocker.mockClear(); + + expect(lease.verifyForCreate()).toBe(false); + expect(lease.finalizeAfterCreate()).toBeNull(); + expect(lease.verify.call({ ...lease })).toBe(false); + expect(lease.retainForRecreate.call({ ...lease })).toBe(false); + expect(lease.abort.call({ ...lease })).toBe(false); + expect(harness.runDocker).not.toHaveBeenCalled(); + + expect(lease.verify()).toBe(true); + expect(lease.retainForRecreate()).toBe(true); + expect(harness.exitListeners.size).toBe(1); + expect(lease.verify()).toBe(false); + expect(lease.retainForRecreate()).toBe(false); + expect(lease.finalizeAfterCreate()).toBeNull(); + expect(lease.verifyForCreate()).toBe(true); + expect(lease.finalizeAfterCreate.call({ ...lease })).toBeNull(); + expect(lease.finalizeAfterCreate()).toEqual({ + mutableTagVerified: true, + registryImageRef: null, + }); + expect(lease.finalizeAfterCreate()).toBeNull(); + expect(harness.exitListeners.size).toBe(0); + expect(lease.dispose()).toBe(true); + expect(commandCalls(harness)).not.toContainEqual(["rmi", IMAGE_ID]); + }); + + it("refuses retention when the directly addressed immutable image no longer agrees", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + harness.runDocker.mockClear(); + harness.state.directImageId = OTHER_IMAGE_ID; + + expect(lease.retainForRecreate()).toBe(false); + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ]); + expect(harness.exitListeners.size).toBe(1); + }); + + it.each([ + ["the named context is repointed", (state: DockerState) => (state.engineId = "engine-b")], + ["the mutable tag is retargeted", (state: DockerState) => (state.tagImageId = OTHER_IMAGE_ID)], + ["the immutable image is removed", (state: DockerState) => (state.directImageId = "")], + ])("refuses creation after %s", (_scenario, mutate) => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + expect(lease.retainForRecreate()).toBe(true); + harness.runDocker.mockClear(); + + mutate(harness.state); + + expect(lease.verifyForCreate()).toBe(false); + expect(lease.finalizeAfterCreate()).toBeNull(); + expect(commandCalls(harness)).not.toContainEqual(["rmi", IMAGE_ID]); + }); + + it.each([ + ["removed", ""], + ["retargeted", OTHER_IMAGE_ID], + ])("suppresses registry cleanup when the mutable tag is %s after create", (_scenario, tagImageId) => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + expect(lease.retainForRecreate()).toBe(true); + expect(lease.verifyForCreate()).toBe(true); + harness.runDocker.mockClear(); + harness.state.tagImageId = tagImageId; + + expect(lease.finalizeAfterCreate()).toEqual({ + mutableTagVerified: false, + registryImageRef: null, + }); + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ]); + expect(lease.abort()).toBe(true); + expect(commandCalls(harness)).not.toContainEqual(["rmi", IMAGE_ID]); + }); + + it("fails finalization on engine drift and keeps exact abort cleanup retryable", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + expect(lease.retainForRecreate()).toBe(true); + expect(lease.verifyForCreate()).toBe(true); + harness.runDocker.mockClear(); + harness.state.engineId = "engine-b"; + + expect(lease.finalizeAfterCreate()).toBeNull(); + expect(lease.abort()).toBe(false); + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["info", "--format", "{{.ID}}"], + ]); + expect(harness.exitListeners.size).toBe(1); + + harness.state.engineId = "engine-a"; + expect(lease.abort()).toBe(true); + expect(commandCalls(harness)).toContainEqual(["rmi", IMAGE_ID]); + expect(harness.exitListeners.size).toBe(0); + }); + + it.each([ + ["removed", ""], + ["retargeted", OTHER_IMAGE_ID], + ])("aborts a retained unused image by exact ID when its tag is %s", (_scenario, tagImageId) => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + expect(lease.retainForRecreate()).toBe(true); + harness.runDocker.mockClear(); + harness.state.tagImageId = tagImageId; + + expect(lease.abort()).toBe(true); + expect(lease.abort()).toBe(true); + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ["rmi", IMAGE_ID], + ]); + expect(harness.exitListeners.size).toBe(0); + }); + + it("cleans an unretained lease by immutable ID exactly once", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + harness.runDocker.mockClear(); + + expect(lease.dispose()).toBe(true); + expect(lease.dispose()).toBe(true); + + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ["rmi", IMAGE_ID], + ]); + expect(harness.exitListeners.size).toBe(0); + }); + + it("disposes a retained lease that never reached finalization", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + expect(lease.retainForRecreate()).toBe(true); + harness.runDocker.mockClear(); + + expect(lease.dispose()).toBe(true); + + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ["rmi", IMAGE_ID], + ]); + expect(harness.exitListeners.size).toBe(0); + }); + + it("refuses cleanup on engine drift and leaves exit cleanup retryable", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + harness.runDocker.mockClear(); + harness.state.engineId = "engine-b"; + + expect(lease.dispose()).toBe(false); + expect(commandCalls(harness)).toEqual([["info", "--format", "{{.ID}}"]]); + expect(harness.exitListeners.size).toBe(1); + + harness.state.engineId = "engine-a"; + const [exitListener] = harness.exitListeners; + exitListener?.(); + expect(commandCalls(harness)).toContainEqual(["rmi", IMAGE_ID]); + expect(harness.exitListeners.size).toBe(0); + }); + + it("cleans a pre-lease image only after same-engine tag and captured direct-ID proof", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + harness.runDocker.mockClear(); + + expect(disposeOpenClawLegacyDockerImage(binding, IMAGE_REF, IMAGE_ID)).toBe(true); + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ["rmi", IMAGE_ID], + ]); + + harness.runDocker.mockClear(); + harness.state.tagImageId = OTHER_IMAGE_ID; + expect(disposeOpenClawLegacyDockerImage(binding, IMAGE_REF, IMAGE_ID)).toBe(false); + expect(commandCalls(harness)).not.toContainEqual(["rmi", IMAGE_ID]); + }); + + it("refuses tag-authorized cleanup when no immutable ID was captured", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + harness.runDocker.mockClear(); + harness.state.tagImageId = OTHER_IMAGE_ID; + + expect(disposeOpenClawLegacyDockerImage(binding, IMAGE_REF)).toBe(false); + expect(commandCalls(harness)).toEqual([]); + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts index 9f8e11183a1..bbd1a923da2 100644 --- a/src/lib/actions/sandbox/rebuild-pipeline.ts +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -29,7 +29,9 @@ import { runRebuildPreflightPhase, } from "./rebuild-preflight-phase"; import { + abortPreparedImageRecreate, disposePreparedBuildContext, + retainPreparedImageForRecreate, verifyPreparedBuildContext, } from "./rebuild-prepared-image-context"; import { @@ -217,14 +219,14 @@ async function rebuildSandboxUnlocked( }; } - // The post-delete create must consume the exact context that passed the - // image preflight. Revalidate at the last safe point so mutation of the - // retained copy cannot cross the destructive boundary. + // The post-delete create must consume the exact context and retained + // image that passed preflight. Revalidate before any destroy-phase + // mutation so drift cannot cross the destructive boundary. if (preparedImage && !verifyPreparedBuildContext(preparedImage)) { printRebuildPreflightFailure( - "the retained replacement image context changed after preflight.", + "the retained replacement image inputs changed after preflight.", "Retry the rebuild so the replacement inputs can be staged again.", - "Replacement sandbox image context changed before delete", + "Replacement sandbox image inputs changed before delete", bail, ); return; @@ -271,6 +273,8 @@ async function rebuildSandboxUnlocked( return; } + const preparedImageForAbort = preparedImage; + const mcpPreparation = await runRebuildDestroyPhase({ sandboxName, sandboxEntry, @@ -282,6 +286,13 @@ async function rebuildSandboxUnlocked( bail, relockShieldsIfNeeded, validateAfterMcpPreparation: async () => { + if (preparedImage && !verifyPreparedBuildContext(preparedImage)) { + return { + ok: false, + message: + "The retained replacement image inputs changed after MCP preparation. Retry the rebuild.", + }; + } const providerReconfigure = recreateOptions.rebuildProviderReconfigure; if (providerReconfigure && !hydrateCredentialEnv(providerReconfigure.credentialEnv)) { return { @@ -314,6 +325,27 @@ async function rebuildSandboxUnlocked( ); }, validateAtDeleteEdge: () => revalidateRebuildRouteBeforeDelete(routePreflightReceipt), + validateDeleteEdge: () => { + if (!preparedImage) return { ok: true }; + if (!verifyPreparedBuildContext(preparedImage)) { + return { + ok: false, + message: + "The retained replacement image inputs changed before sandbox deletion. Retry the rebuild.", + }; + } + if (!retainPreparedImageForRecreate(preparedImage)) { + return { + ok: false, + message: + "The retained replacement image could not be committed to sandbox recreation. Retry the rebuild.", + }; + } + return { ok: true }; + }, + abortPreparedImageRecreate: preparedImageForAbort + ? () => abortPreparedImageRecreate(preparedImageForAbort) + : undefined, onDeleted: () => { sandboxStillExists = false; }, diff --git a/src/lib/actions/sandbox/rebuild-prepared-image-context.ts b/src/lib/actions/sandbox/rebuild-prepared-image-context.ts index c95145e3cbd..95c60f9513d 100644 --- a/src/lib/actions/sandbox/rebuild-prepared-image-context.ts +++ b/src/lib/actions/sandbox/rebuild-prepared-image-context.ts @@ -28,7 +28,29 @@ export function createIdempotentBuildContextCleanup(cleanup: () => boolean): () /** Confirm that a retained private context still matches the prebuilt bytes. */ export function verifyPreparedBuildContext(prepared: FingerprintedPreparedBuildContext): boolean { try { - return fingerprintBuildContext(prepared.buildCtx) === prepared.contextFingerprint; + if (!prepared.verifyBuildCtx.call(prepared)) return false; + if (fingerprintBuildContext(prepared.buildCtx) !== prepared.contextFingerprint) return false; + return prepared.preparedOpenClawLegacyImage?.verify() ?? true; + } catch { + return false; + } +} + +/** Commit a verified retained image to recreation at the synchronous delete edge. */ +export function retainPreparedImageForRecreate( + prepared: FingerprintedPreparedBuildContext, +): boolean { + try { + return prepared.preparedOpenClawLegacyImage?.retainForRecreate() ?? true; + } catch { + return false; + } +} + +/** Release an unused retained image through its bound immutable cleanup path. */ +export function abortPreparedImageRecreate(prepared: FingerprintedPreparedBuildContext): boolean { + try { + return prepared.preparedOpenClawLegacyImage?.abort() ?? true; } catch { return false; } @@ -50,5 +72,12 @@ export function createBuildContextVerifier( /** Dispose retained build inputs after onboarding consumes them or rebuild aborts. */ export function disposePreparedBuildContext(prepared: FingerprintedPreparedBuildContext): boolean { - return prepared.cleanupBuildCtx(); + const imageDisposed = abortPreparedImageRecreate(prepared); + let contextDisposed = false; + try { + contextDisposed = prepared.cleanupBuildCtx(); + } catch { + contextDisposed = false; + } + return imageDisposed && contextDisposed; } diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.ts b/src/lib/actions/sandbox/rebuild-target-runtime.ts index e92f7d3da04..126116c74e2 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.ts @@ -21,6 +21,7 @@ import { readGatewayProviderMetadata, } from "../../onboard/gateway-provider-metadata"; import { resolveSandboxGpuConfig } from "../../onboard/sandbox-gpu-mode"; +import { resolveSandboxPrebuildEnabled } from "../../onboard/sandbox-prebuild"; import { agentSupportsWebSearchProvider } from "../../onboard/web-search-support"; import { redact } from "../../security/redact"; import { @@ -174,8 +175,8 @@ export async function preflightRebuildTargetRuntime( ); return { ok: false }; } + const dockerDriverGateway = isLinuxDockerDriverGatewayEnabled(); try { - const dockerDriverGateway = isLinuxDockerDriverGatewayEnabled(); const selectedRoute = initialDockerGpuRoute( resolveDockerGpuRoutePlan(sandboxGpuConfig, { dockerDriverGateway, @@ -213,6 +214,8 @@ export async function preflightRebuildTargetRuntime( toolDisclosure: target.durableConfig.toolDisclosure, hermesToolGateways: target.hermesToolGateways, sandboxGpuConfig, + sandboxName: sb.name, + localPrebuildEnabled: resolveSandboxPrebuildEnabled(process.env, dockerDriverGateway), preResolvedBaseImageMetadata: recreateOptions.preResolvedBaseImageMetadata ?? null, gatewayPort: recreateOptions.targetGatewayPort, chatUiUrl: managesDashboard diff --git a/src/lib/actions/sandbox/rebuild/openclaw-legacy-image.ts b/src/lib/actions/sandbox/rebuild/openclaw-legacy-image.ts new file mode 100644 index 00000000000..360cc99b455 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild/openclaw-legacy-image.ts @@ -0,0 +1,378 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SpawnSyncOptions } from "node:child_process"; + +import { dockerSpawnSync } from "../../../adapters/docker/exec"; +import type { + PreparedOpenClawLegacyImage, + PreparedOpenClawLegacyImageFinalization, +} from "../../../onboard/build-context-stage"; +import { isImmutableDockerImageId } from "../../../onboard/openshell-docker-sandbox-containers"; +import { dockerBuildSubprocessEnv } from "../../../onboard/sandbox-prebuild"; + +const DOCKER_IDENTITY_TIMEOUT_MS = 30_000; +const SAFE_VALUE_MAX_LENGTH = 4096; + +export interface OpenClawLegacyDockerBinding { + readonly dockerEnv: Readonly>; + readonly engineId: string; +} + +export interface OpenClawLegacyDockerBindingDeps { + cwd: string; + buildDockerEnv?(): Record; + runDocker?: typeof dockerSpawnSync; + addExitListener?(listener: () => void): void; + removeExitListener?(listener: () => void): void; +} + +type InternalBinding = { + readonly dockerOptions: Readonly; + readonly runDocker: typeof dockerSpawnSync; + readonly addExitListener: (listener: () => void) => void; + readonly removeExitListener: (listener: () => void) => void; +}; + +const internalBindings = new WeakMap(); + +function isSafeSingleLineValue(value: string): boolean { + return ( + value.length > 0 && + value.length <= SAFE_VALUE_MAX_LENGTH && + !/[\s\u0000-\u001f\u007f]/.test(value) + ); +} + +function normalizeDockerOutput(output: string | Buffer | null | undefined): string { + return typeof output === "string" ? output.trim() : String(output ?? "").trim(); +} + +function commandSucceeded(result: ReturnType): boolean { + return result.error == null && result.status === 0; +} + +function runDockerCapture(internal: InternalBinding, args: readonly string[]): string | null { + try { + const result = internal.runDocker(args, internal.dockerOptions); + if (!commandSucceeded(result)) return null; + return normalizeDockerOutput(result.stdout); + } catch { + return null; + } +} + +function requireInternalBinding(binding: OpenClawLegacyDockerBinding): InternalBinding { + const internal = internalBindings.get(binding); + if (!internal) { + throw new Error("OpenClaw legacy-image Docker binding is not authentic."); + } + return internal; +} + +function validateMutableImageRef(imageRef: string): string { + if ( + !isSafeSingleLineValue(imageRef) || + isImmutableDockerImageId(imageRef) || + imageRef.includes("@") || + imageRef.lastIndexOf(":") <= imageRef.lastIndexOf("/") + ) { + throw new Error("OpenClaw legacy-image reference must be a mutable Docker tag."); + } + return imageRef; +} + +function validateImageId(imageId: string): string { + const normalized = imageId.trim().toLowerCase(); + if (!isImmutableDockerImageId(normalized)) { + throw new Error("OpenClaw legacy-image ID must be an immutable sha256 identifier."); + } + return normalized; +} + +function currentEngineMatches( + binding: OpenClawLegacyDockerBinding, + internal: InternalBinding, +): boolean { + const currentEngineId = runDockerCapture(internal, ["info", "--format", "{{.ID}}"]); + return currentEngineId !== null && currentEngineId === binding.engineId; +} + +function inspectImageId(internal: InternalBinding, imageSelector: string): string | null { + const output = runDockerCapture(internal, [ + "image", + "inspect", + "--format", + "{{.Id}}", + imageSelector, + ]); + if (output === null || !isImmutableDockerImageId(output)) return null; + return output.toLowerCase(); +} + +function verifyImageIdentity( + binding: OpenClawLegacyDockerBinding, + internal: InternalBinding, + imageRef: string, + imageId: string, +): boolean { + if (!currentEngineMatches(binding, internal)) return false; + if (inspectImageId(internal, imageRef) !== imageId) return false; + return inspectImageId(internal, imageId) === imageId; +} + +/** + * Pin the Docker selector used by a rebuild. The default context is made + * explicit so a later ambient `docker context use` cannot redirect the lease. + */ +export function captureOpenClawLegacyDockerBinding( + deps: OpenClawLegacyDockerBindingDeps, +): OpenClawLegacyDockerBinding { + const runDocker = deps.runDocker ?? dockerSpawnSync; + const initialEnv = (deps.buildDockerEnv ?? dockerBuildSubprocessEnv)(); + const dockerEnv: Record = { ...initialEnv }; + const explicitDockerHost = dockerEnv.DOCKER_HOST; + const explicitDockerContext = dockerEnv.DOCKER_CONTEXT; + + for (const selector of [explicitDockerHost, explicitDockerContext]) { + if (selector !== undefined && !isSafeSingleLineValue(selector)) { + throw new Error("OpenClaw legacy-image Docker selector is malformed."); + } + } + + const addExitListener = + deps.addExitListener ?? ((listener: () => void) => process.once("exit", listener)); + const removeExitListener = + deps.removeExitListener ?? ((listener: () => void) => process.removeListener("exit", listener)); + + if (explicitDockerHost === undefined && explicitDockerContext === undefined) { + const contextOptions: SpawnSyncOptions = Object.freeze({ + cwd: deps.cwd, + encoding: "utf-8", + env: Object.freeze({ ...dockerEnv }), + shell: false, + stdio: ["ignore", "pipe", "pipe"] as SpawnSyncOptions["stdio"], + timeout: DOCKER_IDENTITY_TIMEOUT_MS, + }); + let contextResult: ReturnType; + try { + contextResult = runDocker(["context", "show"], contextOptions); + } catch { + throw new Error("OpenClaw legacy-image Docker context could not be captured."); + } + const context = commandSucceeded(contextResult) + ? normalizeDockerOutput(contextResult.stdout) + : ""; + if (!isSafeSingleLineValue(context)) { + throw new Error("OpenClaw legacy-image Docker context could not be captured."); + } + dockerEnv.DOCKER_CONTEXT = context; + } + + const frozenDockerEnv = Object.freeze({ ...dockerEnv }); + const dockerOptions: Readonly = Object.freeze({ + cwd: deps.cwd, + encoding: "utf-8", + env: frozenDockerEnv, + shell: false, + stdio: ["ignore", "pipe", "pipe"] as SpawnSyncOptions["stdio"], + timeout: DOCKER_IDENTITY_TIMEOUT_MS, + }); + const internal: InternalBinding = Object.freeze({ + dockerOptions, + runDocker, + addExitListener, + removeExitListener, + }); + const engineId = runDockerCapture(internal, ["info", "--format", "{{.ID}}"]); + if (engineId === null || !isSafeSingleLineValue(engineId)) { + throw new Error("OpenClaw legacy-image Docker engine identity could not be captured."); + } + + const binding = Object.freeze({ + dockerEnv: frozenDockerEnv, + engineId, + }); + internalBindings.set(binding, internal); + return binding; +} + +/** Resolve a mutable tag to one directly inspectable immutable ID on the bound engine. */ +export function inspectOpenClawLegacyImageId( + binding: OpenClawLegacyDockerBinding, + imageRef: string, +): string { + const internal = requireInternalBinding(binding); + const validatedImageRef = validateMutableImageRef(imageRef); + if (!currentEngineMatches(binding, internal)) { + throw new Error("OpenClaw legacy-image Docker engine identity changed."); + } + const imageId = inspectImageId(internal, validatedImageRef); + if (imageId === null || inspectImageId(internal, imageId) !== imageId) { + throw new Error("OpenClaw legacy-image identity could not be verified."); + } + return imageId; +} + +/** + * Remove an unleased preflight image only when the bound engine, mutable tag, + * and directly addressed immutable ID still agree. + */ +export function disposeOpenClawLegacyDockerImage( + binding: OpenClawLegacyDockerBinding, + imageRef: string, + imageId?: string, +): boolean { + const internal = requireInternalBinding(binding); + let validatedImageRef: string; + let expectedImageId: string; + try { + validatedImageRef = validateMutableImageRef(imageRef); + if (imageId === undefined) return false; + expectedImageId = validateImageId(imageId); + } catch { + return false; + } + if (!currentEngineMatches(binding, internal)) return false; + const taggedImageId = inspectImageId(internal, validatedImageRef); + if (taggedImageId !== expectedImageId) return false; + if (inspectImageId(internal, expectedImageId) !== expectedImageId) return false; + + try { + const result = internal.runDocker(["rmi", expectedImageId], internal.dockerOptions); + return commandSucceeded(result); + } catch { + return false; + } +} + +/** + * Remove only the expected immutable image from the bound engine. + * + * A retained lease can outlive its mutable tag. Cleanup therefore proves the + * engine and direct immutable ID without consulting or removing whatever the + * tag may reference now. + */ +function disposeRetainedOpenClawLegacyDockerImage( + binding: OpenClawLegacyDockerBinding, + internal: InternalBinding, + imageId: string, +): boolean { + if (!currentEngineMatches(binding, internal)) return false; + if (inspectImageId(internal, imageId) !== imageId) return false; + + try { + const result = internal.runDocker(["rmi", imageId], internal.dockerOptions); + return commandSucceeded(result); + } catch { + return false; + } +} + +/** + * Create a retained-image lease after proving the tag and immutable ID still + * resolve on the exact Docker engine that built them. + */ +export function createPreparedOpenClawLegacyImage( + binding: OpenClawLegacyDockerBinding, + imageRef: string, + imageId: string, +): PreparedOpenClawLegacyImage { + const internal = requireInternalBinding(binding); + const validatedImageRef = validateMutableImageRef(imageRef); + const validatedImageId = validateImageId(imageId); + if (!verifyImageIdentity(binding, internal, validatedImageRef, validatedImageId)) { + throw new Error("OpenClaw legacy-image identity changed before lease creation."); + } + + let state: "prepared" | "retained" | "finalized" | "disposed" = "prepared"; + let verifiedForCreate = false; + let exitListenerRegistered = false; + let lease: PreparedOpenClawLegacyImage; + + const removeExitListener = (): void => { + if (!exitListenerRegistered) return; + internal.removeExitListener(exitListener); + exitListenerRegistered = false; + }; + const exitListener = (): void => { + lease.abort(); + }; + const abortLease = (): boolean => { + if (state === "finalized" || state === "disposed") { + removeExitListener(); + return true; + } + const removed = + state === "retained" + ? disposeRetainedOpenClawLegacyDockerImage(binding, internal, validatedImageId) + : disposeOpenClawLegacyDockerImage(binding, validatedImageRef, validatedImageId); + if (!removed) return false; + state = "disposed"; + removeExitListener(); + return true; + }; + + lease = Object.freeze({ + dockerEnv: binding.dockerEnv, + engineId: binding.engineId, + imageRef: validatedImageRef, + imageId: validatedImageId, + verify(this: PreparedOpenClawLegacyImage): boolean { + return ( + this === lease && + state === "prepared" && + verifyImageIdentity(binding, internal, validatedImageRef, validatedImageId) + ); + }, + retainForRecreate(this: PreparedOpenClawLegacyImage): boolean { + if (this !== lease || state !== "prepared") return false; + if (!verifyImageIdentity(binding, internal, validatedImageRef, validatedImageId)) + return false; + state = "retained"; + return true; + }, + verifyForCreate(this: PreparedOpenClawLegacyImage): boolean { + if (this !== lease || state !== "retained") return false; + verifiedForCreate = verifyImageIdentity( + binding, + internal, + validatedImageRef, + validatedImageId, + ); + return verifiedForCreate; + }, + finalizeAfterCreate( + this: PreparedOpenClawLegacyImage, + ): PreparedOpenClawLegacyImageFinalization | null { + if (this !== lease || state !== "retained" || !verifiedForCreate) return null; + if (!currentEngineMatches(binding, internal)) return null; + if (inspectImageId(internal, validatedImageId) !== validatedImageId) return null; + const mutableTagVerified = inspectImageId(internal, validatedImageRef) === validatedImageId; + + // The registry does not persist the bound Docker selector and engine ID. + // A later ambient context could therefore redirect tag-based deletion. + // Keep the unique tag for maintenance GC, but do not register it for + // per-sandbox cleanup. + const finalization: PreparedOpenClawLegacyImageFinalization = Object.freeze({ + registryImageRef: null, + mutableTagVerified, + }); + state = "finalized"; + removeExitListener(); + return finalization; + }, + abort(this: PreparedOpenClawLegacyImage): boolean { + if (this !== lease) return false; + return abortLease(); + }, + dispose(this: PreparedOpenClawLegacyImage): boolean { + if (this !== lease) return false; + return abortLease(); + }, + }); + + internal.addExitListener(exitListener); + exitListenerRegistered = true; + return lease; +} diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index b625740adb2..26b30471f10 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2244,7 +2244,7 @@ async function createSandboxWithBaseImageResolution( ? { extraProviders: createIntent.extraProviders, staleExtraProviders: [] } : planRegisteredExtraProviders(GATEWAY_NAME, { runOpenshell }); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - const resolvedCreateIntent = createIntent?.resolved ?? (await sandboxCreateIntentResolver.resolve({ sandboxName, inferenceProvider: provider, enabledChannels, webSearchConfig, agent, sandboxGpuConfig: effectiveSandboxGpuConfig, resourceProfile, hermesToolGateways, extraProviders: extraProviderPlan.extraProviders, staleExtraProviders: extraProviderPlan.staleExtraProviders, baselineExclusions: sandboxRegistration.baselineExclusionsForCreate(sandboxName), ...(createIntent?.reuseRegisteredCredentials ? { reuseRegisteredCredentials: true } : {}), ...(createIntent?.policyTier !== undefined ? { policyTier: createIntent.policyTier } : {}) })); + const resolvedCreateIntent = sandboxGpuCreateFlow.resolveRetainedOpenClawCreateIntent(createIntent?.resolved ?? (await sandboxCreateIntentResolver.resolve({ sandboxName, inferenceProvider: provider, enabledChannels, webSearchConfig, agent, sandboxGpuConfig: effectiveSandboxGpuConfig, resourceProfile, hermesToolGateways, extraProviders: extraProviderPlan.extraProviders, staleExtraProviders: extraProviderPlan.staleExtraProviders, baselineExclusions: sandboxRegistration.baselineExclusionsForCreate(sandboxName), ...(createIntent?.reuseRegisteredCredentials ? { reuseRegisteredCredentials: true } : {}), ...(createIntent?.policyTier !== undefined ? { policyTier: createIntent.policyTier } : {}) })), effectiveSandboxGpuConfig, isLinuxDockerDriverGatewayEnabled(), preparedBuildContext?.preparedOpenClawLegacyImage); const messagingCapabilities = await sandboxCreateIntentResolver.rebind( { sandboxName, @@ -2701,7 +2701,8 @@ async function createSandboxWithBaseImageResolution( manageDashboard, openshellShellCommand, openshellArgv, - prebuild: { buildCtx, buildId, dockerDriverGateway, origin }, + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + prebuild: { buildCtx, buildId, dockerDriverGateway, origin, preparedOpenClawLegacyImage: preparedBuildContext?.preparedOpenClawLegacyImage }, }); const restoreBackupPath = pendingStateRestore?.manifest?.backupPath ?? pendingStateRestoreBackupPath; @@ -2712,6 +2713,7 @@ async function createSandboxWithBaseImageResolution( route: selectedGpuRoute, firstCreateOutput, registryImageRef, + retainedImageFinalization, } = await sandboxGpuCreateFlow.runSandboxGpuCreateFlow( { sandboxName, @@ -2794,11 +2796,8 @@ async function createSandboxWithBaseImageResolution( } // openshell tags images with seconds; buildId is ms. Parse actual tag from output. Fixes #2672. - const resolvedImageTag = - registryImageRef ?? - prebuild.imageRef ?? - buildContext.extractBuiltImageRef(`${firstCreateOutput}\n${createResult.output}`) ?? - resolveSandboxImageTagFromCreateOutput(`${firstCreateOutput}\n${createResult.output}`, buildId); + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const resolvedImageTag = sandboxGpuCreateFlow.resolveCreatedSandboxRegistryImageRef(retainedImageFinalization, [registryImageRef, prebuild.imageRef, buildContext.extractBuiltImageRef(`${firstCreateOutput}\n${createResult.output}`), resolveSandboxImageTagFromCreateOutput(`${firstCreateOutput}\n${createResult.output}`, buildId)]); const sandboxRuntimeFields = getSandboxRuntimeRegistryFields(effectiveSandboxGpuConfig); recreateRuntime.recordCreated(); finalizeCreatedSandbox( diff --git a/src/lib/onboard/build-context-stage.ts b/src/lib/onboard/build-context-stage.ts index 994f6d8a308..6a329c9e64d 100644 --- a/src/lib/onboard/build-context-stage.ts +++ b/src/lib/onboard/build-context-stage.ts @@ -37,10 +37,31 @@ export interface CreateSandboxBuildContextResult extends StagedBuildContext { cleanupBuildCtx(): boolean; } +/** Exact local Docker image retained across an OpenClaw rebuild recreation. */ +export interface PreparedOpenClawLegacyImage { + readonly dockerEnv: Readonly>; + readonly engineId: string; + readonly imageRef: string; + readonly imageId: string; + verify(): boolean; + retainForRecreate(): boolean; + verifyForCreate(): boolean; + finalizeAfterCreate(): PreparedOpenClawLegacyImageFinalization | null; + abort(): boolean; + dispose(): boolean; +} + +/** Safe registry bookkeeping produced only after final retained-image proof. */ +export interface PreparedOpenClawLegacyImageFinalization { + readonly registryImageRef: null; + readonly mutableTagVerified: boolean; +} + /** Exact staged and patched context transferred from rebuild preflight to create. */ export interface PreparedSandboxBuildContext extends CreateSandboxBuildContextResult { buildId: string; dashboardRemoteBindPrepared?: boolean; + preparedOpenClawLegacyImage?: PreparedOpenClawLegacyImage; /** Recheck retained bytes at the final one-shot consumption boundary. */ verifyBuildCtx?(): boolean; /** Exact recorded target authorized to consume a generic rebuild handoff. */ diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index 496ee830448..e274ad42ad5 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -8,7 +8,7 @@ import { getDockerGpuPatchNetworkMode, printDockerGpuProofFailure, } from "./docker-gpu-patch"; -import type { DockerGpuPatchMode } from "./docker-gpu-patch-types"; +import type { DockerGpuPatchDeps, DockerGpuPatchMode } from "./docker-gpu-patch-types"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; import type { ManagedBootstrapRuntimePatch } from "./managed-bootstrap/runtime-create"; @@ -390,6 +390,9 @@ export type GpuSandboxAfterReadyOptions = { reportGpuProofFailure?: boolean; selectedMode: ManagedBootstrapRuntimePatch["selectedMode"]; runCaptureOpenshell: (args: string[], opts?: Record) => string; + dockerCapture?: DockerGpuPatchDeps["dockerCapture"]; + dockerLogs?: DockerGpuPatchDeps["dockerLogs"]; + env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform; log?: (message: string) => void; @@ -452,6 +455,8 @@ export async function verifyGpuSandboxAccessAfterReady( asDockerGpuPatchMode(options.selectedMode()), { runCaptureOpenshell: options.runCaptureOpenshell, + dockerCapture: options.dockerCapture, + dockerLogs: options.dockerLogs, additionalSummaryLines: adaptDockerGpuRouteForPatch(options.selectedRoute) .additionalSummaryLines, }, diff --git a/src/lib/onboard/docker-gpu-patch.ts b/src/lib/onboard/docker-gpu-patch.ts index d54847a7af6..ac678042261 100644 --- a/src/lib/onboard/docker-gpu-patch.ts +++ b/src/lib/onboard/docker-gpu-patch.ts @@ -157,23 +157,25 @@ function patchedContainerIdFromContext( } function snapshotInspectDeps( - deps: Pick, -): Pick { + deps: Pick, +): Pick { // `depsWithDefaults` spreads the caller's `deps`, so passing an explicit // `dockerCapture: undefined` would shadow the module's default Docker // adapter and disable downstream `docker ps`/`inspect`/`logs` capture. // Build the inner deps object with only the keys the caller actually // supplied so defaults stay in place. - const inner: Pick = {}; + const inner: Pick = + {}; if (deps.runCaptureOpenshell) inner.runCaptureOpenshell = deps.runCaptureOpenshell; if (deps.dockerCapture) inner.dockerCapture = deps.dockerCapture; + if (deps.dockerLogs) inner.dockerLogs = deps.dockerLogs; return inner; } export function printDockerGpuPatchFailureAndExit( sandboxName: string, error: unknown, - deps: Pick & { + deps: Pick & { context?: DockerGpuPatchFailureContext | null; selectedMode?: DockerGpuPatchMode | null; additionalSummaryLines?: readonly string[]; @@ -228,7 +230,7 @@ export function printDockerGpuPatchFailureAndExit( export function printDockerGpuReadinessFailure( sandboxName: string, selectedMode: DockerGpuPatchMode | null, - deps: Pick & { + deps: Pick & { context?: DockerGpuPatchFailureContext | null; additionalSummaryLines?: readonly string[]; }, @@ -263,7 +265,7 @@ export function printDockerGpuProofFailure( sandboxName: string, error: unknown, selectedMode: DockerGpuPatchMode | null, - deps: Pick & { + deps: Pick & { context?: DockerGpuPatchFailureContext | null; additionalSummaryLines?: readonly string[]; }, diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index 6ddea611aa9..d6c0aa4b335 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -3,6 +3,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import * as dockerAdapters from "../adapters/docker"; +import { createDockerGpuInspectFixture } from "./__test-helpers__/docker-gpu-patch-fixtures"; import type { DockerGpuPatchFailureContext, DockerGpuPatchResult } from "./docker-gpu-patch"; import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; @@ -42,7 +44,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { vi.restoreAllMocks(); }); - it("retains the backup after reconnect and removes it only after post-Ready commit", async () => { + it("defers backup removal until waitForSupervisorReconnectIfNeeded sees supervisorReady=true", () => { const deps = makeDeps(); const result = deferredCreateResult(); const recreatePatch = vi.fn(() => result); @@ -84,57 +86,13 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.waitForSupervisorReconnectIfNeeded(); expect(waitForSupervisor).toHaveBeenCalledTimes(1); - expect(finalizeBackup).not.toHaveBeenCalled(); - - await patch.commitAfterReady(); expect(finalizeBackup).toHaveBeenCalledTimes(1); expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: true }, deps); expect(capturePreRollbackDiagnostics).not.toHaveBeenCalled(); expect(onPatchFailureExit).not.toHaveBeenCalled(); }); - it("reports a failed post-Ready rollback instead of treating it as restored", async () => { - const deps = makeDeps(); - const result = deferredCreateResult(); - const finalizeBackup = vi.fn(() => ({ - backupRemoved: false, - rolledBack: false, - })); - const onPatchFailureExit = vi.fn(); - const patch = createDockerGpuSandboxCreatePatch({ - route: "compatibility", - sandboxName: "alpha", - timeoutSecs: 60, - deps, - overrides: { - findContainerIds: vi.fn(() => ["existing-container"]), - recreatePatch: vi.fn(() => result), - waitForSupervisor: vi.fn(() => true), - finalizeBackup, - onPatchFailureExit, - }, - }); - - patch.maybeApplyDuringCreate(); - patch.waitForSupervisorReconnectIfNeeded(); - await patch.rollbackManagedStartupAfterCreateFailure(); - - expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: false }, deps); - expect(onPatchFailureExit).toHaveBeenCalledWith( - "alpha", - expect.objectContaining({ - message: expect.stringContaining("pre-patch container was not restored"), - }), - expect.objectContaining({ - context: expect.objectContaining({ - backupContainerName: result.backupContainerName, - rolledBack: false, - }), - }), - ); - }); - - it("refuses compatibility success when the backup container cannot be removed", async () => { + it("refuses compatibility success when the backup container cannot be removed", () => { const deps = makeDeps(); const result = deferredCreateResult(); const onPatchFailureExit = vi.fn(); @@ -157,14 +115,11 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); patch.waitForSupervisorReconnectIfNeeded(); - expect(onPatchFailureExit).not.toHaveBeenCalled(); - - await patch.commitAfterReady(); expect(onPatchFailureExit).toHaveBeenCalledOnce(); expect(onPatchFailureExit.mock.calls[0]?.[1]).toEqual( expect.objectContaining({ - message: expect.stringContaining("rollback backup"), + message: expect.stringContaining("backup container"), }), ); expect(onPatchFailureExit.mock.calls[0]?.[2]).toEqual( @@ -292,7 +247,126 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(onPatchFailureExit).not.toHaveBeenCalled(); }); - it("records patchError when recreate throws and exitOnPatchError reports it via printDockerGpuPatchFailureAndExit", async () => { + it("uses the injected Docker capture for default container discovery", () => { + const deps = makeDeps(); + const recreatePatch = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { recreatePatch }, + }); + + patch.maybeApplyDuringCreate(); + + expect(deps.dockerCapture).toHaveBeenCalledWith( + expect.arrayContaining(["ps", "-a", "--filter", "label=openshell.ai/sandbox-name=alpha"]), + expect.objectContaining({ ignoreError: true }), + ); + expect(recreatePatch).not.toHaveBeenCalled(); + }); + + it("keeps polling recreation and rollback on bound engine A when ambient adapters select B", () => { + const failOnEngineB = () => { + throw new Error("ambient engine B must not receive Docker operations"); + }; + const engineBSpies = [ + vi.spyOn(dockerAdapters, "dockerCapture").mockImplementation(failOnEngineB), + vi.spyOn(dockerAdapters, "dockerRun").mockImplementation(failOnEngineB), + vi.spyOn(dockerAdapters, "dockerRunDetached").mockImplementation(failOnEngineB), + vi.spyOn(dockerAdapters, "dockerRename").mockImplementation(failOnEngineB), + vi.spyOn(dockerAdapters, "dockerRm").mockImplementation(failOnEngineB), + vi.spyOn(dockerAdapters, "dockerStart").mockImplementation(failOnEngineB), + vi.spyOn(dockerAdapters, "dockerStop").mockImplementation(failOnEngineB), + ]; + const dockerCaptureOutput: Readonly> = { + ps: "old-container-id\n", + inspect: JSON.stringify([createDockerGpuInspectFixture()]), + }; + const dockerCapture = vi.fn( + (args: readonly string[]) => dockerCaptureOutput[args[0] ?? ""] ?? "", + ); + const dockerRun = vi.fn(() => ({ status: 0, stdout: "engine-a-probe\n" })); + const dockerRunDetached = vi.fn(() => ({ + status: 1, + stderr: "engine A forced recreate failure", + })); + const dockerRename = vi.fn(() => ({ status: 0 })); + const dockerRm = vi.fn(() => ({ status: 0 })); + const dockerStart = vi.fn(() => ({ status: 0 })); + const dockerStop = vi.fn(() => ({ status: 0 })); + const onPatchFailureExit = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + dockerDesktopWsl: false, + deps: { + runOpenshell: vi.fn(() => ({ status: 0 })), + runCaptureOpenshell: vi.fn(() => ""), + sleep: vi.fn(), + dockerCapture, + dockerRun, + dockerRunDetached, + dockerRename, + dockerRm, + dockerStart, + dockerStop, + now: () => new Date("2026-07-29T00:00:00Z"), + detectSandboxFallbackDns: () => null, + readDir: () => null, + readFile: () => null, + }, + overrides: { onPatchFailureExit }, + }); + + patch.maybeApplyDuringCreate(); + patch.exitOnPatchError(); + + expect(dockerCapture).toHaveBeenCalledWith( + ["inspect", "--type", "container", "old-container-id"], + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRun).toHaveBeenCalledWith( + expect.arrayContaining(["create", "--gpus", "all"]), + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerStop).toHaveBeenCalledWith( + "old-container-id", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRename).toHaveBeenCalledWith( + "old-container-id", + expect.stringContaining("openshell-alpha-nemoclaw-gpu-backup-"), + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRunDetached).toHaveBeenCalledWith( + expect.arrayContaining(["--name", "openshell-alpha", "--gpus", "all"]), + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerStop).toHaveBeenCalledWith( + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRm).toHaveBeenCalledWith( + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRename).toHaveBeenCalledWith( + expect.stringContaining("openshell-alpha-nemoclaw-gpu-backup-"), + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerStart).toHaveBeenCalledWith( + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + expect(onPatchFailureExit).toHaveBeenCalledOnce(); + for (const engineBSpy of engineBSpies) expect(engineBSpy).not.toHaveBeenCalled(); + }); + + it("records patchError when recreate throws and exitOnPatchError reports it via printDockerGpuPatchFailureAndExit", () => { const deps = makeDeps(); const recreatePatch = vi.fn(() => { throw new Error("docker rename failed"); @@ -318,7 +392,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); expect(patch.createFailureMessage()).toMatch(/Docker GPU patch failed/); - await patch.exitOnPatchError(); + patch.exitOnPatchError(); expect(onPatchFailureExit).toHaveBeenCalledTimes(1); // Supervisor wait must be skipped because needsSupervisorWait stayed false. patch.waitForSupervisorReconnectIfNeeded(); @@ -326,7 +400,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(finalizeBackup).not.toHaveBeenCalled(); }); - it("hard-stops a structured failed GPU proof on the compatibility route", async () => { + it("hard-stops a structured failed GPU proof on the compatibility route", () => { const deps = makeDeps(); const patch = createDockerGpuSandboxCreatePatch({ route: "compatibility", @@ -338,7 +412,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { }, }); - await expect( + expect(() => patch.verifyGpuOrExit(() => ({ status: "failed", cudaVerified: false, @@ -346,38 +420,6 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { detail: "No devices were found", at: "2026-07-07T00:00:00.000Z", })), - ).rejects.toThrow("Sandbox GPU proof returned failed status: nvidia-smi when available"); - }); - - it("reports a failed rollback after GPU-proof diagnostics", async () => { - const deps = makeDeps(); - const result = deferredCreateResult(); - const patch = createDockerGpuSandboxCreatePatch({ - route: "compatibility", - sandboxName: "alpha", - timeoutSecs: 60, - deps, - overrides: { - findContainerIds: vi.fn(() => ["existing-container"]), - recreatePatch: vi.fn(() => result), - waitForSupervisor: vi.fn(() => true), - finalizeBackup: vi.fn(() => ({ - backupRemoved: false, - rolledBack: false, - })), - }, - }); - - patch.maybeApplyDuringCreate(); - patch.waitForSupervisorReconnectIfNeeded(); - - await expect( - patch.verifyGpuOrExit(() => { - throw new Error("nvidia-smi failed"); - }), - ).rejects.toThrow("nvidia-smi failed"); - expect(console.error).toHaveBeenCalledWith( - expect.stringContaining("pre-patch container was not restored"), - ); + ).toThrow("Sandbox GPU proof returned failed status: nvidia-smi when available"); }); }); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index 52c15d0b5be..9554140f726 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -40,10 +40,8 @@ export { resolveDockerGpuSandboxCreatePlan, } from "./docker-gpu-sandbox-create-plan"; -type DockerGpuSandboxCreateDeps = Pick< - DockerGpuPatchDeps, - "runOpenshell" | "runCaptureOpenshell" | "sleep" | "dockerCapture" | "dockerRun" | "dockerStop" ->; +type DockerGpuSandboxCreateDeps = DockerGpuPatchDeps & + Required>; type WaitSupervisorFn = typeof waitForOpenShellSupervisorReconnect; type FindContainerIdsFn = typeof findOpenShellDockerSandboxContainerIds; @@ -265,10 +263,7 @@ export function createDockerGpuSandboxCreatePatch( ` OpenShell Docker container detected; recreating it with ${patchTarget} before readiness wait...`, ); try { - applyPatch({ - runCaptureOpenshell: options.deps.runCaptureOpenshell, - sleep: options.deps.sleep, - }); + applyPatch(options.deps); } catch (error) { patchError = error; } @@ -469,6 +464,7 @@ export function createDockerGpuSandboxCreatePatch( printDockerGpuReadinessFailure(options.sandboxName, selectedMode(), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, + dockerLogs: options.deps.dockerLogs, context: failureContext(), additionalSummaryLines: routeAdapter.additionalSummaryLines, }); @@ -499,6 +495,7 @@ export function createDockerGpuSandboxCreatePatch( printDockerGpuProofFailure(sandboxName, failure, selectedMode(), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, + dockerLogs: options.deps.dockerLogs, context: currentFailureContext, additionalSummaryLines: routeAdapter.additionalSummaryLines, }); @@ -525,6 +522,7 @@ export function createDockerGpuSandboxCreatePatch( printDockerGpuProofFailure(sandboxName, failure, selectedMode(), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, + dockerLogs: options.deps.dockerLogs, context: routeAdapter.enabled ? currentFailureContext : null, additionalSummaryLines: routeAdapter.additionalSummaryLines, }); diff --git a/src/lib/onboard/gateway-sandbox-reachability.ts b/src/lib/onboard/gateway-sandbox-reachability.ts index 81689206f2f..001ec551138 100644 --- a/src/lib/onboard/gateway-sandbox-reachability.ts +++ b/src/lib/onboard/gateway-sandbox-reachability.ts @@ -86,6 +86,9 @@ export interface SandboxBridgeReachabilityOptions { port?: number; timeoutSec?: number; probeImage?: string; + dockerCaptureImpl?: typeof dockerCapture; + dockerRunImpl?: typeof dockerRun; + ensureImageCachedImpl?: typeof ensureProbeImageCached; runImpl?: (args: readonly string[], timeoutMs: number) => SandboxBridgeProbeRunResult; inspectNetworkImpl?: (networkName: string) => DockerBridgeNetworkInfo | undefined; usesHostGatewayRouteImpl?: () => boolean; @@ -126,25 +129,31 @@ function parseDockerNetworkIpamConfig(raw: string): DockerBridgeNetworkInfo | un ); } -function defaultInspectNetwork(networkName: string): DockerBridgeNetworkInfo | undefined { - const raw = dockerCapture( - ["network", "inspect", "--format", "{{json .IPAM.Config}}", networkName], - { ignoreError: true }, - ); +function defaultInspectNetwork( + networkName: string, + capture: typeof dockerCapture = dockerCapture, +): DockerBridgeNetworkInfo | undefined { + const raw = capture(["network", "inspect", "--format", "{{json .IPAM.Config}}", networkName], { + ignoreError: true, + }); return parseDockerNetworkIpamConfig(raw); } -function defaultUsesHostGatewayRoute(): boolean { +function defaultUsesHostGatewayRoute(capture: typeof dockerCapture = dockerCapture): boolean { if (process.platform !== "linux") return true; - const info = dockerCapture( + const info = capture( ["info", "--format", "{{.OperatingSystem}}\n{{range .Labels}}{{.}}\n{{end}}"], { ignoreError: true }, ); return /Docker Desktop|com\.docker\.desktop\./i.test(info); } -function defaultRunImpl(args: readonly string[], timeoutMs: number): SandboxBridgeProbeRunResult { - const result = dockerRun(args, { +function defaultRunImpl( + args: readonly string[], + timeoutMs: number, + runDocker: typeof dockerRun = dockerRun, +): SandboxBridgeProbeRunResult { + const result = runDocker(args, { timeout: timeoutMs, ignoreError: true, suppressOutput: true, @@ -265,9 +274,15 @@ export async function isSandboxBridgeGatewayReachable( const port = opts.port ?? GATEWAY_PORT; const timeoutSec = opts.timeoutSec ?? DEFAULT_PROBE_TIMEOUT_SEC; const probeImage = opts.probeImage ?? DEFAULT_PROBE_IMAGE; - const inspectNetwork = opts.inspectNetworkImpl ?? defaultInspectNetwork; - const usesHostGatewayRoute = opts.usesHostGatewayRouteImpl ?? defaultUsesHostGatewayRoute; - const runImpl = opts.runImpl ?? defaultRunImpl; + const inspectNetwork = + opts.inspectNetworkImpl ?? + ((name: string) => defaultInspectNetwork(name, opts.dockerCaptureImpl)); + const usesHostGatewayRoute = + opts.usesHostGatewayRouteImpl ?? (() => defaultUsesHostGatewayRoute(opts.dockerCaptureImpl)); + const runImpl = + opts.runImpl ?? + ((args: readonly string[], timeoutMs: number) => + defaultRunImpl(args, timeoutMs, opts.dockerRunImpl)); const network = inspectNetwork(networkName); const route = buildOpenShellDockerRoute(networkName, network, usesHostGatewayRoute()); @@ -293,7 +308,9 @@ export async function isSandboxBridgeGatewayReachable( // skip the pre-pull there unless the test supplies an explicit // ensureImageCachedOverride. if (opts.ensureImageCachedOverride !== undefined || opts.runImpl === undefined) { - const cached = opts.ensureImageCachedOverride ?? ensureProbeImageCached(probeImage); + const cached = + opts.ensureImageCachedOverride ?? + (opts.ensureImageCachedImpl ?? ensureProbeImageCached)(probeImage); if (!cached.ok) { // A wedged docker daemon (inspect_unavailable) is a fatal Docker // outage, not a probe/pull uncertainty — keep onboarding from diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.ts b/src/lib/onboard/managed-bootstrap/runtime-create.ts index 6ffcc028966..613e0d16bca 100644 --- a/src/lib/onboard/managed-bootstrap/runtime-create.ts +++ b/src/lib/onboard/managed-bootstrap/runtime-create.ts @@ -21,12 +21,12 @@ export interface ManagedBootstrapRuntimeCommandResult { } export interface ManagedBootstrapRuntimeDependencies { - readonly runCaptureOpenshell?: (args: string[], options?: Record) => string; - readonly runOpenshell?: ( + readonly runCaptureOpenshell: (args: string[], options?: Record) => string; + readonly runOpenshell: ( args: string[], options?: Record, ) => ManagedBootstrapRuntimeCommandResult; - readonly sleep?: (seconds: number) => void; + readonly sleep: (seconds: number) => void; } export type ManagedBootstrapRuntimeRoute = "none" | "native" | "compatibility"; diff --git a/src/lib/onboard/prepared-dcode-rebuild.test.ts b/src/lib/onboard/prepared-dcode-rebuild.test.ts index 808967611f1..67beb3f9051 100644 --- a/src/lib/onboard/prepared-dcode-rebuild.test.ts +++ b/src/lib/onboard/prepared-dcode-rebuild.test.ts @@ -10,7 +10,10 @@ import { describe, expect, it, vi } from "vitest"; import { createBuildContextVerifier } from "../actions/sandbox/rebuild-prepared-image-context"; import { fingerprintBuildContext } from "../adapters/fs/build-context-fingerprint"; import type { AgentDefinition } from "../agent/defs"; -import type { PreparedSandboxBuildContext } from "./build-context-stage"; +import type { + PreparedOpenClawLegacyImage, + PreparedSandboxBuildContext, +} from "./build-context-stage"; import { createPreparedDcodeRebuildRuntime, type PreparedDcodeRebuildOptions, @@ -62,6 +65,49 @@ const preparedImageOptions: PreparedDcodeRebuildOptions = { gatewayName: "nemoclaw", }, }; +const OPENCLAW_IMAGE_ID = `sha256:${"a".repeat(64)}`; + +function createPreparedOpenClawLegacyImageFixture(): PreparedOpenClawLegacyImage { + return { + dockerEnv: { DOCKER_CONTEXT: "retained-context" }, + engineId: "retained-engine", + imageRef: "nemoclaw-sandbox-local:prepared-openclaw", + imageId: OPENCLAW_IMAGE_ID, + verify: vi.fn(() => true), + retainForRecreate: vi.fn(() => true), + verifyForCreate: vi.fn(() => true), + finalizeAfterCreate: vi.fn(() => ({ + mutableTagVerified: true, + registryImageRef: null, + })), + abort: vi.fn(() => true), + dispose: vi.fn(() => true), + }; +} + +function createPreparedOpenClawImageOptions(): PreparedDcodeRebuildOptions { + return { + resume: true, + recreateSandbox: true, + authoritativeResumeConfig: true, + onboardLockAlreadyHeld: true, + agent: null, + fromDockerfile: null, + preparedImageRebuild: { + gatewayName: "nemoclaw", + buildContext: { + buildCtx: "/tmp/prepared-openclaw", + stagedDockerfile: "/tmp/prepared-openclaw/Dockerfile", + buildId: "openclaw-prepared", + cleanupBuildCtx: () => true, + origin: "generated", + verifyBuildCtx: () => true, + rebuildTarget: { agentName: null, fromDockerfile: null }, + preparedOpenClawLegacyImage: createPreparedOpenClawLegacyImageFixture(), + }, + }, + }; +} const sandboxGpuConfig: SandboxGpuConfig = { mode: "0", hostGpuDetected: false, @@ -159,6 +205,80 @@ describe("prepared DCode rebuild adapter", () => { ); }); + it.each([ + ["resume", { resume: false }], + ["recreation", { recreateSandbox: false }], + ["authoritative configuration", { authoritativeResumeConfig: false }], + ["onboard lock", { onboardLockAlreadyHeld: false }], + ])("rejects a retained OpenClaw image without authoritative %s", (_label, override) => { + expect(() => + createPreparedDcodeRebuildRuntime( + { ...createPreparedOpenClawImageOptions(), ...override }, + "nemoclaw", + ), + ).toThrow(/only be used by authoritative resume recreation/); + }); + + it.each([ + [ + "custom origin", + (options: ReturnType) => { + options.preparedImageRebuild!.buildContext.origin = "custom"; + }, + ], + [ + "custom Dockerfile", + (options: ReturnType) => { + options.fromDockerfile = "/tmp/custom/Dockerfile"; + options.preparedImageRebuild!.buildContext.rebuildTarget = { + agentName: null, + fromDockerfile: "/tmp/custom/Dockerfile", + }; + }, + ], + [ + "Hermes agent", + (options: ReturnType) => { + options.agent = "hermes"; + options.preparedImageRebuild!.buildContext.rebuildTarget = { + agentName: "hermes", + fromDockerfile: null, + }; + }, + ], + ] as const)("rejects a retained OpenClaw image for a %s", (_unsupportedTarget, mutate) => { + const options = createPreparedOpenClawImageOptions(); + mutate(options); + + expect(() => createPreparedDcodeRebuildRuntime(options, "nemoclaw")).toThrow( + /retained legacy image can only be used for a generated OpenClaw rebuild/, + ); + }); + + it("rejects a retained OpenClaw image attached to a prepared DCode handoff", () => { + expect(() => + createPreparedDcodeRebuildRuntime( + { + ...preparedOptions, + preparedDcodeRebuild: { + ...preparedOptions.preparedDcodeRebuild!, + buildContext: { + ...preparedBuildContext, + preparedOpenClawLegacyImage: createPreparedOpenClawLegacyImageFixture(), + }, + }, + }, + "nemoclaw", + ), + ).toThrow(/retained legacy image can only be used for a generated OpenClaw rebuild/); + }); + + it("accepts a retained legacy image for authoritative generated OpenClaw recreation", () => { + expect(() => + createPreparedDcodeRebuildRuntime(createPreparedOpenClawImageOptions(), "nemoclaw"), + ).not.toThrow(); + }); + it("normalizes the exact gateway and clears ordinary ambient selection", () => { const preparedEnv: NodeJS.ProcessEnv = { OPENSHELL_GATEWAY: "ambient" }; createPreparedDcodeRebuildRuntime(preparedOptions, "nemoclaw").applyGatewayEnv(preparedEnv); diff --git a/src/lib/onboard/prepared-dcode-rebuild.ts b/src/lib/onboard/prepared-dcode-rebuild.ts index 0fef7cdb7c4..2e92d69a89d 100644 --- a/src/lib/onboard/prepared-dcode-rebuild.ts +++ b/src/lib/onboard/prepared-dcode-rebuild.ts @@ -117,6 +117,29 @@ function verifyPreparedBuildContextForUse(preparedBuildContext: PreparedSandboxB } } +function assertPreparedOpenClawLegacyImageScope( + preparedBuildContext: PreparedSandboxBuildContext, + agentName: string | null | undefined, + fromDockerfile: string | null | undefined, + handoffKind: "dcode" | "image", +): void { + if (!preparedBuildContext.preparedOpenClawLegacyImage) return; + const target = preparedBuildContext.rebuildTarget; + if ( + handoffKind !== "image" || + preparedBuildContext.origin !== "generated" || + normalizedAgentIdentity(agentName) !== "openclaw" || + normalizedDockerfilePath(fromDockerfile) !== null || + !target || + normalizedAgentIdentity(target.agentName) !== "openclaw" || + target.fromDockerfile !== null + ) { + throw new Error( + "A retained legacy image can only be used for a generated OpenClaw rebuild without a custom Dockerfile.", + ); + } +} + export function assertPreparedDcodeTarget( preparedBuildContext: PreparedSandboxBuildContext | null, agent: AgentDefinition | null | undefined, @@ -162,6 +185,14 @@ export function createPreparedDcodeRebuildRuntime( "A prepared rebuild image can only be used by authoritative resume recreation.", ); } + if (preparedDcode) { + assertPreparedOpenClawLegacyImageScope( + preparedDcode.buildContext, + options.agent, + options.fromDockerfile, + "dcode", + ); + } if (preparedImage) { if (!preparedImage.buildContext.rebuildTarget) { throw new Error("Prepared rebuild image target is missing or invalid."); @@ -174,6 +205,12 @@ export function createPreparedDcodeRebuildRuntime( options.agent ?? null, normalizedDockerfilePath(options.fromDockerfile), ); + assertPreparedOpenClawLegacyImageScope( + preparedImage.buildContext, + options.agent, + options.fromDockerfile, + "image", + ); } const prepared = preparedImage ?? preparedDcode; const preparedLabel = preparedImage ? "Prepared rebuild image" : "Prepared DCode rebuild"; diff --git a/src/lib/onboard/rebuild/retained-openclaw-docker-runtime.ts b/src/lib/onboard/rebuild/retained-openclaw-docker-runtime.ts new file mode 100644 index 00000000000..eb843d7eea8 --- /dev/null +++ b/src/lib/onboard/rebuild/retained-openclaw-docker-runtime.ts @@ -0,0 +1,267 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SpawnSyncOptions } from "node:child_process"; +import path from "node:path"; + +import { isIP } from "node:net"; + +import { dockerSpawnSync } from "../../adapters/docker/exec"; +import type { PreparedOpenClawLegacyImage } from "../build-context-stage"; +import type { DockerGpuPatchDeps } from "../docker-gpu-patch-types"; +import { resolveDockerGpuSandboxCreatePlan } from "../docker-gpu-sandbox-create-plan"; +import { + isSandboxBridgeGatewayReachable, + verifySandboxBridgeGatewayReachableOrExit, +} from "../gateway-sandbox-reachability"; +import { + BUSYBOX_PROBE_IMAGE, + dnsProbeName, + type EnsureProbeImageCachedResult, + ensureProbeImageCached, + type ProbeContainerDnsOpts, + type ProbeExecutionResult, + probeContainerDns, +} from "../preflight"; +import type { SandboxCreateIntent } from "../sandbox-create-intent-types"; +import type { SandboxGpuCreateConfig } from "../sandbox-gpu-create"; +import { detectWslDockerDesktopStatus } from "../wsl-docker-desktop-gpu"; + +const RETAINED_DOCKER_OPERATION_TIMEOUT_MS = 30_000; +const RETAINED_DNS_PROBE_TIMEOUT_MS = 20_000; +const RETAINED_DOCKER_PULL_TIMEOUT_MS = 60_000; +const REPOSITORY_ROOT = path.resolve(__dirname, "..", "..", "..", ".."); + +type DockerRunOptions = Record; +type DockerRunResult = NonNullable>; + +export interface RetainedOpenClawDockerRuntime { + readonly deps: DockerGpuPatchDeps; + dockerDesktopWsl(): boolean; + ensureImageCached(image: string): EnsureProbeImageCachedResult; + reverifyBridgeReachability(port: number): Promise; +} + +export interface RetainedOpenClawDockerRuntimeDeps { + runDocker?: typeof dockerSpawnSync; + platform?: NodeJS.Platform; + env?: NodeJS.ProcessEnv; +} + +/** Re-resolve route policy from the bound engine before policy materialization. */ +export function bindRetainedOpenClawGpuRoute( + intent: SandboxCreateIntent, + config: SandboxGpuCreateConfig, + dockerDriverGateway: boolean, + runtime: RetainedOpenClawDockerRuntime, + options: { env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform } = {}, +): SandboxCreateIntent { + const plan = resolveDockerGpuSandboxCreatePlan(config, { + dockerDriverGateway, + dockerDesktopWsl: runtime.dockerDesktopWsl(), + env: options.env, + platform: options.platform, + }); + return { + ...intent, + gpuRoutePlan: plan.gpuRoutePlan, + sandboxGpuLogMessage: plan.logMessage, + }; +} + +function timeoutFromOptions(options: DockerRunOptions | undefined, fallback: number): number { + const timeout = options?.timeout; + return typeof timeout === "number" && Number.isFinite(timeout) && timeout > 0 + ? timeout + : fallback; +} + +function dockerErrorText(result: DockerRunResult): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}`.trim(); +} + +function toProbeExecution(result: DockerRunResult): ProbeExecutionResult { + const error = result.error as NodeJS.ErrnoException | undefined; + return { + stdout: result.stdout, + stderr: result.stderr, + exitCode: result.status ?? null, + signal: result.signal, + timedOut: error?.code === "ETIMEDOUT", + error: error?.message ?? null, + errorCode: error?.code ?? null, + }; +} + +function validateProbeName(value: string): string { + if (!/^[a-z0-9]([a-z0-9.-]{0,253})$/i.test(value)) { + throw new Error( + `probeName must be a plain DNS name (RFC 1035 label characters), got: ${JSON.stringify(value)}`, + ); + } + return value; +} + +/** + * Bind every Docker operation after destructive OpenClaw rebuild handoff to + * the exact engine and selector that own the retained image. + */ +export function createRetainedOpenClawDockerRuntime( + image: PreparedOpenClawLegacyImage, + injected: RetainedOpenClawDockerRuntimeDeps = {}, +): RetainedOpenClawDockerRuntime { + const runDocker = injected.runDocker ?? dockerSpawnSync; + let cachedDockerDesktopWsl: boolean | null = null; + + const runBoundDocker = ( + args: readonly string[], + options: DockerRunOptions = {}, + ): DockerRunResult => { + if (!image.verifyForCreate()) { + throw new Error( + "Retained OpenClaw rebuild Docker engine or image changed before a Docker operation.", + ); + } + const spawnOptions: SpawnSyncOptions = { + cwd: REPOSITORY_ROOT, + encoding: "utf-8", + env: image.dockerEnv, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + timeout: timeoutFromOptions(options, RETAINED_DOCKER_OPERATION_TIMEOUT_MS), + }; + return runDocker(args, spawnOptions); + }; + + const dockerRun = (args: readonly string[], options: DockerRunOptions = {}): DockerRunResult => + runBoundDocker(args, options); + + const dockerCapture = (args: readonly string[], options: DockerRunOptions = {}): string => { + const result = runBoundDocker(args, options); + if (result.error == null && result.status === 0) return String(result.stdout ?? "").trim(); + if (options.ignoreError === true) return ""; + throw new Error(dockerErrorText(result) || "Bound Docker command failed."); + }; + + const dockerContainer = + (command: string) => + (containerName: string, options: DockerRunOptions = {}): DockerRunResult => + runBoundDocker([command, containerName], options); + + const dockerLogs = ( + containerName: string, + options: { tail?: number; timeout?: number } = {}, + ): string => { + const result = runBoundDocker( + ["logs", "--tail", String(options.tail ?? 30), containerName], + options, + ); + return `${String(result.stdout ?? "")}${String(result.stderr ?? "")}`.trim(); + }; + + const runBoundProbe = ( + command: readonly string[], + options: { timeout?: number } = {}, + ): ProbeExecutionResult => { + if (command[0] !== "docker") { + throw new Error("Retained OpenClaw Docker probe must invoke Docker directly."); + } + return toProbeExecution( + runBoundDocker(command.slice(1), { + timeout: options.timeout, + ignoreError: true, + suppressOutput: true, + }), + ); + }; + + const ensureImageCached = (target: string): EnsureProbeImageCachedResult => + ensureProbeImageCached(target, { + inspectProbeImpl: runBoundProbe, + pullProbeImpl: runBoundProbe, + pullTimeoutMs: RETAINED_DOCKER_PULL_TIMEOUT_MS, + }); + + const probeDns = (options: ProbeContainerDnsOpts = {}) => { + const probeName = validateProbeName(options.probeName ?? dnsProbeName()); + const dnsServer = options.dnsServer ?? null; + if (dnsServer !== null && isIP(dnsServer) === 0) { + throw new Error(`dnsServer must be an IP address, got: ${JSON.stringify(dnsServer)}`); + } + const cached = ensureImageCached(BUSYBOX_PROBE_IMAGE); + if (!cached.ok) { + return probeContainerDns({ + ...options, + probeName, + ensureImageCachedOverride: cached, + executionOverride: { exitCode: 1 }, + }); + } + const dockerArgs = [ + "run", + "--rm", + "--pull=missing", + ...(dnsServer === null ? [] : ["--dns", dnsServer]), + BUSYBOX_PROBE_IMAGE, + "nslookup", + probeName, + ]; + const execution = toProbeExecution( + runBoundDocker(dockerArgs, { + timeout: RETAINED_DNS_PROBE_TIMEOUT_MS, + ignoreError: true, + suppressOutput: true, + }), + ); + return probeContainerDns({ + ...options, + probeName, + command: ["docker", ...dockerArgs], + ensureImageCachedOverride: { ok: true, alreadyCached: cached.alreadyCached }, + executionOverride: execution, + }); + }; + + const deps: DockerGpuPatchDeps = Object.freeze({ + dockerCapture, + dockerRun, + dockerRunDetached: (args, options) => runBoundDocker(["run", "-d", ...args], options), + dockerRename: (oldName, newName, options) => + runBoundDocker(["rename", oldName, newName], options), + dockerRm: dockerContainer("rm"), + dockerStart: dockerContainer("start"), + dockerStop: dockerContainer("stop"), + dockerLogs, + probeContainerDns: probeDns, + }); + + return Object.freeze({ + deps, + dockerDesktopWsl: () => { + if (cachedDockerDesktopWsl === null) { + cachedDockerDesktopWsl = + detectWslDockerDesktopStatus({ + platform: injected.platform, + env: injected.env, + dockerInfoFormat: (format, options) => + dockerCapture(["info", "--format", format], options), + }) === "docker-desktop"; + } + return cachedDockerDesktopWsl; + }, + ensureImageCached, + reverifyBridgeReachability: (port: number) => + verifySandboxBridgeGatewayReachableOrExit(true, { + port, + reachabilityImpl: (selected) => + isSandboxBridgeGatewayReachable({ + port: selected?.port ?? port, + dockerCaptureImpl: dockerCapture, + dockerRunImpl: dockerRun as typeof import("../../adapters/docker/run").dockerRun, + ensureImageCachedImpl: ensureImageCached, + }), + }), + }); +} diff --git a/src/lib/onboard/retained-openclaw-docker-runtime.test.ts b/src/lib/onboard/retained-openclaw-docker-runtime.test.ts new file mode 100644 index 00000000000..1792bffd5c2 --- /dev/null +++ b/src/lib/onboard/retained-openclaw-docker-runtime.test.ts @@ -0,0 +1,189 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SpawnSyncOptions } from "node:child_process"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { dockerSpawnSync } from "../adapters/docker/exec"; +import type { PreparedOpenClawLegacyImage } from "./build-context-stage"; +import { + bindRetainedOpenClawGpuRoute, + createRetainedOpenClawDockerRuntime, +} from "./rebuild/retained-openclaw-docker-runtime"; +import type { SandboxCreateIntent } from "./sandbox-create-intent-types"; + +const IMAGE_ID = `sha256:${"a".repeat(64)}`; + +function createIntent(): SandboxCreateIntent { + return { + sandboxName: "alpha", + inferenceProvider: "nim", + activeMessagingChannels: [], + messagingProviderRequests: [], + reusableMessagingProviders: [], + extraProviders: [], + staleExtraProviders: [], + hermesToolGateways: [], + policy: { + basePolicyPath: "/tmp/policy.yaml", + activeMessagingChannels: [], + options: { + directGpu: true, + additionalPresets: [], + policyTier: null, + baselineExclusions: [], + }, + }, + gpuCreateArgs: ["--gpu"], + resourceCreateArgs: [], + gpuRoutePlan: "native-only", + sandboxGpuLogMessage: "ambient native route", + disabledChannelNames: [], + extraPlaceholderKeys: [], + }; +} + +function dockerResult(stdout = "", status = 0): ReturnType { + return { + error: undefined, + output: [null, stdout, ""], + pid: 123, + signal: null, + status, + stderr: "", + stdout, + }; +} + +function createImage(verifyForCreate: () => boolean = () => true): PreparedOpenClawLegacyImage { + return { + dockerEnv: Object.freeze({ DOCKER_CONTEXT: "engine-a", PATH: "/usr/bin" }), + engineId: "engine-a-id", + imageRef: "nemoclaw-sandbox-local:alpha-rebuild", + imageId: IMAGE_ID, + verify: vi.fn(() => true), + retainForRecreate: vi.fn(() => true), + verifyForCreate: vi.fn(verifyForCreate), + finalizeAfterCreate: vi.fn(() => ({ + registryImageRef: null, + mutableTagVerified: true, + })), + abort: vi.fn(() => true), + dispose: vi.fn(() => true), + }; +} + +afterEach(() => vi.unstubAllEnvs()); + +describe("retained OpenClaw Docker runtime", () => { + it("keeps create-adjacent queries and mutations on engine A when ambient Docker selects B", () => { + vi.stubEnv("DOCKER_CONTEXT", "engine-b"); + vi.stubEnv("WSL_DISTRO_NAME", "Ubuntu"); + const image = createImage(); + const calls: Array<{ args: string[]; options: SpawnSyncOptions }> = []; + const runDocker = vi.fn((args: readonly string[], options: SpawnSyncOptions = {}) => { + calls.push({ args: [...args], options }); + switch (true) { + case args.join(" ") === "info --format {{json .OperatingSystem}}": + return dockerResult('"Docker Desktop"\n'); + case args[0] === "image" && args[1] === "inspect": + return dockerResult("[]\n"); + case args[0] === "logs": + return dockerResult("engine-a logs\n"); + default: + return dockerResult("engine-a-result\n"); + } + }); + const runtime = createRetainedOpenClawDockerRuntime(image, { + runDocker: runDocker as typeof dockerSpawnSync, + platform: "linux", + env: { WSL_DISTRO_NAME: "Ubuntu" }, + }); + const deps = runtime.deps; + + expect(deps.dockerCapture?.(["ps", "-a"], { ignoreError: true })).toBe("engine-a-result"); + expect(deps.dockerRun?.(["inspect", "container-a"], { ignoreError: true }).status).toBe(0); + expect(deps.dockerStop?.("container-a", { ignoreError: true }).status).toBe(0); + expect( + deps.dockerRename?.("container-a", "container-a-backup", { ignoreError: true }).status, + ).toBe(0); + expect( + deps.dockerRunDetached?.(["--name", "container-a", IMAGE_ID], { + ignoreError: true, + }).status, + ).toBe(0); + expect(deps.dockerRm?.("container-a-backup", { ignoreError: true }).status).toBe(0); + expect(deps.dockerStart?.("container-a", { ignoreError: true }).status).toBe(0); + expect(deps.dockerLogs?.("container-a", { tail: 20 })).toBe("engine-a logs"); + expect(runtime.dockerDesktopWsl()).toBe(true); + expect(runtime.dockerDesktopWsl()).toBe(true); + expect(runtime.ensureImageCached("busybox:test")).toEqual({ + ok: true, + alreadyCached: true, + }); + + expect(calls.length).toBeGreaterThan(0); + expect(calls.every(({ options }) => options.env === image.dockerEnv)).toBe(true); + expect(calls.some(({ options }) => options.env?.DOCKER_CONTEXT === "engine-b")).toBe(false); + expect(process.env.DOCKER_CONTEXT).toBe("engine-b"); + expect(image.verifyForCreate).toHaveBeenCalledTimes(calls.length); + }); + + it("rejects engine drift before the first mutation after an engine-A query", () => { + let engineId = "engine-a-id"; + const image = createImage(() => engineId === "engine-a-id"); + const runDocker = vi.fn((_args: readonly string[], _options: SpawnSyncOptions = {}) => + dockerResult("container-a\n"), + ); + const runtime = createRetainedOpenClawDockerRuntime(image, { + runDocker: runDocker as typeof dockerSpawnSync, + platform: "linux", + env: { WSL_DISTRO_NAME: "Ubuntu" }, + }); + + expect(runtime.deps.dockerCapture?.(["ps", "-a"], { ignoreError: true })).toBe("container-a"); + engineId = "engine-b-id"; + + expect(() => runtime.deps.dockerStop?.("container-a", { ignoreError: true })).toThrow( + "Retained OpenClaw rebuild Docker engine or image changed before a Docker operation.", + ); + expect(runDocker).toHaveBeenCalledOnce(); + expect(runDocker.mock.calls[0]?.[0]).toEqual(["ps", "-a"]); + }); + + it("replaces an ambient native plan with engine A's Docker Desktop WSL route", () => { + vi.stubEnv("DOCKER_CONTEXT", "engine-b"); + vi.stubEnv("WSL_DISTRO_NAME", "Ubuntu"); + const image = createImage(); + const runDocker = vi.fn((args: readonly string[], _options: SpawnSyncOptions = {}) => + args.join(" ") === "info --format {{json .OperatingSystem}}" + ? dockerResult('"Docker Desktop"\n') + : dockerResult(), + ); + const runtime = createRetainedOpenClawDockerRuntime(image, { + runDocker: runDocker as typeof dockerSpawnSync, + platform: "linux", + env: { WSL_DISTRO_NAME: "Ubuntu" }, + }); + const ambientIntent = createIntent(); + + const boundIntent = bindRetainedOpenClawGpuRoute( + ambientIntent, + { sandboxGpuEnabled: true, hostGpuDetected: true }, + true, + runtime, + { + env: { NEMOCLAW_DOCKER_GPU_PATCH: "0" }, + platform: "linux", + }, + ); + + expect(ambientIntent.gpuRoutePlan).toBe("native-only"); + expect(boundIntent.gpuRoutePlan).toBe("compatibility-only"); + expect(boundIntent.sandboxGpuLogMessage).toContain("Docker-driver GPU patch active"); + expect(runDocker).toHaveBeenCalledOnce(); + expect(runDocker.mock.calls[0]?.[1]?.env).toBe(image.dockerEnv); + expect(process.env.DOCKER_CONTEXT).toBe("engine-b"); + }); +}); diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index dec7913e208..29f4d46bc8f 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -8,11 +8,9 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; import { loadAgent } from "../agent/defs"; import { SANDBOX_BUILD_CONTEXT_PREFIX } from "../sandbox/build-context"; -import { encodeManagedStartupProfile } from "./managed-startup/profile"; -import { createManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; +import type { PreparedOpenClawLegacyImage } from "./build-context-stage"; import { createOpenshellCliHelpers } from "./openshell-cli"; import { buildSandboxRuntimeEnvArgs, @@ -24,6 +22,28 @@ const disabledHermesDashboardState = { config: null, enabled: false }; const IMAGE_ID = `sha256:${"a".repeat(64)}`; const temporaryBuildContexts: string[] = []; +function createPreparedOpenClawLegacyImageFixture(): PreparedOpenClawLegacyImage { + return { + dockerEnv: { + DOCKER_HOST: "unix:///captured/docker.sock", + DOCKER_CONTEXT: "captured-context", + DOCKER_CONFIG: "/captured/docker-config", + }, + engineId: "retained-engine", + imageRef: "nemoclaw-sandbox-local:retained-openclaw", + imageId: IMAGE_ID, + verify: vi.fn(() => true), + retainForRecreate: vi.fn(() => true), + verifyForCreate: vi.fn(() => true), + finalizeAfterCreate: vi.fn(() => ({ + mutableTagVerified: true, + registryImageRef: null, + })), + abort: vi.fn(() => true), + dispose: vi.fn(() => true), + }; +} + function createTrustedBuildContext(): string { const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), SANDBOX_BUILD_CONTEXT_PREFIX)); temporaryBuildContexts.push(buildCtx); @@ -107,49 +127,6 @@ describe("buildSandboxRuntimeEnvArgs", () => { }); describe("prepareSandboxCreateLaunch", () => { - it.each([ - "openclaw", - "hermes", - "langchain-deepagents-code", - ] as const)("renders one identity-bound held launch for %s without exposing the startup profile", (agentName) => { - const request = createManagedStartupRootApplyRequest({ - agent: agentName, - encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agentName)), - }); - const result = prepareSandboxCreateLaunch({ - agent: loadAgent(agentName), - chatUiUrl: "", - createArgs: ["--name", `${agentName}-sandbox`], - env: {}, - extraPlaceholderKeys: [], - getDashboardForwardPort: () => "0", - hermesDashboardState: disabledHermesDashboardState, - manageDashboard: false, - openshellShellCommand: (args) => args.join(" "), - openshellArgv: (args) => ["openshell", ...args], - buildEnv: () => ({}), - managedStartupRootApplyRequest: request, - }); - - expect(result.intendedSandboxStartupCommand).toEqual([ - "env", - ...result.envArgs, - "nemoclaw-start", - ]); - expect(result.managedBootstrapIdentity).toMatch(/^[a-f0-9]{64}$/u); - expect(result.sandboxStartupCommand).toEqual([ - ...result.intendedSandboxStartupCommand.slice(0, -1), - "/usr/local/bin/nemoclaw-managed-startup-hold", - "--agent", - agentName, - "--profile-fingerprint", - request.profileFingerprint, - "--bootstrap-identity", - result.managedBootstrapIdentity, - ]); - expect(result.createArgv.join("\n")).not.toContain(request.encodedProfile); - }); - it("builds the sandbox create command and runtime env envelope", () => { const openshellShellCommand = vi.fn((args: string[]) => `openshell ${args.join(" ")}`); const result = prepareSandboxCreateLaunch({ @@ -485,6 +462,74 @@ describe("prepareSandboxCreateLaunchWithPrebuild", () => { expect(buildImage).toHaveBeenCalledOnce(); }); + it("launches a retained image by immutable ID on its captured Docker selector", async () => { + const buildCtx = createTrustedBuildContext(); + const dockerfile = path.join(buildCtx, "Dockerfile"); + const preparedOpenClawLegacyImage = createPreparedOpenClawLegacyImageFixture(); + const buildImage = vi.fn(async () => 0); + const result = await prepareSandboxCreateLaunchWithPrebuild({ + agent: null, + chatUiUrl: "", + createArgs: ["--from", dockerfile, "--name", "demo"], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: disabledHermesDashboardState, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + openshellArgv: (args) => ["openshell", ...args], + sandboxName: "demo", + buildEnv: () => ({ + HOME: "/ambient/home", + DOCKER_HOST: "ssh://ambient-engine", + DOCKER_CONTEXT: "ambient-context", + DOCKER_CONFIG: "/ambient/docker-config", + DOCKER_CERT_PATH: "/ambient/certs", + DOCKER_TLS_VERIFY: "1", + DOCKER_API_VERSION: "1.41", + }), + prebuild: { + buildCtx, + buildId: "build-123", + dockerDriverGateway: true, + env: { NEMOCLAW_SANDBOX_PREBUILD: "0" }, + buildImage, + origin: "generated", + preparedOpenClawLegacyImage, + }, + }); + + expect(result.prebuild).toEqual({ + createArgs: ["--from", IMAGE_ID, "--name", "demo"], + imageRef: preparedOpenClawLegacyImage.imageRef, + imageId: IMAGE_ID, + preparedOpenClawLegacyImage, + }); + expect(result.createCommand).toContain(`sandbox create --from ${IMAGE_ID} --name demo`); + expect(result.createArgv).toEqual([ + "openshell", + "sandbox", + "create", + "--from", + IMAGE_ID, + "--name", + "demo", + "--", + "env", + ...result.envArgs, + "nemoclaw-start", + ]); + expect(result.sandboxEnv).toEqual({ + HOME: "/ambient/home", + DOCKER_HOST: "unix:///captured/docker.sock", + DOCKER_CONTEXT: "captured-context", + DOCKER_CONFIG: "/captured/docker-config", + }); + expect(buildImage).not.toHaveBeenCalled(); + expect(preparedOpenClawLegacyImage.verifyForCreate).not.toHaveBeenCalled(); + expect(preparedOpenClawLegacyImage.finalizeAfterCreate).not.toHaveBeenCalled(); + }); + it("renders the original Dockerfile for Hermes after a local build failure", async () => { const buildCtx = createTrustedBuildContext(); const dockerfile = path.join(buildCtx, "Dockerfile"); diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index 09e703f6d4b..3490d2208d8 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -62,6 +62,7 @@ export interface SandboxCreateLaunchInput { openshellShellCommand: OpenshellShellCommand; openshellArgv?: OpenshellArgv; buildEnv?(): Record; + dockerSelectorEnv?: Readonly>; /** Dormant until a complete runtime bundle and durable authority store are selected. */ managedStartupRootApplyRequest?: ManagedStartupRootApplyRequest | null; } diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index f60f0a59abe..8b1f451525e 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -62,6 +62,7 @@ import { setupGpuFlowMocks, VERIFIED_GPU_PROOF as VERIFIED_PROOF, } from "./__test-helpers__/sandbox-gpu-create-flow"; +import type { PreparedOpenClawLegacyImage } from "./build-context-stage"; import { MANAGED_BOOTSTRAP_SCHEMA_VERSION, type ManagedBootstrapRecoveryReport, @@ -79,6 +80,7 @@ import type { import { createRuntimeProviderBundleRegistry } from "./runtime-provider/registry"; import { prepareSandboxCreateLaunch } from "./sandbox-create-launch"; import { + resolveCreatedSandboxRegistryImageRef, runSandboxGpuCreateFlow, type SandboxGpuCreateFlowDeps, type SandboxGpuCreateFlowInput, @@ -98,6 +100,7 @@ const NVIDIA_SMI_FAILED_PROOF: SandboxGpuProofResult = { detail: "Failed to initialize NVML: Driver/library version mismatch", at: "2026-07-06T00:00:00.000Z", }; +const OTHER_IMAGE_ID = `sha256:${"b".repeat(64)}`; const DEFAULT_RUNTIME_SNAPSHOT = { ok: true as const, imageId: IMAGE_ID, @@ -166,6 +169,43 @@ function createSourceInput(): SandboxGpuCreateFlowInput { return input; } +function createRetainedImageInput() { + const input = createInput(); + const finalizeAfterCreate = vi.fn(() => ({ + mutableTagVerified: true, + registryImageRef: null, + })); + const preparedOpenClawLegacyImage = { + dockerEnv: { DOCKER_CONTEXT: "retained-context" }, + engineId: "retained-engine", + imageRef: "nemoclaw-sandbox-local:retained-openclaw", + imageId: IMAGE_ID, + verify: vi.fn(() => true), + retainForRecreate: vi.fn(() => true), + verifyForCreate: vi.fn(() => true), + finalizeAfterCreate, + abort: vi.fn(() => true), + dispose: vi.fn(() => true), + } satisfies PreparedOpenClawLegacyImage; + input.createArgv = [ + "openshell", + "sandbox", + "create", + "--from", + IMAGE_ID, + "--name", + "alpha", + "--gpu", + ]; + input.prebuild = { + createArgs: ["--from", IMAGE_ID, "--name", "alpha", "--gpu"], + imageRef: preparedOpenClawLegacyImage.imageRef, + imageId: IMAGE_ID, + preparedOpenClawLegacyImage, + }; + return { input, preparedOpenClawLegacyImage }; +} + beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); @@ -374,6 +414,240 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { }); }); +describe("runSandboxGpuCreateFlow retained OpenClaw image", () => { + it("verifies immediately before streaming and finalizes only after final success", async () => { + const { input, preparedOpenClawLegacyImage } = createRetainedImageInput(); + + await expect(runSandboxGpuCreateFlow(input, createDeps())).resolves.toMatchObject({ + retainedImageFinalization: { + mutableTagVerified: true, + registryImageRef: null, + }, + route: "native", + }); + + expect(preparedOpenClawLegacyImage.verifyForCreate).toHaveBeenCalledOnce(); + expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); + expect(preparedOpenClawLegacyImage.finalizeAfterCreate).toHaveBeenCalledOnce(); + expect(preparedOpenClawLegacyImage.abort).not.toHaveBeenCalled(); + expect(mocks.streamSandboxCreate.mock.invocationCallOrder[0]).toBe( + preparedOpenClawLegacyImage.verifyForCreate.mock.invocationCallOrder[0] + 1, + ); + expect( + preparedOpenClawLegacyImage.finalizeAfterCreate.mock.invocationCallOrder[0], + ).toBeGreaterThan(mocks.waitForCreatedSandboxReadyWithTrace.mock.invocationCallOrder[0]); + expect( + preparedOpenClawLegacyImage.finalizeAfterCreate.mock.invocationCallOrder[0], + ).toBeGreaterThan(mocks.verifyGpuSandboxAccessAfterReady.mock.invocationCallOrder[0]); + expect(mocks.streamSandboxCreate).toHaveBeenCalledWith( + "openshell", + expect.arrayContaining(["--from", IMAGE_ID]), + input.sandboxEnv, + expect.any(Object), + ); + }); + + it("reverifies for compatibility and finalizes only after the final create succeeds", async () => { + const { input, preparedOpenClawLegacyImage } = createRetainedImageInput(); + failNativeCreate(); + + await expect(runSandboxGpuCreateFlow(input, createDeps())).resolves.toMatchObject({ + route: "compatibility", + }); + + expect(preparedOpenClawLegacyImage.verifyForCreate).toHaveBeenCalledTimes(2); + expect(mocks.streamSandboxCreate).toHaveBeenCalledTimes(2); + expect(preparedOpenClawLegacyImage.finalizeAfterCreate).toHaveBeenCalledOnce(); + const verifyOrders = preparedOpenClawLegacyImage.verifyForCreate.mock.invocationCallOrder; + const streamOrders = mocks.streamSandboxCreate.mock.invocationCallOrder; + const finalizeOrder = + preparedOpenClawLegacyImage.finalizeAfterCreate.mock.invocationCallOrder[0]; + expect(streamOrders[0]).toBe(verifyOrders[0] + 1); + expect(streamOrders[1]).toBe(verifyOrders[1] + 1); + expect(streamOrders[0]).toBeLessThan(verifyOrders[1]); + expect(streamOrders[1]).toBeLessThan(finalizeOrder); + expect(mocks.streamSandboxCreate).toHaveBeenNthCalledWith( + 2, + "openshell", + expect.arrayContaining(["--from", IMAGE_ID]), + input.sandboxEnv, + expect.any(Object), + ); + }); + + it("refuses a native snapshot for image B before rendering or creating compatibility", async () => { + const { input, preparedOpenClawLegacyImage } = createRetainedImageInput(); + mockRuntimeSnapshot({ + imageId: OTHER_IMAGE_ID, + stateError: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + }); + mocks.streamSandboxCreate.mockResolvedValueOnce({ + status: 1, + output: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + sawProgress: true, + }); + const deps = createDeps(); + + await expectFlowExit(input, deps); + + expect(mocks.queryOpenShellDockerSandboxRuntimeSnapshot).toHaveBeenCalledOnce(); + expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); + expect(mocks.streamSandboxCreate.mock.calls.flat()).not.toContain(OTHER_IMAGE_ID); + expect(deps.openshellArgv).not.toHaveBeenCalled(); + expect(errorOutput()).toContain( + "Native sandbox image identity does not match the retained OpenClaw rebuild image", + ); + expect(preparedOpenClawLegacyImage.abort).toHaveBeenCalledOnce(); + }); + + it("uses retained image A when the native snapshot corroborates A", async () => { + const { input } = createRetainedImageInput(); + mockRuntimeSnapshot({ + imageId: IMAGE_ID, + stateError: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + }); + mocks.streamSandboxCreate.mockResolvedValueOnce({ + status: 1, + output: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + sawProgress: true, + }); + const deps = createDeps(); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ + route: "compatibility", + }); + + expect(mocks.queryOpenShellDockerSandboxRuntimeSnapshot).toHaveBeenCalledOnce(); + expect(deps.openshellArgv).toHaveBeenCalledWith(expect.arrayContaining(["--from", IMAGE_ID])); + expect(mocks.streamSandboxCreate).toHaveBeenNthCalledWith( + 2, + "openshell", + expect.arrayContaining(["--from", IMAGE_ID]), + input.sandboxEnv, + expect.any(Object), + ); + }); + + it("uses engine A for create, fallback evidence, cleanup, and patch mutation while ambient selects B", async () => { + vi.stubEnv("DOCKER_CONTEXT", "engine-b"); + const { input } = createRetainedImageInput(); + input.sandboxEnv = { DOCKER_CONTEXT: "engine-a" }; + failNativeCreate(); + const engineAQueries: string[][] = []; + const engineAMutations: string[][] = []; + const engineBCapture = vi.fn(() => ""); + const engineBRun = vi.fn(() => ({ status: 0, stdout: "" })); + const engineBMutations = vi.fn(() => ({ status: 0 })); + const boundDockerDeps = { + dockerCapture: vi.fn((args: readonly string[]) => { + engineAQueries.push([...args]); + return ""; + }), + dockerRun: vi.fn((args: readonly string[]) => { + engineAQueries.push([...args]); + return { status: 0, stdout: "" }; + }), + dockerStop: vi.fn((containerName: string) => { + engineAMutations.push(["stop", containerName]); + return { status: 0 }; + }), + }; + const deps = createDeps(); + deps.dockerCapture = engineBCapture; + deps.dockerRun = engineBRun; + deps.dockerStop = engineBMutations; + deps.createRetainedDockerRuntime = vi.fn(() => ({ + deps: boundDockerDeps, + dockerDesktopWsl: () => false, + ensureImageCached: vi.fn(() => ({ ok: true, alreadyCached: true })), + reverifyBridgeReachability: vi.fn(async () => {}), + })); + mocks.queryOpenShellDockerSandboxContainers.mockImplementation( + (_sandboxName: string, dockerDeps: typeof boundDockerDeps) => { + dockerDeps.dockerRun(["ps", "-a"]); + return { ok: true, ids: [] }; + }, + ); + mocks.createDockerGpuSandboxCreatePatch.mockImplementation((options) => { + options.deps.dockerCapture(["inspect", "container-a"]); + options.deps.dockerStop("container-a"); + return createPatch(); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ + route: "compatibility", + }); + + expect(mocks.streamSandboxCreate).toHaveBeenCalledTimes(2); + expect( + mocks.streamSandboxCreate.mock.calls.every(([, , env]) => env === input.sandboxEnv), + ).toBe(true); + expect(input.sandboxEnv).toEqual({ DOCKER_CONTEXT: "engine-a" }); + expect(process.env.DOCKER_CONTEXT).toBe("engine-b"); + expect(engineAQueries).toContainEqual(["ps", "-a"]); + expect(engineAQueries).toContainEqual(["inspect", "container-a"]); + expect(engineAMutations).toContainEqual(["stop", "container-a"]); + expect(engineBCapture).not.toHaveBeenCalled(); + expect(engineBRun).not.toHaveBeenCalled(); + expect(engineBMutations).not.toHaveBeenCalled(); + }); + + it("fails before create instead of falling back when retained image verification fails", async () => { + const { input, preparedOpenClawLegacyImage } = createRetainedImageInput(); + preparedOpenClawLegacyImage.verifyForCreate.mockReturnValue(false); + + await expect(runSandboxGpuCreateFlow(input, createDeps())).rejects.toThrow( + "Retained OpenClaw rebuild image changed before sandbox creation.", + ); + + expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); + expect(preparedOpenClawLegacyImage.finalizeAfterCreate).not.toHaveBeenCalled(); + expect(preparedOpenClawLegacyImage.abort).toHaveBeenCalledOnce(); + }); + + it("aborts a retained image after a failed final create", async () => { + const { input, preparedOpenClawLegacyImage } = createRetainedImageInput(); + mocks.streamSandboxCreate.mockResolvedValueOnce({ + status: 1, + output: "x509: certificate signed by unknown authority", + sawProgress: true, + }); + mockExit(); + + await expect(runSandboxGpuCreateFlow(input, createDeps())).rejects.toThrow("process.exit:1"); + + expect(preparedOpenClawLegacyImage.verifyForCreate).toHaveBeenCalledOnce(); + expect(preparedOpenClawLegacyImage.finalizeAfterCreate).not.toHaveBeenCalled(); + expect(preparedOpenClawLegacyImage.abort).toHaveBeenCalledOnce(); + }); + + it("aborts when post-create finalization cannot prove the bound engine", async () => { + const { input, preparedOpenClawLegacyImage } = createRetainedImageInput(); + preparedOpenClawLegacyImage.finalizeAfterCreate.mockReturnValue(null); + + await expect(runSandboxGpuCreateFlow(input, createDeps())).rejects.toThrow( + "Retained OpenClaw rebuild image could not be finalized after creation.", + ); + + expect(preparedOpenClawLegacyImage.finalizeAfterCreate).toHaveBeenCalledOnce(); + expect(preparedOpenClawLegacyImage.abort).toHaveBeenCalledOnce(); + }); + + it("keeps retained-image cleanup suppressed when a later ambient Docker context differs", () => { + const retainedImageFinalization = { + mutableTagVerified: true, + registryImageRef: null, + } as const; + + expect( + resolveCreatedSandboxRegistryImageRef(retainedImageFinalization, [ + "nemoclaw-sandbox-local:retargeted-on-ambient-engine", + "openshell/sandbox-from:captured-output", + ]), + ).toBeNull(); + }); +}); + describe("runSandboxGpuCreateFlow proof authorization", () => { it("does not retry compatibility when the native proof throws an exec/policy error (#6110)", async () => { const deps = createDeps(); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 46f34359a2e..d0784adeb37 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -10,6 +10,17 @@ import type { DockerGpuPatchDeps, DockerUlimit } from "./docker-gpu-patch-types" import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { renderCompatibilityFallbackCreateArgs } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; +import type { PreparedOpenClawLegacyImageFinalization } from "./build-context-stage"; + +import { + isImmutableDockerImageId, + queryOpenShellDockerSandboxContainers, +} from "./openshell-docker-sandbox-containers"; +import { + bindRetainedOpenClawGpuRoute, + createRetainedOpenClawDockerRuntime, +} from "./rebuild/retained-openclaw-docker-runtime"; +export { bindRetainedOpenClawGpuRoute, createRetainedOpenClawDockerRuntime }; import { type ManagedBootstrapAdapter, type ManagedBootstrapAgentIdentity, @@ -19,7 +30,6 @@ import { } from "./managed-bootstrap/adapter"; import type { ManagedBootstrapRuntimePatch } from "./managed-bootstrap/runtime-create"; import type { ManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; -import { isImmutableDockerImageId } from "./openshell-docker-sandbox-containers"; import type { RuntimeProviderBootstrapSurface, RuntimeProviderBundle, @@ -98,12 +108,14 @@ export interface SandboxGpuCreateFlowInput { requiredUlimits?: readonly DockerUlimit[] | null; } -export interface SandboxGpuCreateFlowDeps { +export interface SandboxGpuCreateFlowDeps + extends Pick { runOpenshell: RunOpenshell; runCaptureOpenshell: RunCaptureOpenshell; sleep: Sleep; openshellArgv(args: string[]): string[]; verifyDirectSandboxGpu(sandboxName: string): SandboxGpuProofResult; + createRetainedDockerRuntime?: typeof createRetainedOpenClawDockerRuntime; /** Production callers omit this factory and use the runtime provider's adapter. */ createManagedBootstrapAdapter?: () => ManagedBootstrapAdapter; } @@ -115,6 +127,80 @@ export interface SandboxGpuCreateFlowResult { firstCreateOutput: string; /** Mutable tag/reference retained only for registry and image-GC bookkeeping. */ registryImageRef: string | null; + /** Authoritative retained-image result, including deliberate cleanup suppression. */ + retainedImageFinalization: PreparedOpenClawLegacyImageFinalization | null; +} + +function resolveCompatibilityRetryImageId( + prebuild: SandboxPrebuildResult, + nativeSnapshotImageId: string | null, +): string | null { + const prebuildImageId = + prebuild.imageId && isImmutableDockerImageId(prebuild.imageId) + ? prebuild.imageId.toLowerCase() + : null; + const retainedImage = prebuild.preparedOpenClawLegacyImage; + if (!retainedImage) return nativeSnapshotImageId ?? prebuildImageId; + + const retainedImageId = isImmutableDockerImageId(retainedImage.imageId) + ? retainedImage.imageId.toLowerCase() + : null; + if (!retainedImageId || prebuildImageId !== retainedImageId) { + throw new Error("Retained OpenClaw rebuild image identity changed before compatibility retry."); + } + if ( + nativeSnapshotImageId !== null && + (!isImmutableDockerImageId(nativeSnapshotImageId) || + nativeSnapshotImageId.toLowerCase() !== retainedImageId) + ) { + throw new Error( + "Native sandbox image identity does not match the retained OpenClaw rebuild image.", + ); + } + return retainedImageId; +} + +export function resolveCreatedSandboxRegistryImageRef( + finalization: PreparedOpenClawLegacyImageFinalization | null, + candidates: readonly (string | null | undefined)[], +): string | null { + if (finalization) return finalization.registryImageRef; + return candidates.find((candidate): candidate is string => Boolean(candidate)) ?? null; +} + +type RetainedOpenClawCreateIntent = Parameters[0]; +type RetainedOpenClawGpuConfig = Parameters[1]; +type RetainedOpenClawImage = Parameters[0]; + +export function resolveRetainedOpenClawCreateIntent( + baseResolvedCreateIntent: RetainedOpenClawCreateIntent, + sandboxGpuConfig: RetainedOpenClawGpuConfig, + dockerDriverGateway: boolean, + preparedImage: RetainedOpenClawImage | null | undefined, +): RetainedOpenClawCreateIntent { + const retainedDockerRuntime = preparedImage + ? createRetainedOpenClawDockerRuntime(preparedImage) + : null; + return retainedDockerRuntime + ? bindRetainedOpenClawGpuRoute( + baseResolvedCreateIntent, + sandboxGpuConfig, + dockerDriverGateway, + retainedDockerRuntime, + ) + : baseResolvedCreateIntent; +} + +function abortUnusedRetainedImage(input: SandboxGpuCreateFlowInput): void { + const preparedImage = input.prebuild.preparedOpenClawLegacyImage; + if (!preparedImage) return; + try { + if (!preparedImage.abort()) { + console.warn(" Warning: the unused retained OpenClaw rebuild image could not be released."); + } + } catch { + console.warn(" Warning: the unused retained OpenClaw rebuild image could not be released."); + } } /** @@ -133,102 +219,132 @@ export async function runSandboxGpuCreateFlow( input: SandboxGpuCreateFlowInput, deps: SandboxGpuCreateFlowDeps, ): Promise { + const { retainedDockerRuntime, effectiveDeps, attemptRunner } = (() => { + try { + const retainedDockerRuntime = input.prebuild.preparedOpenClawLegacyImage + ? (deps.createRetainedDockerRuntime ?? createRetainedOpenClawDockerRuntime)( + input.prebuild.preparedOpenClawLegacyImage, + ) + : null; + const effectiveDeps: SandboxGpuCreateFlowDeps = retainedDockerRuntime + ? { ...deps, ...retainedDockerRuntime.deps } + : deps; + return { + retainedDockerRuntime, + effectiveDeps, + attemptRunner: createSandboxGpuCreateAttemptRunner(input, effectiveDeps), + }; + } catch (error) { + abortUnusedRetainedImage(input); + throw error; + } + })(); let registryImageRef: string | null = input.prebuild.imageRef; - const attemptRunner = createSandboxGpuCreateAttemptRunner(input, deps); - const gpuCreateOutcome = await sandboxGpuCreateAttempt - .executeSandboxGpuCreatePlan(input.gpuRoutePlan, { - runAttempt: attemptRunner.runAttempt, - captureNativeFailure: (failure) => { - const routeAdapter = adaptDockerGpuRouteForPatch(failure.route); - const diagnostics = collectDockerGpuPatchDiagnostics( - input.sandboxName, - { - error: failure.error, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - }, - { runCaptureOpenshell: deps.runCaptureOpenshell }, - ); - if (diagnostics) console.error(` Native GPU diagnostics saved: ${diagnostics.dir}`); - }, - cleanupNativeFailure: () => - sandboxGpuCreateAttempt.cleanupNativeGpuAttemptForFallback(input.sandboxName, { - runOpenshell: deps.runOpenshell, - sleep: deps.sleep, - }), - prepareCompatibilityAttempt: async () => { - if (!input.compatibilityPolicyPath) { - throw new Error("Compatibility retry policy was not materialized."); - } - const nativeRuntimeSnapshot = attemptRunner.state.nativeRuntimeSnapshot; - if (attemptRunner.managedRouting) { - const prepared = attemptRunner.managedRouting.prepareCompatibilityLaunch({ - createArgs: input.prebuild.createArgs, - currentRegistryImageRef: registryImageRef, - prebuildImageId: input.prebuild.imageId, - allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, - compatibilityPolicyPath: input.compatibilityPolicyPath, - startupCommand: input.sandboxStartupCommand, - runtimeSnapshot: nativeRuntimeSnapshot, - }); - attemptRunner.state.compatibilityArgv = [...prepared.createArgv]; - registryImageRef = prepared.registryImageRef; - } else { - const prebuildImageId = input.prebuild.imageId; - const imageId = - nativeRuntimeSnapshot?.imageId ?? - (prebuildImageId && isImmutableDockerImageId(prebuildImageId) - ? prebuildImageId.toLowerCase() - : null); - if ( - !registryImageRef && - nativeRuntimeSnapshot?.bookkeepingImageRef && - !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) - ) { - registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; - } - const compatibilityArgs = renderCompatibilityFallbackCreateArgs( - input.prebuild.createArgs, - { - imageRef: imageId, - allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, - compatibilityPolicyPath: input.compatibilityPolicyPath, - }, - ); - attemptRunner.state.compatibilityArgv = deps.openshellArgv([ - "sandbox", - "create", - ...compatibilityArgs, - "--", - ...input.sandboxStartupCommand, - ]); - } - if (attemptRunner.state.compatibilityArgv.length === 0) { - throw new Error("Compatibility sandbox create executable is missing."); - } - }, - activateCompatibilityAttempt: async () => { - if (!input.managedBootstrap) { - await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( - input.provider, - input.sandboxGpuConfig, + let gpuCreateOutcome: sandboxGpuCreateAttempt.SandboxGpuCreatePlanResult<{ + createResult: StreamSandboxCreateResult; + runtimePatch: ManagedBootstrapRuntimePatch; + }>; + try { + gpuCreateOutcome = await sandboxGpuCreateAttempt.executeSandboxGpuCreatePlan( + input.gpuRoutePlan, + { + runAttempt: attemptRunner.runAttempt, + captureNativeFailure: (failure) => { + const routeAdapter = adaptDockerGpuRouteForPatch(failure.route); + const diagnostics = collectDockerGpuPatchDiagnostics( + input.sandboxName, { - dockerDriverGateway: input.dockerDriverGateway, - selectedRoute: "compatibility", - gatewayPort: input.gatewayPort, - log: console.log, + error: failure.error, + additionalSummaryLines: routeAdapter.additionalSummaryLines, }, + effectiveDeps, ); - } - input.sandboxGpuConfig.sandboxGpuProof = null; + if (diagnostics) console.error(` Native GPU diagnostics saved: ${diagnostics.dir}`); + }, + cleanupNativeFailure: () => + sandboxGpuCreateAttempt.cleanupNativeGpuAttemptForFallback(input.sandboxName, { + runOpenshell: effectiveDeps.runOpenshell, + queryContainers: (sandboxName) => + queryOpenShellDockerSandboxContainers(sandboxName, effectiveDeps), + sleep: effectiveDeps.sleep, + }), + prepareCompatibilityAttempt: async () => { + if (!input.compatibilityPolicyPath) { + throw new Error("Compatibility retry policy was not materialized."); + } + const nativeRuntimeSnapshot = attemptRunner.state.nativeRuntimeSnapshot; + if (attemptRunner.managedRouting) { + const prepared = attemptRunner.managedRouting.prepareCompatibilityLaunch({ + createArgs: input.prebuild.createArgs, + currentRegistryImageRef: registryImageRef, + prebuildImageId: input.prebuild.imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + startupCommand: input.sandboxStartupCommand, + runtimeSnapshot: nativeRuntimeSnapshot, + }); + attemptRunner.state.compatibilityArgv = [...prepared.createArgv]; + registryImageRef = prepared.registryImageRef; + } else { + const imageId = resolveCompatibilityRetryImageId( + input.prebuild, + nativeRuntimeSnapshot?.imageId ?? null, + ); + if ( + !registryImageRef && + nativeRuntimeSnapshot?.bookkeepingImageRef && + !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) + ) { + registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; + } + const compatibilityArgs = renderCompatibilityFallbackCreateArgs( + input.prebuild.createArgs, + { + imageRef: imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + }, + ); + attemptRunner.state.compatibilityArgv = effectiveDeps.openshellArgv([ + "sandbox", + "create", + ...compatibilityArgs, + "--", + ...input.sandboxStartupCommand, + ]); + } + if (attemptRunner.state.compatibilityArgv.length === 0) { + throw new Error("Compatibility sandbox create executable is missing."); + } + }, + activateCompatibilityAttempt: async () => { + if (!input.managedBootstrap) { + await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( + input.provider, + input.sandboxGpuConfig, + { + dockerDriverGateway: input.dockerDriverGateway, + selectedRoute: "compatibility", + gatewayPort: input.gatewayPort, + log: console.log, + reverifyBridgeReachability: retainedDockerRuntime + ? () => retainedDockerRuntime.reverifyBridgeReachability(input.gatewayPort) + : undefined, + }, + ); + } + input.sandboxGpuConfig.sandboxGpuProof = null; + }, + traceEvent: addTraceEvent, }, - traceEvent: addTraceEvent, - }) - .catch((error: unknown) => { - if (error instanceof ManagedBootstrapRecoveryBlockedError) { - exitForManagedBootstrapRecovery(error); - } - throw error; - }); + ); + } catch (error) { + abortUnusedRetainedImage(input); + if (error instanceof ManagedBootstrapRecoveryBlockedError) { + exitForManagedBootstrapRecovery(error); + } + throw error; + } if (!gpuCreateOutcome.ok) { console.error(""); console.error(" Operator-authorized GPU fallback stopped before compatibility retry."); @@ -243,13 +359,22 @@ export async function runSandboxGpuCreateFlow( ); } console.error(` Manual cleanup: openshell sandbox delete "${input.sandboxName}"`); + abortUnusedRetainedImage(input); process.exit(1); } + const retainedImageFinalization = + input.prebuild.preparedOpenClawLegacyImage?.finalizeAfterCreate() ?? null; + if (input.prebuild.preparedOpenClawLegacyImage && !retainedImageFinalization) { + abortUnusedRetainedImage(input); + throw new Error("Retained OpenClaw rebuild image could not be finalized after creation."); + } + return { ...gpuCreateOutcome.value, route: gpuCreateOutcome.route, firstCreateOutput: attemptRunner.state.firstCreateOutput, registryImageRef, + retainedImageFinalization, }; } diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 49c3271ddd6..793cdbd431e 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -69,14 +69,14 @@ export function createSandboxGpuCreateAttemptRunner( !managedRouting && input.initialGpuRoute === "native" && input.gpuRoutePlan === "native-with-fallback" - ? queryOpenShellDockerSandboxContainers(input.sandboxName) + ? queryOpenShellDockerSandboxContainers(input.sandboxName, deps) : null; const nativeFallbackHasCleanBaseline = managedRouting?.nativeFallbackHasCleanBaseline ?? (nativeFallbackBaseline?.ok === true && nativeFallbackBaseline.ids.length === 0); const inspectNativeRuntime = (): NativeRuntimeSnapshot | null => { if (managedRouting) return managedRouting.inspectNativeRuntime(); - const snapshot = queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName); + const snapshot = queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName, deps); return snapshot.ok ? snapshot : null; }; @@ -153,6 +153,10 @@ export function createSandboxGpuCreateAttemptRunner( await managedLifecycle?.prepareNetwork(); const [createExecutable, ...createExecutableArgs] = managedLifecycle?.launchArgv ?? attemptArgv; if (!createExecutable) throw new Error("Sandbox create executable is missing."); + const preparedImage = input.prebuild.preparedOpenClawLegacyImage; + if (preparedImage && !preparedImage.verifyForCreate()) { + throw new Error("Retained OpenClaw rebuild image changed before sandbox creation."); + } const streamCreate = () => streamSandboxCreate(createExecutable, createExecutableArgs, input.sandboxEnv, { readyCheck: () => { @@ -415,6 +419,8 @@ export function createSandboxGpuCreateAttemptRunner( reportGpuProofFailure: !deferNativeProofFailure, selectedMode: runtimePatch.selectedMode, runCaptureOpenshell: deps.runCaptureOpenshell, + dockerCapture: deps.dockerCapture, + dockerLogs: deps.dockerLogs, log: console.log, }, ); diff --git a/src/lib/onboard/sandbox-prebuild.test.ts b/src/lib/onboard/sandbox-prebuild.test.ts index 961a891f90d..eaf79f9a5f4 100644 --- a/src/lib/onboard/sandbox-prebuild.test.ts +++ b/src/lib/onboard/sandbox-prebuild.test.ts @@ -17,6 +17,7 @@ vi.mock("../adapters/docker/exec", async (importOriginal) => ({ import { withStdoutRedirectedToStderr } from "../cli/stdout-guard"; import { SANDBOX_BUILD_CONTEXT_PREFIX } from "../sandbox/build-context"; +import type { PreparedOpenClawLegacyImage } from "./build-context-stage"; import { dockerBuildSubprocessEnv, prebuildSandboxImageIfEligible, @@ -28,6 +29,24 @@ const BUILD_ID = "1234567890"; const IMAGE_ID = `sha256:${"a".repeat(64)}`; const temporaryDirectories: string[] = []; +function createPreparedOpenClawLegacyImageFixture(): PreparedOpenClawLegacyImage { + return { + dockerEnv: { DOCKER_CONTEXT: "retained-context" }, + engineId: "retained-engine", + imageRef: "nemoclaw-sandbox-local:retained-openclaw", + imageId: IMAGE_ID, + verify: vi.fn(() => true), + retainForRecreate: vi.fn(() => true), + verifyForCreate: vi.fn(() => true), + finalizeAfterCreate: vi.fn(() => ({ + mutableTagVerified: true, + registryImageRef: null, + })), + abort: vi.fn(() => true), + dispose: vi.fn(() => true), + }; +} + function createBuildContext( parent = os.tmpdir(), prefix = SANDBOX_BUILD_CONTEXT_PREFIX, @@ -327,6 +346,59 @@ describe("sandbox BuildKit prebuild", () => { expect(log).toHaveBeenCalledWith(expect.stringContaining("could not be inspected")); }); + it("reuses a retained OpenClaw image by immutable ID without rebuilding", async () => { + const { buildCtx, createArgs } = createBuildContext(); + const preparedOpenClawLegacyImage = createPreparedOpenClawLegacyImageFixture(); + const buildImage = vi.fn(async () => 0); + + await expect( + prebuildSandboxImageIfEligible({ + buildCtx, + buildId: BUILD_ID, + origin: "generated", + createArgs, + sandboxName: "alpha", + dockerDriverGateway: true, + preparedOpenClawLegacyImage, + env: { NEMOCLAW_SANDBOX_PREBUILD: "0" }, + buildImage, + }), + ).resolves.toEqual({ + createArgs: ["--from", IMAGE_ID, "--name", "alpha"], + imageRef: preparedOpenClawLegacyImage.imageRef, + imageId: IMAGE_ID, + preparedOpenClawLegacyImage, + }); + expect(buildImage).not.toHaveBeenCalled(); + expect(preparedOpenClawLegacyImage.verifyForCreate).not.toHaveBeenCalled(); + expect(preparedOpenClawLegacyImage.finalizeAfterCreate).not.toHaveBeenCalled(); + }); + + it.each([ + ["a nonlocal gateway", { dockerDriverGateway: false }], + ["a custom Dockerfile", { origin: "custom" as const }], + ["changed create arguments", { createArgs: ["--from", "/tmp/changed/Dockerfile"] }], + ])("fails closed instead of rebuilding or falling back after %s", async (_label, override) => { + const { buildCtx, createArgs } = createBuildContext(); + const buildImage = vi.fn(async () => 0); + + await expect( + prebuildSandboxImageIfEligible({ + buildCtx, + buildId: BUILD_ID, + origin: "generated", + createArgs, + sandboxName: "alpha", + dockerDriverGateway: true, + preparedOpenClawLegacyImage: createPreparedOpenClawLegacyImageFixture(), + env: { NEMOCLAW_SANDBOX_PREBUILD: "0" }, + buildImage, + ...override, + }), + ).rejects.toThrow(/retained OpenClaw rebuild image/i); + expect(buildImage).not.toHaveBeenCalled(); + }); + it("uses the argv-based Docker helper and returns the local image on success", async () => { const { buildCtx, createArgs, dockerfile } = createBuildContext(); const buildImage = vi.fn(async () => 0); diff --git a/src/lib/onboard/sandbox-prebuild.ts b/src/lib/onboard/sandbox-prebuild.ts index 888b094c1fe..b4da1d34202 100644 --- a/src/lib/onboard/sandbox-prebuild.ts +++ b/src/lib/onboard/sandbox-prebuild.ts @@ -14,12 +14,14 @@ import { type SandboxBuildContextOrigin, } from "../sandbox/build-context"; import { buildSubprocessEnv } from "../subprocess-env"; +import type { PreparedOpenClawLegacyImage } from "./build-context-stage"; import { isImmutableDockerImageId } from "./openshell-docker-sandbox-containers"; const TRUTHY_FLAG_VALUES = new Set(["1", "true", "yes", "on"]); const FALSY_FLAG_VALUES = new Set(["0", "false", "no", "off"]); const LOCAL_IMAGE_REPO = LOCAL_SANDBOX_IMAGE_REPO; -const DOCKER_ENV_NAMES = [ +export const DOCKER_SELECTOR_ENV_NAMES = [ + "DOCKER_HOST", "DOCKER_API_VERSION", "DOCKER_CERT_PATH", "DOCKER_CONFIG", @@ -34,6 +36,7 @@ export interface SandboxPrebuildInput { sandboxName: string; dockerDriverGateway: boolean; origin: SandboxBuildContextOrigin; + preparedOpenClawLegacyImage?: PreparedOpenClawLegacyImage; env?: NodeJS.ProcessEnv; buildImage?: ( args: readonly string[], @@ -48,6 +51,7 @@ export interface SandboxPrebuildResult { imageRef: string | null; /** Immutable local image identity; mutable tags never authorize fallback. */ imageId: string | null; + preparedOpenClawLegacyImage?: PreparedOpenClawLegacyImage; } interface TrustedStagedBuildContext { @@ -96,7 +100,7 @@ function resolveTrustedStagedBuildContext(buildCtx: string): TrustedStagedBuildC /** Restrict the host Docker build to environment values used by Docker itself. */ export function dockerBuildSubprocessEnv(): Record { const env = buildSubprocessEnv(); - for (const key of DOCKER_ENV_NAMES) { + for (const key of DOCKER_SELECTOR_ENV_NAMES) { const value = process.env[key]; if (value !== undefined) env[key] = value; } @@ -153,6 +157,38 @@ export async function prebuildSandboxImageIfEligible( input: SandboxPrebuildInput, ): Promise { const createArgs = [...input.createArgs]; + const preparedImage = input.preparedOpenClawLegacyImage; + if (preparedImage) { + if (!input.dockerDriverGateway) { + throw new Error( + "A retained OpenClaw rebuild image cannot be used by a nonlocal OpenShell gateway.", + ); + } + if (input.origin !== "generated") { + throw new Error("A retained OpenClaw rebuild image cannot be used with a custom Dockerfile."); + } + const fromIndexes = createArgs.flatMap((arg, index) => (arg === "--from" ? [index] : [])); + const fromIndex = fromIndexes[0] ?? -1; + const fromDockerfile = createArgs[fromIndex + 1]; + if ( + fromIndexes.length !== 1 || + !fromDockerfile || + path.resolve(fromDockerfile) !== path.resolve(input.buildCtx, "Dockerfile") + ) { + throw new Error("Retained OpenClaw rebuild image arguments changed before sandbox creation."); + } + if (!isImmutableDockerImageId(preparedImage.imageId)) { + throw new Error("Retained OpenClaw rebuild image identity is invalid."); + } + const imageId = preparedImage.imageId.toLowerCase(); + createArgs[fromIndex + 1] = imageId; + return { + createArgs, + imageRef: preparedImage.imageRef, + imageId, + preparedOpenClawLegacyImage: preparedImage, + }; + } const env = input.env ?? process.env; const log = input.log ?? console.log; if (!resolveSandboxPrebuildEnabled(env, input.dockerDriverGateway)) { diff --git a/test/gateway-state-reconcile-2276.test.ts b/test/gateway-state-reconcile-2276.test.ts index 72adb6ce150..c3a53d1c397 100644 --- a/test/gateway-state-reconcile-2276.test.ts +++ b/test/gateway-state-reconcile-2276.test.ts @@ -164,6 +164,11 @@ if (args[0] === "sandbox" && args[1] === "list") { process.exit(0); } +if (args[0] === "sandbox" && args[1] === "create") { + process.stderr.write("injected sandbox create failure\\n"); + process.exit(1); +} + if (args[0] === "inference" && args[1] === "get") { process.stdout.write("Gateway inference:\\n Provider: nvidia-prod\\n Model: nvidia/nemotron-3-super-120b-a12b\\n"); process.exit(0); @@ -261,6 +266,10 @@ const { isOpenClawSecurityInventoryProbe } = require(${JSON.stringify( path.join(import.meta.dirname, "helpers", "onboard-script-mocks.cjs"), )}); if (a[0] === "info") { + if (a.includes("{{.ID}}")) { + process.stdout.write("engine-a\\n"); + process.exit(0); + } process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); process.exit(0); } diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index 972081810e9..d595f7e4eee 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -39,6 +39,9 @@ const registryPersistence = requireDist("../../state/registry/persistence.js"); const sandboxState = requireDist("../../state/sandbox.js"); const sandboxSession = requireDist("../../state/sandbox-session.js"); const sandboxVersion = requireDist("../../sandbox/version.js"); +const mcpLifecycleLock = requireDist( + "../../state/mcp-lifecycle-lock.js", +) as typeof import("../../src/lib/state/mcp-lifecycle-lock"); const destroy = requireDist("./destroy.js"); const rebuildShields = requireDist("./rebuild-shields.js"); const nim = requireDist("../../inference/nim.js"); @@ -141,6 +144,13 @@ export type RebuildFlowOverrides = { openShieldsWindow?: () => { relocked: boolean; wasLocked: boolean } | null; preflightMessagingConflicts?: () => Promise | void; preflightAuthoritativeRebuildTarget?: (options: Record) => Promise | void; + rebuildImagePreflightResult?: + | { ok: false; detail: string } + | { + ok: true; + imageTag: string | null; + prepared?: unknown; + }; revalidateRebuildRouteBeforeDelete?: ( receipt: Record, ) => { ok: true; receipt: Record } | { ok: false; message: string }; @@ -169,6 +179,7 @@ export type RebuildFlowHarness = { openShieldsSpy: MockInstance; onboardSpy: MockInstance; preflightAuthoritativeRebuildTargetSpy: MockInstance; + preflightRebuildImageSpy: MockInstance; preflightMessagingConflictsSpy: MockInstance; preflightDcodeRouteSpy: MockInstance; prepareManagedDcodeRebuildImageSpy: MockInstance; @@ -313,6 +324,9 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined); + vi.spyOn(mcpLifecycleLock, "withMcpLifecycleLock").mockImplementation( + async (_sandboxName: string, work: () => Promise | T): Promise => await work(), + ); const session = createRebuildFlowSession(onboardSession.MACHINE_SNAPSHOT_VERSION); const rebuildShieldsWindow = { relocked: false, wasLocked: false }; @@ -352,10 +366,9 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): }); vi.spyOn(resolve, "resolveOpenshell").mockReturnValue(null); vi.spyOn(dockerImage, "dockerBuild").mockReturnValue({ status: 0 }); - vi.spyOn(rebuildCustomImagePreflight, "preflightRebuildImage").mockResolvedValue({ - ok: true, - imageTag: null, - }); + const preflightRebuildImageSpy = vi + .spyOn(rebuildCustomImagePreflight, "preflightRebuildImage") + .mockResolvedValue(overrides.rebuildImagePreflightResult ?? { ok: true, imageTag: null }); const imageIdsByRef = new Map([ [agentBaseImageRef, agentBaseImageId], [agentBaseImageId, agentBaseImageId], @@ -829,6 +842,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): openShieldsSpy, onboardSpy, preflightAuthoritativeRebuildTargetSpy, + preflightRebuildImageSpy, preflightMessagingConflictsSpy, preflightDcodeRouteSpy, prepareManagedDcodeRebuildImageSpy, diff --git a/test/helpers/rebuild-flow-target-image-cases.ts b/test/helpers/rebuild-flow-target-image-cases.ts index 3995f68eea5..027f87085cf 100644 --- a/test/helpers/rebuild-flow-target-image-cases.ts +++ b/test/helpers/rebuild-flow-target-image-cases.ts @@ -331,7 +331,7 @@ export function registerRebuildFlowTargetImageTests(): void { try { await expect( harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).rejects.toThrow("Replacement sandbox image context changed before delete"); + ).rejects.toThrow("Replacement sandbox image inputs changed before delete"); expectNoSandboxDelete(harness.runOpenshellSpy); expect(harness.onboardSpy).not.toHaveBeenCalled(); @@ -391,7 +391,7 @@ export function registerRebuildFlowTargetImageTests(): void { try { await expect( harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).rejects.toThrow("Replacement sandbox image context changed before delete"); + ).rejects.toThrow("Replacement sandbox image inputs changed before delete"); expectNoSandboxDelete(harness.runOpenshellSpy); expect(harness.onboardSpy).not.toHaveBeenCalled(); expect(cleanupBuildCtx).toHaveBeenCalledOnce(); diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index 7ba8c428b63..187e36d7c16 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -179,7 +179,11 @@ if (a[0] === "sandbox" && a[1] === "get") { process.stdout.write("Sandbox: ${sandboxName}\\nPhase: Ready\\n"); process.exit(0); } -if (a[0] === "sandbox" && a[1] === "delete") { fs.writeFileSync(${JSON.stringify(deleteMarker)}, "deleted\\n"); process.exit(${sandboxDeleteExitCode}); } +if (a[0] === "sandbox" && a[1] === "delete") { + const exitCode = ${sandboxDeleteExitCode}; + if (exitCode === 0) fs.writeFileSync(${JSON.stringify(deleteMarker)}, "deleted\\n"); + process.exit(exitCode); +} if (a[0] === "sandbox" && a[1] === "get") { if (fs.existsSync(${JSON.stringify(deleteMarker)})) { process.stderr.write("sandbox ${sandboxName} not found\\n"); process.exit(1); } process.stdout.write("${sandboxName} Ready\\n"); @@ -271,7 +275,11 @@ const { isOpenClawSecurityInventoryProbe } = require(${JSON.stringify( )}); const provenancePath = ${JSON.stringify(path.join(tmpDir, "docker-base-provenance"))}; const readProvenance = () => fs.existsSync(provenancePath) ? JSON.parse(fs.readFileSync(provenancePath, "utf8")) : {}; -if (a[0] === "info") { process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); process.exit(0); } +if (a[0] === "info") { + if (a.includes("{{.ID}}")) { process.stdout.write("engine-a\\n"); process.exit(0); } + process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); + process.exit(0); +} if (a[0] === "build") { const labelIndex = a.indexOf("--label"); const tagIndex = a.indexOf("-t"); @@ -393,12 +401,16 @@ describe("atomic rebuild process contracts (#2273)", () => { expect(registryHasSandbox(fixture)).toBe(true); }); - it("accepts trimmed case-insensitive yes input before continuing into backup", () => { + it("accepts trimmed case-insensitive yes input before continuing into backup", { + timeout: testTimeout(60_000), + }, () => { const fixture = createFixture({ savedCredential: { key: "NVIDIA_INFERENCE_API_KEY", value: "nvapi-test-key-for-rebuild", }, + // This contract ends at backup; preserve the sandbox when delete fails. + sandboxDeleteExitCode: 1, }); const result = runRebuild(fixture, {}, { yes: false, input: " YES \n" }); @@ -408,6 +420,8 @@ describe("atomic rebuild process contracts (#2273)", () => { expect(output).not.toContain("Cancelled."); expect(output).not.toContain("preflight failed"); expect(output).toContain("Backing up sandbox state"); + expect(output).not.toContain("Creating new sandbox with current image"); + expect(fs.existsSync(fixture.deleteMarker)).toBe(false); }); it("prints an active SSH session warning before interactive confirmation and cancel", () => { diff --git a/test/rebuild-shields-auto-unlock.test.ts b/test/rebuild-shields-auto-unlock.test.ts index c7504cecc4b..6ab6019103f 100644 --- a/test/rebuild-shields-auto-unlock.test.ts +++ b/test/rebuild-shields-auto-unlock.test.ts @@ -235,6 +235,10 @@ function writeLockState(state) { fs.writeFileSync(lockStatePath, state); } if (a[0]==="info") { + if (a.includes("{{.ID}}")) { + process.stdout.write("engine-a\\n"); + process.exit(0); + } process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); process.exit(0); } diff --git a/test/rebuild-stale-recovery.test.ts b/test/rebuild-stale-recovery.test.ts index 138bee2db30..0fee2393b9d 100644 --- a/test/rebuild-stale-recovery.test.ts +++ b/test/rebuild-stale-recovery.test.ts @@ -173,6 +173,10 @@ const { isOpenClawSecurityInventoryProbe } = require(${JSON.stringify( path.join(REPO_ROOT, "test", "helpers", "onboard-script-mocks.cjs"), )}); if (a[0]==="info") { + if (a.includes("{{.ID}}")) { + process.stdout.write("engine-a\\n"); + process.exit(0); + } process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); process.exit(0); } diff --git a/test/repro-2201.test.ts b/test/repro-2201.test.ts index 3bbafd10aaa..1b533c85810 100644 --- a/test/repro-2201.test.ts +++ b/test/repro-2201.test.ts @@ -277,6 +277,10 @@ const { isOpenClawSecurityInventoryProbe } = require(${JSON.stringify( const provenancePath = ${JSON.stringify(path.join(tmpDir, "docker-base-provenance"))}; const readProvenance = () => fs.existsSync(provenancePath) ? JSON.parse(fs.readFileSync(provenancePath, "utf8")) : {}; if (a[0]==="info") { + if (a.includes("{{.ID}}")) { + process.stdout.write("engine-a\\n"); + process.exit(0); + } process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); process.exit(0); }