From 06361912050bd8b006ddbf90cd79698aea8e1087 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Mon, 20 Jul 2026 11:12:12 -0700 Subject: [PATCH 01/16] fix(rebuild): support hosts without Docker Buildx Detect the canonical missing-Buildx failure and verify it with an independent probe. Retry generated managed OpenClaw contexts once with the legacy Docker builder. Seal the builder and Docker endpoint through rebuild and fail closed on drift. Co-authored-by: Tinson Lai Signed-off-by: Apurv Kumaria --- docs/reference/commands.mdx | 5 + .../rebuild-custom-image-preflight.test.ts | 321 ++++++++++++++++++ .../sandbox/rebuild-custom-image-preflight.ts | 217 ++++++++++-- .../sandbox/rebuild-prepared-image-context.ts | 5 +- .../actions/sandbox/rebuild-target-runtime.ts | 4 +- src/lib/onboard.ts | 3 +- src/lib/onboard/build-context-stage.ts | 4 + .../onboard/prepared-dcode-rebuild.test.ts | 19 ++ src/lib/onboard/prepared-dcode-rebuild.ts | 28 ++ src/lib/onboard/sandbox-create-launch.test.ts | 12 +- src/lib/onboard/sandbox-prebuild.test.ts | 118 +++++++ src/lib/onboard/sandbox-prebuild.ts | 82 ++++- 12 files changed, 780 insertions(+), 38 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 05307e1bd37..a03113d3a20 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -2477,6 +2477,11 @@ A rebuild preserves the recorded Deep Agents Code auto-approval capability unles A sandbox onboarded with an explicit GPU opt-out (stored as `sandboxGpuMode: "0"`, plus legacy registry entries that only record `gpuEnabled: false`) is recreated with the same opt-out, so the inner `onboard --resume` skips the Docker CDI GPU preflight on hosts without an NVIDIA GPU. Auto-mode sandboxes remain auto. +Before backup or deletion, `rebuild` stages and validates the replacement image. +When recreation can consume a local Docker image and Docker reports that Buildx is missing or broken while validating the managed OpenClaw context generated by NemoClaw, NemoClaw prints a warning, retries that exact fingerprinted context once with Docker's legacy builder, and carries the verified builder and Docker endpoint configuration into recreation. +The local preflight does not retry a user-supplied `--from` context with a different builder. +If validation still fails or the staged context changes, rebuild stops and leaves the existing sandbox intact. + ```bash $$nemoclaw my-assistant rebuild [--yes|-y|--force] [--verbose|-v] [--tool-disclosure ] [--dcode-auto-approval ] [--observability|--no-observability] ``` diff --git a/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts index 2f6abe52a39..074d3450de6 100644 --- a/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts +++ b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts @@ -42,12 +42,24 @@ function input(fromDockerfile: string | null) { sandboxGpuDevice: null, errors: [], }, + localPrebuildEnabled: true, gatewayPort: 8080, chatUiUrl: "http://127.0.0.1:18789", }; } describe("preflightRebuildImage", () => { + it("keeps the generated OpenClaw Dockerfile compatible with the legacy fallback (#7111)", () => { + const dockerfile = fs.readFileSync(path.join(ROOT, "Dockerfile"), "utf8"); + for (const buildKitOnlySyntax of [ + /^\s*RUN\s+--mount=/mu, + /^\s*(?:COPY|ADD)\s+--link(?:=|\s)/mu, + /^\s*RUN\s+<<-?\w+/mu, + ]) { + expect(dockerfile).not.toMatch(buildKitOnlySyntax); + } + }); + it("prebuilds the managed OpenClaw image instead of deferring its first build until delete", async () => { const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-preflight-")); const stagedDockerfile = path.join(buildCtx, "Dockerfile"); @@ -82,8 +94,312 @@ describe("preflightRebuildImage", () => { ); expect(buildImage).toHaveBeenCalledOnce(); expect(cleanupBuildCtx).not.toHaveBeenCalled(); + expect(result.prepared.prebuildBuilder).toBeUndefined(); + expect(result.prepared.prebuildDockerEnv).toBeUndefined(); + expect(verifyPreparedBuildContext(result.prepared)).toBe(true); + expect(disposePreparedBuildContext(result.prepared)).toBe(true); + expect(cleanupBuildCtx).toHaveBeenCalledOnce(); + } finally { + fs.rmSync(buildCtx, { recursive: true, force: true }); + } + }); + + it("retries an exact generated image once with the legacy builder when Buildx is unavailable (#7111)", async () => { + const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-fallback-")); + const stagedDockerfile = path.join(buildCtx, "Dockerfile"); + fs.writeFileSync(stagedDockerfile, "FROM scratch\n"); + const cleanupBuildCtx = vi.fn(() => { + fs.rmSync(buildCtx, { recursive: true, force: true }); + return true; + }); + const buildImage = vi + .fn() + .mockReturnValueOnce({ + status: 1, + stderr: Buffer.from( + "ERROR: BuildKit is enabled but the buildx component is missing or broken.", + ), + } as never) + .mockReturnValueOnce({ status: 0 } as never); + const dockerEnv = Object.freeze({ + DOCKER_CONFIG: "/home/test/.docker", + DOCKER_CONTEXT: "verified-builder", + }); + const buildxAvailable = vi.fn(() => false); + const removeImage = vi.fn(() => ({ status: 0 }) as never); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + try { + const result = successful( + await preflightRebuildImage(input(null), { + stageBuildContext: vi.fn(() => ({ + buildCtx, + stagedDockerfile, + cleanupBuildCtx, + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "buildx-fallback", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + buildxAvailable, + buildDockerEnv: () => dockerEnv, + removeImage, + }), + ); + + expect(buildImage).toHaveBeenCalledTimes(2); + expect(buildImage.mock.calls[0]?.[3]).toEqual( + expect.objectContaining({ cwd: ROOT, env: dockerEnv }), + ); + expect(buildImage.mock.calls[1]?.[3]).toEqual( + expect.objectContaining({ + cwd: ROOT, + env: { ...dockerEnv, DOCKER_BUILDKIT: "0" }, + }), + ); + expect(buildxAvailable).toHaveBeenCalledWith({ cwd: ROOT, env: dockerEnv }); + expect(removeImage).toHaveBeenCalledWith( + expect.stringMatching(/^nemoclaw-rebuild-preflight:/), + expect.objectContaining({ cwd: ROOT, env: dockerEnv }), + ); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("legacy builder")); + expect(result.prepared.prebuildBuilder).toBe("legacy"); + expect(result.prepared.prebuildDockerEnv).toEqual(dockerEnv); + expect(result.prepared.prebuildDockerEnv).not.toBe(dockerEnv); expect(verifyPreparedBuildContext(result.prepared)).toBe(true); + const clonedPrepared = { ...result.prepared }; + expect(clonedPrepared.verifyBuildCtx()).toBe(false); + expect(verifyPreparedBuildContext(clonedPrepared)).toBe(false); + (result.prepared as { prebuildBuilder?: string }).prebuildBuilder = "buildkit"; + expect(result.prepared.verifyBuildCtx()).toBe(false); + (result.prepared as { prebuildBuilder?: string }).prebuildBuilder = "legacy"; expect(disposePreparedBuildContext(result.prepared)).toBe(true); + } finally { + warn.mockRestore(); + fs.rmSync(buildCtx, { recursive: true, force: true }); + } + }); + + it.each([ + ["the independent Buildx probe succeeds", null, true, true], + ["the generated target is not managed OpenClaw", { name: "hermes" }, false, true], + ["local image prebuild is unavailable", null, false, false], + ])("does not downgrade the builder when %s", async (_label, agent, available, localPrebuildEnabled) => { + const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-no-downgrade-")); + const stagedDockerfile = path.join(buildCtx, "Dockerfile"); + fs.writeFileSync(stagedDockerfile, "FROM scratch\n"); + const cleanupBuildCtx = vi.fn(() => { + fs.rmSync(buildCtx, { recursive: true, force: true }); + return true; + }); + const buildImage = vi.fn( + () => + ({ + status: 1, + stderr: "ERROR: BuildKit is enabled but the buildx component is missing or broken.", + }) as never, + ); + + try { + const result = await preflightRebuildImage( + { ...input(null), agent: agent as never, localPrebuildEnabled }, + { + stageBuildContext: vi.fn(() => ({ + buildCtx, + stagedDockerfile, + cleanupBuildCtx, + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "no-builder-downgrade", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + buildxAvailable: () => available, + removeImage: vi.fn(() => ({ status: 0 }) as never), + }, + ); + + expect(result).toEqual({ + ok: false, + detail: "ERROR: BuildKit is enabled but the buildx component is missing or broken.", + }); + expect(buildImage).toHaveBeenCalledOnce(); + expect(cleanupBuildCtx).toHaveBeenCalledOnce(); + } finally { + fs.rmSync(buildCtx, { recursive: true, force: true }); + } + }); + + it("does not retry a custom Dockerfile when Buildx is unavailable", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-custom-buildx-")); + const dockerfile = path.join(dir, "Dockerfile.custom"); + fs.writeFileSync(dockerfile, "FROM scratch\n"); + const buildImage = vi.fn( + () => + ({ + status: 1, + stderr: "ERROR: BuildKit is enabled but the buildx component is missing or broken.", + }) as never, + ); + + try { + const result = await preflightRebuildImage(input(dockerfile), { + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "custom-buildx", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + removeImage: vi.fn(() => ({ status: 0 }) as never), + }); + + expect(result).toEqual({ + ok: false, + detail: "ERROR: BuildKit is enabled but the buildx component is missing or broken.", + }); + expect(buildImage).toHaveBeenCalledOnce(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects generated build-context drift before a legacy-builder retry", async () => { + const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-drift-")); + const stagedDockerfile = path.join(buildCtx, "Dockerfile"); + fs.writeFileSync(stagedDockerfile, "FROM scratch\n"); + const cleanupBuildCtx = vi.fn(() => { + fs.rmSync(buildCtx, { recursive: true, force: true }); + return true; + }); + const buildImage = vi.fn(() => { + fs.writeFileSync(path.join(buildCtx, "mutated"), "changed\n"); + return { + status: 1, + stderr: "ERROR: BuildKit is enabled but the buildx component is missing or broken.", + } as never; + }); + + try { + await expect( + preflightRebuildImage(input(null), { + stageBuildContext: vi.fn(() => ({ + buildCtx, + stagedDockerfile, + cleanupBuildCtx, + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "buildx-drift", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + buildxAvailable: () => false, + removeImage: vi.fn(() => ({ status: 0 }) as never), + }), + ).resolves.toEqual({ + ok: false, + detail: "replacement build context changed during preflight", + }); + expect(buildImage).toHaveBeenCalledOnce(); + expect(cleanupBuildCtx).toHaveBeenCalledOnce(); + } finally { + fs.rmSync(buildCtx, { recursive: true, force: true }); + } + }); + + it("prioritizes the redacted legacy failure when both generated-image attempts fail", async () => { + const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-double-fail-")); + const stagedDockerfile = path.join(buildCtx, "Dockerfile"); + const credential = ["legacy", "retry", "credential"].join("-"); + fs.writeFileSync(stagedDockerfile, "FROM scratch\n"); + const cleanupBuildCtx = vi.fn(() => { + fs.rmSync(buildCtx, { recursive: true, force: true }); + return true; + }); + const buildImage = vi + .fn() + .mockReturnValueOnce({ + status: 1, + stderr: + "ERROR: BuildKit is enabled but the buildx component is missing or broken.\n" + + "x".repeat(9_000), + } as never) + .mockReturnValueOnce({ + status: 1, + stderr: + `legacy build could not read ${os.homedir()}/private-context\n` + + `Authorization: Bearer ${credential}`, + } as never); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + try { + const result = await preflightRebuildImage(input(null), { + stageBuildContext: vi.fn(() => ({ + buildCtx, + stagedDockerfile, + cleanupBuildCtx, + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "buildx-double-fail", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + buildxAvailable: () => false, + removeImage: vi.fn(() => ({ status: 0 }) as never), + }); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected failed rebuild preflight"); + expect(result.detail).toContain("Legacy-builder retry failed"); + expect(result.detail).toContain("legacy build could not read ~/private-context"); + expect(result.detail).toContain("Authorization: Bearer "); + expect(result.detail).not.toContain(credential); + expect(result.detail.length).toBeLessThan(8_100); + expect(buildImage).toHaveBeenCalledTimes(2); + expect(cleanupBuildCtx).toHaveBeenCalledOnce(); + } finally { + warn.mockRestore(); + fs.rmSync(buildCtx, { recursive: true, force: true }); + } + }); + + it("does not retry an unrelated generated-image build failure", async () => { + const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-build-no-retry-")); + const stagedDockerfile = path.join(buildCtx, "Dockerfile"); + fs.writeFileSync(stagedDockerfile, "FROM scratch\n"); + const cleanupBuildCtx = vi.fn(() => { + fs.rmSync(buildCtx, { recursive: true, force: true }); + return true; + }); + const buildImage = vi.fn(() => ({ status: 1, stderr: "registry timeout" }) as never); + + try { + await expect( + preflightRebuildImage(input(null), { + stageBuildContext: vi.fn(() => ({ + buildCtx, + stagedDockerfile, + cleanupBuildCtx, + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "unrelated-build-failure", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + removeImage: vi.fn(() => ({ status: 0 }) as never), + }), + ).resolves.toEqual({ ok: false, detail: "registry timeout" }); + expect(buildImage).toHaveBeenCalledOnce(); expect(cleanupBuildCtx).toHaveBeenCalledOnce(); } finally { fs.rmSync(buildCtx, { recursive: true, force: true }); @@ -277,6 +593,7 @@ describe("preflightRebuildImage", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-custom-preflight-cleanup-")); const dockerfile = path.join(dir, "Dockerfile.custom"); fs.writeFileSync(dockerfile, "FROM scratch\n"); + const dockerEnv = Object.freeze({ DOCKER_CONTEXT: "cleanup-builder" }); const removeImage = vi .fn() .mockReturnValueOnce({ status: 1 } as never) @@ -296,6 +613,7 @@ describe("preflightRebuildImage", () => { resolvedBaseImage: null, })), buildImage: vi.fn(() => ({ status: 0 }) as never), + buildDockerEnv: () => dockerEnv, removeImage, }), ); @@ -305,6 +623,9 @@ describe("preflightRebuildImage", () => { ); expect(processOnce).toHaveBeenCalledWith("exit", expect.any(Function)); expect(removeImage).toHaveBeenCalledTimes(2); + for (const call of removeImage.mock.calls) { + expect(call[1]).toEqual(expect.objectContaining({ cwd: ROOT, env: dockerEnv })); + } expect(result.prepared.dashboardRemoteBindPrepared).toBe(true); expect(disposePreparedBuildContext(result.prepared)).toBe(true); } finally { diff --git a/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts b/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts index 514e1b9e74c..9f7011886e9 100644 --- a/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts @@ -3,7 +3,8 @@ import path from "node:path"; -import { dockerBuild, dockerRmi } from "../../adapters/docker"; +import type { DockerBuildOptions, DockerRunOptions, DockerRunResult } from "../../adapters/docker"; +import { dockerSpawnSync } from "../../adapters/docker/exec"; import { fingerprintBuildContext } from "../../adapters/fs/build-context-fingerprint"; import type { AgentDefinition } from "../../agent/defs"; import { createAgentSandbox } from "../../agent/onboard"; @@ -11,6 +12,7 @@ import type { WebSearchConfig } from "../../inference/web-search"; import { stageCreateSandboxBuildContext } from "../../onboard/build-context-stage"; import { prepareSandboxDockerfilePatch } from "../../onboard/sandbox-dockerfile-patch-flow"; import type { SandboxGpuConfig } from "../../onboard/sandbox-gpu-mode"; +import { dockerBuildSubprocessEnv } from "../../onboard/sandbox-prebuild"; import { ROOT } from "../../runner"; import { formatBuildFailureDiagnostics, @@ -35,6 +37,8 @@ type PreflightInput = { toolDisclosure: ToolDisclosure; hermesToolGateways: string[]; sandboxGpuConfig: SandboxGpuConfig; + /** Whether recreation can consume an image built by the same host Docker daemon. */ + localPrebuildEnabled: boolean; gatewayPort: number; chatUiUrl: string; }; @@ -42,8 +46,24 @@ type PreflightInput = { type PreflightDeps = { stageBuildContext?: typeof stageCreateSandboxBuildContext; prepareDockerfilePatch?: typeof prepareSandboxDockerfilePatch; - buildImage?: typeof dockerBuild; - removeImage?: typeof dockerRmi; + buildImage?: BuildImage; + removeImage?: RemoveImage; + buildxAvailable?: (process: DockerProofProcess) => boolean; + buildDockerEnv?: () => Record; +}; + +type BuildImage = ( + dockerfilePath: string, + tag: string, + contextDir: string, + options: DockerBuildOptions, +) => DockerRunResult; + +type RemoveImage = (imageRef: string, options: NonNullable) => DockerRunResult; + +type DockerProofProcess = { + cwd: string; + env: NodeJS.ProcessEnv; }; export type PreparedRebuildImage = FingerprintedPreparedBuildContext & { @@ -69,18 +89,108 @@ function resultDetail(result: { ); } +const BUILDX_UNAVAILABLE_DIAGNOSTIC = + "BuildKit is enabled but the buildx component is missing or broken"; + +function hasBuildxUnavailableDiagnostic(result: { + error?: unknown; + stderr?: unknown; + stdout?: unknown; +}): boolean { + return [result.error, result.stderr, result.stdout].some((stream) => { + if (stream == null) return false; + const text = Buffer.isBuffer(stream) ? stream.toString("utf8") : String(stream); + return text.includes(BUILDX_UNAVAILABLE_DIAGNOSTIC); + }); +} + +function legacyRetryFailureDetail( + buildKitResult: Parameters[0], + legacyResult: Parameters[0], +): string { + return formatBuildFailureDiagnostics({ + stderr: + `Legacy-builder retry failed:\n${resultDetail(legacyResult)}\n` + + `Initial BuildKit attempt failed:\n${resultDetail(buildKitResult)}`, + }); +} + +function exactDockerBuild( + dockerfilePath: string, + tag: string, + contextDir: string, + options: DockerBuildOptions, +): DockerRunResult { + const { + env, + ignoreError: _ignoreError, + quiet, + stdio, + suppressOutput: _suppressOutput, + ...spawnOptions + } = options; + return dockerSpawnSync( + ["build", ...(quiet ? ["--quiet"] : []), "-f", dockerfilePath, "-t", tag, contextDir], + { + ...spawnOptions, + cwd: ROOT, + env: { ...env, DOCKER_BUILDKIT: env?.DOCKER_BUILDKIT ?? "1" }, + shell: false, + stdio: stdio ?? ["ignore", "pipe", "pipe"], + }, + ); +} + +function exactDockerRemoveImage( + imageRef: string, + options: NonNullable, +): DockerRunResult { + const { + env, + ignoreError: _ignoreError, + stdio, + suppressOutput: _suppressOutput, + ...spawnOptions + } = options; + return dockerSpawnSync(["rmi", imageRef], { + ...spawnOptions, + cwd: ROOT, + env, + shell: false, + stdio: stdio ?? ["ignore", "pipe", "pipe"], + }); +} + +function defaultBuildxAvailable(process: DockerProofProcess): boolean { + try { + return ( + dockerSpawnSync(["buildx", "version"], { + cwd: process.cwd, + env: process.env, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }).status === 0 + ); + } catch { + return false; + } +} + export async function preflightRebuildImage( input: PreflightInput, deps: PreflightDeps = {}, ): Promise { const stage = deps.stageBuildContext ?? stageCreateSandboxBuildContext; const preparePatch = deps.prepareDockerfilePatch ?? prepareSandboxDockerfilePatch; - const buildImage = deps.buildImage ?? dockerBuild; - const removeImage = deps.removeImage ?? dockerRmi; + const buildImage = deps.buildImage ?? exactDockerBuild; + const removeImage = deps.removeImage ?? exactDockerRemoveImage; + const buildxAvailable = deps.buildxAvailable ?? defaultBuildxAvailable; + const buildDockerEnv = deps.buildDockerEnv ?? dockerBuildSubprocessEnv; let cleanup: (() => boolean) | null = null; let imageTag: string | null = null; let imageBuilt = false; let retainBuildContext = false; + let dockerEnv: Readonly> | null = null; const previousReasoning = process.env.NEMOCLAW_REASONING; try { if (input.provider === "compatible-endpoint") { @@ -120,33 +230,87 @@ export async function preflightRebuildImage( warn: () => {}, }); const contextFingerprint = fingerprintBuildContext(staged.buildCtx); + dockerEnv = Object.freeze({ ...buildDockerEnv() }); imageTag = `nemoclaw-rebuild-preflight:${String(process.pid)}-${String(Date.now())}`; - const result = buildImage(staged.stagedDockerfile, imageTag, staged.buildCtx, { + const buildOptions: DockerBuildOptions = { + cwd: ROOT, + env: dockerEnv, ignoreError: true, suppressOutput: true, stdio: ["ignore", "pipe", "pipe"], - }); + }; + const buildKitResult = buildImage( + staged.stagedDockerfile, + imageTag, + staged.buildCtx, + buildOptions, + ); + let result = buildKitResult; + let usedLegacyFallback = false; + if ( + result.status !== 0 && + staged.origin === "generated" && + input.agent === null && + input.localPrebuildEnabled && + hasBuildxUnavailableDiagnostic(result) && + !buildxAvailable({ cwd: ROOT, env: dockerEnv }) + ) { + // SOURCE_OF_TRUTH_REVIEW (#7111): the generated OpenClaw final-image + // Dockerfile does not use BuildKit-only instructions. Retry its exact + // fingerprinted bytes once with Docker's compatibility builder only + // after an independent buildx probe confirms the host CLI lacks it. + // Dockerfile.base, other agents, and custom --from contexts never enter + // this fallback. Remove it when the supported Docker floor no longer + // provides the legacy builder. + if (fingerprintBuildContext(staged.buildCtx) !== contextFingerprint) { + return { ok: false, detail: "replacement build context changed during preflight" }; + } + console.warn( + " Warning: Docker Buildx is unavailable; retrying the generated rebuild image with Docker's legacy builder.", + ); + usedLegacyFallback = true; + result = buildImage(staged.stagedDockerfile, imageTag, staged.buildCtx, { + ...buildOptions, + env: { ...dockerEnv, DOCKER_BUILDKIT: "0" }, + }); + } + if (result.status !== 0 && usedLegacyFallback) { + return { ok: false, detail: legacyRetryFailureDetail(buildKitResult, result) }; + } if (result.status !== 0) return { ok: false, detail: resultDetail(result) }; imageBuilt = true; if (fingerprintBuildContext(staged.buildCtx) !== contextFingerprint) { return { ok: false, detail: "replacement build context changed during preflight" }; } retainBuildContext = true; + const prebuildBuilder = usedLegacyFallback ? "legacy" : undefined; + const prebuildDockerEnv = usedLegacyFallback ? dockerEnv : undefined; + const verifyFingerprint = createBuildContextVerifier(staged.buildCtx, contextFingerprint); + const prepared: PreparedRebuildImage = { + ...staged, + cleanupBuildCtx: cleanup, + buildId, + dashboardRemoteBindPrepared, + contextFingerprint, + prebuildBuilder, + prebuildDockerEnv, + verifyBuildCtx(this: PreparedRebuildImage) { + return ( + this === prepared && + this.prebuildBuilder === prebuildBuilder && + this.prebuildDockerEnv === prebuildDockerEnv && + verifyFingerprint() + ); + }, + rebuildTarget: { + agentName: input.agent?.name ?? null, + fromDockerfile: input.fromDockerfile ? path.resolve(input.fromDockerfile) : null, + }, + }; return { ok: true, imageTag, - prepared: { - ...staged, - cleanupBuildCtx: cleanup, - buildId, - dashboardRemoteBindPrepared, - contextFingerprint, - verifyBuildCtx: createBuildContextVerifier(staged.buildCtx, contextFingerprint), - rebuildTarget: { - agentName: input.agent?.name ?? null, - fromDockerfile: input.fromDockerfile ? path.resolve(input.fromDockerfile) : null, - }, - }, + prepared, }; } catch (err) { return { ok: false, detail: err instanceof Error ? err.message : String(err) }; @@ -155,18 +319,29 @@ export async function preflightRebuildImage( try { imageRemoved = imageTag !== null && - removeImage(imageTag, { ignoreError: true, suppressOutput: true }).status === 0; + removeImage(imageTag, { + cwd: ROOT, + env: dockerEnv ?? undefined, + ignoreError: true, + suppressOutput: true, + }).status === 0; } catch { // Best effort; retained-context ownership and environment restoration must continue. } if (imageBuilt && imageTag && !imageRemoved) { const retainedImageTag = imageTag; + const retainedDockerEnv = dockerEnv; console.warn( ` Warning: failed to remove temporary rebuild preflight image '${retainedImageTag}'.`, ); process.once("exit", () => { try { - removeImage(retainedImageTag, { ignoreError: true, suppressOutput: true }); + removeImage(retainedImageTag, { + cwd: ROOT, + env: retainedDockerEnv ?? undefined, + ignoreError: true, + suppressOutput: true, + }); } catch { // Best effort process-exit retry. } diff --git a/src/lib/actions/sandbox/rebuild-prepared-image-context.ts b/src/lib/actions/sandbox/rebuild-prepared-image-context.ts index c95145e3cbd..ff222249bd1 100644 --- a/src/lib/actions/sandbox/rebuild-prepared-image-context.ts +++ b/src/lib/actions/sandbox/rebuild-prepared-image-context.ts @@ -28,7 +28,10 @@ export function createIdempotentBuildContextCleanup(cleanup: () => boolean): () /** Confirm that a retained private context still matches the prebuilt bytes. */ export function verifyPreparedBuildContext(prepared: FingerprintedPreparedBuildContext): boolean { try { - return fingerprintBuildContext(prepared.buildCtx) === prepared.contextFingerprint; + return ( + prepared.verifyBuildCtx() && + fingerprintBuildContext(prepared.buildCtx) === prepared.contextFingerprint + ); } catch { return false; } diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.ts b/src/lib/actions/sandbox/rebuild-target-runtime.ts index 108c3ca197b..f8eb9911826 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.ts @@ -21,6 +21,7 @@ import { readGatewayProviderMetadata, } from "../../onboard/gateway-provider-metadata"; import { resolveSandboxGpuConfig } from "../../onboard/sandbox-gpu-mode"; +import { resolveSandboxPrebuildEnabled } from "../../onboard/sandbox-prebuild"; import { agentSupportsWebSearchProvider } from "../../onboard/web-search-support"; import { redact } from "../../security/redact"; import { @@ -174,8 +175,8 @@ export async function preflightRebuildTargetRuntime( ); return { ok: false }; } + const dockerDriverGateway = isLinuxDockerDriverGatewayEnabled(); try { - const dockerDriverGateway = isLinuxDockerDriverGatewayEnabled(); const selectedRoute = initialDockerGpuRoute( resolveDockerGpuRoutePlan(sandboxGpuConfig, { dockerDriverGateway, @@ -212,6 +213,7 @@ export async function preflightRebuildTargetRuntime( toolDisclosure: target.durableConfig.toolDisclosure, hermesToolGateways: target.hermesToolGateways, sandboxGpuConfig, + localPrebuildEnabled: resolveSandboxPrebuildEnabled(process.env, dockerDriverGateway), gatewayPort: recreateOptions.targetGatewayPort, chatUiUrl: managesDashboard ? `http://127.0.0.1:${String(recreateOptions.controlUiPort)}` diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 12ae08d5718..2de028a6b3f 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2758,7 +2758,8 @@ async function createSandboxWithBaseImageResolution( manageDashboard, openshellShellCommand, openshellArgv, - prebuild: { buildCtx, buildId, dockerDriverGateway, origin }, + // biome-ignore format: keep src/lib/onboard.ts growth bounded. + prebuild: { buildCtx, buildId, dockerDriverGateway, origin, builder: preparedBuildContext?.prebuildBuilder, dockerEnv: preparedBuildContext?.prebuildDockerEnv }, }); const restoreBackupPath = pendingStateRestore?.manifest?.backupPath ?? pendingStateRestoreBackupPath; diff --git a/src/lib/onboard/build-context-stage.ts b/src/lib/onboard/build-context-stage.ts index e5aec698dbc..ba8c223be62 100644 --- a/src/lib/onboard/build-context-stage.ts +++ b/src/lib/onboard/build-context-stage.ts @@ -41,6 +41,10 @@ export interface CreateSandboxBuildContextResult extends StagedBuildContext { export interface PreparedSandboxBuildContext extends CreateSandboxBuildContextResult { buildId: string; dashboardRemoteBindPrepared?: boolean; + /** Host builder that successfully validated this exact retained context. */ + readonly prebuildBuilder?: "legacy"; + /** Sanitized Docker endpoint/config environment bound to that builder proof. */ + readonly prebuildDockerEnv?: Readonly>; /** Recheck retained bytes at the final one-shot consumption boundary. */ verifyBuildCtx?(): boolean; /** Exact recorded target authorized to consume a generic rebuild handoff. */ diff --git a/src/lib/onboard/prepared-dcode-rebuild.test.ts b/src/lib/onboard/prepared-dcode-rebuild.test.ts index 808967611f1..0fe9aa9657c 100644 --- a/src/lib/onboard/prepared-dcode-rebuild.test.ts +++ b/src/lib/onboard/prepared-dcode-rebuild.test.ts @@ -239,6 +239,25 @@ describe("prepared DCode rebuild adapter", () => { expect(create).not.toHaveBeenCalled(); }); + it("rejects a legacy builder handoff outside generated OpenClaw rebuilds", () => { + expect(() => + createPreparedDcodeRebuildRuntime( + { + ...preparedImageOptions, + preparedImageRebuild: { + ...preparedImageOptions.preparedImageRebuild!, + buildContext: { + ...preparedImageBuildContext, + prebuildBuilder: "legacy", + prebuildDockerEnv: Object.freeze({ DOCKER_CONTEXT: "verified-builder" }), + }, + }, + }, + "nemoclaw", + ), + ).toThrow(/only be reused for a generated OpenClaw rebuild image/); + }); + it.runIf(process.platform !== "win32").each(oneShotContextMutations)( "rejects $label at the post-delete one-shot boundary", async ({ arrange, mutate, label }) => { diff --git a/src/lib/onboard/prepared-dcode-rebuild.ts b/src/lib/onboard/prepared-dcode-rebuild.ts index 0fef7cdb7c4..bfbf9fe18bc 100644 --- a/src/lib/onboard/prepared-dcode-rebuild.ts +++ b/src/lib/onboard/prepared-dcode-rebuild.ts @@ -88,11 +88,39 @@ function normalizedAgentIdentity(agentName: string | null | undefined): string { return agentName?.trim() || "openclaw"; } +function assertPreparedBuilderSelection(preparedBuildContext: PreparedSandboxBuildContext): void { + const builder = preparedBuildContext.prebuildBuilder; + const dockerEnv = preparedBuildContext.prebuildDockerEnv; + if (builder !== undefined && builder !== "legacy") { + throw new Error("Prepared rebuild image builder is missing or invalid."); + } + if ( + (builder === "legacy" && + (!dockerEnv || + !Object.isFrozen(dockerEnv) || + Object.values(dockerEnv).some((value) => typeof value !== "string"))) || + (builder !== "legacy" && dockerEnv !== undefined) + ) { + throw new Error("Prepared rebuild image Docker environment is missing or invalid."); + } + if ( + builder === "legacy" && + (preparedBuildContext.origin !== "generated" || + normalizedAgentIdentity(preparedBuildContext.rebuildTarget?.agentName) !== "openclaw" || + preparedBuildContext.rebuildTarget?.fromDockerfile !== null) + ) { + throw new Error( + "Docker's legacy builder can only be reused for a generated OpenClaw rebuild image.", + ); + } +} + function assertPreparedTargetIdentity( preparedBuildContext: PreparedSandboxBuildContext, agentName: string | null, fromDockerfile: string | null, ): void { + assertPreparedBuilderSelection(preparedBuildContext); const target = preparedBuildContext.rebuildTarget; if (target) { if ( diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index fef755927fc..7bb7c57df46 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -355,7 +355,7 @@ describe("prepareSandboxCreateLaunch", () => { }); describe("prepareSandboxCreateLaunchWithPrebuild", () => { - it("hands the build-qualified image to the canonical launch renderer", async () => { + it("hands the legacy-builder image to the canonical launch renderer", async () => { const buildCtx = createTrustedBuildContext(); const dockerfile = path.join(buildCtx, "Dockerfile"); const buildImage = vi.fn(async () => 0); @@ -374,6 +374,8 @@ describe("prepareSandboxCreateLaunchWithPrebuild", () => { prebuild: { buildCtx, buildId: "build-123", + builder: "legacy", + dockerEnv: Object.freeze({ DOCKER_CONTEXT: "verified-builder" }), dockerDriverGateway: true, env: { NEMOCLAW_SANDBOX_PREBUILD: "1" }, buildImage, @@ -391,7 +393,13 @@ describe("prepareSandboxCreateLaunchWithPrebuild", () => { expect(result.createCommand).toContain( "sandbox create --from nemoclaw-sandbox-local:demo-build-123 --name demo", ); - expect(buildImage).toHaveBeenCalledOnce(); + expect(buildImage).toHaveBeenCalledWith( + expect.any(Array), + expect.objectContaining({ + cwd: expect.any(String), + env: expect.objectContaining({ DOCKER_BUILDKIT: "0" }), + }), + ); }); it("renders the original Dockerfile after a local build failure", async () => { diff --git a/src/lib/onboard/sandbox-prebuild.test.ts b/src/lib/onboard/sandbox-prebuild.test.ts index 519b63b87ad..6fd13237096 100644 --- a/src/lib/onboard/sandbox-prebuild.test.ts +++ b/src/lib/onboard/sandbox-prebuild.test.ts @@ -7,6 +7,7 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { ROOT } from "../runner"; import { SANDBOX_BUILD_CONTEXT_PREFIX } from "../sandbox/build-context"; import { dockerBuildSubprocessEnv, @@ -17,6 +18,10 @@ import { const BUILD_ID = "1234567890"; const IMAGE_ID = `sha256:${"a".repeat(64)}`; +const VERIFIED_DOCKER_ENV = Object.freeze({ + DOCKER_CONFIG: "/home/test/.docker", + DOCKER_CONTEXT: "verified-builder", +}); const temporaryDirectories: string[] = []; function createBuildContext( @@ -345,6 +350,7 @@ describe("sandbox BuildKit prebuild", () => { resolvedBuildCtx, ], expect.objectContaining({ + cwd: ROOT, env: expect.objectContaining({ DOCKER_BUILDKIT: "1" }), stdio: "inherit", }), @@ -356,6 +362,118 @@ describe("sandbox BuildKit prebuild", () => { }); }); + it("reuses the legacy builder proven by rebuild preflight", async () => { + const { buildCtx, createArgs } = createBuildContext(); + const buildImage = vi.fn(async () => 0); + const inspectImageId = vi.fn(() => IMAGE_ID); + const log = vi.fn(); + const result = await prebuildSandboxImageIfEligible({ + buildCtx, + buildId: BUILD_ID, + origin: "generated", + builder: "legacy", + dockerEnv: VERIFIED_DOCKER_ENV, + createArgs, + sandboxName: "alpha", + dockerDriverGateway: true, + env: {}, + buildImage, + inspectImageId, + log, + }); + + expect(buildImage).toHaveBeenCalledWith( + expect.arrayContaining(["build", "nemoclaw-sandbox-local:alpha-1234567890"]), + expect.objectContaining({ + cwd: ROOT, + env: { ...VERIFIED_DOCKER_ENV, DOCKER_BUILDKIT: "0" }, + stdio: "inherit", + }), + ); + expect(log).toHaveBeenCalledWith(expect.stringContaining("matches rebuild preflight")); + expect(inspectImageId).toHaveBeenCalledWith("nemoclaw-sandbox-local:alpha-1234567890", { + cwd: ROOT, + env: VERIFIED_DOCKER_ENV, + }); + expect(result.imageRef).toBe("nemoclaw-sandbox-local:alpha-1234567890"); + }); + + it.each([ + ["local prebuild is disabled", true, { NEMOCLAW_SANDBOX_PREBUILD: "0" }], + ["the gateway cannot consume a local image", false, { NEMOCLAW_SANDBOX_PREBUILD: "1" }], + ])("fails a prepared builder closed when %s", async (_label, dockerDriverGateway, env) => { + const { buildCtx, createArgs } = createBuildContext(); + await expect( + prebuildSandboxImageIfEligible({ + buildCtx, + buildId: BUILD_ID, + origin: "generated", + builder: "legacy", + dockerEnv: VERIFIED_DOCKER_ENV, + createArgs, + sandboxName: "alpha", + dockerDriverGateway, + env, + }), + ).rejects.toThrow(/verified local Docker builder is not enabled/); + }); + + it("fails a prepared builder closed when create arguments drift", async () => { + const { buildCtx } = createBuildContext(); + await expect( + prebuildSandboxImageIfEligible({ + buildCtx, + buildId: BUILD_ID, + origin: "generated", + builder: "legacy", + dockerEnv: VERIFIED_DOCKER_ENV, + createArgs: ["--from", "/other/Dockerfile"], + sandboxName: "alpha", + dockerDriverGateway: true, + env: {}, + }), + ).rejects.toThrow(/arguments no longer select the retained Dockerfile/); + }); + + it("fails a prepared builder closed when context trust validation drifts", async () => { + const { buildCtx, createArgs } = createBuildContext(); + fs.chmodSync(buildCtx, 0o770); + await expect( + prebuildSandboxImageIfEligible({ + buildCtx, + buildId: BUILD_ID, + origin: "generated", + builder: "legacy", + dockerEnv: VERIFIED_DOCKER_ENV, + createArgs, + sandboxName: "alpha", + dockerDriverGateway: true, + env: {}, + }), + ).rejects.toThrow(/context failed trust validation/); + }); + + it.each([ + ["exits nonzero", async () => 1], + ["cannot start", async () => Promise.reject(new Error("daemon unavailable"))], + ])("fails a prepared builder closed when its repeated build %s", async (_label, buildImage) => { + const { buildCtx, createArgs } = createBuildContext(); + await expect( + prebuildSandboxImageIfEligible({ + buildCtx, + buildId: BUILD_ID, + origin: "generated", + builder: "legacy", + dockerEnv: VERIFIED_DOCKER_ENV, + createArgs, + sandboxName: "alpha", + dockerDriverGateway: true, + env: {}, + buildImage, + }), + ).rejects.toThrow(/verified legacy builder/); + }); + it.each([ ["nonzero result", async () => 1], ["missing exit status", async () => null], diff --git a/src/lib/onboard/sandbox-prebuild.ts b/src/lib/onboard/sandbox-prebuild.ts index 5ed957227e0..6e0b4df90a8 100644 --- a/src/lib/onboard/sandbox-prebuild.ts +++ b/src/lib/onboard/sandbox-prebuild.ts @@ -5,9 +5,9 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { dockerImageInspectFormat } from "../adapters/docker"; -import { dockerSpawn } from "../adapters/docker/exec"; +import { dockerSpawn, dockerSpawnSync } from "../adapters/docker/exec"; import { LOCAL_SANDBOX_IMAGE_REPO } from "../domain/sandbox/image-tag"; +import { ROOT } from "../runner"; import { SANDBOX_BUILD_CONTEXT_PREFIX, type SandboxBuildContextOrigin, @@ -33,12 +33,16 @@ export interface SandboxPrebuildInput { sandboxName: string; dockerDriverGateway: boolean; origin: SandboxBuildContextOrigin; + /** Builder already proven against this retained rebuild context. */ + builder?: "legacy"; + /** Sanitized Docker endpoint/config environment used by that builder proof. */ + dockerEnv?: Readonly>; env?: NodeJS.ProcessEnv; buildImage?: ( args: readonly string[], - options: { env: NodeJS.ProcessEnv; stdio: "inherit" }, + options: { cwd: string; env: NodeJS.ProcessEnv; stdio: "inherit" }, ) => Promise; - inspectImageId?: (imageRef: string) => string; + inspectImageId?: (imageRef: string, options: { cwd: string; env: NodeJS.ProcessEnv }) => string; log?: (message: string) => void; } @@ -144,7 +148,8 @@ export function sandboxLocalImageRef(sandboxName: string, buildId: string): stri /** * Build a NemoClaw-generated staged context with BuildKit on the shared local * Docker daemon. User-supplied Dockerfiles stay on the OpenShell gateway - * builder trust boundary, and any failure preserves that original build path. + * builder trust boundary. Ordinary onboarding preserves that path on local + * build failure; a builder proven by rebuild preflight instead fails closed. * Remove this bridge once OpenShell uses BuildKit for this local-driver path; * extraction and observable retirement criteria are tracked by #6258. */ @@ -154,10 +159,29 @@ export async function prebuildSandboxImageIfEligible( const createArgs = [...input.createArgs]; const env = input.env ?? process.env; const log = input.log ?? console.log; + const requiredBuilder = input.builder ?? null; + const failPreparedBuild = (detail: string): never => { + throw new Error(`Prepared rebuild image cannot be recreated safely: ${detail}`); + }; + if ( + (requiredBuilder && + (!input.dockerEnv || + !Object.isFrozen(input.dockerEnv) || + Object.values(input.dockerEnv).some((value) => typeof value !== "string"))) || + (!requiredBuilder && input.dockerEnv) + ) { + failPreparedBuild("the verified Docker environment is missing or invalid"); + } if (!resolveSandboxPrebuildEnabled(env, input.dockerDriverGateway)) { + if (requiredBuilder) { + failPreparedBuild("the verified local Docker builder is not enabled"); + } return { createArgs, imageRef: null, imageId: null }; } if (input.origin !== "generated") { + if (requiredBuilder) { + failPreparedBuild("the retained build context is not NemoClaw-generated"); + } log( " Local BuildKit build skipped for a custom Dockerfile; using the gateway builder instead.", ); @@ -170,6 +194,9 @@ export async function prebuildSandboxImageIfEligible( !fromDockerfile || path.resolve(fromDockerfile) !== path.resolve(input.buildCtx, "Dockerfile") ) { + if (requiredBuilder) { + failPreparedBuild("sandbox create arguments no longer select the retained Dockerfile"); + } return { createArgs, imageRef: null, imageId: null }; } let trustedContext: TrustedStagedBuildContext | null; @@ -177,12 +204,18 @@ export async function prebuildSandboxImageIfEligible( trustedContext = resolveTrustedStagedBuildContext(input.buildCtx); } catch (error) { const detail = error instanceof Error ? error.message : String(error); + if (requiredBuilder) { + failPreparedBuild(`the retained build context could not be inspected (${detail})`); + } log( ` Local BuildKit build skipped: staged build context could not be inspected (${detail}); using the gateway builder instead.`, ); return { createArgs, imageRef: null, imageId: null }; } if (!trustedContext) { + if (requiredBuilder) { + failPreparedBuild("the retained build context failed trust validation"); + } log( " Local BuildKit build skipped: staged build context failed trust validation; using the gateway builder instead.", ); @@ -190,6 +223,9 @@ export async function prebuildSandboxImageIfEligible( } const imageRef = sandboxLocalImageRef(input.sandboxName, input.buildId); + const dockerEnv = requiredBuilder + ? (input.dockerEnv as Readonly>) + : dockerBuildSubprocessEnv(); const buildImage = input.buildImage ?? ((args, options) => @@ -198,25 +234,40 @@ export async function prebuildSandboxImageIfEligible( child.once("error", reject); child.once("close", resolve); })); - log(" Building sandbox image with BuildKit (skips the slower in-gateway builder)..."); + const builder = requiredBuilder ?? "buildkit"; + log( + builder === "legacy" + ? " Building sandbox image with Docker's legacy builder (matches rebuild preflight)..." + : " Building sandbox image with BuildKit (skips the slower in-gateway builder)...", + ); let status: number | null; try { status = await buildImage( ["build", "-t", imageRef, "-f", trustedContext.dockerfile, trustedContext.buildCtx], { - env: { ...dockerBuildSubprocessEnv(), DOCKER_BUILDKIT: "1" }, + cwd: ROOT, + env: { + ...dockerEnv, + DOCKER_BUILDKIT: builder === "legacy" ? "0" : "1", + }, stdio: "inherit", }, ); } catch (error) { const detail = error instanceof Error ? error.message : String(error); + if (requiredBuilder) { + failPreparedBuild(`the verified ${builder} builder could not start (${detail})`); + } log(` Local BuildKit build could not start (${detail}); using the gateway builder instead.`); return { createArgs, imageRef: null, imageId: null }; } if (status !== 0) { const detail = status === null ? " without an exit status" : ` (exit ${status})`; + if (requiredBuilder) { + failPreparedBuild(`the verified ${builder} builder failed${detail}`); + } log(` Local BuildKit build failed${detail}; using the gateway builder instead.`); return { createArgs, imageRef: null, imageId: null }; } @@ -224,13 +275,20 @@ export async function prebuildSandboxImageIfEligible( createArgs[fromIndex + 1] = imageRef; const inspectImageId = input.inspectImageId ?? - ((ref: string) => - dockerImageInspectFormat("{{.Id}}", ref, { - ignoreError: true, - }).trim()); + ((ref: string, options: { cwd: string; env: NodeJS.ProcessEnv }) => { + const inspected = dockerSpawnSync(["image", "inspect", "--format", "{{.Id}}", ref], { + ...options, + encoding: "utf8", + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }); + return inspected.status === 0 && !inspected.error + ? String(inspected.stdout ?? "").trim() + : ""; + }); let imageId: string | null = null; try { - const inspected = inspectImageId(imageRef).trim(); + const inspected = inspectImageId(imageRef, { cwd: ROOT, env: dockerEnv }).trim(); if (isImmutableDockerImageId(inspected)) imageId = inspected.toLowerCase(); } catch { // Native creation can still use the local tag. Automatic compatibility From 0e9a222886234953bacc6e83c9128a5fa13562ee Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Mon, 20 Jul 2026 11:28:30 -0700 Subject: [PATCH 02/16] refactor(onboard): preserve entrypoint growth budget Move prepared rebuild proof binding into the sandbox launch coordinator. Keep the top-level onboard entrypoint net-neutral and avoid documenting an unsupported rebuild flag. Co-authored-by: Tinson Lai Signed-off-by: Apurv Kumaria --- docs/reference/commands.mdx | 2 +- src/lib/onboard.ts | 3 +-- src/lib/onboard/sandbox-create-launch.test.ts | 6 ++++-- src/lib/onboard/sandbox-create-launch.ts | 10 ++++++++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index a03113d3a20..083ab816226 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -2479,7 +2479,7 @@ Auto-mode sandboxes remain auto. Before backup or deletion, `rebuild` stages and validates the replacement image. When recreation can consume a local Docker image and Docker reports that Buildx is missing or broken while validating the managed OpenClaw context generated by NemoClaw, NemoClaw prints a warning, retries that exact fingerprinted context once with Docker's legacy builder, and carries the verified builder and Docker endpoint configuration into recreation. -The local preflight does not retry a user-supplied `--from` context with a different builder. +The local preflight does not retry a user-supplied custom Dockerfile context with a different builder. If validation still fails or the staged context changes, rebuild stops and leaves the existing sandbox intact. ```bash diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 2de028a6b3f..1cacfbeb9a6 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2758,8 +2758,7 @@ async function createSandboxWithBaseImageResolution( manageDashboard, openshellShellCommand, openshellArgv, - // biome-ignore format: keep src/lib/onboard.ts growth bounded. - prebuild: { buildCtx, buildId, dockerDriverGateway, origin, builder: preparedBuildContext?.prebuildBuilder, dockerEnv: preparedBuildContext?.prebuildDockerEnv }, + prebuild: { buildCtx, buildId, dockerDriverGateway, origin, prepared: preparedBuildContext }, }); const restoreBackupPath = pendingStateRestore?.manifest?.backupPath ?? pendingStateRestoreBackupPath; diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index 7bb7c57df46..bc06f6c8e45 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -374,14 +374,16 @@ describe("prepareSandboxCreateLaunchWithPrebuild", () => { prebuild: { buildCtx, buildId: "build-123", - builder: "legacy", - dockerEnv: Object.freeze({ DOCKER_CONTEXT: "verified-builder" }), dockerDriverGateway: true, env: { NEMOCLAW_SANDBOX_PREBUILD: "1" }, buildImage, inspectImageId: () => IMAGE_ID, log: vi.fn(), origin: "generated", + prepared: { + prebuildBuilder: "legacy", + prebuildDockerEnv: Object.freeze({ DOCKER_CONTEXT: "verified-builder" }), + }, }, }); diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index 1e5431bb479..7a5fe2839ee 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -4,6 +4,7 @@ import type { AgentDefinition } from "../agent/defs"; import { formatEnvAssignment } from "../core/url-utils"; import { buildSubprocessEnv } from "../subprocess-env"; +import type { PreparedSandboxBuildContext } from "./build-context-stage"; import { isValidProxyHost, isValidProxyPort } from "./dockerfile-patch"; import { appendExtraPlaceholderKeysEnvArg } from "./extra-placeholder-keys"; import type { HermesDashboardOnboardState } from "./hermes-dashboard"; @@ -68,7 +69,9 @@ export interface SandboxCreateLaunch { export interface SandboxCreateLaunchWithPrebuildInput extends SandboxCreateLaunchInput { sandboxName: string; - prebuild: Omit; + prebuild: Omit & { + prepared?: Pick | null; + }; } export interface SandboxCreateLaunchWithPrebuild extends SandboxCreateLaunch { @@ -220,9 +223,12 @@ export async function prepareSandboxCreateLaunchWithPrebuild( input: SandboxCreateLaunchWithPrebuildInput, ): Promise { const { prebuild: prebuildInput, ...launchInput } = input; + const { prepared, ...prebuildOptions } = prebuildInput; const prebuild = await prebuildSandboxImageIfEligible({ - ...prebuildInput, + ...prebuildOptions, + builder: prepared?.prebuildBuilder, createArgs: input.createArgs, + dockerEnv: prepared?.prebuildDockerEnv, sandboxName: input.sandboxName, }); return { From 5d07406ca9c56bbd918ded79e3f792857a0ba8c1 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Mon, 20 Jul 2026 11:32:53 -0700 Subject: [PATCH 03/16] test(rebuild): keep failure assertion linear Narrow the failed preflight result with an explicit type assertion. Keep the regression test free of conditional control flow. Co-authored-by: Tinson Lai Signed-off-by: Apurv Kumaria --- .../sandbox/rebuild-custom-image-preflight.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts index 074d3450de6..1bf4d406848 100644 --- a/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts +++ b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts @@ -357,12 +357,12 @@ describe("preflightRebuildImage", () => { }); expect(result.ok).toBe(false); - if (result.ok) throw new Error("expected failed rebuild preflight"); - expect(result.detail).toContain("Legacy-builder retry failed"); - expect(result.detail).toContain("legacy build could not read ~/private-context"); - expect(result.detail).toContain("Authorization: Bearer "); - expect(result.detail).not.toContain(credential); - expect(result.detail.length).toBeLessThan(8_100); + const failure = result as Extract; + expect(failure.detail).toContain("Legacy-builder retry failed"); + expect(failure.detail).toContain("legacy build could not read ~/private-context"); + expect(failure.detail).toContain("Authorization: Bearer "); + expect(failure.detail).not.toContain(credential); + expect(failure.detail.length).toBeLessThan(8_100); expect(buildImage).toHaveBeenCalledTimes(2); expect(cleanupBuildCtx).toHaveBeenCalledOnce(); } finally { From 33bcd9d9fec9baa0933c428c8bffadc5ff1c2ee8 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 12:03:15 -0400 Subject: [PATCH 04/16] fix(rebuild): require Docker Buildx Signed-off-by: Julie Yaunches --- ci/source-architecture-budget.json | 2 +- docs/reference/commands.mdx | 15 +- .../rebuild-custom-image-preflight.test.ts | 368 +++--------------- .../sandbox/rebuild-custom-image-preflight.ts | 208 ++-------- .../sandbox/rebuild-prepared-image-context.ts | 5 +- .../actions/sandbox/rebuild-target-runtime.ts | 4 +- src/lib/onboard.ts | 2 +- src/lib/onboard/build-context-stage.ts | 4 - .../onboard/prepared-dcode-rebuild.test.ts | 19 - src/lib/onboard/prepared-dcode-rebuild.ts | 28 -- src/lib/onboard/sandbox-create-launch.test.ts | 14 +- src/lib/onboard/sandbox-create-launch.ts | 10 +- src/lib/onboard/sandbox-prebuild.test.ts | 118 ------ src/lib/onboard/sandbox-prebuild.ts | 79 +--- 14 files changed, 100 insertions(+), 776 deletions(-) diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 0fa1d747759..b1cb940c153 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -5,7 +5,7 @@ "maxByFile": { "src/lib/actions/sandbox/mcp-bridge-contracts.ts": 26, "src/lib/actions/sandbox/process-recovery.ts": 26, - "src/lib/adapters/docker/index.ts": 43, + "src/lib/adapters/docker/index.ts": 45, "src/lib/adapters/openshell/client.ts": 22, "src/lib/adapters/openshell/resolve.ts": 28, "src/lib/adapters/openshell/runtime.ts": 50, diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index bac917fb813..3d05a9756e6 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -2689,17 +2689,10 @@ A rebuild preserves the recorded Deep Agents Code auto-approval capability unles A sandbox onboarded with an explicit GPU opt-out (stored as `sandboxGpuMode: "0"`, plus legacy registry entries that only record `gpuEnabled: false`) is recreated with the same opt-out, so the inner `onboard --resume` skips the Docker CDI GPU preflight on hosts without an NVIDIA GPU. Auto-mode sandboxes remain auto. -Before backup or deletion, `rebuild` stages and validates the replacement image. - - -If the local Docker path is enabled and reports a Buildx failure for a generated OpenClaw context, NemoClaw verifies the failure with `docker buildx version`. -When that probe also fails, NemoClaw retries the exact fingerprinted context once with Docker's legacy builder. -It carries the verified builder and Docker endpoint configuration into recreation. -The local preflight does not retry a user-supplied custom Dockerfile context with a different builder. - - - -If validation still fails or the staged context changes, rebuild stops and leaves the existing sandbox intact. +Before backup or deletion, `rebuild` builds and validates the replacement image with Docker Buildx. +If Buildx is missing or broken, NemoClaw stops and leaves the existing sandbox intact. +Install or repair Docker Buildx, then verify it with `docker buildx version`. +Rerun the rebuild after that command succeeds. ```bash $$nemoclaw my-assistant rebuild [--yes|-y|--force] [--verbose|-v] [--tool-disclosure ] [--dcode-auto-approval ] [--observability|--no-observability] diff --git a/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts index 84cc9f1410c..be213cf8086 100644 --- a/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts +++ b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts @@ -44,24 +44,12 @@ function input(fromDockerfile: string | null) { sandboxGpuDevice: null, errors: [], }, - localPrebuildEnabled: true, gatewayPort: 8080, chatUiUrl: "http://127.0.0.1:18789", }; } describe("preflightRebuildImage", () => { - it("keeps the generated OpenClaw Dockerfile compatible with the legacy fallback (#7111)", () => { - const dockerfile = fs.readFileSync(path.join(ROOT, "Dockerfile"), "utf8"); - for (const buildKitOnlySyntax of [ - /^\s*RUN\s+--mount=/mu, - /^\s*(?:COPY|ADD)\s+--link(?:=|\s)/mu, - /^\s*RUN\s+<<-?\w+/mu, - ]) { - expect(dockerfile).not.toMatch(buildKitOnlySyntax); - } - }); - it("carries verified base provenance into the retained managed context (#7144)", async () => { const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-provenance-")); const stagedDockerfile = path.join(buildCtx, "Dockerfile"); @@ -151,8 +139,6 @@ describe("preflightRebuildImage", () => { ); expect(buildImage).toHaveBeenCalledOnce(); expect(cleanupBuildCtx).not.toHaveBeenCalled(); - expect(result.prepared.prebuildBuilder).toBeUndefined(); - expect(result.prepared.prebuildDockerEnv).toBeUndefined(); expect(verifyPreparedBuildContext(result.prepared)).toBe(true); expect(disposePreparedBuildContext(result.prepared)).toBe(true); expect(cleanupBuildCtx).toHaveBeenCalledOnce(); @@ -161,308 +147,6 @@ describe("preflightRebuildImage", () => { } }); - it("retries an exact generated image once with the legacy builder when Buildx is unavailable (#7111)", async () => { - const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-fallback-")); - const stagedDockerfile = path.join(buildCtx, "Dockerfile"); - fs.writeFileSync(stagedDockerfile, "FROM scratch\n"); - const cleanupBuildCtx = vi.fn(() => { - fs.rmSync(buildCtx, { recursive: true, force: true }); - return true; - }); - const buildImage = vi - .fn() - .mockReturnValueOnce({ - status: 1, - stderr: Buffer.from( - "ERROR: BuildKit is enabled but the buildx component is missing or broken.", - ), - } as never) - .mockReturnValueOnce({ status: 0 } as never); - const dockerEnv = Object.freeze({ - DOCKER_CONFIG: "/home/test/.docker", - DOCKER_CONTEXT: "verified-builder", - }); - const buildxAvailable = vi.fn(() => false); - const removeImage = vi.fn(() => ({ status: 0 }) as never); - const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); - - try { - const result = successful( - await preflightRebuildImage(input(null), { - stageBuildContext: vi.fn(() => ({ - buildCtx, - stagedDockerfile, - cleanupBuildCtx, - origin: "generated" as const, - })), - prepareDockerfilePatch: vi.fn(async () => ({ - buildId: "buildx-fallback", - dashboardRemoteBindPrepared: false, - resolvedBaseImage: null, - })), - buildImage, - buildxAvailable, - buildDockerEnv: () => dockerEnv, - removeImage, - }), - ); - - expect(buildImage).toHaveBeenCalledTimes(2); - expect(buildImage.mock.calls[0]?.[3]).toEqual( - expect.objectContaining({ cwd: ROOT, env: dockerEnv }), - ); - expect(buildImage.mock.calls[1]?.[3]).toEqual( - expect.objectContaining({ - cwd: ROOT, - env: { ...dockerEnv, DOCKER_BUILDKIT: "0" }, - }), - ); - expect(buildxAvailable).toHaveBeenCalledWith({ cwd: ROOT, env: dockerEnv }); - expect(removeImage).toHaveBeenCalledWith( - expect.stringMatching(/^nemoclaw-rebuild-preflight:/), - expect.objectContaining({ cwd: ROOT, env: dockerEnv }), - ); - expect(warn).toHaveBeenCalledWith(expect.stringContaining("legacy builder")); - expect(result.prepared.prebuildBuilder).toBe("legacy"); - expect(result.prepared.prebuildDockerEnv).toEqual(dockerEnv); - expect(result.prepared.prebuildDockerEnv).not.toBe(dockerEnv); - expect(verifyPreparedBuildContext(result.prepared)).toBe(true); - const clonedPrepared = { ...result.prepared }; - expect(clonedPrepared.verifyBuildCtx()).toBe(false); - expect(verifyPreparedBuildContext(clonedPrepared)).toBe(false); - (result.prepared as { prebuildBuilder?: string }).prebuildBuilder = "buildkit"; - expect(result.prepared.verifyBuildCtx()).toBe(false); - (result.prepared as { prebuildBuilder?: string }).prebuildBuilder = "legacy"; - expect(disposePreparedBuildContext(result.prepared)).toBe(true); - } finally { - warn.mockRestore(); - fs.rmSync(buildCtx, { recursive: true, force: true }); - } - }); - - it.each([ - ["the independent Buildx probe succeeds", null, true, true], - ["the generated target is not managed OpenClaw", { name: "hermes" }, false, true], - ["local image prebuild is unavailable", null, false, false], - ])("does not downgrade the builder when %s", async (_label, agent, available, localPrebuildEnabled) => { - const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-no-downgrade-")); - const stagedDockerfile = path.join(buildCtx, "Dockerfile"); - fs.writeFileSync(stagedDockerfile, "FROM scratch\n"); - const cleanupBuildCtx = vi.fn(() => { - fs.rmSync(buildCtx, { recursive: true, force: true }); - return true; - }); - const buildImage = vi.fn( - () => - ({ - status: 1, - stderr: "ERROR: BuildKit is enabled but the buildx component is missing or broken.", - }) as never, - ); - - try { - const result = await preflightRebuildImage( - { ...input(null), agent: agent as never, localPrebuildEnabled }, - { - stageBuildContext: vi.fn(() => ({ - buildCtx, - stagedDockerfile, - cleanupBuildCtx, - origin: "generated" as const, - })), - prepareDockerfilePatch: vi.fn(async () => ({ - buildId: "no-builder-downgrade", - dashboardRemoteBindPrepared: false, - resolvedBaseImage: null, - })), - buildImage, - buildxAvailable: () => available, - removeImage: vi.fn(() => ({ status: 0 }) as never), - }, - ); - - expect(result).toEqual({ - ok: false, - detail: "ERROR: BuildKit is enabled but the buildx component is missing or broken.", - }); - expect(buildImage).toHaveBeenCalledOnce(); - expect(cleanupBuildCtx).toHaveBeenCalledOnce(); - } finally { - fs.rmSync(buildCtx, { recursive: true, force: true }); - } - }); - - it("does not retry a custom Dockerfile when Buildx is unavailable", async () => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-custom-buildx-")); - const dockerfile = path.join(dir, "Dockerfile.custom"); - fs.writeFileSync(dockerfile, "FROM scratch\n"); - const buildImage = vi.fn( - () => - ({ - status: 1, - stderr: "ERROR: BuildKit is enabled but the buildx component is missing or broken.", - }) as never, - ); - - try { - const result = await preflightRebuildImage(input(dockerfile), { - prepareDockerfilePatch: vi.fn(async () => ({ - buildId: "custom-buildx", - dashboardRemoteBindPrepared: false, - resolvedBaseImage: null, - })), - buildImage, - removeImage: vi.fn(() => ({ status: 0 }) as never), - }); - - expect(result).toEqual({ - ok: false, - detail: "ERROR: BuildKit is enabled but the buildx component is missing or broken.", - }); - expect(buildImage).toHaveBeenCalledOnce(); - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } - }); - - it("rejects generated build-context drift before a legacy-builder retry", async () => { - const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-drift-")); - const stagedDockerfile = path.join(buildCtx, "Dockerfile"); - fs.writeFileSync(stagedDockerfile, "FROM scratch\n"); - const cleanupBuildCtx = vi.fn(() => { - fs.rmSync(buildCtx, { recursive: true, force: true }); - return true; - }); - const buildImage = vi.fn(() => { - fs.writeFileSync(path.join(buildCtx, "mutated"), "changed\n"); - return { - status: 1, - stderr: "ERROR: BuildKit is enabled but the buildx component is missing or broken.", - } as never; - }); - - try { - await expect( - preflightRebuildImage(input(null), { - stageBuildContext: vi.fn(() => ({ - buildCtx, - stagedDockerfile, - cleanupBuildCtx, - origin: "generated" as const, - })), - prepareDockerfilePatch: vi.fn(async () => ({ - buildId: "buildx-drift", - dashboardRemoteBindPrepared: false, - resolvedBaseImage: null, - })), - buildImage, - buildxAvailable: () => false, - removeImage: vi.fn(() => ({ status: 0 }) as never), - }), - ).resolves.toEqual({ - ok: false, - detail: "replacement build context changed during preflight", - }); - expect(buildImage).toHaveBeenCalledOnce(); - expect(cleanupBuildCtx).toHaveBeenCalledOnce(); - } finally { - fs.rmSync(buildCtx, { recursive: true, force: true }); - } - }); - - it("prioritizes the redacted legacy failure when both generated-image attempts fail", async () => { - const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-double-fail-")); - const stagedDockerfile = path.join(buildCtx, "Dockerfile"); - const credential = ["legacy", "retry", "credential"].join("-"); - fs.writeFileSync(stagedDockerfile, "FROM scratch\n"); - const cleanupBuildCtx = vi.fn(() => { - fs.rmSync(buildCtx, { recursive: true, force: true }); - return true; - }); - const buildImage = vi - .fn() - .mockReturnValueOnce({ - status: 1, - stderr: - "ERROR: BuildKit is enabled but the buildx component is missing or broken.\n" + - "x".repeat(9_000), - } as never) - .mockReturnValueOnce({ - status: 1, - stderr: - `legacy build could not read ${os.homedir()}/private-context\n` + - `Authorization: Bearer ${credential}`, - } as never); - const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); - - try { - const result = await preflightRebuildImage(input(null), { - stageBuildContext: vi.fn(() => ({ - buildCtx, - stagedDockerfile, - cleanupBuildCtx, - origin: "generated" as const, - })), - prepareDockerfilePatch: vi.fn(async () => ({ - buildId: "buildx-double-fail", - dashboardRemoteBindPrepared: false, - resolvedBaseImage: null, - })), - buildImage, - buildxAvailable: () => false, - removeImage: vi.fn(() => ({ status: 0 }) as never), - }); - - expect(result.ok).toBe(false); - const failure = result as Extract; - expect(failure.detail).toContain("Legacy-builder retry failed"); - expect(failure.detail).toContain("legacy build could not read ~/private-context"); - expect(failure.detail).toContain("Authorization: Bearer "); - expect(failure.detail).not.toContain(credential); - expect(failure.detail.length).toBeLessThan(8_100); - expect(buildImage).toHaveBeenCalledTimes(2); - expect(cleanupBuildCtx).toHaveBeenCalledOnce(); - } finally { - warn.mockRestore(); - fs.rmSync(buildCtx, { recursive: true, force: true }); - } - }); - - it("does not retry an unrelated generated-image build failure", async () => { - const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-build-no-retry-")); - const stagedDockerfile = path.join(buildCtx, "Dockerfile"); - fs.writeFileSync(stagedDockerfile, "FROM scratch\n"); - const cleanupBuildCtx = vi.fn(() => { - fs.rmSync(buildCtx, { recursive: true, force: true }); - return true; - }); - const buildImage = vi.fn(() => ({ status: 1, stderr: "registry timeout" }) as never); - - try { - await expect( - preflightRebuildImage(input(null), { - stageBuildContext: vi.fn(() => ({ - buildCtx, - stagedDockerfile, - cleanupBuildCtx, - origin: "generated" as const, - })), - prepareDockerfilePatch: vi.fn(async () => ({ - buildId: "unrelated-build-failure", - dashboardRemoteBindPrepared: false, - resolvedBaseImage: null, - })), - buildImage, - removeImage: vi.fn(() => ({ status: 0 }) as never), - }), - ).resolves.toEqual({ ok: false, detail: "registry timeout" }); - expect(buildImage).toHaveBeenCalledOnce(); - expect(cleanupBuildCtx).toHaveBeenCalledOnce(); - } finally { - fs.rmSync(buildCtx, { recursive: true, force: true }); - } - }); - it.runIf(process.platform !== "win32")( "rejects a symlinked build-context root before the preflight build", async () => { @@ -569,6 +253,53 @@ describe("preflightRebuildImage", () => { } }); + it("requires Buildx repair before deleting the existing sandbox (#7111)", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-required-")); + const dockerfile = path.join(dir, "Dockerfile.custom"); + fs.writeFileSync(dockerfile, "FROM scratch\n"); + const buildImage = vi.fn( + () => + ({ + status: 1, + stderr: Buffer.from( + "ERROR: BuildKit is enabled but the buildx component is missing or broken.", + ), + }) as never, + ); + const cleanupBuildCtx = vi.fn(() => true); + + try { + const result = await preflightRebuildImage(input(dockerfile), { + stageBuildContext: vi.fn(() => ({ + buildCtx: dir, + stagedDockerfile: dockerfile, + cleanupBuildCtx, + origin: "custom" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "buildx-required", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + removeImage: vi.fn(() => ({ status: 0 }) as never), + }); + + expect(result).toEqual({ + ok: false, + detail: + "Docker Buildx is required for sandbox rebuilds. " + + "Install or repair Docker Buildx, then verify it with 'docker buildx version'. " + + "Rerun the rebuild after that command succeeds. " + + "NemoClaw stopped before deleting the existing sandbox.", + }); + expect(buildImage).toHaveBeenCalledOnce(); + expect(cleanupBuildCtx).toHaveBeenCalledOnce(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + it("builds and removes the exact staged custom context on success", async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-custom-preflight-")); const dockerfile = path.join(dir, "Dockerfile.custom"); @@ -650,7 +381,6 @@ describe("preflightRebuildImage", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-custom-preflight-cleanup-")); const dockerfile = path.join(dir, "Dockerfile.custom"); fs.writeFileSync(dockerfile, "FROM scratch\n"); - const dockerEnv = Object.freeze({ DOCKER_CONTEXT: "cleanup-builder" }); const removeImage = vi .fn() .mockReturnValueOnce({ status: 1 } as never) @@ -670,7 +400,6 @@ describe("preflightRebuildImage", () => { resolvedBaseImage: null, })), buildImage: vi.fn(() => ({ status: 0 }) as never), - buildDockerEnv: () => dockerEnv, removeImage, }), ); @@ -680,9 +409,6 @@ describe("preflightRebuildImage", () => { ); expect(processOnce).toHaveBeenCalledWith("exit", expect.any(Function)); expect(removeImage).toHaveBeenCalledTimes(2); - for (const call of removeImage.mock.calls) { - expect(call[1]).toEqual(expect.objectContaining({ cwd: ROOT, env: dockerEnv })); - } expect(result.prepared.dashboardRemoteBindPrepared).toBe(true); expect(disposePreparedBuildContext(result.prepared)).toBe(true); } finally { diff --git a/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts b/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts index 655ea3168db..51fd4877fcd 100644 --- a/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts @@ -3,8 +3,7 @@ import path from "node:path"; -import type { DockerBuildOptions, DockerRunOptions, DockerRunResult } from "../../adapters/docker"; -import { dockerSpawnSync } from "../../adapters/docker/exec"; +import { dockerBuild, dockerRmi } from "../../adapters/docker"; import { fingerprintBuildContext } from "../../adapters/fs/build-context-fingerprint"; import type { AgentDefinition } from "../../agent/defs"; import { createAgentSandbox } from "../../agent/onboard"; @@ -12,7 +11,6 @@ import type { WebSearchConfig } from "../../inference/web-search"; import { stageCreateSandboxBuildContext } from "../../onboard/build-context-stage"; import { prepareSandboxDockerfilePatch } from "../../onboard/sandbox-dockerfile-patch-flow"; import type { SandboxGpuConfig } from "../../onboard/sandbox-gpu-mode"; -import { dockerBuildSubprocessEnv } from "../../onboard/sandbox-prebuild"; import { ROOT } from "../../runner"; import { formatBuildFailureDiagnostics, @@ -44,8 +42,6 @@ type PreflightInput = { toolDisclosure: ToolDisclosure; hermesToolGateways: string[]; sandboxGpuConfig: SandboxGpuConfig; - /** Whether recreation can consume an image built by the same host Docker daemon. */ - localPrebuildEnabled: boolean; gatewayPort: number; chatUiUrl: string; preResolvedBaseImageMetadata?: SandboxBaseImageResolutionMetadata | null; @@ -54,24 +50,8 @@ type PreflightInput = { type PreflightDeps = { stageBuildContext?: typeof stageCreateSandboxBuildContext; prepareDockerfilePatch?: typeof prepareSandboxDockerfilePatch; - buildImage?: BuildImage; - removeImage?: RemoveImage; - buildxAvailable?: (process: DockerProofProcess) => boolean; - buildDockerEnv?: () => Record; -}; - -type BuildImage = ( - dockerfilePath: string, - tag: string, - contextDir: string, - options: DockerBuildOptions, -) => DockerRunResult; - -type RemoveImage = (imageRef: string, options: NonNullable) => DockerRunResult; - -type DockerProofProcess = { - cwd: string; - env: NodeJS.ProcessEnv; + buildImage?: typeof dockerBuild; + removeImage?: typeof dockerRmi; }; export type PreparedRebuildImage = FingerprintedPreparedBuildContext & { @@ -100,7 +80,7 @@ function resultDetail(result: { const BUILDX_UNAVAILABLE_DIAGNOSTIC = "BuildKit is enabled but the buildx component is missing or broken"; -function hasBuildxUnavailableDiagnostic(result: { +function requiresBuildxRepair(result: { error?: unknown; stderr?: unknown; stdout?: unknown; @@ -112,93 +92,27 @@ function hasBuildxUnavailableDiagnostic(result: { }); } -function legacyRetryFailureDetail( - buildKitResult: Parameters[0], - legacyResult: Parameters[0], -): string { - return formatBuildFailureDiagnostics({ - stderr: - `Legacy-builder retry failed:\n${resultDetail(legacyResult)}\n` + - `Initial BuildKit attempt failed:\n${resultDetail(buildKitResult)}`, - }); -} - -function exactDockerBuild( - dockerfilePath: string, - tag: string, - contextDir: string, - options: DockerBuildOptions, -): DockerRunResult { - const { - env, - ignoreError: _ignoreError, - quiet, - stdio, - suppressOutput: _suppressOutput, - ...spawnOptions - } = options; - return dockerSpawnSync( - ["build", ...(quiet ? ["--quiet"] : []), "-f", dockerfilePath, "-t", tag, contextDir], - { - ...spawnOptions, - cwd: ROOT, - env: { ...env, DOCKER_BUILDKIT: env?.DOCKER_BUILDKIT ?? "1" }, - shell: false, - stdio: stdio ?? ["ignore", "pipe", "pipe"], - }, +function buildxRepairDetail(): string { + return ( + "Docker Buildx is required for sandbox rebuilds. " + + "Install or repair Docker Buildx, then verify it with 'docker buildx version'. " + + "Rerun the rebuild after that command succeeds. " + + "NemoClaw stopped before deleting the existing sandbox." ); } -function exactDockerRemoveImage( - imageRef: string, - options: NonNullable, -): DockerRunResult { - const { - env, - ignoreError: _ignoreError, - stdio, - suppressOutput: _suppressOutput, - ...spawnOptions - } = options; - return dockerSpawnSync(["rmi", imageRef], { - ...spawnOptions, - cwd: ROOT, - env, - shell: false, - stdio: stdio ?? ["ignore", "pipe", "pipe"], - }); -} - -function defaultBuildxAvailable(process: DockerProofProcess): boolean { - try { - return ( - dockerSpawnSync(["buildx", "version"], { - cwd: process.cwd, - env: process.env, - shell: false, - stdio: ["ignore", "pipe", "pipe"], - }).status === 0 - ); - } catch { - return false; - } -} - export async function preflightRebuildImage( input: PreflightInput, deps: PreflightDeps = {}, ): Promise { const stage = deps.stageBuildContext ?? stageCreateSandboxBuildContext; const preparePatch = deps.prepareDockerfilePatch ?? prepareSandboxDockerfilePatch; - const buildImage = deps.buildImage ?? exactDockerBuild; - const removeImage = deps.removeImage ?? exactDockerRemoveImage; - const buildxAvailable = deps.buildxAvailable ?? defaultBuildxAvailable; - const buildDockerEnv = deps.buildDockerEnv ?? dockerBuildSubprocessEnv; + const buildImage = deps.buildImage ?? dockerBuild; + const removeImage = deps.removeImage ?? dockerRmi; let cleanup: (() => boolean) | null = null; let imageTag: string | null = null; let imageBuilt = false; let retainBuildContext = false; - let dockerEnv: Readonly> | null = null; const previousReasoning = process.env.NEMOCLAW_REASONING; const previousReasoningEffort = process.env[REASONING_EFFORT_ENV]; try { @@ -242,52 +156,14 @@ export async function preflightRebuildImage( warn: () => {}, }); const contextFingerprint = fingerprintBuildContext(staged.buildCtx); - dockerEnv = Object.freeze({ ...buildDockerEnv() }); imageTag = `nemoclaw-rebuild-preflight:${String(process.pid)}-${String(Date.now())}`; - const buildOptions: DockerBuildOptions = { - cwd: ROOT, - env: dockerEnv, + const result = buildImage(staged.stagedDockerfile, imageTag, staged.buildCtx, { ignoreError: true, suppressOutput: true, stdio: ["ignore", "pipe", "pipe"], - }; - const buildKitResult = buildImage( - staged.stagedDockerfile, - imageTag, - staged.buildCtx, - buildOptions, - ); - let result = buildKitResult; - let usedLegacyFallback = false; - if ( - result.status !== 0 && - staged.origin === "generated" && - input.agent === null && - input.localPrebuildEnabled && - hasBuildxUnavailableDiagnostic(result) && - !buildxAvailable({ cwd: ROOT, env: dockerEnv }) - ) { - // SOURCE_OF_TRUTH_REVIEW (#7111): the generated OpenClaw final-image - // Dockerfile does not use BuildKit-only instructions. Retry its exact - // fingerprinted bytes once with Docker's compatibility builder only - // after an independent buildx probe confirms the host CLI lacks it. - // Dockerfile.base, other agents, and custom --from contexts never enter - // this fallback. Remove it when the supported Docker floor no longer - // provides the legacy builder. - if (fingerprintBuildContext(staged.buildCtx) !== contextFingerprint) { - return { ok: false, detail: "replacement build context changed during preflight" }; - } - console.warn( - " Warning: Docker Buildx is unavailable; retrying the generated rebuild image with Docker's legacy builder.", - ); - usedLegacyFallback = true; - result = buildImage(staged.stagedDockerfile, imageTag, staged.buildCtx, { - ...buildOptions, - env: { ...dockerEnv, DOCKER_BUILDKIT: "0" }, - }); - } - if (result.status !== 0 && usedLegacyFallback) { - return { ok: false, detail: legacyRetryFailureDetail(buildKitResult, result) }; + }); + if (result.status !== 0 && requiresBuildxRepair(result)) { + return { ok: false, detail: buildxRepairDetail() }; } if (result.status !== 0) return { ok: false, detail: resultDetail(result) }; imageBuilt = true; @@ -295,34 +171,21 @@ export async function preflightRebuildImage( return { ok: false, detail: "replacement build context changed during preflight" }; } retainBuildContext = true; - const prebuildBuilder = usedLegacyFallback ? "legacy" : undefined; - const prebuildDockerEnv = usedLegacyFallback ? dockerEnv : undefined; - const verifyFingerprint = createBuildContextVerifier(staged.buildCtx, contextFingerprint); - const prepared: PreparedRebuildImage = { - ...staged, - cleanupBuildCtx: cleanup, - buildId, - dashboardRemoteBindPrepared, - contextFingerprint, - prebuildBuilder, - prebuildDockerEnv, - verifyBuildCtx(this: PreparedRebuildImage) { - return ( - this === prepared && - this.prebuildBuilder === prebuildBuilder && - this.prebuildDockerEnv === prebuildDockerEnv && - verifyFingerprint() - ); - }, - rebuildTarget: { - agentName: input.agent?.name ?? null, - fromDockerfile: input.fromDockerfile ? path.resolve(input.fromDockerfile) : null, - }, - }; return { ok: true, imageTag, - prepared, + prepared: { + ...staged, + cleanupBuildCtx: cleanup, + buildId, + dashboardRemoteBindPrepared, + contextFingerprint, + verifyBuildCtx: createBuildContextVerifier(staged.buildCtx, contextFingerprint), + rebuildTarget: { + agentName: input.agent?.name ?? null, + fromDockerfile: input.fromDockerfile ? path.resolve(input.fromDockerfile) : null, + }, + }, }; } catch (err) { return { ok: false, detail: err instanceof Error ? err.message : String(err) }; @@ -331,29 +194,18 @@ export async function preflightRebuildImage( try { imageRemoved = imageTag !== null && - removeImage(imageTag, { - cwd: ROOT, - env: dockerEnv ?? undefined, - ignoreError: true, - suppressOutput: true, - }).status === 0; + removeImage(imageTag, { ignoreError: true, suppressOutput: true }).status === 0; } catch { // Best effort; retained-context ownership and environment restoration must continue. } if (imageBuilt && imageTag && !imageRemoved) { const retainedImageTag = imageTag; - const retainedDockerEnv = dockerEnv; console.warn( ` Warning: failed to remove temporary rebuild preflight image '${retainedImageTag}'.`, ); process.once("exit", () => { try { - removeImage(retainedImageTag, { - cwd: ROOT, - env: retainedDockerEnv ?? undefined, - ignoreError: true, - suppressOutput: true, - }); + removeImage(retainedImageTag, { ignoreError: true, suppressOutput: true }); } catch { // Best effort process-exit retry. } diff --git a/src/lib/actions/sandbox/rebuild-prepared-image-context.ts b/src/lib/actions/sandbox/rebuild-prepared-image-context.ts index ff222249bd1..c95145e3cbd 100644 --- a/src/lib/actions/sandbox/rebuild-prepared-image-context.ts +++ b/src/lib/actions/sandbox/rebuild-prepared-image-context.ts @@ -28,10 +28,7 @@ export function createIdempotentBuildContextCleanup(cleanup: () => boolean): () /** Confirm that a retained private context still matches the prebuilt bytes. */ export function verifyPreparedBuildContext(prepared: FingerprintedPreparedBuildContext): boolean { try { - return ( - prepared.verifyBuildCtx() && - fingerprintBuildContext(prepared.buildCtx) === prepared.contextFingerprint - ); + return fingerprintBuildContext(prepared.buildCtx) === prepared.contextFingerprint; } catch { return false; } diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.ts b/src/lib/actions/sandbox/rebuild-target-runtime.ts index 049d2cb71d2..e92f7d3da04 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.ts @@ -21,7 +21,6 @@ import { readGatewayProviderMetadata, } from "../../onboard/gateway-provider-metadata"; import { resolveSandboxGpuConfig } from "../../onboard/sandbox-gpu-mode"; -import { resolveSandboxPrebuildEnabled } from "../../onboard/sandbox-prebuild"; import { agentSupportsWebSearchProvider } from "../../onboard/web-search-support"; import { redact } from "../../security/redact"; import { @@ -175,8 +174,8 @@ export async function preflightRebuildTargetRuntime( ); return { ok: false }; } - const dockerDriverGateway = isLinuxDockerDriverGatewayEnabled(); try { + const dockerDriverGateway = isLinuxDockerDriverGatewayEnabled(); const selectedRoute = initialDockerGpuRoute( resolveDockerGpuRoutePlan(sandboxGpuConfig, { dockerDriverGateway, @@ -214,7 +213,6 @@ export async function preflightRebuildTargetRuntime( toolDisclosure: target.durableConfig.toolDisclosure, hermesToolGateways: target.hermesToolGateways, sandboxGpuConfig, - localPrebuildEnabled: resolveSandboxPrebuildEnabled(process.env, dockerDriverGateway), preResolvedBaseImageMetadata: recreateOptions.preResolvedBaseImageMetadata ?? null, gatewayPort: recreateOptions.targetGatewayPort, chatUiUrl: managesDashboard diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 75038af3562..1b70da187fc 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2708,7 +2708,7 @@ async function createSandboxWithBaseImageResolution( manageDashboard, openshellShellCommand, openshellArgv, - prebuild: { buildCtx, buildId, dockerDriverGateway, origin, prepared: preparedBuildContext }, + prebuild: { buildCtx, buildId, dockerDriverGateway, origin }, }); const restoreBackupPath = pendingStateRestore?.manifest?.backupPath ?? pendingStateRestoreBackupPath; diff --git a/src/lib/onboard/build-context-stage.ts b/src/lib/onboard/build-context-stage.ts index 8868ff49ff1..994f6d8a308 100644 --- a/src/lib/onboard/build-context-stage.ts +++ b/src/lib/onboard/build-context-stage.ts @@ -41,10 +41,6 @@ export interface CreateSandboxBuildContextResult extends StagedBuildContext { export interface PreparedSandboxBuildContext extends CreateSandboxBuildContextResult { buildId: string; dashboardRemoteBindPrepared?: boolean; - /** Host builder that successfully validated this exact retained context. */ - readonly prebuildBuilder?: "legacy"; - /** Sanitized Docker endpoint/config environment bound to that builder proof. */ - readonly prebuildDockerEnv?: Readonly>; /** Recheck retained bytes at the final one-shot consumption boundary. */ verifyBuildCtx?(): boolean; /** Exact recorded target authorized to consume a generic rebuild handoff. */ diff --git a/src/lib/onboard/prepared-dcode-rebuild.test.ts b/src/lib/onboard/prepared-dcode-rebuild.test.ts index 0fe9aa9657c..808967611f1 100644 --- a/src/lib/onboard/prepared-dcode-rebuild.test.ts +++ b/src/lib/onboard/prepared-dcode-rebuild.test.ts @@ -239,25 +239,6 @@ describe("prepared DCode rebuild adapter", () => { expect(create).not.toHaveBeenCalled(); }); - it("rejects a legacy builder handoff outside generated OpenClaw rebuilds", () => { - expect(() => - createPreparedDcodeRebuildRuntime( - { - ...preparedImageOptions, - preparedImageRebuild: { - ...preparedImageOptions.preparedImageRebuild!, - buildContext: { - ...preparedImageBuildContext, - prebuildBuilder: "legacy", - prebuildDockerEnv: Object.freeze({ DOCKER_CONTEXT: "verified-builder" }), - }, - }, - }, - "nemoclaw", - ), - ).toThrow(/only be reused for a generated OpenClaw rebuild image/); - }); - it.runIf(process.platform !== "win32").each(oneShotContextMutations)( "rejects $label at the post-delete one-shot boundary", async ({ arrange, mutate, label }) => { diff --git a/src/lib/onboard/prepared-dcode-rebuild.ts b/src/lib/onboard/prepared-dcode-rebuild.ts index bfbf9fe18bc..0fef7cdb7c4 100644 --- a/src/lib/onboard/prepared-dcode-rebuild.ts +++ b/src/lib/onboard/prepared-dcode-rebuild.ts @@ -88,39 +88,11 @@ function normalizedAgentIdentity(agentName: string | null | undefined): string { return agentName?.trim() || "openclaw"; } -function assertPreparedBuilderSelection(preparedBuildContext: PreparedSandboxBuildContext): void { - const builder = preparedBuildContext.prebuildBuilder; - const dockerEnv = preparedBuildContext.prebuildDockerEnv; - if (builder !== undefined && builder !== "legacy") { - throw new Error("Prepared rebuild image builder is missing or invalid."); - } - if ( - (builder === "legacy" && - (!dockerEnv || - !Object.isFrozen(dockerEnv) || - Object.values(dockerEnv).some((value) => typeof value !== "string"))) || - (builder !== "legacy" && dockerEnv !== undefined) - ) { - throw new Error("Prepared rebuild image Docker environment is missing or invalid."); - } - if ( - builder === "legacy" && - (preparedBuildContext.origin !== "generated" || - normalizedAgentIdentity(preparedBuildContext.rebuildTarget?.agentName) !== "openclaw" || - preparedBuildContext.rebuildTarget?.fromDockerfile !== null) - ) { - throw new Error( - "Docker's legacy builder can only be reused for a generated OpenClaw rebuild image.", - ); - } -} - function assertPreparedTargetIdentity( preparedBuildContext: PreparedSandboxBuildContext, agentName: string | null, fromDockerfile: string | null, ): void { - assertPreparedBuilderSelection(preparedBuildContext); const target = preparedBuildContext.rebuildTarget; if (target) { if ( diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index 51cec958e38..e687681e3f4 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -356,7 +356,7 @@ describe("prepareSandboxCreateLaunch", () => { }); describe("prepareSandboxCreateLaunchWithPrebuild", () => { - it("hands the legacy-builder image to the canonical launch renderer", async () => { + it("hands the build-qualified image to the canonical launch renderer", async () => { const buildCtx = createTrustedBuildContext(); const dockerfile = path.join(buildCtx, "Dockerfile"); const buildImage = vi.fn(async () => 0); @@ -381,10 +381,6 @@ describe("prepareSandboxCreateLaunchWithPrebuild", () => { inspectImageId: () => IMAGE_ID, log: vi.fn(), origin: "generated", - prepared: { - prebuildBuilder: "legacy", - prebuildDockerEnv: Object.freeze({ DOCKER_CONTEXT: "verified-builder" }), - }, }, }); @@ -396,13 +392,7 @@ describe("prepareSandboxCreateLaunchWithPrebuild", () => { expect(result.createCommand).toContain( "sandbox create --from nemoclaw-sandbox-local:demo-build-123 --name demo", ); - expect(buildImage).toHaveBeenCalledWith( - expect.any(Array), - expect.objectContaining({ - cwd: expect.any(String), - env: expect.objectContaining({ DOCKER_BUILDKIT: "0" }), - }), - ); + expect(buildImage).toHaveBeenCalledOnce(); }); it("renders the original Dockerfile for Hermes after a local build failure", async () => { diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index 7a5fe2839ee..1e5431bb479 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -4,7 +4,6 @@ import type { AgentDefinition } from "../agent/defs"; import { formatEnvAssignment } from "../core/url-utils"; import { buildSubprocessEnv } from "../subprocess-env"; -import type { PreparedSandboxBuildContext } from "./build-context-stage"; import { isValidProxyHost, isValidProxyPort } from "./dockerfile-patch"; import { appendExtraPlaceholderKeysEnvArg } from "./extra-placeholder-keys"; import type { HermesDashboardOnboardState } from "./hermes-dashboard"; @@ -69,9 +68,7 @@ export interface SandboxCreateLaunch { export interface SandboxCreateLaunchWithPrebuildInput extends SandboxCreateLaunchInput { sandboxName: string; - prebuild: Omit & { - prepared?: Pick | null; - }; + prebuild: Omit; } export interface SandboxCreateLaunchWithPrebuild extends SandboxCreateLaunch { @@ -223,12 +220,9 @@ export async function prepareSandboxCreateLaunchWithPrebuild( input: SandboxCreateLaunchWithPrebuildInput, ): Promise { const { prebuild: prebuildInput, ...launchInput } = input; - const { prepared, ...prebuildOptions } = prebuildInput; const prebuild = await prebuildSandboxImageIfEligible({ - ...prebuildOptions, - builder: prepared?.prebuildBuilder, + ...prebuildInput, createArgs: input.createArgs, - dockerEnv: prepared?.prebuildDockerEnv, sandboxName: input.sandboxName, }); return { diff --git a/src/lib/onboard/sandbox-prebuild.test.ts b/src/lib/onboard/sandbox-prebuild.test.ts index 716a4c8a85f..961a891f90d 100644 --- a/src/lib/onboard/sandbox-prebuild.test.ts +++ b/src/lib/onboard/sandbox-prebuild.test.ts @@ -16,7 +16,6 @@ vi.mock("../adapters/docker/exec", async (importOriginal) => ({ })); import { withStdoutRedirectedToStderr } from "../cli/stdout-guard"; -import { ROOT } from "../runner"; import { SANDBOX_BUILD_CONTEXT_PREFIX } from "../sandbox/build-context"; import { dockerBuildSubprocessEnv, @@ -27,10 +26,6 @@ import { const BUILD_ID = "1234567890"; const IMAGE_ID = `sha256:${"a".repeat(64)}`; -const VERIFIED_DOCKER_ENV = Object.freeze({ - DOCKER_CONFIG: "/home/test/.docker", - DOCKER_CONTEXT: "verified-builder", -}); const temporaryDirectories: string[] = []; function createBuildContext( @@ -360,7 +355,6 @@ describe("sandbox BuildKit prebuild", () => { resolvedBuildCtx, ], expect.objectContaining({ - cwd: ROOT, env: expect.objectContaining({ DOCKER_BUILDKIT: "1" }), stdio: "inherit", }), @@ -372,118 +366,6 @@ describe("sandbox BuildKit prebuild", () => { }); }); - it("reuses the legacy builder proven by rebuild preflight", async () => { - const { buildCtx, createArgs } = createBuildContext(); - const buildImage = vi.fn(async () => 0); - const inspectImageId = vi.fn(() => IMAGE_ID); - const log = vi.fn(); - const result = await prebuildSandboxImageIfEligible({ - buildCtx, - buildId: BUILD_ID, - origin: "generated", - builder: "legacy", - dockerEnv: VERIFIED_DOCKER_ENV, - createArgs, - sandboxName: "alpha", - dockerDriverGateway: true, - env: {}, - buildImage, - inspectImageId, - log, - }); - - expect(buildImage).toHaveBeenCalledWith( - expect.arrayContaining(["build", "nemoclaw-sandbox-local:alpha-1234567890"]), - expect.objectContaining({ - cwd: ROOT, - env: { ...VERIFIED_DOCKER_ENV, DOCKER_BUILDKIT: "0" }, - stdio: "inherit", - }), - ); - expect(log).toHaveBeenCalledWith(expect.stringContaining("matches rebuild preflight")); - expect(inspectImageId).toHaveBeenCalledWith("nemoclaw-sandbox-local:alpha-1234567890", { - cwd: ROOT, - env: VERIFIED_DOCKER_ENV, - }); - expect(result.imageRef).toBe("nemoclaw-sandbox-local:alpha-1234567890"); - }); - - it.each([ - ["local prebuild is disabled", true, { NEMOCLAW_SANDBOX_PREBUILD: "0" }], - ["the gateway cannot consume a local image", false, { NEMOCLAW_SANDBOX_PREBUILD: "1" }], - ])("fails a prepared builder closed when %s", async (_label, dockerDriverGateway, env) => { - const { buildCtx, createArgs } = createBuildContext(); - await expect( - prebuildSandboxImageIfEligible({ - buildCtx, - buildId: BUILD_ID, - origin: "generated", - builder: "legacy", - dockerEnv: VERIFIED_DOCKER_ENV, - createArgs, - sandboxName: "alpha", - dockerDriverGateway, - env, - }), - ).rejects.toThrow(/verified local Docker builder is not enabled/); - }); - - it("fails a prepared builder closed when create arguments drift", async () => { - const { buildCtx } = createBuildContext(); - await expect( - prebuildSandboxImageIfEligible({ - buildCtx, - buildId: BUILD_ID, - origin: "generated", - builder: "legacy", - dockerEnv: VERIFIED_DOCKER_ENV, - createArgs: ["--from", "/other/Dockerfile"], - sandboxName: "alpha", - dockerDriverGateway: true, - env: {}, - }), - ).rejects.toThrow(/arguments no longer select the retained Dockerfile/); - }); - - it("fails a prepared builder closed when context trust validation drifts", async () => { - const { buildCtx, createArgs } = createBuildContext(); - fs.chmodSync(buildCtx, 0o770); - await expect( - prebuildSandboxImageIfEligible({ - buildCtx, - buildId: BUILD_ID, - origin: "generated", - builder: "legacy", - dockerEnv: VERIFIED_DOCKER_ENV, - createArgs, - sandboxName: "alpha", - dockerDriverGateway: true, - env: {}, - }), - ).rejects.toThrow(/context failed trust validation/); - }); - - it.each([ - ["exits nonzero", async () => 1], - ["cannot start", async () => Promise.reject(new Error("daemon unavailable"))], - ])("fails a prepared builder closed when its repeated build %s", async (_label, buildImage) => { - const { buildCtx, createArgs } = createBuildContext(); - await expect( - prebuildSandboxImageIfEligible({ - buildCtx, - buildId: BUILD_ID, - origin: "generated", - builder: "legacy", - dockerEnv: VERIFIED_DOCKER_ENV, - createArgs, - sandboxName: "alpha", - dockerDriverGateway: true, - env: {}, - buildImage, - }), - ).rejects.toThrow(/verified legacy builder/); - }); - it("routes default Docker build stdout only while JSONL owns stdout (#6403)", async () => { const { buildCtx, createArgs } = createBuildContext(); mocks.dockerSpawn.mockImplementation(() => { diff --git a/src/lib/onboard/sandbox-prebuild.ts b/src/lib/onboard/sandbox-prebuild.ts index c170c768dee..888b094c1fe 100644 --- a/src/lib/onboard/sandbox-prebuild.ts +++ b/src/lib/onboard/sandbox-prebuild.ts @@ -5,14 +5,14 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { dockerSpawn, dockerSpawnSync } from "../adapters/docker/exec"; +import { dockerImageInspectFormat } from "../adapters/docker"; +import { dockerSpawn } from "../adapters/docker/exec"; import { redirectInheritedChildStdoutToStderr } from "../cli/stdout-guard"; import { LOCAL_SANDBOX_IMAGE_REPO } from "../domain/sandbox/image-tag"; import { SANDBOX_BUILD_CONTEXT_PREFIX, type SandboxBuildContextOrigin, } from "../sandbox/build-context"; -import { ROOT } from "../state/paths"; import { buildSubprocessEnv } from "../subprocess-env"; import { isImmutableDockerImageId } from "./openshell-docker-sandbox-containers"; @@ -34,16 +34,12 @@ export interface SandboxPrebuildInput { sandboxName: string; dockerDriverGateway: boolean; origin: SandboxBuildContextOrigin; - /** Builder already proven against this retained rebuild context. */ - builder?: "legacy"; - /** Sanitized Docker endpoint/config environment used by that builder proof. */ - dockerEnv?: Readonly>; env?: NodeJS.ProcessEnv; buildImage?: ( args: readonly string[], - options: { cwd: string; env: NodeJS.ProcessEnv; stdio: "inherit" }, + options: { env: NodeJS.ProcessEnv; stdio: "inherit" }, ) => Promise; - inspectImageId?: (imageRef: string, options: { cwd: string; env: NodeJS.ProcessEnv }) => string; + inspectImageId?: (imageRef: string) => string; log?: (message: string) => void; } @@ -159,29 +155,10 @@ export async function prebuildSandboxImageIfEligible( const createArgs = [...input.createArgs]; const env = input.env ?? process.env; const log = input.log ?? console.log; - const requiredBuilder = input.builder ?? null; - const failPreparedBuild = (detail: string): never => { - throw new Error(`Prepared rebuild image cannot be recreated safely: ${detail}`); - }; - if ( - (requiredBuilder && - (!input.dockerEnv || - !Object.isFrozen(input.dockerEnv) || - Object.values(input.dockerEnv).some((value) => typeof value !== "string"))) || - (!requiredBuilder && input.dockerEnv) - ) { - failPreparedBuild("the verified Docker environment is missing or invalid"); - } if (!resolveSandboxPrebuildEnabled(env, input.dockerDriverGateway)) { - if (requiredBuilder) { - failPreparedBuild("the verified local Docker builder is not enabled"); - } return { createArgs, imageRef: null, imageId: null }; } if (input.origin !== "generated") { - if (requiredBuilder) { - failPreparedBuild("the retained build context is not NemoClaw-generated"); - } log( " Local BuildKit build skipped for a custom Dockerfile; using the gateway builder instead.", ); @@ -194,9 +171,6 @@ export async function prebuildSandboxImageIfEligible( !fromDockerfile || path.resolve(fromDockerfile) !== path.resolve(input.buildCtx, "Dockerfile") ) { - if (requiredBuilder) { - failPreparedBuild("sandbox create arguments no longer select the retained Dockerfile"); - } return { createArgs, imageRef: null, imageId: null }; } let trustedContext: TrustedStagedBuildContext | null; @@ -204,18 +178,12 @@ export async function prebuildSandboxImageIfEligible( trustedContext = resolveTrustedStagedBuildContext(input.buildCtx); } catch (error) { const detail = error instanceof Error ? error.message : String(error); - if (requiredBuilder) { - failPreparedBuild(`the retained build context could not be inspected (${detail})`); - } log( ` Local BuildKit build skipped: staged build context could not be inspected (${detail}); using the gateway builder instead.`, ); return { createArgs, imageRef: null, imageId: null }; } if (!trustedContext) { - if (requiredBuilder) { - failPreparedBuild("the retained build context failed trust validation"); - } log( " Local BuildKit build skipped: staged build context failed trust validation; using the gateway builder instead.", ); @@ -223,9 +191,6 @@ export async function prebuildSandboxImageIfEligible( } const imageRef = sandboxLocalImageRef(input.sandboxName, input.buildId); - const dockerEnv = requiredBuilder - ? (input.dockerEnv as Readonly>) - : dockerBuildSubprocessEnv(); const buildImage = input.buildImage ?? ((args, options) => @@ -238,40 +203,25 @@ export async function prebuildSandboxImageIfEligible( child.once("error", reject); child.once("close", resolve); })); - const builder = requiredBuilder ?? "buildkit"; - log( - builder === "legacy" - ? " Building sandbox image with Docker's legacy builder (matches rebuild preflight)..." - : " Building sandbox image with BuildKit (skips the slower in-gateway builder)...", - ); + log(" Building sandbox image with BuildKit (skips the slower in-gateway builder)..."); let status: number | null; try { status = await buildImage( ["build", "-t", imageRef, "-f", trustedContext.dockerfile, trustedContext.buildCtx], { - cwd: ROOT, - env: { - ...dockerEnv, - DOCKER_BUILDKIT: builder === "legacy" ? "0" : "1", - }, + env: { ...dockerBuildSubprocessEnv(), DOCKER_BUILDKIT: "1" }, stdio: "inherit", }, ); } catch (error) { const detail = error instanceof Error ? error.message : String(error); - if (requiredBuilder) { - failPreparedBuild(`the verified ${builder} builder could not start (${detail})`); - } log(` Local BuildKit build could not start (${detail}); using the gateway builder instead.`); return { createArgs, imageRef: null, imageId: null }; } if (status !== 0) { const detail = status === null ? " without an exit status" : ` (exit ${status})`; - if (requiredBuilder) { - failPreparedBuild(`the verified ${builder} builder failed${detail}`); - } log(` Local BuildKit build failed${detail}; using the gateway builder instead.`); return { createArgs, imageRef: null, imageId: null }; } @@ -279,20 +229,13 @@ export async function prebuildSandboxImageIfEligible( createArgs[fromIndex + 1] = imageRef; const inspectImageId = input.inspectImageId ?? - ((ref: string, options: { cwd: string; env: NodeJS.ProcessEnv }) => { - const inspected = dockerSpawnSync(["image", "inspect", "--format", "{{.Id}}", ref], { - ...options, - encoding: "utf8", - shell: false, - stdio: ["ignore", "pipe", "pipe"], - }); - return inspected.status === 0 && !inspected.error - ? String(inspected.stdout ?? "").trim() - : ""; - }); + ((ref: string) => + dockerImageInspectFormat("{{.Id}}", ref, { + ignoreError: true, + }).trim()); let imageId: string | null = null; try { - const inspected = inspectImageId(imageRef, { cwd: ROOT, env: dockerEnv }).trim(); + const inspected = inspectImageId(imageRef).trim(); if (isImmutableDockerImageId(inspected)) imageId = inspected.toLowerCase(); } catch { // Native creation can still use the local tag. Automatic compatibility From 4c04584f10017e964c729723aff50876baf8049e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 12:20:21 -0400 Subject: [PATCH 05/16] chore(ci): ratchet shell quote fan-in Signed-off-by: Julie Yaunches --- ci/source-architecture-budget.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index b1cb940c153..3168eb519ba 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -16,7 +16,7 @@ "src/lib/cli/terminal-style.ts": 45, "src/lib/core/json-types.ts": 37, "src/lib/core/ports.ts": 86, - "src/lib/core/shell-quote.ts": 27, + "src/lib/core/shell-quote.ts": 26, "src/lib/core/url-utils.ts": 27, "src/lib/core/wait.ts": 35, "src/lib/credentials/store.ts": 44, From 0a077a3cd325300c6fbf0a5987b970f66fbd13a3 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 12:57:23 -0400 Subject: [PATCH 06/16] test(rebuild): cover Buildx diagnostic streams --- .../sandbox/rebuild-custom-image-preflight.test.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts index be213cf8086..90733dc1805 100644 --- a/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts +++ b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts @@ -253,17 +253,23 @@ describe("preflightRebuildImage", () => { } }); - it("requires Buildx repair before deleting the existing sandbox (#7111)", async () => { + it.each([ + ["error string", "error", false], + ["error buffer", "error", true], + ["stderr string", "stderr", false], + ["stderr buffer", "stderr", true], + ["stdout string", "stdout", false], + ["stdout buffer", "stdout", true], + ] as const)("requires Buildx repair when %s contains the diagnostic before sandbox deletion (#7111)", async (_case, stream, buffered) => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-required-")); const dockerfile = path.join(dir, "Dockerfile.custom"); fs.writeFileSync(dockerfile, "FROM scratch\n"); + const diagnostic = "ERROR: BuildKit is enabled but the buildx component is missing or broken."; const buildImage = vi.fn( () => ({ status: 1, - stderr: Buffer.from( - "ERROR: BuildKit is enabled but the buildx component is missing or broken.", - ), + [stream]: buffered ? Buffer.from(diagnostic) : diagnostic, }) as never, ); const cleanupBuildCtx = vi.fn(() => true); From ed9f04e8f9c2d3491ff4f447ad516af4e6ec5f25 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 13:38:51 -0400 Subject: [PATCH 07/16] test(rebuild): guard Buildx mutation boundary --- docs/reference/commands.mdx | 2 + .../rebuild-buildx-mutation-boundary.test.ts | 52 +++++++++++++++++++ test/helpers/rebuild-flow-harness.ts | 10 ++-- 3 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 src/lib/actions/sandbox/rebuild-buildx-mutation-boundary.test.ts diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 5d8944a6fc6..bcaa13cb622 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -2702,10 +2702,12 @@ A rebuild preserves the recorded Deep Agents Code auto-approval capability unles A sandbox onboarded with an explicit GPU opt-out (stored as `sandboxGpuMode: "0"`, plus legacy registry entries that only record `gpuEnabled: false`) is recreated with the same opt-out, so the inner `onboard --resume` skips the Docker CDI GPU preflight on hosts without an NVIDIA GPU. Auto-mode sandboxes remain auto. + Before backup or deletion, `rebuild` builds and validates the replacement image with Docker Buildx. If Buildx is missing or broken, NemoClaw stops and leaves the existing sandbox intact. Install or repair Docker Buildx, then verify it with `docker buildx version`. Rerun the rebuild after that command succeeds. + ```bash $$nemoclaw my-assistant rebuild [--yes|-y|--force] [--verbose|-v] [--tool-disclosure ] [--dcode-auto-approval ] [--observability|--no-observability] diff --git a/src/lib/actions/sandbox/rebuild-buildx-mutation-boundary.test.ts b/src/lib/actions/sandbox/rebuild-buildx-mutation-boundary.test.ts new file mode 100644 index 00000000000..430f77abb7f --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-buildx-mutation-boundary.test.ts @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { expectNoSandboxDelete } from "../../../../test/helpers/rebuild-delete-assertions"; +import { + createRebuildFlowHarness, + resetRebuildFlowTestEnvironment, + restoreRebuildFlowTestEnvironment, +} from "../../../../test/helpers/rebuild-flow-harness"; + +const BUILDX_REPAIR_DETAIL = + "Docker Buildx is required for sandbox rebuilds. " + + "Install or repair Docker Buildx, then verify it with 'docker buildx version'. " + + "Rerun the rebuild after that command succeeds. " + + "NemoClaw stopped before deleting the existing sandbox."; + +describe("rebuildSandbox Buildx mutation boundary", () => { + beforeEach(resetRebuildFlowTestEnvironment); + afterEach(restoreRebuildFlowTestEnvironment); + + it("keeps the sandbox intact until Buildx is repaired, then allows a retry (#7111)", async () => { + const blocked = createRebuildFlowHarness({ + rebuildImagePreflightResult: { ok: false, detail: BUILDX_REPAIR_DETAIL }, + }); + + await expect( + blocked.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Replacement sandbox image preflight failed"); + + expect(blocked.preflightRebuildImageSpy).toHaveBeenCalledOnce(); + expect(blocked.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(blocked.registryUpdateSpy).not.toHaveBeenCalled(); + expectNoSandboxDelete(blocked.runOpenshellSpy); + expect(blocked.removeSandboxRegistryEntrySpy).not.toHaveBeenCalled(); + expect(blocked.onboardSpy).not.toHaveBeenCalled(); + + restoreRebuildFlowTestEnvironment(); + resetRebuildFlowTestEnvironment(); + const repaired = createRebuildFlowHarness(); + + await expect( + repaired.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(repaired.preflightRebuildImageSpy).toHaveBeenCalledOnce(); + expect(repaired.registryUpdateSpy).toHaveBeenCalled(); + expect(repaired.backupSandboxStateSpy).toHaveBeenCalledOnce(); + expect(repaired.removeSandboxRegistryEntrySpy).toHaveBeenCalledOnce(); + expect(repaired.onboardSpy).toHaveBeenCalledOnce(); + }); +}); diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index ef2846b26c6..325d6c99087 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -137,6 +137,7 @@ export type RebuildFlowOverrides = { openShieldsWindow?: () => { relocked: boolean; wasLocked: boolean } | null; preflightMessagingConflicts?: () => Promise | void; preflightAuthoritativeRebuildTarget?: (options: Record) => Promise | void; + rebuildImagePreflightResult?: { ok: false; detail: string } | { ok: true; imageTag: null }; mcpPreparation?: { entries: Array>; detachedProviderEntries: Array>; @@ -162,6 +163,7 @@ export type RebuildFlowHarness = { openShieldsSpy: MockInstance; onboardSpy: MockInstance; preflightAuthoritativeRebuildTargetSpy: MockInstance; + preflightRebuildImageSpy: MockInstance; preflightMessagingConflictsSpy: MockInstance; preflightDcodeRouteSpy: MockInstance; prepareManagedDcodeRebuildImageSpy: MockInstance; @@ -313,10 +315,9 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): }); vi.spyOn(resolve, "resolveOpenshell").mockReturnValue(null); vi.spyOn(dockerImage, "dockerBuild").mockReturnValue({ status: 0 }); - vi.spyOn(rebuildCustomImagePreflight, "preflightRebuildImage").mockResolvedValue({ - ok: true, - imageTag: null, - }); + const preflightRebuildImageSpy = vi + .spyOn(rebuildCustomImagePreflight, "preflightRebuildImage") + .mockResolvedValue(overrides.rebuildImagePreflightResult ?? { ok: true, imageTag: null }); const imageIdsByRef = new Map([ [agentBaseImageRef, agentBaseImageId], [agentBaseImageId, agentBaseImageId], @@ -743,6 +744,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): openShieldsSpy, onboardSpy, preflightAuthoritativeRebuildTargetSpy, + preflightRebuildImageSpy, preflightMessagingConflictsSpy, preflightDcodeRouteSpy, prepareManagedDcodeRebuildImageSpy, From 89d5de675a3903da0e69cfc7f7e764eb84cc6beb Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 18:09:19 -0400 Subject: [PATCH 08/16] fix(rebuild): support OpenClaw without Buildx Signed-off-by: Julie Yaunches --- ci/source-architecture-budget.json | 2 +- docs/reference/commands.mdx | 29 +- .../rebuild-buildx-mutation-boundary.test.ts | 149 +++++- .../rebuild-custom-image-preflight.test.ts | 470 ++++++++++++++-- .../sandbox/rebuild-custom-image-preflight.ts | 292 ++++++++-- .../sandbox/rebuild-destroy-phase.test.ts | 133 +++++ .../actions/sandbox/rebuild-destroy-phase.ts | 47 ++ .../rebuild-openclaw-legacy-image.test.ts | 500 ++++++++++++++++++ src/lib/actions/sandbox/rebuild-pipeline.ts | 40 +- .../sandbox/rebuild-prepared-image-context.ts | 33 +- .../actions/sandbox/rebuild-target-runtime.ts | 5 +- .../sandbox/rebuild/openclaw-legacy-image.ts | 378 +++++++++++++ src/lib/onboard.ts | 44 +- src/lib/onboard/build-context-stage.ts | 21 + src/lib/onboard/docker-gpu-local-inference.ts | 6 +- src/lib/onboard/docker-gpu-patch.ts | 14 +- ...ocker-gpu-sandbox-create-lifecycle.test.ts | 119 +++++ src/lib/onboard/docker-gpu-sandbox-create.ts | 21 +- .../onboard/gateway-sandbox-reachability.ts | 43 +- .../onboard/prepared-dcode-rebuild.test.ts | 111 +++- src/lib/onboard/prepared-dcode-rebuild.ts | 37 ++ .../retained-openclaw-docker-runtime.ts | 265 ++++++++++ .../retained-openclaw-docker-runtime.test.ts | 185 +++++++ src/lib/onboard/sandbox-create-launch.test.ts | 91 ++++ src/lib/onboard/sandbox-create-launch.ts | 15 +- .../onboard/sandbox-gpu-create-flow.test.ts | 274 ++++++++++ src/lib/onboard/sandbox-gpu-create-flow.ts | 255 ++++++--- .../onboard/sandbox-gpu-create-run-attempt.ts | 13 +- src/lib/onboard/sandbox-prebuild.test.ts | 72 +++ src/lib/onboard/sandbox-prebuild.ts | 40 +- test/helpers/rebuild-flow-harness.ts | 14 +- .../rebuild-flow-target-image-cases.ts | 4 +- 32 files changed, 3494 insertions(+), 228 deletions(-) create mode 100644 src/lib/actions/sandbox/rebuild-openclaw-legacy-image.test.ts create mode 100644 src/lib/actions/sandbox/rebuild/openclaw-legacy-image.ts create mode 100644 src/lib/onboard/rebuild/retained-openclaw-docker-runtime.ts create mode 100644 src/lib/onboard/retained-openclaw-docker-runtime.test.ts diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 3168eb519ba..07bc74c2543 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -5,7 +5,7 @@ "maxByFile": { "src/lib/actions/sandbox/mcp-bridge-contracts.ts": 26, "src/lib/actions/sandbox/process-recovery.ts": 26, - "src/lib/adapters/docker/index.ts": 45, + "src/lib/adapters/docker/index.ts": 44, "src/lib/adapters/openshell/client.ts": 22, "src/lib/adapters/openshell/resolve.ts": 28, "src/lib/adapters/openshell/runtime.ts": 50, diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index bcaa13cb622..c64d2edb952 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -2702,11 +2702,38 @@ A rebuild preserves the recorded Deep Agents Code auto-approval capability unles A sandbox onboarded with an explicit GPU opt-out (stored as `sandboxGpuMode: "0"`, plus legacy registry entries that only record `gpuEnabled: false`) is recreated with the same opt-out, so the inner `onboard --resume` skips the Docker CDI GPU preflight on hosts without an NVIDIA GPU. Auto-mode sandboxes remain auto. - + + +Before backup or deletion, `rebuild` builds and validates the replacement image with Docker's BuildKit path. +For a NemoClaw-generated OpenClaw image on a local Docker-driver gateway, NemoClaw can use one compatibility retry when Docker reports `BuildKit is enabled but the buildx component is missing or broken`. +NemoClaw runs `docker buildx version` independently and uses the compatibility retry only when that command also fails. +The retry rebuilds the same verified build context once with Docker's legacy builder. +NemoClaw then binds the retained image to the verified Docker engine and its immutable image ID. +It verifies the build context, Docker engine, mutable tag, and immutable image ID again at the final delete boundary. +Any failed build or pre-delete verification stops the rebuild without deleting the existing sandbox. +After deletion, Docker operations for replacement creation remain bound to the retained engine and immutable image ID. +If either identity changes during replacement creation, NemoClaw stops instead of operating on a different Docker target. + +The legacy-builder retry does not apply in the following cases: + +- The sandbox was recorded with a custom `--from` Dockerfile. +- The OpenShell gateway is not local. +- The host-side local prebuild is disabled. +- The build has an unrelated failure. +- The independent `docker buildx version` command succeeds. + +When the retry does not apply and Buildx is missing or broken, NemoClaw stops and leaves the existing sandbox intact. +Install or repair Docker Buildx, then verify it with `docker buildx version`. +Rerun the rebuild after that command succeeds. + + + + Before backup or deletion, `rebuild` builds and validates the replacement image with Docker Buildx. If Buildx is missing or broken, NemoClaw stops and leaves the existing sandbox intact. Install or repair Docker Buildx, then verify it with `docker buildx version`. Rerun the rebuild after that command succeeds. + ```bash diff --git a/src/lib/actions/sandbox/rebuild-buildx-mutation-boundary.test.ts b/src/lib/actions/sandbox/rebuild-buildx-mutation-boundary.test.ts index 430f77abb7f..ddd39293888 100644 --- a/src/lib/actions/sandbox/rebuild-buildx-mutation-boundary.test.ts +++ b/src/lib/actions/sandbox/rebuild-buildx-mutation-boundary.test.ts @@ -1,52 +1,147 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { expectNoSandboxDelete } from "../../../../test/helpers/rebuild-delete-assertions"; import { createRebuildFlowHarness, resetRebuildFlowTestEnvironment, restoreRebuildFlowTestEnvironment, } from "../../../../test/helpers/rebuild-flow-harness"; +import { fingerprintBuildContext } from "../../adapters/fs/build-context-fingerprint"; +import type { + PreparedOpenClawLegacyImage, + PreparedSandboxBuildContext, +} from "../../onboard/build-context-stage"; -const BUILDX_REPAIR_DETAIL = - "Docker Buildx is required for sandbox rebuilds. " + - "Install or repair Docker Buildx, then verify it with 'docker buildx version'. " + - "Rerun the rebuild after that command succeeds. " + - "NemoClaw stopped before deleting the existing sandbox."; +type RetainedImageFixture = { + buildContext: PreparedSandboxBuildContext & { + contextFingerprint: string; + verifyBuildCtx(): boolean; + }; + lease: PreparedOpenClawLegacyImage; + verifyImage: ReturnType; + retainForRecreate: ReturnType; +}; -describe("rebuildSandbox Buildx mutation boundary", () => { +function createRetainedImageFixture(imageVerificationResults: boolean[]): RetainedImageFixture { + const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-boundary-")); + const stagedDockerfile = path.join(buildCtx, "Dockerfile"); + fs.writeFileSync(stagedDockerfile, "FROM scratch\n"); + const contextFingerprint = fingerprintBuildContext(buildCtx); + const verifyImage = vi.fn(() => imageVerificationResults.shift() ?? true); + const retainForRecreate = vi.fn(() => true); + const lease = Object.freeze({ + dockerEnv: Object.freeze({ DOCKER_CONTEXT: "verified-builder" }), + engineId: "verified-engine", + imageRef: "nemoclaw-sandbox-local:alpha-rebuild-preflight", + imageId: `sha256:${"d".repeat(64)}`, + verify: verifyImage, + retainForRecreate, + verifyForCreate: vi.fn(() => true), + finalizeAfterCreate: vi.fn(() => ({ + mutableTagVerified: true, + registryImageRef: null, + })), + abort: vi.fn(() => true), + dispose: vi.fn(() => true), + }); + let buildContext: RetainedImageFixture["buildContext"]; + buildContext = { + buildCtx, + stagedDockerfile, + cleanupBuildCtx: vi.fn(() => { + fs.rmSync(buildCtx, { recursive: true, force: true }); + return true; + }), + origin: "generated", + buildId: "retained-build", + contextFingerprint, + preparedOpenClawLegacyImage: lease, + verifyBuildCtx(this: RetainedImageFixture["buildContext"]) { + return ( + this === buildContext && + this.preparedOpenClawLegacyImage === lease && + fingerprintBuildContext(buildCtx) === contextFingerprint + ); + }, + rebuildTarget: { + agentName: null, + fromDockerfile: null, + }, + }; + return { buildContext, lease, verifyImage, retainForRecreate }; +} + +function sandboxDeleteCallOrder(runOpenshellSpy: ReturnType): number { + const deleteCallIndex = runOpenshellSpy.mock.calls.findIndex( + ([args]) => Array.isArray(args) && args.join(" ") === "sandbox delete -g nemoclaw alpha", + ); + expect(deleteCallIndex).toBeGreaterThanOrEqual(0); + return runOpenshellSpy.mock.invocationCallOrder[deleteCallIndex] ?? Number.POSITIVE_INFINITY; +} + +describe("rebuildSandbox retained OpenClaw image mutation boundary", () => { beforeEach(resetRebuildFlowTestEnvironment); afterEach(restoreRebuildFlowTestEnvironment); - it("keeps the sandbox intact until Buildx is repaired, then allows a retry (#7111)", async () => { - const blocked = createRebuildFlowHarness({ - rebuildImagePreflightResult: { ok: false, detail: BUILDX_REPAIR_DETAIL }, + it("keeps the sandbox intact when the retained image drifts at the final delete edge (#7253)", async () => { + const fixture = createRetainedImageFixture([true, true, false]); + const harness = createRebuildFlowHarness({ + rebuildImagePreflightResult: { + ok: true, + imageTag: fixture.lease.imageRef, + prepared: fixture.buildContext, + }, }); await expect( - blocked.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).rejects.toThrow("Replacement sandbox image preflight failed"); + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow( + "The retained replacement image inputs changed before sandbox deletion. Retry the rebuild.", + ); - expect(blocked.preflightRebuildImageSpy).toHaveBeenCalledOnce(); - expect(blocked.backupSandboxStateSpy).not.toHaveBeenCalled(); - expect(blocked.registryUpdateSpy).not.toHaveBeenCalled(); - expectNoSandboxDelete(blocked.runOpenshellSpy); - expect(blocked.removeSandboxRegistryEntrySpy).not.toHaveBeenCalled(); - expect(blocked.onboardSpy).not.toHaveBeenCalled(); + expect(harness.preflightRebuildImageSpy).toHaveBeenCalledOnce(); + expect(harness.backupSandboxStateSpy).toHaveBeenCalledOnce(); + expect(fixture.verifyImage).toHaveBeenCalledTimes(3); + expect(fixture.retainForRecreate).not.toHaveBeenCalled(); + expect(harness.reattachMcpProvidersAfterRebuildAbortSpy).toHaveBeenCalledOnce(); + expect(harness.relockSpy).toHaveBeenCalled(); + expectNoSandboxDelete(harness.runOpenshellSpy); + expect(harness.removeSandboxRegistryEntrySpy).not.toHaveBeenCalled(); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + }); - restoreRebuildFlowTestEnvironment(); - resetRebuildFlowTestEnvironment(); - const repaired = createRebuildFlowHarness(); + it("commits the exact retained image immediately before delete and never rebuilds it afterward (#7253)", async () => { + const fixture = createRetainedImageFixture([true, true, true]); + const harness = createRebuildFlowHarness({ + rebuildImagePreflightResult: { + ok: true, + imageTag: fixture.lease.imageRef, + prepared: fixture.buildContext, + }, + }); await expect( - repaired.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), ).resolves.toBeUndefined(); - expect(repaired.preflightRebuildImageSpy).toHaveBeenCalledOnce(); - expect(repaired.registryUpdateSpy).toHaveBeenCalled(); - expect(repaired.backupSandboxStateSpy).toHaveBeenCalledOnce(); - expect(repaired.removeSandboxRegistryEntrySpy).toHaveBeenCalledOnce(); - expect(repaired.onboardSpy).toHaveBeenCalledOnce(); + expect(harness.preflightRebuildImageSpy).toHaveBeenCalledOnce(); + expect(harness.backupSandboxStateSpy).toHaveBeenCalledOnce(); + expect(fixture.verifyImage).toHaveBeenCalledTimes(3); + expect(fixture.retainForRecreate).toHaveBeenCalledOnce(); + expect(fixture.verifyImage.mock.invocationCallOrder.at(-1)).toBeLessThan( + fixture.retainForRecreate.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + const deleteOrder = sandboxDeleteCallOrder(harness.runOpenshellSpy as never); + expect(fixture.retainForRecreate.mock.invocationCallOrder[0]).toBeLessThan(deleteOrder); + expect(deleteOrder).toBeLessThan( + harness.onboardSpy.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + expect(harness.removeSandboxRegistryEntrySpy).toHaveBeenCalledOnce(); + expect(harness.onboardSpy).toHaveBeenCalledOnce(); }); }); diff --git a/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts index 90733dc1805..58550159585 100644 --- a/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts +++ b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts @@ -6,8 +6,10 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import type { PreparedOpenClawLegacyImage } from "../../onboard/build-context-stage"; import { ROOT } from "../../runner"; import type { SandboxBaseImageResolutionMetadata } from "../../sandbox-base-image"; +import type { OpenClawLegacyDockerBinding } from "./rebuild/openclaw-legacy-image"; import { preflightRebuildImage, type RebuildImagePreflightResult, @@ -18,6 +20,7 @@ import { } from "./rebuild-prepared-image-context"; type SuccessfulPreflight = Extract; +const LEGACY_IMAGE_ID = `sha256:${"d".repeat(64)}`; function successful(result: RebuildImagePreflightResult): SuccessfulPreflight { expect(result.ok).toBe(true); @@ -44,11 +47,44 @@ function input(fromDockerfile: string | null) { sandboxGpuDevice: null, errors: [], }, + sandboxName: "alpha", + localPrebuildEnabled: false, gatewayPort: 8080, chatUiUrl: "http://127.0.0.1:18789", }; } +function createLegacyBinding(): OpenClawLegacyDockerBinding { + return Object.freeze({ + dockerEnv: Object.freeze({ + DOCKER_CONFIG: "/home/test/.docker", + DOCKER_CONTEXT: "verified-builder", + }), + engineId: "verified-engine", + }); +} + +function createLegacyLease( + binding: OpenClawLegacyDockerBinding, + imageRef: string, +): PreparedOpenClawLegacyImage { + return Object.freeze({ + dockerEnv: binding.dockerEnv, + engineId: binding.engineId, + imageRef, + imageId: LEGACY_IMAGE_ID, + verify: vi.fn(() => true), + retainForRecreate: vi.fn(() => true), + verifyForCreate: vi.fn(() => true), + finalizeAfterCreate: vi.fn(() => ({ + mutableTagVerified: true, + registryImageRef: null, + })), + abort: vi.fn(() => true), + dispose: vi.fn(() => true), + }); +} + describe("preflightRebuildImage", () => { it("carries verified base provenance into the retained managed context (#7144)", async () => { const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-provenance-")); @@ -147,6 +183,52 @@ describe("preflightRebuildImage", () => { } }); + it("removes a successful BuildKit candidate only with its captured same-engine image ID", async () => { + const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-buildkit-")); + const stagedDockerfile = path.join(buildCtx, "Dockerfile"); + fs.writeFileSync(stagedDockerfile, "FROM scratch\n"); + const binding = createLegacyBinding(); + const buildImage = vi.fn(() => ({ status: 0 }) as never); + const disposeLegacyImage = vi.fn(() => true); + + try { + const result = successful( + await preflightRebuildImage( + { ...input(null), localPrebuildEnabled: true }, + { + stageBuildContext: vi.fn(() => ({ + buildCtx, + stagedDockerfile, + cleanupBuildCtx: vi.fn(() => true), + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "buildkit-success", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + captureLegacyDockerBinding: vi.fn(() => binding), + inspectLegacyImageId: vi.fn(() => LEGACY_IMAGE_ID), + disposeLegacyImage, + }, + ), + ); + + expect(buildImage).toHaveBeenCalledWith( + stagedDockerfile, + result.imageTag, + buildCtx, + expect.objectContaining({ env: binding.dockerEnv }), + ); + expect(result.prepared.preparedOpenClawLegacyImage).toBeUndefined(); + expect(disposeLegacyImage).toHaveBeenCalledWith(binding, result.imageTag, LEGACY_IMAGE_ID); + expect(disposePreparedBuildContext(result.prepared)).toBe(true); + } finally { + fs.rmSync(buildCtx, { recursive: true, force: true }); + } + }); + it.runIf(process.platform !== "win32")( "rejects a symlinked build-context root before the preflight build", async () => { @@ -260,48 +342,374 @@ describe("preflightRebuildImage", () => { ["stderr buffer", "stderr", true], ["stdout string", "stdout", false], ["stdout buffer", "stdout", true], - ] as const)("requires Buildx repair when %s contains the diagnostic before sandbox deletion (#7111)", async (_case, stream, buffered) => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-required-")); - const dockerfile = path.join(dir, "Dockerfile.custom"); + ] as const)("retains one exact generated OpenClaw legacy image when %s contains the diagnostic (#7111)", async (_case, stream, buffered) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-fallback-")); + const dockerfile = path.join(dir, "Dockerfile"); fs.writeFileSync(dockerfile, "FROM scratch\n"); const diagnostic = "ERROR: BuildKit is enabled but the buildx component is missing or broken."; - const buildImage = vi.fn( - () => - ({ - status: 1, - [stream]: buffered ? Buffer.from(diagnostic) : diagnostic, - }) as never, - ); + const buildImage = vi + .fn() + .mockReturnValueOnce({ + status: 1, + [stream]: buffered ? Buffer.from(diagnostic) : diagnostic, + } as never) + .mockReturnValueOnce({ status: 0 } as never); const cleanupBuildCtx = vi.fn(() => true); + const binding = createLegacyBinding(); + let lease: PreparedOpenClawLegacyImage | null = null; + const createLegacyImage = vi.fn((_binding: OpenClawLegacyDockerBinding, imageRef: string) => { + lease = createLegacyLease(binding, imageRef); + return lease; + }); + const removeImage = vi.fn(() => ({ status: 0 }) as never); + const disposeLegacyImage = vi.fn(() => true); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); try { - const result = await preflightRebuildImage(input(dockerfile), { - stageBuildContext: vi.fn(() => ({ - buildCtx: dir, - stagedDockerfile: dockerfile, - cleanupBuildCtx, - origin: "custom" as const, - })), - prepareDockerfilePatch: vi.fn(async () => ({ - buildId: "buildx-required", - dashboardRemoteBindPrepared: false, - resolvedBaseImage: null, - })), - buildImage, - removeImage: vi.fn(() => ({ status: 0 }) as never), + const result = successful( + await preflightRebuildImage( + { ...input(null), localPrebuildEnabled: true }, + { + stageBuildContext: vi.fn(() => ({ + buildCtx: dir, + stagedDockerfile: dockerfile, + cleanupBuildCtx, + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "buildx-fallback", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + removeImage, + buildxAvailable: vi.fn(() => false), + captureLegacyDockerBinding: vi.fn(() => binding), + inspectLegacyImageId: vi.fn(() => LEGACY_IMAGE_ID), + createLegacyImage, + disposeLegacyImage, + }, + ), + ); + + expect(buildImage).toHaveBeenCalledTimes(2); + expect(buildImage.mock.calls[0]?.slice(0, 3)).toEqual(buildImage.mock.calls[1]?.slice(0, 3)); + expect(buildImage.mock.calls[0]?.[3]).toEqual( + expect.objectContaining({ cwd: ROOT, env: binding.dockerEnv }), + ); + expect(buildImage.mock.calls[1]?.[3]).toEqual( + expect.objectContaining({ + cwd: ROOT, + env: { ...binding.dockerEnv, DOCKER_BUILDKIT: "0" }, + }), + ); + expect(result.imageTag).toMatch(/^nemoclaw-sandbox-local:/); + expect(result.prepared.preparedOpenClawLegacyImage).toBe(lease); + expect(createLegacyImage).toHaveBeenCalledWith(binding, result.imageTag, LEGACY_IMAGE_ID); + expect(removeImage).not.toHaveBeenCalled(); + expect(disposeLegacyImage).not.toHaveBeenCalled(); + expect(cleanupBuildCtx).not.toHaveBeenCalled(); + expect(verifyPreparedBuildContext(result.prepared)).toBe(true); + expect(disposePreparedBuildContext(result.prepared)).toBe(true); + } finally { + warn.mockRestore(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it.each([ + { + label: "the independent Buildx probe succeeds", + agent: null, + fromDockerfile: null, + origin: "generated" as const, + localPrebuildEnabled: true, + buildxAvailable: true, + capturesBinding: true, + }, + { + label: "the generated target is Hermes", + agent: { name: "hermes" }, + fromDockerfile: null, + origin: "generated" as const, + localPrebuildEnabled: true, + buildxAvailable: false, + capturesBinding: false, + }, + { + label: "local image consumption is unavailable", + agent: null, + fromDockerfile: null, + origin: "generated" as const, + localPrebuildEnabled: false, + buildxAvailable: false, + capturesBinding: false, + }, + { + label: "the Dockerfile is user supplied", + agent: null, + fromDockerfile: "/tmp/Dockerfile.custom", + origin: "custom" as const, + localPrebuildEnabled: true, + buildxAvailable: false, + capturesBinding: false, + }, + ])("does not retry when $label (#7111)", async (testCase) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-no-fallback-")); + const dockerfile = path.join(dir, "Dockerfile"); + fs.writeFileSync(dockerfile, "FROM scratch\n"); + const diagnostic = "ERROR: BuildKit is enabled but the buildx component is missing or broken."; + const buildImage = vi.fn(() => ({ status: 1, stderr: diagnostic }) as never); + const cleanupBuildCtx = vi.fn(() => true); + const binding = createLegacyBinding(); + const captureLegacyDockerBinding = vi.fn(() => binding); + const buildxAvailable = vi.fn(() => testCase.buildxAvailable); + + try { + const result = await preflightRebuildImage( + { + ...input(testCase.fromDockerfile), + agent: testCase.agent as never, + localPrebuildEnabled: testCase.localPrebuildEnabled, + }, + { + stageBuildContext: vi.fn(() => ({ + buildCtx: dir, + stagedDockerfile: dockerfile, + cleanupBuildCtx, + origin: testCase.origin, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "no-fallback", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + removeImage: vi.fn(() => ({ status: 0 }) as never), + buildxAvailable, + captureLegacyDockerBinding, + disposeLegacyImage: vi.fn(() => true), + }, + ); + + expect(result).toEqual({ ok: false, detail: diagnostic }); + expect(buildImage).toHaveBeenCalledOnce(); + expect(captureLegacyDockerBinding).toHaveBeenCalledTimes(testCase.capturesBinding ? 1 : 0); + if (testCase.capturesBinding) { + expect(captureLegacyDockerBinding).toHaveBeenCalledWith({ + buildDockerEnv: expect.any(Function), + cwd: ROOT, + }); + } + expect(buildxAvailable).toHaveBeenCalledTimes(testCase.capturesBinding ? 1 : 0); + expect(cleanupBuildCtx).toHaveBeenCalledOnce(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects generated build-context drift before the legacy retry (#7111)", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-drift-")); + const dockerfile = path.join(dir, "Dockerfile"); + fs.writeFileSync(dockerfile, "FROM scratch\n"); + const buildImage = vi.fn(() => { + fs.writeFileSync(path.join(dir, "mutated"), "changed\n"); + return { + status: 1, + stderr: "ERROR: BuildKit is enabled but the buildx component is missing or broken.", + } as never; + }); + const cleanupBuildCtx = vi.fn(() => true); + + try { + await expect( + preflightRebuildImage( + { ...input(null), localPrebuildEnabled: true }, + { + stageBuildContext: vi.fn(() => ({ + buildCtx: dir, + stagedDockerfile: dockerfile, + cleanupBuildCtx, + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "buildx-drift", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + buildxAvailable: vi.fn(() => false), + captureLegacyDockerBinding: vi.fn(() => createLegacyBinding()), + disposeLegacyImage: vi.fn(() => true), + }, + ), + ).resolves.toEqual({ + ok: false, + detail: "replacement build context changed during preflight", }); + expect(buildImage).toHaveBeenCalledOnce(); + expect(cleanupBuildCtx).toHaveBeenCalledOnce(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("leaves a unique GC-visible tag when legacy image identity cannot be established (#7253)", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-identity-failure-")); + const dockerfile = path.join(dir, "Dockerfile"); + fs.writeFileSync(dockerfile, "FROM scratch\n"); + const binding = createLegacyBinding(); + const buildImage = vi + .fn() + .mockReturnValueOnce({ + status: 1, + stderr: "ERROR: BuildKit is enabled but the buildx component is missing or broken.", + } as never) + .mockReturnValueOnce({ status: 0 } as never); + const disposeLegacyImage = vi.fn(() => true); + const removeImage = vi.fn(() => ({ status: 0 }) as never); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + try { + const result = await preflightRebuildImage( + { ...input(null), localPrebuildEnabled: true }, + { + stageBuildContext: vi.fn(() => ({ + buildCtx: dir, + stagedDockerfile: dockerfile, + cleanupBuildCtx: vi.fn(() => true), + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "identity-failure", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + removeImage, + buildxAvailable: vi.fn(() => false), + captureLegacyDockerBinding: vi.fn(() => binding), + inspectLegacyImageId: vi.fn(() => { + throw new Error("OpenClaw legacy-image identity could not be verified."); + }), + disposeLegacyImage, + }, + ); expect(result).toEqual({ ok: false, - detail: - "Docker Buildx is required for sandbox rebuilds. " + - "Install or repair Docker Buildx, then verify it with 'docker buildx version'. " + - "Rerun the rebuild after that command succeeds. " + - "NemoClaw stopped before deleting the existing sandbox.", + detail: "OpenClaw legacy-image identity could not be verified.", }); + expect(disposeLegacyImage).not.toHaveBeenCalled(); + expect(removeImage).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + expect.stringMatching( + /nemoclaw-sandbox-local:.*no verified immutable cleanup identity.*maintenance cleanup/, + ), + ); + } finally { + warn.mockRestore(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("does not retry an unrelated generated-image build failure (#7111)", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-build-no-retry-")); + const dockerfile = path.join(dir, "Dockerfile"); + fs.writeFileSync(dockerfile, "FROM scratch\n"); + const buildImage = vi.fn(() => ({ status: 1, stderr: "registry timeout" }) as never); + const buildxAvailable = vi.fn(() => false); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + try { + await expect( + preflightRebuildImage( + { ...input(null), localPrebuildEnabled: true }, + { + stageBuildContext: vi.fn(() => ({ + buildCtx: dir, + stagedDockerfile: dockerfile, + cleanupBuildCtx: vi.fn(() => true), + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "unrelated-build-failure", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + buildxAvailable, + captureLegacyDockerBinding: vi.fn(() => createLegacyBinding()), + disposeLegacyImage: vi.fn(() => true), + }, + ), + ).resolves.toEqual({ ok: false, detail: "registry timeout" }); expect(buildImage).toHaveBeenCalledOnce(); - expect(cleanupBuildCtx).toHaveBeenCalledOnce(); + expect(buildxAvailable).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + expect.stringMatching( + /nemoclaw-sandbox-local:.*no verified immutable cleanup identity.*maintenance cleanup/, + ), + ); } finally { + warn.mockRestore(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("reports the redacted legacy retry failure before the BuildKit diagnostic (#7111)", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-buildx-double-fail-")); + const dockerfile = path.join(dir, "Dockerfile"); + const credential = ["legacy", "retry", "credential"].join("-"); + fs.writeFileSync(dockerfile, "FROM scratch\n"); + const buildImage = vi + .fn() + .mockReturnValueOnce({ + status: 1, + stderr: + "ERROR: BuildKit is enabled but the buildx component is missing or broken.\n" + + "x".repeat(9_000), + } as never) + .mockReturnValueOnce({ + status: 1, + stderr: + `legacy build could not read ${os.homedir()}/private-context\n` + + `Authorization: Bearer ${credential}`, + } as never); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + try { + const result = await preflightRebuildImage( + { ...input(null), localPrebuildEnabled: true }, + { + stageBuildContext: vi.fn(() => ({ + buildCtx: dir, + stagedDockerfile: dockerfile, + cleanupBuildCtx: vi.fn(() => true), + origin: "generated" as const, + })), + prepareDockerfilePatch: vi.fn(async () => ({ + buildId: "double-failure", + dashboardRemoteBindPrepared: false, + resolvedBaseImage: null, + })), + buildImage, + buildxAvailable: vi.fn(() => false), + captureLegacyDockerBinding: vi.fn(() => createLegacyBinding()), + disposeLegacyImage: vi.fn(() => true), + }, + ); + + expect(result.ok).toBe(false); + const failure = result as Extract; + expect(failure.detail).toContain("Legacy-builder retry failed"); + expect(failure.detail).toContain("legacy build could not read ~/private-context"); + expect(failure.detail).toContain("Authorization: Bearer "); + expect(failure.detail).not.toContain(credential); + expect(failure.detail.length).toBeLessThan(8_100); + expect(buildImage).toHaveBeenCalledTimes(2); + } finally { + warn.mockRestore(); fs.rmSync(dir, { recursive: true, force: true }); } }); @@ -326,7 +734,7 @@ describe("preflightRebuildImage", () => { ); expect(buildImage).toHaveBeenCalledWith( expect.stringContaining("Dockerfile"), - expect.stringMatching(/^nemoclaw-rebuild-preflight:/), + expect.stringMatching(/^nemoclaw-sandbox-local:/), expect.any(String), expect.objectContaining({ ignoreError: true }), ); diff --git a/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts b/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts index 51fd4877fcd..abbfca5e99f 100644 --- a/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts @@ -3,14 +3,21 @@ import path from "node:path"; -import { dockerBuild, dockerRmi } from "../../adapters/docker"; +import type { DockerBuildOptions, DockerRunOptions, DockerRunResult } from "../../adapters/docker"; +import { dockerSpawnSync } from "../../adapters/docker/exec"; import { fingerprintBuildContext } from "../../adapters/fs/build-context-fingerprint"; import type { AgentDefinition } from "../../agent/defs"; import { createAgentSandbox } from "../../agent/onboard"; import type { WebSearchConfig } from "../../inference/web-search"; import { stageCreateSandboxBuildContext } from "../../onboard/build-context-stage"; +import { + applyReasoningEffortEnv, + REASONING_EFFORT_ENV, + type ReasoningEffort, +} from "../../onboard/reasoning-mode"; import { prepareSandboxDockerfilePatch } from "../../onboard/sandbox-dockerfile-patch-flow"; import type { SandboxGpuConfig } from "../../onboard/sandbox-gpu-mode"; +import { dockerBuildSubprocessEnv, sandboxLocalImageRef } from "../../onboard/sandbox-prebuild"; import { ROOT } from "../../runner"; import { formatBuildFailureDiagnostics, @@ -18,12 +25,14 @@ import { SANDBOX_BASE_TAG, type SandboxBaseImageResolutionMetadata, } from "../../sandbox-base-image"; -import { - applyReasoningEffortEnv, - REASONING_EFFORT_ENV, - type ReasoningEffort, -} from "../../onboard/reasoning-mode"; import type { ToolDisclosure } from "../../tool-disclosure"; +import { + captureOpenClawLegacyDockerBinding, + createPreparedOpenClawLegacyImage, + disposeOpenClawLegacyDockerImage, + inspectOpenClawLegacyImageId, + type OpenClawLegacyDockerBinding, +} from "./rebuild/openclaw-legacy-image"; import { createBuildContextVerifier, createIdempotentBuildContextCleanup, @@ -42,6 +51,9 @@ type PreflightInput = { toolDisclosure: ToolDisclosure; hermesToolGateways: string[]; sandboxGpuConfig: SandboxGpuConfig; + sandboxName: string; + /** Whether recreation can consume an image built by the host Docker engine. */ + localPrebuildEnabled: boolean; gatewayPort: number; chatUiUrl: string; preResolvedBaseImageMetadata?: SandboxBaseImageResolutionMetadata | null; @@ -50,8 +62,28 @@ type PreflightInput = { type PreflightDeps = { stageBuildContext?: typeof stageCreateSandboxBuildContext; prepareDockerfilePatch?: typeof prepareSandboxDockerfilePatch; - buildImage?: typeof dockerBuild; - removeImage?: typeof dockerRmi; + buildImage?: BuildImage; + removeImage?: RemoveImage; + buildxAvailable?: (process: DockerProofProcess) => boolean; + buildDockerEnv?: () => Record; + captureLegacyDockerBinding?: typeof captureOpenClawLegacyDockerBinding; + inspectLegacyImageId?: typeof inspectOpenClawLegacyImageId; + createLegacyImage?: typeof createPreparedOpenClawLegacyImage; + disposeLegacyImage?: typeof disposeOpenClawLegacyDockerImage; +}; + +type BuildImage = ( + dockerfilePath: string, + tag: string, + contextDir: string, + options: DockerBuildOptions, +) => DockerRunResult; + +type RemoveImage = (imageRef: string, options: NonNullable) => DockerRunResult; + +type DockerProofProcess = { + cwd: string; + env: NodeJS.ProcessEnv; }; export type PreparedRebuildImage = FingerprintedPreparedBuildContext & { @@ -80,7 +112,7 @@ function resultDetail(result: { const BUILDX_UNAVAILABLE_DIAGNOSTIC = "BuildKit is enabled but the buildx component is missing or broken"; -function requiresBuildxRepair(result: { +function hasBuildxUnavailableDiagnostic(result: { error?: unknown; stderr?: unknown; stdout?: unknown; @@ -92,27 +124,101 @@ function requiresBuildxRepair(result: { }); } -function buildxRepairDetail(): string { - return ( - "Docker Buildx is required for sandbox rebuilds. " + - "Install or repair Docker Buildx, then verify it with 'docker buildx version'. " + - "Rerun the rebuild after that command succeeds. " + - "NemoClaw stopped before deleting the existing sandbox." +function legacyRetryFailureDetail( + buildKitResult: Parameters[0], + legacyResult: Parameters[0], +): string { + return formatBuildFailureDiagnostics({ + stderr: + `Legacy-builder retry failed:\n${resultDetail(legacyResult)}\n` + + `Initial BuildKit attempt failed:\n${resultDetail(buildKitResult)}`, + }); +} + +function exactDockerBuild( + dockerfilePath: string, + tag: string, + contextDir: string, + options: DockerBuildOptions, +): DockerRunResult { + const { + env, + ignoreError: _ignoreError, + quiet, + stdio, + suppressOutput: _suppressOutput, + ...spawnOptions + } = options; + return dockerSpawnSync( + ["build", ...(quiet ? ["--quiet"] : []), "-f", dockerfilePath, "-t", tag, contextDir], + { + ...spawnOptions, + cwd: ROOT, + env: { ...env, DOCKER_BUILDKIT: env?.DOCKER_BUILDKIT ?? "1" }, + shell: false, + stdio: stdio ?? ["ignore", "pipe", "pipe"], + }, ); } +function exactDockerRemoveImage( + imageRef: string, + options: NonNullable, +): DockerRunResult { + const { + env, + ignoreError: _ignoreError, + stdio, + suppressOutput: _suppressOutput, + ...spawnOptions + } = options; + return dockerSpawnSync(["rmi", imageRef], { + ...spawnOptions, + cwd: ROOT, + env, + shell: false, + stdio: stdio ?? ["ignore", "pipe", "pipe"], + }); +} + +function defaultBuildxAvailable(process: DockerProofProcess): boolean { + try { + return ( + dockerSpawnSync(["buildx", "version"], { + cwd: process.cwd, + env: process.env, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }).status === 0 + ); + } catch { + return false; + } +} + export async function preflightRebuildImage( input: PreflightInput, deps: PreflightDeps = {}, ): Promise { const stage = deps.stageBuildContext ?? stageCreateSandboxBuildContext; const preparePatch = deps.prepareDockerfilePatch ?? prepareSandboxDockerfilePatch; - const buildImage = deps.buildImage ?? dockerBuild; - const removeImage = deps.removeImage ?? dockerRmi; + const buildImage = deps.buildImage ?? exactDockerBuild; + const removeImage = deps.removeImage ?? exactDockerRemoveImage; + const buildxAvailable = deps.buildxAvailable ?? defaultBuildxAvailable; + const buildDockerEnv = deps.buildDockerEnv ?? dockerBuildSubprocessEnv; + const captureLegacyDockerBinding = + deps.captureLegacyDockerBinding ?? captureOpenClawLegacyDockerBinding; + const inspectLegacyImageId = deps.inspectLegacyImageId ?? inspectOpenClawLegacyImageId; + const createLegacyImage = deps.createLegacyImage ?? createPreparedOpenClawLegacyImage; + const disposeLegacyImage = deps.disposeLegacyImage ?? disposeOpenClawLegacyDockerImage; let cleanup: (() => boolean) | null = null; let imageTag: string | null = null; let imageBuilt = false; let retainBuildContext = false; + let dockerEnv: Readonly> | null = null; + let legacyBinding: OpenClawLegacyDockerBinding | null = null; + let legacyImageId: string | null = null; + let preparedLegacyImage: ReturnType | null = null; const previousReasoning = process.env.NEMOCLAW_REASONING; const previousReasoningEffort = process.env[REASONING_EFFORT_ENV]; try { @@ -156,56 +262,158 @@ export async function preflightRebuildImage( warn: () => {}, }); const contextFingerprint = fingerprintBuildContext(staged.buildCtx); - imageTag = `nemoclaw-rebuild-preflight:${String(process.pid)}-${String(Date.now())}`; - const result = buildImage(staged.stagedDockerfile, imageTag, staged.buildCtx, { + const legacyFallbackEligible = + staged.origin === "generated" && + input.agent === null && + input.fromDockerfile === null && + input.localPrebuildEnabled; + if (legacyFallbackEligible) { + legacyBinding = captureLegacyDockerBinding({ buildDockerEnv, cwd: ROOT }); + dockerEnv = legacyBinding.dockerEnv; + } else { + dockerEnv = Object.freeze({ ...buildDockerEnv() }); + } + imageTag = sandboxLocalImageRef( + input.sandboxName, + `rebuild-preflight-${buildId}-${String(process.pid)}-${String(Date.now())}`, + ); + const buildOptions: DockerBuildOptions = { + cwd: ROOT, + env: dockerEnv, ignoreError: true, suppressOutput: true, stdio: ["ignore", "pipe", "pipe"], - }); - if (result.status !== 0 && requiresBuildxRepair(result)) { - return { ok: false, detail: buildxRepairDetail() }; + }; + const buildKitResult = buildImage( + staged.stagedDockerfile, + imageTag, + staged.buildCtx, + buildOptions, + ); + let result = buildKitResult; + let usedLegacyFallback = false; + if ( + result.status !== 0 && + legacyFallbackEligible && + legacyBinding !== null && + hasBuildxUnavailableDiagnostic(result) && + !buildxAvailable({ cwd: ROOT, env: legacyBinding.dockerEnv }) + ) { + // SOURCE_OF_TRUTH_REVIEW (#7111): the generated OpenClaw final-image + // Dockerfile does not use BuildKit-only instructions. Retry its exact + // fingerprinted bytes once with Docker's compatibility builder only + // after an independent Buildx probe confirms the host CLI lacks it. + // Dockerfile.base, other agents, and custom --from contexts never enter + // this fallback. Remove it when the supported Docker floor no longer + // provides the legacy builder. + if (fingerprintBuildContext(staged.buildCtx) !== contextFingerprint) { + return { ok: false, detail: "replacement build context changed during preflight" }; + } + console.warn( + " Warning: Docker Buildx is unavailable; retrying the generated OpenClaw rebuild image with Docker's legacy builder.", + ); + usedLegacyFallback = true; + result = buildImage(staged.stagedDockerfile, imageTag, staged.buildCtx, { + ...buildOptions, + env: { ...legacyBinding.dockerEnv, DOCKER_BUILDKIT: "0" }, + }); + } + if (result.status !== 0 && usedLegacyFallback) { + return { ok: false, detail: legacyRetryFailureDetail(buildKitResult, result) }; } if (result.status !== 0) return { ok: false, detail: resultDetail(result) }; imageBuilt = true; if (fingerprintBuildContext(staged.buildCtx) !== contextFingerprint) { return { ok: false, detail: "replacement build context changed during preflight" }; } + if (legacyBinding) { + legacyImageId = inspectLegacyImageId(legacyBinding, imageTag); + } + if (usedLegacyFallback && legacyBinding && legacyImageId) { + preparedLegacyImage = createLegacyImage(legacyBinding, imageTag, legacyImageId); + } retainBuildContext = true; + const verifyFingerprint = createBuildContextVerifier(staged.buildCtx, contextFingerprint); + const prepared: PreparedRebuildImage = { + ...staged, + cleanupBuildCtx: cleanup, + buildId, + dashboardRemoteBindPrepared, + preparedOpenClawLegacyImage: preparedLegacyImage ?? undefined, + contextFingerprint, + verifyBuildCtx(this: PreparedRebuildImage) { + return ( + this === prepared && + this.preparedOpenClawLegacyImage === (preparedLegacyImage ?? undefined) && + verifyFingerprint() + ); + }, + rebuildTarget: { + agentName: input.agent?.name ?? null, + fromDockerfile: input.fromDockerfile ? path.resolve(input.fromDockerfile) : null, + }, + }; return { ok: true, imageTag, - prepared: { - ...staged, - cleanupBuildCtx: cleanup, - buildId, - dashboardRemoteBindPrepared, - contextFingerprint, - verifyBuildCtx: createBuildContextVerifier(staged.buildCtx, contextFingerprint), - rebuildTarget: { - agentName: input.agent?.name ?? null, - fromDockerfile: input.fromDockerfile ? path.resolve(input.fromDockerfile) : null, - }, - }, + prepared, }; } catch (err) { return { ok: false, detail: err instanceof Error ? err.message : String(err) }; } finally { let imageRemoved = false; - try { - imageRemoved = - imageTag !== null && - removeImage(imageTag, { ignoreError: true, suppressOutput: true }).status === 0; - } catch { - // Best effort; retained-context ownership and environment restoration must continue. + if (preparedLegacyImage === null && imageTag !== null) { + try { + imageRemoved = legacyBinding + ? legacyImageId !== null && disposeLegacyImage(legacyBinding, imageTag, legacyImageId) + : removeImage(imageTag, { + cwd: ROOT, + env: dockerEnv ?? undefined, + ignoreError: true, + suppressOutput: true, + }).status === 0; + } catch { + // Best effort; retained-context ownership and environment restoration must continue. + } + } + if ( + preparedLegacyImage === null && + imageTag !== null && + legacyBinding !== null && + legacyImageId === null + ) { + console.warn( + ` Warning: temporary rebuild preflight image '${imageTag}' has no verified immutable cleanup identity; leaving its unique tag for maintenance cleanup.`, + ); } - if (imageBuilt && imageTag && !imageRemoved) { + if ( + imageBuilt && + preparedLegacyImage === null && + imageTag && + !imageRemoved && + (legacyBinding === null || legacyImageId !== null) + ) { const retainedImageTag = imageTag; + const retainedDockerEnv = dockerEnv; + const retainedLegacyBinding = legacyBinding; + const retainedLegacyImageId = legacyImageId; console.warn( ` Warning: failed to remove temporary rebuild preflight image '${retainedImageTag}'.`, ); process.once("exit", () => { try { - removeImage(retainedImageTag, { ignoreError: true, suppressOutput: true }); + if (retainedLegacyBinding) { + if (retainedLegacyImageId !== null) { + disposeLegacyImage(retainedLegacyBinding, retainedImageTag, retainedLegacyImageId); + } + } else { + removeImage(retainedImageTag, { + cwd: ROOT, + env: retainedDockerEnv ?? undefined, + ignoreError: true, + suppressOutput: true, + }); + } } catch { // Best effort process-exit retry. } diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index 993d3a6a92a..a81fcf6fcda 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -308,6 +308,131 @@ describe("rebuild destroy phase", () => { expect(relockShieldsIfNeeded).toHaveBeenCalledWith(true); }); + it("reattaches MCP state and keeps the sandbox when the synchronous delete-edge gate fails (#7253)", async () => { + const revalidateBeforeDelete = vi.fn().mockResolvedValue(undefined); + const assertDeleteEdgeUnchanged = vi.fn(); + const validateDeleteEdge = vi.fn(() => ({ + ok: false as const, + message: "Replacement image changed before sandbox deletion.", + code: 7, + })); + const detachedProviderEntries = [{ server: "github" }]; + const scrubbedAdapterEntries = [{ server: "filesystem" }]; + mocks.prepareMcpForRebuild.mockResolvedValue({ + entries: detachedProviderEntries, + detachedProviderEntries, + scrubbedAdapterEntries, + revalidateBeforeDelete, + assertDeleteEdgeUnchanged, + }); + const onDeleted = vi.fn(); + const relockShieldsIfNeeded = vi.fn(() => true); + const bail = vi.fn((message: string): never => { + throw new Error(message); + }); + + await expect( + runRebuildDestroyPhase({ + sandboxName: "alpha", + sandboxEntry: { name: "alpha", agent: "openclaw" }, + staleRecovery: false, + backupManifest: null, + force: true, + log: vi.fn(), + bail, + relockShieldsIfNeeded, + validateDeleteEdge, + onDeleted, + }), + ).rejects.toThrow("Replacement image changed before sandbox deletion."); + + expect(revalidateBeforeDelete).toHaveBeenCalledOnce(); + expect(assertDeleteEdgeUnchanged).toHaveBeenCalledOnce(); + expect(validateDeleteEdge).toHaveBeenCalledOnce(); + expect(revalidateBeforeDelete.mock.invocationCallOrder[0]).toBeLessThan( + assertDeleteEdgeUnchanged.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + expect(assertDeleteEdgeUnchanged.mock.invocationCallOrder[0]).toBeLessThan( + mocks.getSandbox.mock.invocationCallOrder[1] ?? Number.POSITIVE_INFINITY, + ); + expect(mocks.getSandbox.mock.invocationCallOrder[1]).toBeLessThan( + validateDeleteEdge.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + expect(mocks.runOpenshell).not.toHaveBeenCalled(); + expect(mocks.reattachMcpAfterDeleteFailure).toHaveBeenCalledWith( + "alpha", + detachedProviderEntries, + scrubbedAdapterEntries, + ); + expect(relockShieldsIfNeeded).toHaveBeenCalledWith(true); + expect(bail).toHaveBeenCalledWith("Replacement image changed before sandbox deletion.", 7); + expect(mocks.removeSandboxRegistryEntryWithReceipt).not.toHaveBeenCalled(); + expect(mocks.stopNimContainer).not.toHaveBeenCalled(); + expect(mocks.stopNimContainerByName).not.toHaveBeenCalled(); + expect(onDeleted).not.toHaveBeenCalled(); + }); + + it("runs the synchronous delete-edge gate after registry validation and before delete (#7253)", async () => { + const deleteEdgeEvents: string[] = []; + const revalidateBeforeDelete = vi.fn().mockResolvedValue(undefined); + const assertDeleteEdgeUnchanged = vi.fn(); + const validateDeleteEdge = vi.fn(() => { + deleteEdgeEvents.push("validate"); + return { ok: true as const }; + }); + const log = vi.fn((message: string) => { + if (message.startsWith("Running: openshell sandbox delete")) { + deleteEdgeEvents.push("log"); + } + }); + mocks.runOpenshell.mockImplementation((args: string[]) => { + deleteEdgeEvents.push(args[1] ?? "unknown"); + return { status: 0, stdout: "", stderr: "" }; + }); + mocks.prepareMcpForRebuild.mockResolvedValue({ + entries: [], + detachedProviderEntries: [], + scrubbedAdapterEntries: [], + revalidateBeforeDelete, + assertDeleteEdgeUnchanged, + }); + + await runRebuildDestroyPhase({ + sandboxName: "alpha", + sandboxEntry: { name: "alpha", agent: "openclaw" }, + staleRecovery: false, + backupManifest: null, + force: true, + log, + bail: vi.fn((message: string): never => { + throw new Error(message); + }), + relockShieldsIfNeeded: vi.fn(() => true), + validateDeleteEdge, + onDeleted: vi.fn(), + }); + + expect(revalidateBeforeDelete).toHaveBeenCalledOnce(); + expect(assertDeleteEdgeUnchanged).toHaveBeenCalledOnce(); + expect(validateDeleteEdge).toHaveBeenCalledOnce(); + expect(assertDeleteEdgeUnchanged.mock.invocationCallOrder[0]).toBeLessThan( + mocks.getSandbox.mock.invocationCallOrder[1] ?? Number.POSITIVE_INFINITY, + ); + expect(mocks.getSandbox.mock.invocationCallOrder[1]).toBeLessThan( + validateDeleteEdge.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + expect(validateDeleteEdge.mock.invocationCallOrder[0]).toBeLessThan( + mocks.runOpenshell.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + expect(mocks.runOpenshell).toHaveBeenNthCalledWith( + 1, + ["sandbox", "delete", "-g", "nemoclaw", "alpha"], + expect.any(Object), + ); + expect(mocks.reattachMcpAfterDeleteFailure).not.toHaveBeenCalled(); + expect(deleteEdgeEvents).toEqual(["log", "validate", "delete"]); + }); + it("refuses sandbox deletion when read-only MCP state drifts at the delete edge (#7062)", async () => { const revalidateBeforeDelete = vi.fn().mockRejectedValue(new Error("live policy drifted")); mocks.prepareMcpForRebuild.mockResolvedValue({ @@ -359,6 +484,7 @@ describe("rebuild destroy phase", () => { .mockReturnValueOnce({ status: 9, stdout: "", stderr: "delete failed" }) .mockReturnValueOnce({ status: 0, stdout: "Phase: Ready\n", stderr: "" }); const onDeleted = vi.fn(); + const abortPreparedImageRecreate = vi.fn(() => true); const relockShieldsIfNeeded = vi.fn(() => true); const bail = vi.fn((message: string): never => { throw new Error(message); @@ -374,6 +500,8 @@ describe("rebuild destroy phase", () => { log: vi.fn(), bail, relockShieldsIfNeeded, + validateDeleteEdge: vi.fn(() => ({ ok: true as const })), + abortPreparedImageRecreate, onDeleted, }), ).rejects.toThrow("Failed to delete sandbox."); @@ -388,6 +516,7 @@ describe("rebuild destroy phase", () => { expect(mocks.stopNimContainer).not.toHaveBeenCalled(); expect(mocks.stopNimContainerByName).not.toHaveBeenCalled(); expect(relockShieldsIfNeeded).toHaveBeenCalledWith(true); + expect(abortPreparedImageRecreate).toHaveBeenCalledOnce(); expect(mocks.runOpenshell).toHaveBeenNthCalledWith( 2, ["sandbox", "get", "-g", "nemoclaw", "alpha"], @@ -575,6 +704,7 @@ describe("rebuild destroy phase", () => { .mockReturnValueOnce({ status: 0, stdout: "Phase: Terminating\n", stderr: "" }); const onDeleted = vi.fn(); const onDeleteStateAmbiguous = vi.fn(); + const abortPreparedImageRecreate = vi.fn(() => true); const relockShieldsIfNeeded = vi.fn(() => true); await expect( @@ -589,6 +719,8 @@ describe("rebuild destroy phase", () => { throw new Error(message); }), relockShieldsIfNeeded, + validateDeleteEdge: vi.fn(() => ({ ok: true as const })), + abortPreparedImageRecreate, onDeleted, onDeleteStateAmbiguous, }), @@ -596,6 +728,7 @@ describe("rebuild destroy phase", () => { expect(onDeleted).not.toHaveBeenCalled(); expect(onDeleteStateAmbiguous).toHaveBeenCalledOnce(); + expect(abortPreparedImageRecreate).toHaveBeenCalledOnce(); expect(mocks.stopNimContainer).not.toHaveBeenCalled(); expect(mocks.stopNimContainerByName).not.toHaveBeenCalled(); expect(mocks.reattachMcpAfterDeleteFailure).not.toHaveBeenCalled(); diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.ts index 9226de56d55..0b32b8ca1c9 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.ts @@ -39,6 +39,8 @@ export interface RebuildDestroyPhaseInput { relockShieldsIfNeeded: (sandboxStillExists: boolean) => boolean; force?: boolean; validateAfterMcpPreparation?: () => Promise; + validateDeleteEdge?: () => RebuildDeleteValidationResult; + abortPreparedImageRecreate?: () => boolean; onDeleted: () => void; onDeleteStateAmbiguous?: () => void; } @@ -229,6 +231,7 @@ export async function runRebuildDestroyPhase( bail, relockShieldsIfNeeded, validateAfterMcpPreparation, + validateDeleteEdge, onDeleted, } = input; const deleteTarget = resolveRebuildDeleteTarget(sandboxName, input.sandboxEntry); @@ -315,6 +318,15 @@ export async function runRebuildDestroyPhase( const rebuildMcpEntries = mcpPreparation.entries; const rebuildDetachedMcpProviderEntries = mcpPreparation.detachedProviderEntries; const rebuildScrubbedMcpAdapterEntries = mcpPreparation.scrubbedAdapterEntries; + const abortPreparedImageRecreate = (): void => { + try { + if (input.abortPreparedImageRecreate?.() === false) { + log("The unused retained replacement image could not be released safely."); + } + } catch { + log("The unused retained replacement image could not be released safely."); + } + }; // Exec-unavailable recovery deliberately made no MCP mutation during // preparation. Re-prove target, policy, provider, and registry state while @@ -356,6 +368,38 @@ export async function runRebuildDestroyPhase( } log(`Running: openshell sandbox delete -g ${gatewayName} ${sandboxName}`); + + // Keep this synchronous and at the final delete edge. A successful check + // must be the last executed code before OpenShell receives the delete + // command. + if (validateDeleteEdge) { + let validation: RebuildDeleteValidationResult; + try { + validation = validateDeleteEdge(); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + log(`Unexpected rebuild delete-edge validation failure: ${redactFull(detail)}`); + validation = { + ok: false, + message: "Replacement image validation failed before sandbox deletion.", + }; + } + if (!validation.ok) { + const mcpRecoveryFailure = await reattachMcpAfterDeleteFailure( + sandboxName, + rebuildDetachedMcpProviderEntries, + rebuildScrubbedMcpAdapterEntries, + ); + relockShieldsIfNeeded(true); + bail( + mcpRecoveryFailure + ? `${validation.message} MCP provider recovery also failed: ${mcpRecoveryFailure}` + : validation.message, + validation.code, + ); + return null; + } + } const deleteResult = runOpenshell(["sandbox", "delete", "-g", gatewayName, sandboxName], { ignoreError: true, stdio: ["ignore", "pipe", "pipe"], @@ -387,6 +431,7 @@ export async function runRebuildDestroyPhase( console.error(" State backup is preserved at: " + backupManifest.backupPath); } relockShieldsIfNeeded(true); + abortPreparedImageRecreate(); bail( mcpRecoveryFailure ? `Failed to delete sandbox; MCP provider recovery also failed: ${mcpRecoveryFailure}` @@ -405,6 +450,7 @@ export async function runRebuildDestroyPhase( console.error(" State backup is preserved at: " + backupManifest.backupPath); } input.onDeleteStateAmbiguous?.(); + abortPreparedImageRecreate(); bail( "Sandbox delete failed and exact post-delete state is ambiguous; recovery state was preserved.", deleteResult.status || 1, @@ -422,6 +468,7 @@ export async function runRebuildDestroyPhase( console.error(" State backup is preserved at: " + backupManifest.backupPath); } input.onDeleteStateAmbiguous?.(); + abortPreparedImageRecreate(); bail("Sandbox deletion could not be confirmed."); return null; } diff --git a/src/lib/actions/sandbox/rebuild-openclaw-legacy-image.test.ts b/src/lib/actions/sandbox/rebuild-openclaw-legacy-image.test.ts new file mode 100644 index 00000000000..0965b46e86b --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-openclaw-legacy-image.test.ts @@ -0,0 +1,500 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SpawnSyncOptions } from "node:child_process"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { dockerSpawnSync } from "../../adapters/docker/exec"; +import { + captureOpenClawLegacyDockerBinding, + createPreparedOpenClawLegacyImage, + disposeOpenClawLegacyDockerImage, + inspectOpenClawLegacyImageId, + type OpenClawLegacyDockerBindingDeps, +} from "./rebuild/openclaw-legacy-image"; + +const IMAGE_ID = `sha256:${"a".repeat(64)}`; +const OTHER_IMAGE_ID = `sha256:${"b".repeat(64)}`; +const IMAGE_REF = "nemoclaw-sandbox-local:alpha-rebuild"; +const CANONICAL_CWD = "/canonical/nemoclaw"; + +type DockerState = { + context: string; + contextStatus: number; + engineId: string; + infoStatus: number; + tagImageId: string; + directImageId: string; + rmiStatus: number; +}; + +function dockerResult(stdout: string, status = 0): ReturnType { + return { + error: undefined, + output: [null, stdout, ""], + pid: 123, + signal: null, + status, + stderr: "", + stdout, + }; +} + +function createDockerHarness(overrides: Partial = {}) { + const state: DockerState = { + context: "desktop-linux", + contextStatus: 0, + engineId: "engine-a", + infoStatus: 0, + tagImageId: IMAGE_ID, + directImageId: IMAGE_ID, + rmiStatus: 0, + ...overrides, + }; + const exitListeners = new Set<() => void>(); + const runDocker = vi.fn((args: readonly string[], _options: SpawnSyncOptions = {}) => { + if (args.join(" ") === "context show") { + return dockerResult(state.context, state.contextStatus); + } + if (args.join(" ") === "info --format {{.ID}}") { + return dockerResult(state.engineId, state.infoStatus); + } + if ( + args[0] === "image" && + args[1] === "inspect" && + args[2] === "--format" && + args[3] === "{{.Id}}" + ) { + const selector = args[4]; + if (selector === IMAGE_REF) return dockerResult(state.tagImageId); + if (selector === IMAGE_ID) return dockerResult(state.directImageId); + if (selector === OTHER_IMAGE_ID) return dockerResult(OTHER_IMAGE_ID); + return dockerResult("", 1); + } + if (args[0] === "rmi") return dockerResult("", state.rmiStatus); + throw new Error(`Unexpected Docker command: ${args.join(" ")}`); + }); + const deps: OpenClawLegacyDockerBindingDeps = { + cwd: CANONICAL_CWD, + buildDockerEnv: () => ({ PATH: "/usr/bin" }), + runDocker: runDocker as typeof dockerSpawnSync, + addExitListener: (listener) => { + exitListeners.add(listener); + }, + removeExitListener: (listener) => { + exitListeners.delete(listener); + }, + }; + return { deps, exitListeners, runDocker, state }; +} + +function commandCalls(harness: ReturnType): string[][] { + return harness.runDocker.mock.calls.map(([args]) => [...args]); +} + +afterEach(() => vi.unstubAllEnvs()); + +describe("OpenClaw legacy-image Docker binding", () => { + it("pins the current Docker context when no selector is explicit", () => { + const harness = createDockerHarness(); + + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + + expect(binding).toEqual({ + dockerEnv: { DOCKER_CONTEXT: "desktop-linux", PATH: "/usr/bin" }, + engineId: "engine-a", + }); + expect(Object.isFrozen(binding)).toBe(true); + expect(Object.isFrozen(binding.dockerEnv)).toBe(true); + expect(commandCalls(harness)).toEqual([ + ["context", "show"], + ["info", "--format", "{{.ID}}"], + ]); + const contextOptions = harness.runDocker.mock.calls[0]?.[1] as SpawnSyncOptions; + const infoOptions = harness.runDocker.mock.calls[1]?.[1] as SpawnSyncOptions; + expect(Object.isFrozen(contextOptions.env)).toBe(true); + expect(infoOptions.env).toBe(binding.dockerEnv); + expect(infoOptions).toMatchObject({ + encoding: "utf-8", + shell: false, + stdio: ["ignore", "pipe", "pipe"], + timeout: 30_000, + }); + }); + + it("preserves an explicit Docker host without consulting ambient context", () => { + const harness = createDockerHarness(); + harness.deps.buildDockerEnv = () => ({ + DOCKER_HOST: "unix:///var/run/docker.sock", + PATH: "/usr/bin", + }); + + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + + expect(binding.dockerEnv).toEqual({ + DOCKER_HOST: "unix:///var/run/docker.sock", + PATH: "/usr/bin", + }); + expect(commandCalls(harness)).toEqual([["info", "--format", "{{.ID}}"]]); + }); + + it("uses the Docker subprocess allowlist before freezing the selector", () => { + vi.stubEnv("DOCKER_HOST", "unix:///var/run/docker.sock"); + vi.stubEnv("KUBECONFIG", "/secret/kubeconfig"); + vi.stubEnv("OPENSHELL_SECRET", "secret"); + vi.stubEnv("GITHUB_TOKEN", "secret"); + const harness = createDockerHarness(); + delete harness.deps.buildDockerEnv; + + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + + expect(binding.dockerEnv.DOCKER_HOST).toBe("unix:///var/run/docker.sock"); + expect(binding.dockerEnv).not.toHaveProperty("KUBECONFIG"); + expect(binding.dockerEnv).not.toHaveProperty("OPENSHELL_SECRET"); + expect(binding.dockerEnv).not.toHaveProperty("GITHUB_TOKEN"); + }); + + it("keeps a relative Docker config on canonical engine A for identity and cleanup", () => { + const harness = createDockerHarness(); + const runEngineA = harness.runDocker.getMockImplementation(); + if (!runEngineA) throw new Error("engine A Docker harness is missing"); + const engineBCalls: string[][] = []; + harness.deps.buildDockerEnv = () => ({ + DOCKER_CONFIG: "relative/docker-config", + PATH: "/usr/bin", + }); + harness.runDocker.mockImplementation( + (args: readonly string[], options: SpawnSyncOptions = {}) => { + if (options.cwd === CANONICAL_CWD) return runEngineA(args, options); + engineBCalls.push([...args]); + if (args.join(" ") === "context show") return dockerResult("ambient-context"); + if (args.join(" ") === "info --format {{.ID}}") return dockerResult("engine-b"); + if (args[0] === "image" && args[1] === "inspect") return dockerResult(OTHER_IMAGE_ID); + if (args[0] === "rmi") return dockerResult(""); + throw new Error(`Unexpected engine B Docker command: ${args.join(" ")}`); + }, + ); + + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const imageId = inspectOpenClawLegacyImageId(binding, IMAGE_REF); + const removed = disposeOpenClawLegacyDockerImage(binding, IMAGE_REF, imageId); + + expect(binding).toEqual({ + dockerEnv: { + DOCKER_CONFIG: "relative/docker-config", + DOCKER_CONTEXT: "desktop-linux", + PATH: "/usr/bin", + }, + engineId: "engine-a", + }); + expect(imageId).toBe(IMAGE_ID); + expect(removed).toBe(true); + expect(engineBCalls).toEqual([]); + expect(commandCalls(harness)).toEqual([ + ["context", "show"], + ["info", "--format", "{{.ID}}"], + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ["rmi", IMAGE_ID], + ]); + for (const [, options] of harness.runDocker.mock.calls) { + expect(options?.cwd).toBe(CANONICAL_CWD); + } + }); + + it.each([ + ["a malformed explicit selector", { DOCKER_CONTEXT: "bad\ncontext" }, {}, /malformed/], + ["a failed context query", {}, { contextStatus: 1 }, /context could not be captured/], + ["an empty context", {}, { context: "" }, /context could not be captured/], + ["a failed engine query", {}, { infoStatus: 1 }, /engine identity/], + ["a malformed engine identity", {}, { engineId: "engine a" }, /engine identity/], + ] as const)("fails closed for %s", (_scenario, env, overrides, expected) => { + const harness = createDockerHarness(overrides); + harness.deps.buildDockerEnv = () => ({ ...env }); + + expect(() => captureOpenClawLegacyDockerBinding(harness.deps)).toThrow(expected); + }); +}); + +describe("OpenClaw legacy-image identity and lifecycle", () => { + it("inspects a mutable tag and its direct immutable ID using the exact bound environment", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + harness.runDocker.mockClear(); + + expect(inspectOpenClawLegacyImageId(binding, IMAGE_REF)).toBe(IMAGE_ID); + + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ]); + for (const [, options] of harness.runDocker.mock.calls) { + expect(options?.env).toBe(binding.dockerEnv); + } + }); + + it("constructs only a verified frozen lease from an authentic binding", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + harness.runDocker.mockClear(); + + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID.toUpperCase()); + + expect(lease).toMatchObject({ + dockerEnv: binding.dockerEnv, + engineId: "engine-a", + imageId: IMAGE_ID, + imageRef: IMAGE_REF, + }); + expect(Object.isFrozen(lease)).toBe(true); + expect(harness.exitListeners.size).toBe(1); + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ]); + }); + + it("rejects counterfeit bindings, non-tag references, and mismatched image identity", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + + expect(() => + createPreparedOpenClawLegacyImage( + Object.freeze({ dockerEnv: Object.freeze({}), engineId: "engine-a" }), + IMAGE_REF, + IMAGE_ID, + ), + ).toThrow(/not authentic/); + expect(() => createPreparedOpenClawLegacyImage(binding, IMAGE_ID, IMAGE_ID)).toThrow( + /mutable Docker tag/, + ); + harness.state.tagImageId = OTHER_IMAGE_ID; + expect(() => createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID)).toThrow( + /identity changed/, + ); + expect(harness.exitListeners.size).toBe(0); + }); + + it("guards method receivers and finalizes only after create verification", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + harness.runDocker.mockClear(); + + expect(lease.verifyForCreate()).toBe(false); + expect(lease.finalizeAfterCreate()).toBeNull(); + expect(lease.verify.call({ ...lease })).toBe(false); + expect(lease.retainForRecreate.call({ ...lease })).toBe(false); + expect(lease.abort.call({ ...lease })).toBe(false); + expect(harness.runDocker).not.toHaveBeenCalled(); + + expect(lease.verify()).toBe(true); + expect(lease.retainForRecreate()).toBe(true); + expect(harness.exitListeners.size).toBe(1); + expect(lease.verify()).toBe(false); + expect(lease.retainForRecreate()).toBe(false); + expect(lease.finalizeAfterCreate()).toBeNull(); + expect(lease.verifyForCreate()).toBe(true); + expect(lease.finalizeAfterCreate.call({ ...lease })).toBeNull(); + expect(lease.finalizeAfterCreate()).toEqual({ + mutableTagVerified: true, + registryImageRef: null, + }); + expect(lease.finalizeAfterCreate()).toBeNull(); + expect(harness.exitListeners.size).toBe(0); + expect(lease.dispose()).toBe(true); + expect(commandCalls(harness)).not.toContainEqual(["rmi", IMAGE_ID]); + }); + + it("refuses retention when the directly addressed immutable image no longer agrees", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + harness.runDocker.mockClear(); + harness.state.directImageId = OTHER_IMAGE_ID; + + expect(lease.retainForRecreate()).toBe(false); + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ]); + expect(harness.exitListeners.size).toBe(1); + }); + + it.each([ + ["the named context is repointed", (state: DockerState) => (state.engineId = "engine-b")], + ["the mutable tag is retargeted", (state: DockerState) => (state.tagImageId = OTHER_IMAGE_ID)], + ["the immutable image is removed", (state: DockerState) => (state.directImageId = "")], + ])("refuses creation after %s", (_scenario, mutate) => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + expect(lease.retainForRecreate()).toBe(true); + harness.runDocker.mockClear(); + + mutate(harness.state); + + expect(lease.verifyForCreate()).toBe(false); + expect(lease.finalizeAfterCreate()).toBeNull(); + expect(commandCalls(harness)).not.toContainEqual(["rmi", IMAGE_ID]); + }); + + it.each([ + ["removed", ""], + ["retargeted", OTHER_IMAGE_ID], + ])("suppresses registry cleanup when the mutable tag is %s after create", (_scenario, tagImageId) => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + expect(lease.retainForRecreate()).toBe(true); + expect(lease.verifyForCreate()).toBe(true); + harness.runDocker.mockClear(); + harness.state.tagImageId = tagImageId; + + expect(lease.finalizeAfterCreate()).toEqual({ + mutableTagVerified: false, + registryImageRef: null, + }); + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ]); + expect(lease.abort()).toBe(true); + expect(commandCalls(harness)).not.toContainEqual(["rmi", IMAGE_ID]); + }); + + it("fails finalization on engine drift and keeps exact abort cleanup retryable", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + expect(lease.retainForRecreate()).toBe(true); + expect(lease.verifyForCreate()).toBe(true); + harness.runDocker.mockClear(); + harness.state.engineId = "engine-b"; + + expect(lease.finalizeAfterCreate()).toBeNull(); + expect(lease.abort()).toBe(false); + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["info", "--format", "{{.ID}}"], + ]); + expect(harness.exitListeners.size).toBe(1); + + harness.state.engineId = "engine-a"; + expect(lease.abort()).toBe(true); + expect(commandCalls(harness)).toContainEqual(["rmi", IMAGE_ID]); + expect(harness.exitListeners.size).toBe(0); + }); + + it.each([ + ["removed", ""], + ["retargeted", OTHER_IMAGE_ID], + ])("aborts a retained unused image by exact ID when its tag is %s", (_scenario, tagImageId) => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + expect(lease.retainForRecreate()).toBe(true); + harness.runDocker.mockClear(); + harness.state.tagImageId = tagImageId; + + expect(lease.abort()).toBe(true); + expect(lease.abort()).toBe(true); + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ["rmi", IMAGE_ID], + ]); + expect(harness.exitListeners.size).toBe(0); + }); + + it("cleans an unretained lease by immutable ID exactly once", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + harness.runDocker.mockClear(); + + expect(lease.dispose()).toBe(true); + expect(lease.dispose()).toBe(true); + + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ["rmi", IMAGE_ID], + ]); + expect(harness.exitListeners.size).toBe(0); + }); + + it("disposes a retained lease that never reached finalization", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + expect(lease.retainForRecreate()).toBe(true); + harness.runDocker.mockClear(); + + expect(lease.dispose()).toBe(true); + + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ["rmi", IMAGE_ID], + ]); + expect(harness.exitListeners.size).toBe(0); + }); + + it("refuses cleanup on engine drift and leaves exit cleanup retryable", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + const lease = createPreparedOpenClawLegacyImage(binding, IMAGE_REF, IMAGE_ID); + harness.runDocker.mockClear(); + harness.state.engineId = "engine-b"; + + expect(lease.dispose()).toBe(false); + expect(commandCalls(harness)).toEqual([["info", "--format", "{{.ID}}"]]); + expect(harness.exitListeners.size).toBe(1); + + harness.state.engineId = "engine-a"; + const [exitListener] = harness.exitListeners; + exitListener?.(); + expect(commandCalls(harness)).toContainEqual(["rmi", IMAGE_ID]); + expect(harness.exitListeners.size).toBe(0); + }); + + it("cleans a pre-lease image only after same-engine tag and captured direct-ID proof", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + harness.runDocker.mockClear(); + + expect(disposeOpenClawLegacyDockerImage(binding, IMAGE_REF, IMAGE_ID)).toBe(true); + expect(commandCalls(harness)).toEqual([ + ["info", "--format", "{{.ID}}"], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_REF], + ["image", "inspect", "--format", "{{.Id}}", IMAGE_ID], + ["rmi", IMAGE_ID], + ]); + + harness.runDocker.mockClear(); + harness.state.tagImageId = OTHER_IMAGE_ID; + expect(disposeOpenClawLegacyDockerImage(binding, IMAGE_REF, IMAGE_ID)).toBe(false); + expect(commandCalls(harness)).not.toContainEqual(["rmi", IMAGE_ID]); + }); + + it("refuses tag-authorized cleanup when no immutable ID was captured", () => { + const harness = createDockerHarness(); + const binding = captureOpenClawLegacyDockerBinding(harness.deps); + harness.runDocker.mockClear(); + harness.state.tagImageId = OTHER_IMAGE_ID; + + expect(disposeOpenClawLegacyDockerImage(binding, IMAGE_REF)).toBe(false); + expect(commandCalls(harness)).toEqual([]); + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts index 1f973340629..22f618c9b14 100644 --- a/src/lib/actions/sandbox/rebuild-pipeline.ts +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -23,7 +23,9 @@ import { printRebuildPreflightFailure } from "./rebuild-preflight-error"; import { blockRebuildOnPendingBaselineTransition } from "./rebuild-preflight-guards"; import { runRebuildPreflightPhase } from "./rebuild-preflight-phase"; import { + abortPreparedImageRecreate, disposePreparedBuildContext, + retainPreparedImageForRecreate, verifyPreparedBuildContext, } from "./rebuild-prepared-image-context"; import { @@ -182,14 +184,14 @@ async function rebuildSandboxUnlocked( }); if (!backup) return; - // The post-delete create must consume the exact context that passed the - // image preflight. Revalidate at the last safe point so mutation of the - // retained copy cannot cross the destructive boundary. + // The post-delete create must consume the exact context and retained + // image that passed preflight. Revalidate before any destroy-phase + // mutation so drift cannot cross the destructive boundary. if (preparedImage && !verifyPreparedBuildContext(preparedImage)) { printRebuildPreflightFailure( - "the retained replacement image context changed after preflight.", + "the retained replacement image inputs changed after preflight.", "Retry the rebuild so the replacement inputs can be staged again.", - "Replacement sandbox image context changed before delete", + "Replacement sandbox image inputs changed before delete", bail, ); return; @@ -220,6 +222,13 @@ async function rebuildSandboxUnlocked( bail, relockShieldsIfNeeded, validateAfterMcpPreparation: async () => { + if (preparedImage && !verifyPreparedBuildContext(preparedImage)) { + return { + ok: false, + message: + "The retained replacement image inputs changed after MCP preparation. Retry the rebuild.", + }; + } const providerReconfigure = recreateOptions.rebuildProviderReconfigure; if (providerReconfigure && !hydrateCredentialEnv(providerReconfigure.credentialEnv)) { return { @@ -251,6 +260,27 @@ async function rebuildSandboxUnlocked( recreateOptions.targetGatewayPort, ); }, + validateDeleteEdge: () => { + if (!preparedImage) return { ok: true }; + if (!verifyPreparedBuildContext(preparedImage)) { + return { + ok: false, + message: + "The retained replacement image inputs changed before sandbox deletion. Retry the rebuild.", + }; + } + if (!retainPreparedImageForRecreate(preparedImage)) { + return { + ok: false, + message: + "The retained replacement image could not be committed to sandbox recreation. Retry the rebuild.", + }; + } + return { ok: true }; + }, + abortPreparedImageRecreate: preparedImage + ? () => abortPreparedImageRecreate(preparedImage) + : undefined, onDeleted: () => { sandboxStillExists = false; }, diff --git a/src/lib/actions/sandbox/rebuild-prepared-image-context.ts b/src/lib/actions/sandbox/rebuild-prepared-image-context.ts index c95145e3cbd..95c60f9513d 100644 --- a/src/lib/actions/sandbox/rebuild-prepared-image-context.ts +++ b/src/lib/actions/sandbox/rebuild-prepared-image-context.ts @@ -28,7 +28,29 @@ export function createIdempotentBuildContextCleanup(cleanup: () => boolean): () /** Confirm that a retained private context still matches the prebuilt bytes. */ export function verifyPreparedBuildContext(prepared: FingerprintedPreparedBuildContext): boolean { try { - return fingerprintBuildContext(prepared.buildCtx) === prepared.contextFingerprint; + if (!prepared.verifyBuildCtx.call(prepared)) return false; + if (fingerprintBuildContext(prepared.buildCtx) !== prepared.contextFingerprint) return false; + return prepared.preparedOpenClawLegacyImage?.verify() ?? true; + } catch { + return false; + } +} + +/** Commit a verified retained image to recreation at the synchronous delete edge. */ +export function retainPreparedImageForRecreate( + prepared: FingerprintedPreparedBuildContext, +): boolean { + try { + return prepared.preparedOpenClawLegacyImage?.retainForRecreate() ?? true; + } catch { + return false; + } +} + +/** Release an unused retained image through its bound immutable cleanup path. */ +export function abortPreparedImageRecreate(prepared: FingerprintedPreparedBuildContext): boolean { + try { + return prepared.preparedOpenClawLegacyImage?.abort() ?? true; } catch { return false; } @@ -50,5 +72,12 @@ export function createBuildContextVerifier( /** Dispose retained build inputs after onboarding consumes them or rebuild aborts. */ export function disposePreparedBuildContext(prepared: FingerprintedPreparedBuildContext): boolean { - return prepared.cleanupBuildCtx(); + const imageDisposed = abortPreparedImageRecreate(prepared); + let contextDisposed = false; + try { + contextDisposed = prepared.cleanupBuildCtx(); + } catch { + contextDisposed = false; + } + return imageDisposed && contextDisposed; } diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.ts b/src/lib/actions/sandbox/rebuild-target-runtime.ts index e92f7d3da04..126116c74e2 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.ts @@ -21,6 +21,7 @@ import { readGatewayProviderMetadata, } from "../../onboard/gateway-provider-metadata"; import { resolveSandboxGpuConfig } from "../../onboard/sandbox-gpu-mode"; +import { resolveSandboxPrebuildEnabled } from "../../onboard/sandbox-prebuild"; import { agentSupportsWebSearchProvider } from "../../onboard/web-search-support"; import { redact } from "../../security/redact"; import { @@ -174,8 +175,8 @@ export async function preflightRebuildTargetRuntime( ); return { ok: false }; } + const dockerDriverGateway = isLinuxDockerDriverGatewayEnabled(); try { - const dockerDriverGateway = isLinuxDockerDriverGatewayEnabled(); const selectedRoute = initialDockerGpuRoute( resolveDockerGpuRoutePlan(sandboxGpuConfig, { dockerDriverGateway, @@ -213,6 +214,8 @@ export async function preflightRebuildTargetRuntime( toolDisclosure: target.durableConfig.toolDisclosure, hermesToolGateways: target.hermesToolGateways, sandboxGpuConfig, + sandboxName: sb.name, + localPrebuildEnabled: resolveSandboxPrebuildEnabled(process.env, dockerDriverGateway), preResolvedBaseImageMetadata: recreateOptions.preResolvedBaseImageMetadata ?? null, gatewayPort: recreateOptions.targetGatewayPort, chatUiUrl: managesDashboard diff --git a/src/lib/actions/sandbox/rebuild/openclaw-legacy-image.ts b/src/lib/actions/sandbox/rebuild/openclaw-legacy-image.ts new file mode 100644 index 00000000000..360cc99b455 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild/openclaw-legacy-image.ts @@ -0,0 +1,378 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SpawnSyncOptions } from "node:child_process"; + +import { dockerSpawnSync } from "../../../adapters/docker/exec"; +import type { + PreparedOpenClawLegacyImage, + PreparedOpenClawLegacyImageFinalization, +} from "../../../onboard/build-context-stage"; +import { isImmutableDockerImageId } from "../../../onboard/openshell-docker-sandbox-containers"; +import { dockerBuildSubprocessEnv } from "../../../onboard/sandbox-prebuild"; + +const DOCKER_IDENTITY_TIMEOUT_MS = 30_000; +const SAFE_VALUE_MAX_LENGTH = 4096; + +export interface OpenClawLegacyDockerBinding { + readonly dockerEnv: Readonly>; + readonly engineId: string; +} + +export interface OpenClawLegacyDockerBindingDeps { + cwd: string; + buildDockerEnv?(): Record; + runDocker?: typeof dockerSpawnSync; + addExitListener?(listener: () => void): void; + removeExitListener?(listener: () => void): void; +} + +type InternalBinding = { + readonly dockerOptions: Readonly; + readonly runDocker: typeof dockerSpawnSync; + readonly addExitListener: (listener: () => void) => void; + readonly removeExitListener: (listener: () => void) => void; +}; + +const internalBindings = new WeakMap(); + +function isSafeSingleLineValue(value: string): boolean { + return ( + value.length > 0 && + value.length <= SAFE_VALUE_MAX_LENGTH && + !/[\s\u0000-\u001f\u007f]/.test(value) + ); +} + +function normalizeDockerOutput(output: string | Buffer | null | undefined): string { + return typeof output === "string" ? output.trim() : String(output ?? "").trim(); +} + +function commandSucceeded(result: ReturnType): boolean { + return result.error == null && result.status === 0; +} + +function runDockerCapture(internal: InternalBinding, args: readonly string[]): string | null { + try { + const result = internal.runDocker(args, internal.dockerOptions); + if (!commandSucceeded(result)) return null; + return normalizeDockerOutput(result.stdout); + } catch { + return null; + } +} + +function requireInternalBinding(binding: OpenClawLegacyDockerBinding): InternalBinding { + const internal = internalBindings.get(binding); + if (!internal) { + throw new Error("OpenClaw legacy-image Docker binding is not authentic."); + } + return internal; +} + +function validateMutableImageRef(imageRef: string): string { + if ( + !isSafeSingleLineValue(imageRef) || + isImmutableDockerImageId(imageRef) || + imageRef.includes("@") || + imageRef.lastIndexOf(":") <= imageRef.lastIndexOf("/") + ) { + throw new Error("OpenClaw legacy-image reference must be a mutable Docker tag."); + } + return imageRef; +} + +function validateImageId(imageId: string): string { + const normalized = imageId.trim().toLowerCase(); + if (!isImmutableDockerImageId(normalized)) { + throw new Error("OpenClaw legacy-image ID must be an immutable sha256 identifier."); + } + return normalized; +} + +function currentEngineMatches( + binding: OpenClawLegacyDockerBinding, + internal: InternalBinding, +): boolean { + const currentEngineId = runDockerCapture(internal, ["info", "--format", "{{.ID}}"]); + return currentEngineId !== null && currentEngineId === binding.engineId; +} + +function inspectImageId(internal: InternalBinding, imageSelector: string): string | null { + const output = runDockerCapture(internal, [ + "image", + "inspect", + "--format", + "{{.Id}}", + imageSelector, + ]); + if (output === null || !isImmutableDockerImageId(output)) return null; + return output.toLowerCase(); +} + +function verifyImageIdentity( + binding: OpenClawLegacyDockerBinding, + internal: InternalBinding, + imageRef: string, + imageId: string, +): boolean { + if (!currentEngineMatches(binding, internal)) return false; + if (inspectImageId(internal, imageRef) !== imageId) return false; + return inspectImageId(internal, imageId) === imageId; +} + +/** + * Pin the Docker selector used by a rebuild. The default context is made + * explicit so a later ambient `docker context use` cannot redirect the lease. + */ +export function captureOpenClawLegacyDockerBinding( + deps: OpenClawLegacyDockerBindingDeps, +): OpenClawLegacyDockerBinding { + const runDocker = deps.runDocker ?? dockerSpawnSync; + const initialEnv = (deps.buildDockerEnv ?? dockerBuildSubprocessEnv)(); + const dockerEnv: Record = { ...initialEnv }; + const explicitDockerHost = dockerEnv.DOCKER_HOST; + const explicitDockerContext = dockerEnv.DOCKER_CONTEXT; + + for (const selector of [explicitDockerHost, explicitDockerContext]) { + if (selector !== undefined && !isSafeSingleLineValue(selector)) { + throw new Error("OpenClaw legacy-image Docker selector is malformed."); + } + } + + const addExitListener = + deps.addExitListener ?? ((listener: () => void) => process.once("exit", listener)); + const removeExitListener = + deps.removeExitListener ?? ((listener: () => void) => process.removeListener("exit", listener)); + + if (explicitDockerHost === undefined && explicitDockerContext === undefined) { + const contextOptions: SpawnSyncOptions = Object.freeze({ + cwd: deps.cwd, + encoding: "utf-8", + env: Object.freeze({ ...dockerEnv }), + shell: false, + stdio: ["ignore", "pipe", "pipe"] as SpawnSyncOptions["stdio"], + timeout: DOCKER_IDENTITY_TIMEOUT_MS, + }); + let contextResult: ReturnType; + try { + contextResult = runDocker(["context", "show"], contextOptions); + } catch { + throw new Error("OpenClaw legacy-image Docker context could not be captured."); + } + const context = commandSucceeded(contextResult) + ? normalizeDockerOutput(contextResult.stdout) + : ""; + if (!isSafeSingleLineValue(context)) { + throw new Error("OpenClaw legacy-image Docker context could not be captured."); + } + dockerEnv.DOCKER_CONTEXT = context; + } + + const frozenDockerEnv = Object.freeze({ ...dockerEnv }); + const dockerOptions: Readonly = Object.freeze({ + cwd: deps.cwd, + encoding: "utf-8", + env: frozenDockerEnv, + shell: false, + stdio: ["ignore", "pipe", "pipe"] as SpawnSyncOptions["stdio"], + timeout: DOCKER_IDENTITY_TIMEOUT_MS, + }); + const internal: InternalBinding = Object.freeze({ + dockerOptions, + runDocker, + addExitListener, + removeExitListener, + }); + const engineId = runDockerCapture(internal, ["info", "--format", "{{.ID}}"]); + if (engineId === null || !isSafeSingleLineValue(engineId)) { + throw new Error("OpenClaw legacy-image Docker engine identity could not be captured."); + } + + const binding = Object.freeze({ + dockerEnv: frozenDockerEnv, + engineId, + }); + internalBindings.set(binding, internal); + return binding; +} + +/** Resolve a mutable tag to one directly inspectable immutable ID on the bound engine. */ +export function inspectOpenClawLegacyImageId( + binding: OpenClawLegacyDockerBinding, + imageRef: string, +): string { + const internal = requireInternalBinding(binding); + const validatedImageRef = validateMutableImageRef(imageRef); + if (!currentEngineMatches(binding, internal)) { + throw new Error("OpenClaw legacy-image Docker engine identity changed."); + } + const imageId = inspectImageId(internal, validatedImageRef); + if (imageId === null || inspectImageId(internal, imageId) !== imageId) { + throw new Error("OpenClaw legacy-image identity could not be verified."); + } + return imageId; +} + +/** + * Remove an unleased preflight image only when the bound engine, mutable tag, + * and directly addressed immutable ID still agree. + */ +export function disposeOpenClawLegacyDockerImage( + binding: OpenClawLegacyDockerBinding, + imageRef: string, + imageId?: string, +): boolean { + const internal = requireInternalBinding(binding); + let validatedImageRef: string; + let expectedImageId: string; + try { + validatedImageRef = validateMutableImageRef(imageRef); + if (imageId === undefined) return false; + expectedImageId = validateImageId(imageId); + } catch { + return false; + } + if (!currentEngineMatches(binding, internal)) return false; + const taggedImageId = inspectImageId(internal, validatedImageRef); + if (taggedImageId !== expectedImageId) return false; + if (inspectImageId(internal, expectedImageId) !== expectedImageId) return false; + + try { + const result = internal.runDocker(["rmi", expectedImageId], internal.dockerOptions); + return commandSucceeded(result); + } catch { + return false; + } +} + +/** + * Remove only the expected immutable image from the bound engine. + * + * A retained lease can outlive its mutable tag. Cleanup therefore proves the + * engine and direct immutable ID without consulting or removing whatever the + * tag may reference now. + */ +function disposeRetainedOpenClawLegacyDockerImage( + binding: OpenClawLegacyDockerBinding, + internal: InternalBinding, + imageId: string, +): boolean { + if (!currentEngineMatches(binding, internal)) return false; + if (inspectImageId(internal, imageId) !== imageId) return false; + + try { + const result = internal.runDocker(["rmi", imageId], internal.dockerOptions); + return commandSucceeded(result); + } catch { + return false; + } +} + +/** + * Create a retained-image lease after proving the tag and immutable ID still + * resolve on the exact Docker engine that built them. + */ +export function createPreparedOpenClawLegacyImage( + binding: OpenClawLegacyDockerBinding, + imageRef: string, + imageId: string, +): PreparedOpenClawLegacyImage { + const internal = requireInternalBinding(binding); + const validatedImageRef = validateMutableImageRef(imageRef); + const validatedImageId = validateImageId(imageId); + if (!verifyImageIdentity(binding, internal, validatedImageRef, validatedImageId)) { + throw new Error("OpenClaw legacy-image identity changed before lease creation."); + } + + let state: "prepared" | "retained" | "finalized" | "disposed" = "prepared"; + let verifiedForCreate = false; + let exitListenerRegistered = false; + let lease: PreparedOpenClawLegacyImage; + + const removeExitListener = (): void => { + if (!exitListenerRegistered) return; + internal.removeExitListener(exitListener); + exitListenerRegistered = false; + }; + const exitListener = (): void => { + lease.abort(); + }; + const abortLease = (): boolean => { + if (state === "finalized" || state === "disposed") { + removeExitListener(); + return true; + } + const removed = + state === "retained" + ? disposeRetainedOpenClawLegacyDockerImage(binding, internal, validatedImageId) + : disposeOpenClawLegacyDockerImage(binding, validatedImageRef, validatedImageId); + if (!removed) return false; + state = "disposed"; + removeExitListener(); + return true; + }; + + lease = Object.freeze({ + dockerEnv: binding.dockerEnv, + engineId: binding.engineId, + imageRef: validatedImageRef, + imageId: validatedImageId, + verify(this: PreparedOpenClawLegacyImage): boolean { + return ( + this === lease && + state === "prepared" && + verifyImageIdentity(binding, internal, validatedImageRef, validatedImageId) + ); + }, + retainForRecreate(this: PreparedOpenClawLegacyImage): boolean { + if (this !== lease || state !== "prepared") return false; + if (!verifyImageIdentity(binding, internal, validatedImageRef, validatedImageId)) + return false; + state = "retained"; + return true; + }, + verifyForCreate(this: PreparedOpenClawLegacyImage): boolean { + if (this !== lease || state !== "retained") return false; + verifiedForCreate = verifyImageIdentity( + binding, + internal, + validatedImageRef, + validatedImageId, + ); + return verifiedForCreate; + }, + finalizeAfterCreate( + this: PreparedOpenClawLegacyImage, + ): PreparedOpenClawLegacyImageFinalization | null { + if (this !== lease || state !== "retained" || !verifiedForCreate) return null; + if (!currentEngineMatches(binding, internal)) return null; + if (inspectImageId(internal, validatedImageId) !== validatedImageId) return null; + const mutableTagVerified = inspectImageId(internal, validatedImageRef) === validatedImageId; + + // The registry does not persist the bound Docker selector and engine ID. + // A later ambient context could therefore redirect tag-based deletion. + // Keep the unique tag for maintenance GC, but do not register it for + // per-sandbox cleanup. + const finalization: PreparedOpenClawLegacyImageFinalization = Object.freeze({ + registryImageRef: null, + mutableTagVerified, + }); + state = "finalized"; + removeExitListener(); + return finalization; + }, + abort(this: PreparedOpenClawLegacyImage): boolean { + if (this !== lease) return false; + return abortLease(); + }, + dispose(this: PreparedOpenClawLegacyImage): boolean { + if (this !== lease) return false; + return abortLease(); + }, + }); + + internal.addExitListener(exitListener); + exitListenerRegistered = true; + return lease; +} diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 1b70da187fc..be84219b901 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2246,7 +2246,20 @@ async function createSandboxWithBaseImageResolution( ? { extraProviders: createIntent.extraProviders, staleExtraProviders: [] } : planRegisteredExtraProviders(GATEWAY_NAME, { runOpenshell }); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - const resolvedCreateIntent = createIntent?.resolved ?? (await sandboxCreateIntentResolver.resolve({ sandboxName, inferenceProvider: provider, enabledChannels, webSearchConfig, agent, sandboxGpuConfig: effectiveSandboxGpuConfig, resourceProfile, hermesToolGateways, extraProviders: extraProviderPlan.extraProviders, staleExtraProviders: extraProviderPlan.staleExtraProviders, baselineExclusions: sandboxRegistration.baselineExclusionsForCreate(sandboxName), ...(createIntent?.reuseRegisteredCredentials ? { reuseRegisteredCredentials: true } : {}), ...(createIntent?.policyTier !== undefined ? { policyTier: createIntent.policyTier } : {}) })); + const baseResolvedCreateIntent = createIntent?.resolved ?? (await sandboxCreateIntentResolver.resolve({ sandboxName, inferenceProvider: provider, enabledChannels, webSearchConfig, agent, sandboxGpuConfig: effectiveSandboxGpuConfig, resourceProfile, hermesToolGateways, extraProviders: extraProviderPlan.extraProviders, staleExtraProviders: extraProviderPlan.staleExtraProviders, baselineExclusions: sandboxRegistration.baselineExclusionsForCreate(sandboxName), ...(createIntent?.reuseRegisteredCredentials ? { reuseRegisteredCredentials: true } : {}), ...(createIntent?.policyTier !== undefined ? { policyTier: createIntent.policyTier } : {}) })); + const retainedDockerRuntime = preparedBuildContext?.preparedOpenClawLegacyImage + ? sandboxGpuCreateFlow.createRetainedOpenClawDockerRuntime( + preparedBuildContext.preparedOpenClawLegacyImage, + ) + : null; + const resolvedCreateIntent = retainedDockerRuntime + ? sandboxGpuCreateFlow.bindRetainedOpenClawGpuRoute( + baseResolvedCreateIntent, + effectiveSandboxGpuConfig, + isLinuxDockerDriverGatewayEnabled(), + retainedDockerRuntime, + ) + : baseResolvedCreateIntent; const messagingCapabilities = await sandboxCreateIntentResolver.rebind( { sandboxName, @@ -2708,7 +2721,13 @@ async function createSandboxWithBaseImageResolution( manageDashboard, openshellShellCommand, openshellArgv, - prebuild: { buildCtx, buildId, dockerDriverGateway, origin }, + prebuild: { + buildCtx, + buildId, + dockerDriverGateway, + origin, + preparedOpenClawLegacyImage: preparedBuildContext?.preparedOpenClawLegacyImage, + }, }); const restoreBackupPath = pendingStateRestore?.manifest?.backupPath ?? pendingStateRestoreBackupPath; @@ -2719,6 +2738,7 @@ async function createSandboxWithBaseImageResolution( route: selectedGpuRoute, firstCreateOutput, registryImageRef, + retainedImageFinalization, } = await sandboxGpuCreateFlow.runSandboxGpuCreateFlow( { sandboxName, @@ -2744,6 +2764,9 @@ async function createSandboxWithBaseImageResolution( sleep: sleepSeconds, openshellArgv, verifyDirectSandboxGpu, + ...(retainedDockerRuntime + ? { createRetainedDockerRuntime: () => retainedDockerRuntime } + : {}), }, ); @@ -2802,11 +2825,18 @@ async function createSandboxWithBaseImageResolution( } // openshell tags images with seconds; buildId is ms. Parse actual tag from output. Fixes #2672. - const resolvedImageTag = - registryImageRef ?? - prebuild.imageRef ?? - buildContext.extractBuiltImageRef(`${firstCreateOutput}\n${createResult.output}`) ?? - resolveSandboxImageTagFromCreateOutput(`${firstCreateOutput}\n${createResult.output}`, buildId); + const resolvedImageTag = sandboxGpuCreateFlow.resolveCreatedSandboxRegistryImageRef( + retainedImageFinalization, + [ + registryImageRef, + prebuild.imageRef, + buildContext.extractBuiltImageRef(`${firstCreateOutput}\n${createResult.output}`), + resolveSandboxImageTagFromCreateOutput( + `${firstCreateOutput}\n${createResult.output}`, + buildId, + ), + ], + ); const sandboxRuntimeFields = getSandboxRuntimeRegistryFields(effectiveSandboxGpuConfig); recreateRuntime.recordCreated(); finalizeCreatedSandbox( diff --git a/src/lib/onboard/build-context-stage.ts b/src/lib/onboard/build-context-stage.ts index 994f6d8a308..6a329c9e64d 100644 --- a/src/lib/onboard/build-context-stage.ts +++ b/src/lib/onboard/build-context-stage.ts @@ -37,10 +37,31 @@ export interface CreateSandboxBuildContextResult extends StagedBuildContext { cleanupBuildCtx(): boolean; } +/** Exact local Docker image retained across an OpenClaw rebuild recreation. */ +export interface PreparedOpenClawLegacyImage { + readonly dockerEnv: Readonly>; + readonly engineId: string; + readonly imageRef: string; + readonly imageId: string; + verify(): boolean; + retainForRecreate(): boolean; + verifyForCreate(): boolean; + finalizeAfterCreate(): PreparedOpenClawLegacyImageFinalization | null; + abort(): boolean; + dispose(): boolean; +} + +/** Safe registry bookkeeping produced only after final retained-image proof. */ +export interface PreparedOpenClawLegacyImageFinalization { + readonly registryImageRef: null; + readonly mutableTagVerified: boolean; +} + /** Exact staged and patched context transferred from rebuild preflight to create. */ export interface PreparedSandboxBuildContext extends CreateSandboxBuildContextResult { buildId: string; dashboardRemoteBindPrepared?: boolean; + preparedOpenClawLegacyImage?: PreparedOpenClawLegacyImage; /** Recheck retained bytes at the final one-shot consumption boundary. */ verifyBuildCtx?(): boolean; /** Exact recorded target authorized to consume a generic rebuild handoff. */ diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index da58f8c44d8..54332ac1ce5 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -8,7 +8,7 @@ import { getDockerGpuPatchNetworkMode, printDockerGpuProofFailure, } from "./docker-gpu-patch"; -import type { DockerGpuPatchMode } from "./docker-gpu-patch-types"; +import type { DockerGpuPatchDeps, DockerGpuPatchMode } from "./docker-gpu-patch-types"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; import { executeSandboxCommandForVerification } from "./sandbox-verification-exec"; @@ -389,6 +389,8 @@ export type GpuSandboxAfterReadyOptions = { reportGpuProofFailure?: boolean; selectedMode: () => DockerGpuPatchMode | null; runCaptureOpenshell: (args: string[], opts?: Record) => string; + dockerCapture?: DockerGpuPatchDeps["dockerCapture"]; + dockerLogs?: DockerGpuPatchDeps["dockerLogs"]; env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform; log?: (message: string) => void; @@ -433,6 +435,8 @@ export function verifyGpuSandboxAccessAfterReady( if (!options.verifyGpuOrExit && options.reportGpuProofFailure !== false) { printDockerGpuProofFailure(options.sandboxName, error, options.selectedMode(), { runCaptureOpenshell: options.runCaptureOpenshell, + dockerCapture: options.dockerCapture, + dockerLogs: options.dockerLogs, additionalSummaryLines: adaptDockerGpuRouteForPatch(options.selectedRoute) .additionalSummaryLines, }); diff --git a/src/lib/onboard/docker-gpu-patch.ts b/src/lib/onboard/docker-gpu-patch.ts index d54847a7af6..ac678042261 100644 --- a/src/lib/onboard/docker-gpu-patch.ts +++ b/src/lib/onboard/docker-gpu-patch.ts @@ -157,23 +157,25 @@ function patchedContainerIdFromContext( } function snapshotInspectDeps( - deps: Pick, -): Pick { + deps: Pick, +): Pick { // `depsWithDefaults` spreads the caller's `deps`, so passing an explicit // `dockerCapture: undefined` would shadow the module's default Docker // adapter and disable downstream `docker ps`/`inspect`/`logs` capture. // Build the inner deps object with only the keys the caller actually // supplied so defaults stay in place. - const inner: Pick = {}; + const inner: Pick = + {}; if (deps.runCaptureOpenshell) inner.runCaptureOpenshell = deps.runCaptureOpenshell; if (deps.dockerCapture) inner.dockerCapture = deps.dockerCapture; + if (deps.dockerLogs) inner.dockerLogs = deps.dockerLogs; return inner; } export function printDockerGpuPatchFailureAndExit( sandboxName: string, error: unknown, - deps: Pick & { + deps: Pick & { context?: DockerGpuPatchFailureContext | null; selectedMode?: DockerGpuPatchMode | null; additionalSummaryLines?: readonly string[]; @@ -228,7 +230,7 @@ export function printDockerGpuPatchFailureAndExit( export function printDockerGpuReadinessFailure( sandboxName: string, selectedMode: DockerGpuPatchMode | null, - deps: Pick & { + deps: Pick & { context?: DockerGpuPatchFailureContext | null; additionalSummaryLines?: readonly string[]; }, @@ -263,7 +265,7 @@ export function printDockerGpuProofFailure( sandboxName: string, error: unknown, selectedMode: DockerGpuPatchMode | null, - deps: Pick & { + deps: Pick & { context?: DockerGpuPatchFailureContext | null; additionalSummaryLines?: readonly string[]; }, diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index fb94eba175e..936457078ec 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -3,6 +3,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import * as dockerAdapters from "../adapters/docker"; +import { createDockerGpuInspectFixture } from "./__test-helpers__/docker-gpu-patch-fixtures"; import type { DockerGpuPatchFailureContext, DockerGpuPatchResult } from "./docker-gpu-patch"; import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; @@ -245,6 +247,123 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(onPatchFailureExit).not.toHaveBeenCalled(); }); + it("uses the injected Docker capture for default container discovery", () => { + const deps = makeDeps(); + const recreatePatch = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { recreatePatch }, + }); + + patch.maybeApplyDuringCreate(); + + expect(deps.dockerCapture).toHaveBeenCalledWith( + expect.arrayContaining(["ps", "-a", "--filter", "label=openshell.ai/sandbox-name=alpha"]), + expect.objectContaining({ ignoreError: true }), + ); + expect(recreatePatch).not.toHaveBeenCalled(); + }); + + it("keeps polling recreation and rollback on bound engine A when ambient adapters select B", () => { + const failOnEngineB = () => { + throw new Error("ambient engine B must not receive Docker operations"); + }; + const engineBSpies = [ + vi.spyOn(dockerAdapters, "dockerCapture").mockImplementation(failOnEngineB), + vi.spyOn(dockerAdapters, "dockerRun").mockImplementation(failOnEngineB), + vi.spyOn(dockerAdapters, "dockerRunDetached").mockImplementation(failOnEngineB), + vi.spyOn(dockerAdapters, "dockerRename").mockImplementation(failOnEngineB), + vi.spyOn(dockerAdapters, "dockerRm").mockImplementation(failOnEngineB), + vi.spyOn(dockerAdapters, "dockerStart").mockImplementation(failOnEngineB), + vi.spyOn(dockerAdapters, "dockerStop").mockImplementation(failOnEngineB), + ]; + const dockerCapture = vi.fn((args: readonly string[]) => { + if (args[0] === "ps") return "old-container-id\n"; + if (args[0] === "inspect") return JSON.stringify([createDockerGpuInspectFixture()]); + return ""; + }); + const dockerRun = vi.fn(() => ({ status: 0, stdout: "engine-a-probe\n" })); + const dockerRunDetached = vi.fn(() => ({ + status: 1, + stderr: "engine A forced recreate failure", + })); + const dockerRename = vi.fn(() => ({ status: 0 })); + const dockerRm = vi.fn(() => ({ status: 0 })); + const dockerStart = vi.fn(() => ({ status: 0 })); + const dockerStop = vi.fn(() => ({ status: 0 })); + const onPatchFailureExit = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + dockerDesktopWsl: false, + deps: { + runOpenshell: vi.fn(() => ({ status: 0 })), + runCaptureOpenshell: vi.fn(() => ""), + sleep: vi.fn(), + dockerCapture, + dockerRun, + dockerRunDetached, + dockerRename, + dockerRm, + dockerStart, + dockerStop, + now: () => new Date("2026-07-29T00:00:00Z"), + detectSandboxFallbackDns: () => null, + readDir: () => null, + readFile: () => null, + }, + overrides: { onPatchFailureExit }, + }); + + patch.maybeApplyDuringCreate(); + patch.exitOnPatchError(); + + expect(dockerCapture).toHaveBeenCalledWith( + ["inspect", "--type", "container", "old-container-id"], + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRun).toHaveBeenCalledWith( + expect.arrayContaining(["create", "--gpus", "all"]), + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerStop).toHaveBeenCalledWith( + "old-container-id", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRename).toHaveBeenCalledWith( + "old-container-id", + expect.stringContaining("openshell-alpha-nemoclaw-gpu-backup-"), + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRunDetached).toHaveBeenCalledWith( + expect.arrayContaining(["--name", "openshell-alpha", "--gpus", "all"]), + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerStop).toHaveBeenCalledWith( + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRm).toHaveBeenCalledWith( + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRename).toHaveBeenCalledWith( + expect.stringContaining("openshell-alpha-nemoclaw-gpu-backup-"), + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerStart).toHaveBeenCalledWith( + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + expect(onPatchFailureExit).toHaveBeenCalledOnce(); + for (const engineBSpy of engineBSpies) expect(engineBSpy).not.toHaveBeenCalled(); + }); + it("records patchError when recreate throws and exitOnPatchError reports it via printDockerGpuPatchFailureAndExit", () => { const deps = makeDeps(); const recreatePatch = vi.fn(() => { diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index b683cbb1c77..2ae3d857c1f 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -40,10 +40,8 @@ export { resolveDockerGpuSandboxCreatePlan, } from "./docker-gpu-sandbox-create-plan"; -type DockerGpuSandboxCreateDeps = Pick< - DockerGpuPatchDeps, - "runOpenshell" | "runCaptureOpenshell" | "sleep" | "dockerCapture" ->; +type DockerGpuSandboxCreateDeps = DockerGpuPatchDeps & + Required>; type WaitSupervisorFn = typeof waitForOpenShellSupervisorReconnect; type FindContainerIdsFn = typeof findOpenShellDockerSandboxContainerIds; @@ -125,7 +123,8 @@ export function createDockerGpuSandboxCreatePatch( let needsSupervisorWait = false; const findContainerIds = - options.overrides?.findContainerIds ?? findOpenShellDockerSandboxContainerIds; + options.overrides?.findContainerIds ?? + ((sandboxName: string) => findOpenShellDockerSandboxContainerIds(sandboxName, options.deps)); const recreatePatch = options.overrides?.recreatePatch ?? recreateOpenShellDockerSandboxWithGpu; const recreateStartupPatch = options.overrides?.recreateStartupPatch; const waitForSupervisor = @@ -165,10 +164,7 @@ export function createDockerGpuSandboxCreatePatch( ` OpenShell Docker container detected; recreating it with ${patchTarget} before readiness wait...`, ); try { - result = recreateSelectedPatch(false, { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - sleep: options.deps.sleep, - }); + result = recreateSelectedPatch(false, options.deps); needsSupervisorWait = true; console.log(` ✓ Docker container mode selected: ${result.mode.label}`); } catch (error) { @@ -188,6 +184,7 @@ export function createDockerGpuSandboxCreatePatch( onPatchFailureExit(options.sandboxName, patchError, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, + dockerLogs: options.deps.dockerLogs, additionalSummaryLines: routeAdapter.additionalSummaryLines, }); }, @@ -203,6 +200,7 @@ export function createDockerGpuSandboxCreatePatch( onPatchFailureExit(options.sandboxName, error, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, + dockerLogs: options.deps.dockerLogs, additionalSummaryLines: routeAdapter.additionalSummaryLines, }); } @@ -251,6 +249,7 @@ export function createDockerGpuSandboxCreatePatch( { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, + dockerLogs: options.deps.dockerLogs, additionalSummaryLines: routeAdapter.additionalSummaryLines, context: { sandboxName: options.sandboxName, @@ -276,6 +275,7 @@ export function createDockerGpuSandboxCreatePatch( onPatchFailureExit(options.sandboxName, new Error(failureMessage), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, + dockerLogs: options.deps.dockerLogs, additionalSummaryLines: routeAdapter.additionalSummaryLines, context: { sandboxName: options.sandboxName, @@ -297,6 +297,7 @@ export function createDockerGpuSandboxCreatePatch( printDockerGpuReadinessFailure(options.sandboxName, result?.mode ?? null, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, + dockerLogs: options.deps.dockerLogs, context: buildFailureContext(options.sandboxName, result), additionalSummaryLines: routeAdapter.additionalSummaryLines, }); @@ -330,6 +331,7 @@ export function createDockerGpuSandboxCreatePatch( { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, + dockerLogs: options.deps.dockerLogs, context: failureContext, additionalSummaryLines: routeAdapter.additionalSummaryLines, }, @@ -349,6 +351,7 @@ export function createDockerGpuSandboxCreatePatch( printDockerGpuProofFailure(sandboxName, error, result?.mode ?? null, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, + dockerLogs: options.deps.dockerLogs, context: routeAdapter.enabled ? failureContext : null, additionalSummaryLines: routeAdapter.additionalSummaryLines, }); diff --git a/src/lib/onboard/gateway-sandbox-reachability.ts b/src/lib/onboard/gateway-sandbox-reachability.ts index 81689206f2f..001ec551138 100644 --- a/src/lib/onboard/gateway-sandbox-reachability.ts +++ b/src/lib/onboard/gateway-sandbox-reachability.ts @@ -86,6 +86,9 @@ export interface SandboxBridgeReachabilityOptions { port?: number; timeoutSec?: number; probeImage?: string; + dockerCaptureImpl?: typeof dockerCapture; + dockerRunImpl?: typeof dockerRun; + ensureImageCachedImpl?: typeof ensureProbeImageCached; runImpl?: (args: readonly string[], timeoutMs: number) => SandboxBridgeProbeRunResult; inspectNetworkImpl?: (networkName: string) => DockerBridgeNetworkInfo | undefined; usesHostGatewayRouteImpl?: () => boolean; @@ -126,25 +129,31 @@ function parseDockerNetworkIpamConfig(raw: string): DockerBridgeNetworkInfo | un ); } -function defaultInspectNetwork(networkName: string): DockerBridgeNetworkInfo | undefined { - const raw = dockerCapture( - ["network", "inspect", "--format", "{{json .IPAM.Config}}", networkName], - { ignoreError: true }, - ); +function defaultInspectNetwork( + networkName: string, + capture: typeof dockerCapture = dockerCapture, +): DockerBridgeNetworkInfo | undefined { + const raw = capture(["network", "inspect", "--format", "{{json .IPAM.Config}}", networkName], { + ignoreError: true, + }); return parseDockerNetworkIpamConfig(raw); } -function defaultUsesHostGatewayRoute(): boolean { +function defaultUsesHostGatewayRoute(capture: typeof dockerCapture = dockerCapture): boolean { if (process.platform !== "linux") return true; - const info = dockerCapture( + const info = capture( ["info", "--format", "{{.OperatingSystem}}\n{{range .Labels}}{{.}}\n{{end}}"], { ignoreError: true }, ); return /Docker Desktop|com\.docker\.desktop\./i.test(info); } -function defaultRunImpl(args: readonly string[], timeoutMs: number): SandboxBridgeProbeRunResult { - const result = dockerRun(args, { +function defaultRunImpl( + args: readonly string[], + timeoutMs: number, + runDocker: typeof dockerRun = dockerRun, +): SandboxBridgeProbeRunResult { + const result = runDocker(args, { timeout: timeoutMs, ignoreError: true, suppressOutput: true, @@ -265,9 +274,15 @@ export async function isSandboxBridgeGatewayReachable( const port = opts.port ?? GATEWAY_PORT; const timeoutSec = opts.timeoutSec ?? DEFAULT_PROBE_TIMEOUT_SEC; const probeImage = opts.probeImage ?? DEFAULT_PROBE_IMAGE; - const inspectNetwork = opts.inspectNetworkImpl ?? defaultInspectNetwork; - const usesHostGatewayRoute = opts.usesHostGatewayRouteImpl ?? defaultUsesHostGatewayRoute; - const runImpl = opts.runImpl ?? defaultRunImpl; + const inspectNetwork = + opts.inspectNetworkImpl ?? + ((name: string) => defaultInspectNetwork(name, opts.dockerCaptureImpl)); + const usesHostGatewayRoute = + opts.usesHostGatewayRouteImpl ?? (() => defaultUsesHostGatewayRoute(opts.dockerCaptureImpl)); + const runImpl = + opts.runImpl ?? + ((args: readonly string[], timeoutMs: number) => + defaultRunImpl(args, timeoutMs, opts.dockerRunImpl)); const network = inspectNetwork(networkName); const route = buildOpenShellDockerRoute(networkName, network, usesHostGatewayRoute()); @@ -293,7 +308,9 @@ export async function isSandboxBridgeGatewayReachable( // skip the pre-pull there unless the test supplies an explicit // ensureImageCachedOverride. if (opts.ensureImageCachedOverride !== undefined || opts.runImpl === undefined) { - const cached = opts.ensureImageCachedOverride ?? ensureProbeImageCached(probeImage); + const cached = + opts.ensureImageCachedOverride ?? + (opts.ensureImageCachedImpl ?? ensureProbeImageCached)(probeImage); if (!cached.ok) { // A wedged docker daemon (inspect_unavailable) is a fatal Docker // outage, not a probe/pull uncertainty — keep onboarding from diff --git a/src/lib/onboard/prepared-dcode-rebuild.test.ts b/src/lib/onboard/prepared-dcode-rebuild.test.ts index 808967611f1..abfc4472dca 100644 --- a/src/lib/onboard/prepared-dcode-rebuild.test.ts +++ b/src/lib/onboard/prepared-dcode-rebuild.test.ts @@ -10,7 +10,10 @@ import { describe, expect, it, vi } from "vitest"; import { createBuildContextVerifier } from "../actions/sandbox/rebuild-prepared-image-context"; import { fingerprintBuildContext } from "../adapters/fs/build-context-fingerprint"; import type { AgentDefinition } from "../agent/defs"; -import type { PreparedSandboxBuildContext } from "./build-context-stage"; +import type { + PreparedOpenClawLegacyImage, + PreparedSandboxBuildContext, +} from "./build-context-stage"; import { createPreparedDcodeRebuildRuntime, type PreparedDcodeRebuildOptions, @@ -62,6 +65,49 @@ const preparedImageOptions: PreparedDcodeRebuildOptions = { gatewayName: "nemoclaw", }, }; +const OPENCLAW_IMAGE_ID = `sha256:${"a".repeat(64)}`; + +function createPreparedOpenClawLegacyImageFixture(): PreparedOpenClawLegacyImage { + return { + dockerEnv: { DOCKER_CONTEXT: "retained-context" }, + engineId: "retained-engine", + imageRef: "nemoclaw-sandbox-local:prepared-openclaw", + imageId: OPENCLAW_IMAGE_ID, + verify: vi.fn(() => true), + retainForRecreate: vi.fn(() => true), + verifyForCreate: vi.fn(() => true), + finalizeAfterCreate: vi.fn(() => ({ + mutableTagVerified: true, + registryImageRef: null, + })), + abort: vi.fn(() => true), + dispose: vi.fn(() => true), + }; +} + +function createPreparedOpenClawImageOptions(): PreparedDcodeRebuildOptions { + return { + resume: true, + recreateSandbox: true, + authoritativeResumeConfig: true, + onboardLockAlreadyHeld: true, + agent: null, + fromDockerfile: null, + preparedImageRebuild: { + gatewayName: "nemoclaw", + buildContext: { + buildCtx: "/tmp/prepared-openclaw", + stagedDockerfile: "/tmp/prepared-openclaw/Dockerfile", + buildId: "openclaw-prepared", + cleanupBuildCtx: () => true, + origin: "generated", + verifyBuildCtx: () => true, + rebuildTarget: { agentName: null, fromDockerfile: null }, + preparedOpenClawLegacyImage: createPreparedOpenClawLegacyImageFixture(), + }, + }, + }; +} const sandboxGpuConfig: SandboxGpuConfig = { mode: "0", hostGpuDetected: false, @@ -159,6 +205,69 @@ describe("prepared DCode rebuild adapter", () => { ); }); + it.each([ + ["resume", { resume: false }], + ["recreation", { recreateSandbox: false }], + ["authoritative configuration", { authoritativeResumeConfig: false }], + ["onboard lock", { onboardLockAlreadyHeld: false }], + ])("rejects a retained OpenClaw image without authoritative %s", (_label, override) => { + expect(() => + createPreparedDcodeRebuildRuntime( + { ...createPreparedOpenClawImageOptions(), ...override }, + "nemoclaw", + ), + ).toThrow(/only be used by authoritative resume recreation/); + }); + + it.each([ + "custom origin", + "custom Dockerfile", + "Hermes agent", + ])("rejects a retained OpenClaw image for a %s", (unsupportedTarget) => { + const options = createPreparedOpenClawImageOptions(); + const buildContext = options.preparedImageRebuild!.buildContext; + if (unsupportedTarget === "custom origin") { + buildContext.origin = "custom"; + } else if (unsupportedTarget === "custom Dockerfile") { + options.fromDockerfile = "/tmp/custom/Dockerfile"; + buildContext.rebuildTarget = { + agentName: null, + fromDockerfile: "/tmp/custom/Dockerfile", + }; + } else { + options.agent = "hermes"; + buildContext.rebuildTarget = { agentName: "hermes", fromDockerfile: null }; + } + + expect(() => createPreparedDcodeRebuildRuntime(options, "nemoclaw")).toThrow( + /retained legacy image can only be used for a generated OpenClaw rebuild/, + ); + }); + + it("rejects a retained OpenClaw image attached to a prepared DCode handoff", () => { + expect(() => + createPreparedDcodeRebuildRuntime( + { + ...preparedOptions, + preparedDcodeRebuild: { + ...preparedOptions.preparedDcodeRebuild!, + buildContext: { + ...preparedBuildContext, + preparedOpenClawLegacyImage: createPreparedOpenClawLegacyImageFixture(), + }, + }, + }, + "nemoclaw", + ), + ).toThrow(/retained legacy image can only be used for a generated OpenClaw rebuild/); + }); + + it("accepts a retained legacy image for authoritative generated OpenClaw recreation", () => { + expect(() => + createPreparedDcodeRebuildRuntime(createPreparedOpenClawImageOptions(), "nemoclaw"), + ).not.toThrow(); + }); + it("normalizes the exact gateway and clears ordinary ambient selection", () => { const preparedEnv: NodeJS.ProcessEnv = { OPENSHELL_GATEWAY: "ambient" }; createPreparedDcodeRebuildRuntime(preparedOptions, "nemoclaw").applyGatewayEnv(preparedEnv); diff --git a/src/lib/onboard/prepared-dcode-rebuild.ts b/src/lib/onboard/prepared-dcode-rebuild.ts index 0fef7cdb7c4..2e92d69a89d 100644 --- a/src/lib/onboard/prepared-dcode-rebuild.ts +++ b/src/lib/onboard/prepared-dcode-rebuild.ts @@ -117,6 +117,29 @@ function verifyPreparedBuildContextForUse(preparedBuildContext: PreparedSandboxB } } +function assertPreparedOpenClawLegacyImageScope( + preparedBuildContext: PreparedSandboxBuildContext, + agentName: string | null | undefined, + fromDockerfile: string | null | undefined, + handoffKind: "dcode" | "image", +): void { + if (!preparedBuildContext.preparedOpenClawLegacyImage) return; + const target = preparedBuildContext.rebuildTarget; + if ( + handoffKind !== "image" || + preparedBuildContext.origin !== "generated" || + normalizedAgentIdentity(agentName) !== "openclaw" || + normalizedDockerfilePath(fromDockerfile) !== null || + !target || + normalizedAgentIdentity(target.agentName) !== "openclaw" || + target.fromDockerfile !== null + ) { + throw new Error( + "A retained legacy image can only be used for a generated OpenClaw rebuild without a custom Dockerfile.", + ); + } +} + export function assertPreparedDcodeTarget( preparedBuildContext: PreparedSandboxBuildContext | null, agent: AgentDefinition | null | undefined, @@ -162,6 +185,14 @@ export function createPreparedDcodeRebuildRuntime( "A prepared rebuild image can only be used by authoritative resume recreation.", ); } + if (preparedDcode) { + assertPreparedOpenClawLegacyImageScope( + preparedDcode.buildContext, + options.agent, + options.fromDockerfile, + "dcode", + ); + } if (preparedImage) { if (!preparedImage.buildContext.rebuildTarget) { throw new Error("Prepared rebuild image target is missing or invalid."); @@ -174,6 +205,12 @@ export function createPreparedDcodeRebuildRuntime( options.agent ?? null, normalizedDockerfilePath(options.fromDockerfile), ); + assertPreparedOpenClawLegacyImageScope( + preparedImage.buildContext, + options.agent, + options.fromDockerfile, + "image", + ); } const prepared = preparedImage ?? preparedDcode; const preparedLabel = preparedImage ? "Prepared rebuild image" : "Prepared DCode rebuild"; diff --git a/src/lib/onboard/rebuild/retained-openclaw-docker-runtime.ts b/src/lib/onboard/rebuild/retained-openclaw-docker-runtime.ts new file mode 100644 index 00000000000..b4c8d75ab60 --- /dev/null +++ b/src/lib/onboard/rebuild/retained-openclaw-docker-runtime.ts @@ -0,0 +1,265 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SpawnSyncOptions } from "node:child_process"; +import { isIP } from "node:net"; + +import { dockerSpawnSync } from "../../adapters/docker/exec"; +import { ROOT } from "../../state/paths"; +import type { PreparedOpenClawLegacyImage } from "../build-context-stage"; +import type { DockerGpuPatchDeps } from "../docker-gpu-patch-types"; +import { resolveDockerGpuSandboxCreatePlan } from "../docker-gpu-sandbox-create-plan"; +import { + isSandboxBridgeGatewayReachable, + verifySandboxBridgeGatewayReachableOrExit, +} from "../gateway-sandbox-reachability"; +import { + BUSYBOX_PROBE_IMAGE, + dnsProbeName, + type EnsureProbeImageCachedResult, + ensureProbeImageCached, + type ProbeContainerDnsOpts, + type ProbeExecutionResult, + probeContainerDns, +} from "../preflight"; +import type { SandboxCreateIntent } from "../sandbox-create-intent-types"; +import type { SandboxGpuCreateConfig } from "../sandbox-gpu-create"; +import { detectWslDockerDesktopStatus } from "../wsl-docker-desktop-gpu"; + +const RETAINED_DOCKER_OPERATION_TIMEOUT_MS = 30_000; +const RETAINED_DNS_PROBE_TIMEOUT_MS = 20_000; +const RETAINED_DOCKER_PULL_TIMEOUT_MS = 60_000; + +type DockerRunOptions = Record; +type DockerRunResult = NonNullable>; + +export interface RetainedOpenClawDockerRuntime { + readonly deps: DockerGpuPatchDeps; + dockerDesktopWsl(): boolean; + ensureImageCached(image: string): EnsureProbeImageCachedResult; + reverifyBridgeReachability(port: number): Promise; +} + +export interface RetainedOpenClawDockerRuntimeDeps { + runDocker?: typeof dockerSpawnSync; + platform?: NodeJS.Platform; + env?: NodeJS.ProcessEnv; +} + +/** Re-resolve route policy from the bound engine before policy materialization. */ +export function bindRetainedOpenClawGpuRoute( + intent: SandboxCreateIntent, + config: SandboxGpuCreateConfig, + dockerDriverGateway: boolean, + runtime: RetainedOpenClawDockerRuntime, + options: { env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform } = {}, +): SandboxCreateIntent { + const plan = resolveDockerGpuSandboxCreatePlan(config, { + dockerDriverGateway, + dockerDesktopWsl: runtime.dockerDesktopWsl(), + env: options.env, + platform: options.platform, + }); + return { + ...intent, + gpuRoutePlan: plan.gpuRoutePlan, + sandboxGpuLogMessage: plan.logMessage, + }; +} + +function timeoutFromOptions(options: DockerRunOptions | undefined, fallback: number): number { + const timeout = options?.timeout; + return typeof timeout === "number" && Number.isFinite(timeout) && timeout > 0 + ? timeout + : fallback; +} + +function dockerErrorText(result: DockerRunResult): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}`.trim(); +} + +function toProbeExecution(result: DockerRunResult): ProbeExecutionResult { + const error = result.error as NodeJS.ErrnoException | undefined; + return { + stdout: result.stdout, + stderr: result.stderr, + exitCode: result.status ?? null, + signal: result.signal, + timedOut: error?.code === "ETIMEDOUT", + error: error?.message ?? null, + errorCode: error?.code ?? null, + }; +} + +function validateProbeName(value: string): string { + if (!/^[a-z0-9]([a-z0-9.-]{0,253})$/i.test(value)) { + throw new Error( + `probeName must be a plain DNS name (RFC 1035 label characters), got: ${JSON.stringify(value)}`, + ); + } + return value; +} + +/** + * Bind every Docker operation after destructive OpenClaw rebuild handoff to + * the exact engine and selector that own the retained image. + */ +export function createRetainedOpenClawDockerRuntime( + image: PreparedOpenClawLegacyImage, + injected: RetainedOpenClawDockerRuntimeDeps = {}, +): RetainedOpenClawDockerRuntime { + const runDocker = injected.runDocker ?? dockerSpawnSync; + let cachedDockerDesktopWsl: boolean | null = null; + + const runBoundDocker = ( + args: readonly string[], + options: DockerRunOptions = {}, + ): DockerRunResult => { + if (!image.verifyForCreate()) { + throw new Error( + "Retained OpenClaw rebuild Docker engine or image changed before a Docker operation.", + ); + } + const spawnOptions: SpawnSyncOptions = { + cwd: ROOT, + encoding: "utf-8", + env: image.dockerEnv, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + timeout: timeoutFromOptions(options, RETAINED_DOCKER_OPERATION_TIMEOUT_MS), + }; + return runDocker(args, spawnOptions); + }; + + const dockerRun = (args: readonly string[], options: DockerRunOptions = {}): DockerRunResult => + runBoundDocker(args, options); + + const dockerCapture = (args: readonly string[], options: DockerRunOptions = {}): string => { + const result = runBoundDocker(args, options); + if (result.error == null && result.status === 0) return String(result.stdout ?? "").trim(); + if (options.ignoreError === true) return ""; + throw new Error(dockerErrorText(result) || "Bound Docker command failed."); + }; + + const dockerContainer = + (command: string) => + (containerName: string, options: DockerRunOptions = {}): DockerRunResult => + runBoundDocker([command, containerName], options); + + const dockerLogs = ( + containerName: string, + options: { tail?: number; timeout?: number } = {}, + ): string => { + const result = runBoundDocker( + ["logs", "--tail", String(options.tail ?? 30), containerName], + options, + ); + return `${String(result.stdout ?? "")}${String(result.stderr ?? "")}`.trim(); + }; + + const runBoundProbe = ( + command: readonly string[], + options: { timeout?: number } = {}, + ): ProbeExecutionResult => { + if (command[0] !== "docker") { + throw new Error("Retained OpenClaw Docker probe must invoke Docker directly."); + } + return toProbeExecution( + runBoundDocker(command.slice(1), { + timeout: options.timeout, + ignoreError: true, + suppressOutput: true, + }), + ); + }; + + const ensureImageCached = (target: string): EnsureProbeImageCachedResult => + ensureProbeImageCached(target, { + inspectProbeImpl: runBoundProbe, + pullProbeImpl: runBoundProbe, + pullTimeoutMs: RETAINED_DOCKER_PULL_TIMEOUT_MS, + }); + + const probeDns = (options: ProbeContainerDnsOpts = {}) => { + const probeName = validateProbeName(options.probeName ?? dnsProbeName()); + const dnsServer = options.dnsServer ?? null; + if (dnsServer !== null && isIP(dnsServer) === 0) { + throw new Error(`dnsServer must be an IP address, got: ${JSON.stringify(dnsServer)}`); + } + const cached = ensureImageCached(BUSYBOX_PROBE_IMAGE); + if (!cached.ok) { + return probeContainerDns({ + ...options, + probeName, + ensureImageCachedOverride: cached, + executionOverride: { exitCode: 1 }, + }); + } + const dockerArgs = [ + "run", + "--rm", + "--pull=missing", + ...(dnsServer === null ? [] : ["--dns", dnsServer]), + BUSYBOX_PROBE_IMAGE, + "nslookup", + probeName, + ]; + const execution = toProbeExecution( + runBoundDocker(dockerArgs, { + timeout: RETAINED_DNS_PROBE_TIMEOUT_MS, + ignoreError: true, + suppressOutput: true, + }), + ); + return probeContainerDns({ + ...options, + probeName, + command: ["docker", ...dockerArgs], + ensureImageCachedOverride: { ok: true, alreadyCached: cached.alreadyCached }, + executionOverride: execution, + }); + }; + + const deps: DockerGpuPatchDeps = Object.freeze({ + dockerCapture, + dockerRun, + dockerRunDetached: (args, options) => runBoundDocker(["run", "-d", ...args], options), + dockerRename: (oldName, newName, options) => + runBoundDocker(["rename", oldName, newName], options), + dockerRm: dockerContainer("rm"), + dockerStart: dockerContainer("start"), + dockerStop: dockerContainer("stop"), + dockerLogs, + probeContainerDns: probeDns, + }); + + return Object.freeze({ + deps, + dockerDesktopWsl: () => { + if (cachedDockerDesktopWsl === null) { + cachedDockerDesktopWsl = + detectWslDockerDesktopStatus({ + platform: injected.platform, + env: injected.env, + dockerInfoFormat: (format, options) => + dockerCapture(["info", "--format", format], options), + }) === "docker-desktop"; + } + return cachedDockerDesktopWsl; + }, + ensureImageCached, + reverifyBridgeReachability: (port: number) => + verifySandboxBridgeGatewayReachableOrExit(true, { + port, + reachabilityImpl: (selected) => + isSandboxBridgeGatewayReachable({ + port: selected?.port ?? port, + dockerCaptureImpl: dockerCapture, + dockerRunImpl: dockerRun as typeof import("../../adapters/docker/run").dockerRun, + ensureImageCachedImpl: ensureImageCached, + }), + }), + }); +} diff --git a/src/lib/onboard/retained-openclaw-docker-runtime.test.ts b/src/lib/onboard/retained-openclaw-docker-runtime.test.ts new file mode 100644 index 00000000000..ef2f2dea72a --- /dev/null +++ b/src/lib/onboard/retained-openclaw-docker-runtime.test.ts @@ -0,0 +1,185 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SpawnSyncOptions } from "node:child_process"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { dockerSpawnSync } from "../adapters/docker/exec"; +import type { PreparedOpenClawLegacyImage } from "./build-context-stage"; +import { + bindRetainedOpenClawGpuRoute, + createRetainedOpenClawDockerRuntime, +} from "./rebuild/retained-openclaw-docker-runtime"; +import type { SandboxCreateIntent } from "./sandbox-create-intent-types"; + +const IMAGE_ID = `sha256:${"a".repeat(64)}`; + +function createIntent(): SandboxCreateIntent { + return { + sandboxName: "alpha", + inferenceProvider: "nim", + activeMessagingChannels: [], + messagingProviderRequests: [], + reusableMessagingProviders: [], + extraProviders: [], + staleExtraProviders: [], + hermesToolGateways: [], + policy: { + basePolicyPath: "/tmp/policy.yaml", + activeMessagingChannels: [], + options: { + directGpu: true, + additionalPresets: [], + policyTier: null, + baselineExclusions: [], + }, + }, + gpuCreateArgs: ["--gpu"], + resourceCreateArgs: [], + gpuRoutePlan: "native-only", + sandboxGpuLogMessage: "ambient native route", + disabledChannelNames: [], + extraPlaceholderKeys: [], + }; +} + +function dockerResult(stdout = "", status = 0): ReturnType { + return { + error: undefined, + output: [null, stdout, ""], + pid: 123, + signal: null, + status, + stderr: "", + stdout, + }; +} + +function createImage(verifyForCreate: () => boolean = () => true): PreparedOpenClawLegacyImage { + return { + dockerEnv: Object.freeze({ DOCKER_CONTEXT: "engine-a", PATH: "/usr/bin" }), + engineId: "engine-a-id", + imageRef: "nemoclaw-sandbox-local:alpha-rebuild", + imageId: IMAGE_ID, + verify: vi.fn(() => true), + retainForRecreate: vi.fn(() => true), + verifyForCreate: vi.fn(verifyForCreate), + finalizeAfterCreate: vi.fn(() => ({ + registryImageRef: null, + mutableTagVerified: true, + })), + abort: vi.fn(() => true), + dispose: vi.fn(() => true), + }; +} + +afterEach(() => vi.unstubAllEnvs()); + +describe("retained OpenClaw Docker runtime", () => { + it("keeps create-adjacent queries and mutations on engine A when ambient Docker selects B", () => { + vi.stubEnv("DOCKER_CONTEXT", "engine-b"); + vi.stubEnv("WSL_DISTRO_NAME", "Ubuntu"); + const image = createImage(); + const calls: Array<{ args: string[]; options: SpawnSyncOptions }> = []; + const runDocker = vi.fn((args: readonly string[], options: SpawnSyncOptions = {}) => { + calls.push({ args: [...args], options }); + if (args.join(" ") === "info --format {{json .OperatingSystem}}") { + return dockerResult('"Docker Desktop"\n'); + } + if (args[0] === "image" && args[1] === "inspect") return dockerResult("[]\n"); + if (args[0] === "logs") return dockerResult("engine-a logs\n"); + return dockerResult("engine-a-result\n"); + }); + const runtime = createRetainedOpenClawDockerRuntime(image, { + runDocker: runDocker as typeof dockerSpawnSync, + platform: "linux", + env: { WSL_DISTRO_NAME: "Ubuntu" }, + }); + const deps = runtime.deps; + + expect(deps.dockerCapture?.(["ps", "-a"], { ignoreError: true })).toBe("engine-a-result"); + expect(deps.dockerRun?.(["inspect", "container-a"], { ignoreError: true }).status).toBe(0); + expect(deps.dockerStop?.("container-a", { ignoreError: true }).status).toBe(0); + expect( + deps.dockerRename?.("container-a", "container-a-backup", { ignoreError: true }).status, + ).toBe(0); + expect( + deps.dockerRunDetached?.(["--name", "container-a", IMAGE_ID], { + ignoreError: true, + }).status, + ).toBe(0); + expect(deps.dockerRm?.("container-a-backup", { ignoreError: true }).status).toBe(0); + expect(deps.dockerStart?.("container-a", { ignoreError: true }).status).toBe(0); + expect(deps.dockerLogs?.("container-a", { tail: 20 })).toBe("engine-a logs"); + expect(runtime.dockerDesktopWsl()).toBe(true); + expect(runtime.dockerDesktopWsl()).toBe(true); + expect(runtime.ensureImageCached("busybox:test")).toEqual({ + ok: true, + alreadyCached: true, + }); + + expect(calls.length).toBeGreaterThan(0); + expect(calls.every(({ options }) => options.env === image.dockerEnv)).toBe(true); + expect(calls.some(({ options }) => options.env?.DOCKER_CONTEXT === "engine-b")).toBe(false); + expect(process.env.DOCKER_CONTEXT).toBe("engine-b"); + expect(image.verifyForCreate).toHaveBeenCalledTimes(calls.length); + }); + + it("rejects engine drift before the first mutation after an engine-A query", () => { + let engineId = "engine-a-id"; + const image = createImage(() => engineId === "engine-a-id"); + const runDocker = vi.fn((_args: readonly string[], _options: SpawnSyncOptions = {}) => + dockerResult("container-a\n"), + ); + const runtime = createRetainedOpenClawDockerRuntime(image, { + runDocker: runDocker as typeof dockerSpawnSync, + platform: "linux", + env: { WSL_DISTRO_NAME: "Ubuntu" }, + }); + + expect(runtime.deps.dockerCapture?.(["ps", "-a"], { ignoreError: true })).toBe("container-a"); + engineId = "engine-b-id"; + + expect(() => runtime.deps.dockerStop?.("container-a", { ignoreError: true })).toThrow( + "Retained OpenClaw rebuild Docker engine or image changed before a Docker operation.", + ); + expect(runDocker).toHaveBeenCalledOnce(); + expect(runDocker.mock.calls[0]?.[0]).toEqual(["ps", "-a"]); + }); + + it("replaces an ambient native plan with engine A's Docker Desktop WSL route", () => { + vi.stubEnv("DOCKER_CONTEXT", "engine-b"); + vi.stubEnv("WSL_DISTRO_NAME", "Ubuntu"); + const image = createImage(); + const runDocker = vi.fn((args: readonly string[], _options: SpawnSyncOptions = {}) => + args.join(" ") === "info --format {{json .OperatingSystem}}" + ? dockerResult('"Docker Desktop"\n') + : dockerResult(), + ); + const runtime = createRetainedOpenClawDockerRuntime(image, { + runDocker: runDocker as typeof dockerSpawnSync, + platform: "linux", + env: { WSL_DISTRO_NAME: "Ubuntu" }, + }); + const ambientIntent = createIntent(); + + const boundIntent = bindRetainedOpenClawGpuRoute( + ambientIntent, + { sandboxGpuEnabled: true, hostGpuDetected: true }, + true, + runtime, + { + env: { NEMOCLAW_DOCKER_GPU_PATCH: "0" }, + platform: "linux", + }, + ); + + expect(ambientIntent.gpuRoutePlan).toBe("native-only"); + expect(boundIntent.gpuRoutePlan).toBe("compatibility-only"); + expect(boundIntent.sandboxGpuLogMessage).toContain("Docker-driver GPU patch active"); + expect(runDocker).toHaveBeenCalledOnce(); + expect(runDocker.mock.calls[0]?.[1]?.env).toBe(image.dockerEnv); + expect(process.env.DOCKER_CONTEXT).toBe("engine-b"); + }); +}); diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index e687681e3f4..2d088555d21 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -10,6 +10,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { loadAgent } from "../agent/defs"; import { SANDBOX_BUILD_CONTEXT_PREFIX } from "../sandbox/build-context"; +import type { PreparedOpenClawLegacyImage } from "./build-context-stage"; import { createOpenshellCliHelpers } from "./openshell-cli"; import { buildSandboxRuntimeEnvArgs, @@ -21,6 +22,28 @@ const disabledHermesDashboardState = { config: null, enabled: false }; const IMAGE_ID = `sha256:${"a".repeat(64)}`; const temporaryBuildContexts: string[] = []; +function createPreparedOpenClawLegacyImageFixture(): PreparedOpenClawLegacyImage { + return { + dockerEnv: { + DOCKER_HOST: "unix:///captured/docker.sock", + DOCKER_CONTEXT: "captured-context", + DOCKER_CONFIG: "/captured/docker-config", + }, + engineId: "retained-engine", + imageRef: "nemoclaw-sandbox-local:retained-openclaw", + imageId: IMAGE_ID, + verify: vi.fn(() => true), + retainForRecreate: vi.fn(() => true), + verifyForCreate: vi.fn(() => true), + finalizeAfterCreate: vi.fn(() => ({ + mutableTagVerified: true, + registryImageRef: null, + })), + abort: vi.fn(() => true), + dispose: vi.fn(() => true), + }; +} + function createTrustedBuildContext(): string { const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), SANDBOX_BUILD_CONTEXT_PREFIX)); temporaryBuildContexts.push(buildCtx); @@ -395,6 +418,74 @@ describe("prepareSandboxCreateLaunchWithPrebuild", () => { expect(buildImage).toHaveBeenCalledOnce(); }); + it("launches a retained image by immutable ID on its captured Docker selector", async () => { + const buildCtx = createTrustedBuildContext(); + const dockerfile = path.join(buildCtx, "Dockerfile"); + const preparedOpenClawLegacyImage = createPreparedOpenClawLegacyImageFixture(); + const buildImage = vi.fn(async () => 0); + const result = await prepareSandboxCreateLaunchWithPrebuild({ + agent: null, + chatUiUrl: "", + createArgs: ["--from", dockerfile, "--name", "demo"], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: disabledHermesDashboardState, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + openshellArgv: (args) => ["openshell", ...args], + sandboxName: "demo", + buildEnv: () => ({ + HOME: "/ambient/home", + DOCKER_HOST: "ssh://ambient-engine", + DOCKER_CONTEXT: "ambient-context", + DOCKER_CONFIG: "/ambient/docker-config", + DOCKER_CERT_PATH: "/ambient/certs", + DOCKER_TLS_VERIFY: "1", + DOCKER_API_VERSION: "1.41", + }), + prebuild: { + buildCtx, + buildId: "build-123", + dockerDriverGateway: true, + env: { NEMOCLAW_SANDBOX_PREBUILD: "0" }, + buildImage, + origin: "generated", + preparedOpenClawLegacyImage, + }, + }); + + expect(result.prebuild).toEqual({ + createArgs: ["--from", IMAGE_ID, "--name", "demo"], + imageRef: preparedOpenClawLegacyImage.imageRef, + imageId: IMAGE_ID, + preparedOpenClawLegacyImage, + }); + expect(result.createCommand).toContain(`sandbox create --from ${IMAGE_ID} --name demo`); + expect(result.createArgv).toEqual([ + "openshell", + "sandbox", + "create", + "--from", + IMAGE_ID, + "--name", + "demo", + "--", + "env", + ...result.envArgs, + "nemoclaw-start", + ]); + expect(result.sandboxEnv).toEqual({ + HOME: "/ambient/home", + DOCKER_HOST: "unix:///captured/docker.sock", + DOCKER_CONTEXT: "captured-context", + DOCKER_CONFIG: "/captured/docker-config", + }); + expect(buildImage).not.toHaveBeenCalled(); + expect(preparedOpenClawLegacyImage.verifyForCreate).not.toHaveBeenCalled(); + expect(preparedOpenClawLegacyImage.finalizeAfterCreate).not.toHaveBeenCalled(); + }); + it("renders the original Dockerfile for Hermes after a local build failure", async () => { const buildCtx = createTrustedBuildContext(); const dockerfile = path.join(buildCtx, "Dockerfile"); diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index 1e5431bb479..a42b62e0ce6 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -11,6 +11,7 @@ import { appendHermesDashboardEnvArgs } from "./hermes-dashboard"; import { appendHostProxyEnvArgs } from "./host-proxy-env"; import { appendOpenClawRuntimeEnvArgs } from "./openclaw-runtime-env"; import { + DOCKER_SELECTOR_ENV_NAMES, prebuildSandboxImageIfEligible, type SandboxPrebuildInput, type SandboxPrebuildResult, @@ -55,6 +56,7 @@ export interface SandboxCreateLaunchInput { openshellShellCommand: OpenshellShellCommand; openshellArgv?: OpenshellArgv; buildEnv?(): Record; + dockerSelectorEnv?: Readonly>; } export interface SandboxCreateLaunch { @@ -189,6 +191,13 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San // permits for host-side processes but that must not enter the sandbox. delete sandboxEnv.KUBECONFIG; delete sandboxEnv.SSH_AUTH_SOCK; + if (input.dockerSelectorEnv) { + for (const key of DOCKER_SELECTOR_ENV_NAMES) { + delete sandboxEnv[key]; + const value = input.dockerSelectorEnv[key]; + if (value !== undefined) sandboxEnv[key] = value; + } + } // Run without piping through awk; the pipe masked non-zero exit codes // from openshell because bash returns the status of the last pipeline @@ -226,7 +235,11 @@ export async function prepareSandboxCreateLaunchWithPrebuild( sandboxName: input.sandboxName, }); return { - ...prepareSandboxCreateLaunch({ ...launchInput, createArgs: prebuild.createArgs }), + ...prepareSandboxCreateLaunch({ + ...launchInput, + createArgs: prebuild.createArgs, + dockerSelectorEnv: prebuild.preparedOpenClawLegacyImage?.dockerEnv, + }), prebuild, }; } diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index f11e1e21142..c71feba73c5 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -59,7 +59,9 @@ import { setupGpuFlowMocks, VERIFIED_GPU_PROOF as VERIFIED_PROOF, } from "./__test-helpers__/sandbox-gpu-create-flow"; +import type { PreparedOpenClawLegacyImage } from "./build-context-stage"; import { + resolveCreatedSandboxRegistryImageRef, runSandboxGpuCreateFlow, type SandboxGpuCreateFlowDeps, type SandboxGpuCreateFlowInput, @@ -79,6 +81,7 @@ const NVIDIA_SMI_FAILED_PROOF: SandboxGpuProofResult = { detail: "Failed to initialize NVML: Driver/library version mismatch", at: "2026-07-06T00:00:00.000Z", }; +const OTHER_IMAGE_ID = `sha256:${"b".repeat(64)}`; const DEFAULT_RUNTIME_SNAPSHOT = { ok: true as const, imageId: IMAGE_ID, @@ -143,9 +146,280 @@ function createSourceInput(): SandboxGpuCreateFlowInput { return input; } +function createRetainedImageInput() { + const input = createInput(); + const finalizeAfterCreate = vi.fn(() => ({ + mutableTagVerified: true, + registryImageRef: null, + })); + const preparedOpenClawLegacyImage = { + dockerEnv: { DOCKER_CONTEXT: "retained-context" }, + engineId: "retained-engine", + imageRef: "nemoclaw-sandbox-local:retained-openclaw", + imageId: IMAGE_ID, + verify: vi.fn(() => true), + retainForRecreate: vi.fn(() => true), + verifyForCreate: vi.fn(() => true), + finalizeAfterCreate, + abort: vi.fn(() => true), + dispose: vi.fn(() => true), + } satisfies PreparedOpenClawLegacyImage; + input.createArgv = [ + "openshell", + "sandbox", + "create", + "--from", + IMAGE_ID, + "--name", + "alpha", + "--gpu", + ]; + input.prebuild = { + createArgs: ["--from", IMAGE_ID, "--name", "alpha", "--gpu"], + imageRef: preparedOpenClawLegacyImage.imageRef, + imageId: IMAGE_ID, + preparedOpenClawLegacyImage, + }; + return { input, preparedOpenClawLegacyImage }; +} + beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); +describe("runSandboxGpuCreateFlow retained OpenClaw image", () => { + it("verifies immediately before streaming and finalizes only after final success", async () => { + const { input, preparedOpenClawLegacyImage } = createRetainedImageInput(); + + await expect(runSandboxGpuCreateFlow(input, createDeps())).resolves.toMatchObject({ + retainedImageFinalization: { + mutableTagVerified: true, + registryImageRef: null, + }, + route: "native", + }); + + expect(preparedOpenClawLegacyImage.verifyForCreate).toHaveBeenCalledOnce(); + expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); + expect(preparedOpenClawLegacyImage.finalizeAfterCreate).toHaveBeenCalledOnce(); + expect(preparedOpenClawLegacyImage.abort).not.toHaveBeenCalled(); + expect(mocks.streamSandboxCreate.mock.invocationCallOrder[0]).toBe( + preparedOpenClawLegacyImage.verifyForCreate.mock.invocationCallOrder[0] + 1, + ); + expect( + preparedOpenClawLegacyImage.finalizeAfterCreate.mock.invocationCallOrder[0], + ).toBeGreaterThan(mocks.waitForCreatedSandboxReadyWithTrace.mock.invocationCallOrder[0]); + expect( + preparedOpenClawLegacyImage.finalizeAfterCreate.mock.invocationCallOrder[0], + ).toBeGreaterThan(mocks.verifyGpuSandboxAccessAfterReady.mock.invocationCallOrder[0]); + expect(mocks.streamSandboxCreate).toHaveBeenCalledWith( + "openshell", + expect.arrayContaining(["--from", IMAGE_ID]), + input.sandboxEnv, + expect.any(Object), + ); + }); + + it("reverifies for compatibility and finalizes only after the final create succeeds", async () => { + const { input, preparedOpenClawLegacyImage } = createRetainedImageInput(); + failNativeCreate(); + + await expect(runSandboxGpuCreateFlow(input, createDeps())).resolves.toMatchObject({ + route: "compatibility", + }); + + expect(preparedOpenClawLegacyImage.verifyForCreate).toHaveBeenCalledTimes(2); + expect(mocks.streamSandboxCreate).toHaveBeenCalledTimes(2); + expect(preparedOpenClawLegacyImage.finalizeAfterCreate).toHaveBeenCalledOnce(); + const verifyOrders = preparedOpenClawLegacyImage.verifyForCreate.mock.invocationCallOrder; + const streamOrders = mocks.streamSandboxCreate.mock.invocationCallOrder; + const finalizeOrder = + preparedOpenClawLegacyImage.finalizeAfterCreate.mock.invocationCallOrder[0]; + expect(streamOrders[0]).toBe(verifyOrders[0] + 1); + expect(streamOrders[1]).toBe(verifyOrders[1] + 1); + expect(streamOrders[0]).toBeLessThan(verifyOrders[1]); + expect(streamOrders[1]).toBeLessThan(finalizeOrder); + expect(mocks.streamSandboxCreate).toHaveBeenNthCalledWith( + 2, + "openshell", + expect.arrayContaining(["--from", IMAGE_ID]), + input.sandboxEnv, + expect.any(Object), + ); + }); + + it("refuses a native snapshot for image B before rendering or creating compatibility", async () => { + const { input, preparedOpenClawLegacyImage } = createRetainedImageInput(); + mockRuntimeSnapshot({ + imageId: OTHER_IMAGE_ID, + stateError: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + }); + mocks.streamSandboxCreate.mockResolvedValueOnce({ + status: 1, + output: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + sawProgress: true, + }); + const deps = createDeps(); + + await expectFlowExit(input, deps); + + expect(mocks.queryOpenShellDockerSandboxRuntimeSnapshot).toHaveBeenCalledOnce(); + expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); + expect(mocks.streamSandboxCreate.mock.calls.flat()).not.toContain(OTHER_IMAGE_ID); + expect(deps.openshellArgv).not.toHaveBeenCalled(); + expect(errorOutput()).toContain( + "Native sandbox image identity does not match the retained OpenClaw rebuild image", + ); + expect(preparedOpenClawLegacyImage.abort).toHaveBeenCalledOnce(); + }); + + it("uses retained image A when the native snapshot corroborates A", async () => { + const { input } = createRetainedImageInput(); + mockRuntimeSnapshot({ + imageId: IMAGE_ID, + stateError: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + }); + mocks.streamSandboxCreate.mockResolvedValueOnce({ + status: 1, + output: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + sawProgress: true, + }); + const deps = createDeps(); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ + route: "compatibility", + }); + + expect(mocks.queryOpenShellDockerSandboxRuntimeSnapshot).toHaveBeenCalledOnce(); + expect(deps.openshellArgv).toHaveBeenCalledWith(expect.arrayContaining(["--from", IMAGE_ID])); + expect(mocks.streamSandboxCreate).toHaveBeenNthCalledWith( + 2, + "openshell", + expect.arrayContaining(["--from", IMAGE_ID]), + input.sandboxEnv, + expect.any(Object), + ); + }); + + it("uses engine A for create, fallback evidence, cleanup, and patch mutation while ambient selects B", async () => { + vi.stubEnv("DOCKER_CONTEXT", "engine-b"); + const { input } = createRetainedImageInput(); + input.sandboxEnv = { DOCKER_CONTEXT: "engine-a" }; + failNativeCreate(); + const engineAQueries: string[][] = []; + const engineAMutations: string[][] = []; + const engineBCapture = vi.fn(() => ""); + const engineBRun = vi.fn(() => ({ status: 0, stdout: "" })); + const engineBMutations = vi.fn(() => ({ status: 0 })); + const boundDockerDeps = { + dockerCapture: vi.fn((args: readonly string[]) => { + engineAQueries.push([...args]); + return ""; + }), + dockerRun: vi.fn((args: readonly string[]) => { + engineAQueries.push([...args]); + return { status: 0, stdout: "" }; + }), + dockerStop: vi.fn((containerName: string) => { + engineAMutations.push(["stop", containerName]); + return { status: 0 }; + }), + }; + const deps = createDeps(); + deps.dockerCapture = engineBCapture; + deps.dockerRun = engineBRun; + deps.dockerStop = engineBMutations; + deps.createRetainedDockerRuntime = vi.fn(() => ({ + deps: boundDockerDeps, + dockerDesktopWsl: () => false, + ensureImageCached: vi.fn(() => ({ ok: true, alreadyCached: true })), + reverifyBridgeReachability: vi.fn(async () => {}), + })); + mocks.queryOpenShellDockerSandboxContainers.mockImplementation( + (_sandboxName: string, dockerDeps: typeof boundDockerDeps) => { + dockerDeps.dockerRun(["ps", "-a"]); + return { ok: true, ids: [] }; + }, + ); + mocks.createDockerGpuSandboxCreatePatch.mockImplementation((options) => { + options.deps.dockerCapture(["inspect", "container-a"]); + options.deps.dockerStop("container-a"); + return createPatch(); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ + route: "compatibility", + }); + + expect(mocks.streamSandboxCreate).toHaveBeenCalledTimes(2); + expect( + mocks.streamSandboxCreate.mock.calls.every(([, , env]) => env === input.sandboxEnv), + ).toBe(true); + expect(input.sandboxEnv).toEqual({ DOCKER_CONTEXT: "engine-a" }); + expect(process.env.DOCKER_CONTEXT).toBe("engine-b"); + expect(engineAQueries).toContainEqual(["ps", "-a"]); + expect(engineAQueries).toContainEqual(["inspect", "container-a"]); + expect(engineAMutations).toContainEqual(["stop", "container-a"]); + expect(engineBCapture).not.toHaveBeenCalled(); + expect(engineBRun).not.toHaveBeenCalled(); + expect(engineBMutations).not.toHaveBeenCalled(); + }); + + it("fails before create instead of falling back when retained image verification fails", async () => { + const { input, preparedOpenClawLegacyImage } = createRetainedImageInput(); + preparedOpenClawLegacyImage.verifyForCreate.mockReturnValue(false); + + await expect(runSandboxGpuCreateFlow(input, createDeps())).rejects.toThrow( + "Retained OpenClaw rebuild image changed before sandbox creation.", + ); + + expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); + expect(preparedOpenClawLegacyImage.finalizeAfterCreate).not.toHaveBeenCalled(); + expect(preparedOpenClawLegacyImage.abort).toHaveBeenCalledOnce(); + }); + + it("aborts a retained image after a failed final create", async () => { + const { input, preparedOpenClawLegacyImage } = createRetainedImageInput(); + mocks.streamSandboxCreate.mockResolvedValueOnce({ + status: 1, + output: "x509: certificate signed by unknown authority", + sawProgress: true, + }); + mockExit(); + + await expect(runSandboxGpuCreateFlow(input, createDeps())).rejects.toThrow("process.exit:1"); + + expect(preparedOpenClawLegacyImage.verifyForCreate).toHaveBeenCalledOnce(); + expect(preparedOpenClawLegacyImage.finalizeAfterCreate).not.toHaveBeenCalled(); + expect(preparedOpenClawLegacyImage.abort).toHaveBeenCalledOnce(); + }); + + it("aborts when post-create finalization cannot prove the bound engine", async () => { + const { input, preparedOpenClawLegacyImage } = createRetainedImageInput(); + preparedOpenClawLegacyImage.finalizeAfterCreate.mockReturnValue(null); + + await expect(runSandboxGpuCreateFlow(input, createDeps())).rejects.toThrow( + "Retained OpenClaw rebuild image could not be finalized after creation.", + ); + + expect(preparedOpenClawLegacyImage.finalizeAfterCreate).toHaveBeenCalledOnce(); + expect(preparedOpenClawLegacyImage.abort).toHaveBeenCalledOnce(); + }); + + it("keeps retained-image cleanup suppressed when a later ambient Docker context differs", () => { + const retainedImageFinalization = { + mutableTagVerified: true, + registryImageRef: null, + } as const; + + expect( + resolveCreatedSandboxRegistryImageRef(retainedImageFinalization, [ + "nemoclaw-sandbox-local:retargeted-on-ambient-engine", + "openshell/sandbox-from:captured-output", + ]), + ).toBeNull(); + }); +}); + describe("runSandboxGpuCreateFlow proof authorization", () => { it("does not retry compatibility when the native proof throws an exec/policy error (#6110)", async () => { const deps = createDeps(); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 98574891e0b..f6438be45c3 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -4,6 +4,7 @@ import type { StreamSandboxCreateResult } from "../sandbox/create-stream"; import { redactFull } from "../security/redact"; import type { SandboxGpuProofResult } from "../state/registry"; +import type { PreparedOpenClawLegacyImageFinalization } from "./build-context-stage"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import { collectDockerGpuPatchDiagnostics } from "./docker-gpu-patch"; import type { DockerGpuPatchDeps, DockerUlimit } from "./docker-gpu-patch-types"; @@ -11,7 +12,14 @@ import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { renderCompatibilityFallbackCreateArgs } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; import type { DockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; -import { isImmutableDockerImageId } from "./openshell-docker-sandbox-containers"; +import { + isImmutableDockerImageId, + queryOpenShellDockerSandboxContainers, +} from "./openshell-docker-sandbox-containers"; +import { + bindRetainedOpenClawGpuRoute, + createRetainedOpenClawDockerRuntime, +} from "./rebuild/retained-openclaw-docker-runtime"; import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; import { createSandboxGpuCreateAttemptRunner } from "./sandbox-gpu-create-run-attempt"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; @@ -19,6 +27,7 @@ import type { SandboxPrebuildResult } from "./sandbox-prebuild"; import { addTraceEvent } from "./tracing"; export { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; +export { bindRetainedOpenClawGpuRoute, createRetainedOpenClawDockerRuntime }; type RunOpenshell = NonNullable; type RunCaptureOpenshell = NonNullable; @@ -44,12 +53,13 @@ export interface SandboxGpuCreateFlowInput { requiredUlimits?: readonly DockerUlimit[] | null; } -export interface SandboxGpuCreateFlowDeps { +export interface SandboxGpuCreateFlowDeps extends DockerGpuPatchDeps { runOpenshell: RunOpenshell; runCaptureOpenshell: RunCaptureOpenshell; sleep: Sleep; openshellArgv(args: string[]): string[]; verifyDirectSandboxGpu(sandboxName: string): SandboxGpuProofResult; + createRetainedDockerRuntime?: typeof createRetainedOpenClawDockerRuntime; } export interface SandboxGpuCreateFlowResult { @@ -59,6 +69,58 @@ export interface SandboxGpuCreateFlowResult { firstCreateOutput: string; /** Mutable tag/reference retained only for registry and image-GC bookkeeping. */ registryImageRef: string | null; + /** Authoritative retained-image result, including deliberate cleanup suppression. */ + retainedImageFinalization: PreparedOpenClawLegacyImageFinalization | null; +} + +/** Do not infer tag cleanup when retained-image finalization suppresses it. */ +export function resolveCreatedSandboxRegistryImageRef( + retainedImageFinalization: PreparedOpenClawLegacyImageFinalization | null, + fallbackImageRefs: readonly (string | null | undefined)[], +): string | null { + if (retainedImageFinalization) return retainedImageFinalization.registryImageRef; + return fallbackImageRefs.find((imageRef): imageRef is string => Boolean(imageRef)) ?? null; +} + +function resolveCompatibilityRetryImageId( + prebuild: SandboxPrebuildResult, + nativeSnapshotImageId: string | null, +): string | null { + const prebuildImageId = + prebuild.imageId && isImmutableDockerImageId(prebuild.imageId) + ? prebuild.imageId.toLowerCase() + : null; + const retainedImage = prebuild.preparedOpenClawLegacyImage; + if (!retainedImage) return nativeSnapshotImageId ?? prebuildImageId; + + const retainedImageId = isImmutableDockerImageId(retainedImage.imageId) + ? retainedImage.imageId.toLowerCase() + : null; + if (!retainedImageId || prebuildImageId !== retainedImageId) { + throw new Error("Retained OpenClaw rebuild image identity changed before compatibility retry."); + } + if ( + nativeSnapshotImageId !== null && + (!isImmutableDockerImageId(nativeSnapshotImageId) || + nativeSnapshotImageId.toLowerCase() !== retainedImageId) + ) { + throw new Error( + "Native sandbox image identity does not match the retained OpenClaw rebuild image.", + ); + } + return retainedImageId; +} + +function abortUnusedRetainedImage(input: SandboxGpuCreateFlowInput): void { + const preparedImage = input.prebuild.preparedOpenClawLegacyImage; + if (!preparedImage) return; + try { + if (!preparedImage.abort()) { + console.warn(" Warning: the unused retained OpenClaw rebuild image could not be released."); + } + } catch { + console.warn(" Warning: the unused retained OpenClaw rebuild image could not be released."); + } } /** @@ -77,79 +139,117 @@ export async function runSandboxGpuCreateFlow( input: SandboxGpuCreateFlowInput, deps: SandboxGpuCreateFlowDeps, ): Promise { + const { retainedDockerRuntime, effectiveDeps, attemptRunner } = (() => { + try { + const retainedDockerRuntime = input.prebuild.preparedOpenClawLegacyImage + ? (deps.createRetainedDockerRuntime ?? createRetainedOpenClawDockerRuntime)( + input.prebuild.preparedOpenClawLegacyImage, + ) + : null; + const effectiveDeps: SandboxGpuCreateFlowDeps = retainedDockerRuntime + ? { ...deps, ...retainedDockerRuntime.deps } + : deps; + return { + retainedDockerRuntime, + effectiveDeps, + attemptRunner: createSandboxGpuCreateAttemptRunner( + input, + effectiveDeps, + retainedDockerRuntime?.dockerDesktopWsl(), + ), + }; + } catch (error) { + abortUnusedRetainedImage(input); + throw error; + } + })(); let registryImageRef: string | null = input.prebuild.imageRef; - const attemptRunner = createSandboxGpuCreateAttemptRunner(input, deps); - const gpuCreateOutcome = await sandboxGpuCreateAttempt.executeSandboxGpuCreatePlan( - input.gpuRoutePlan, - { - runAttempt: attemptRunner.runAttempt, - captureNativeFailure: (failure) => { - const routeAdapter = adaptDockerGpuRouteForPatch(failure.route); - const diagnostics = collectDockerGpuPatchDiagnostics( - input.sandboxName, - { - error: failure.error, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - }, - { runCaptureOpenshell: deps.runCaptureOpenshell }, - ); - if (diagnostics) console.error(` Native GPU diagnostics saved: ${diagnostics.dir}`); - }, - cleanupNativeFailure: () => - sandboxGpuCreateAttempt.cleanupNativeGpuAttemptForFallback(input.sandboxName, { - runOpenshell: deps.runOpenshell, - sleep: deps.sleep, - }), - prepareCompatibilityAttempt: async () => { - if (!input.compatibilityPolicyPath) { - throw new Error("Compatibility retry policy was not materialized."); - } - const nativeRuntimeSnapshot = attemptRunner.state.nativeRuntimeSnapshot; - const prebuildImageId = input.prebuild.imageId; - const imageId = - nativeRuntimeSnapshot?.imageId ?? - (prebuildImageId && isImmutableDockerImageId(prebuildImageId) - ? prebuildImageId.toLowerCase() - : null); - if ( - !registryImageRef && - nativeRuntimeSnapshot?.bookkeepingImageRef && - !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) - ) { - registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; - } - const compatibilityArgs = renderCompatibilityFallbackCreateArgs(input.prebuild.createArgs, { - imageRef: imageId, - allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, - compatibilityPolicyPath: input.compatibilityPolicyPath, - }); - attemptRunner.state.compatibilityArgv = deps.openshellArgv([ - "sandbox", - "create", - ...compatibilityArgs, - "--", - ...input.sandboxStartupCommand, - ]); - if (attemptRunner.state.compatibilityArgv.length === 0) { - throw new Error("Compatibility sandbox create executable is missing."); - } + let gpuCreateOutcome: sandboxGpuCreateAttempt.SandboxGpuCreatePlanResult<{ + createResult: StreamSandboxCreateResult; + dockerGpuCreatePatch: DockerGpuSandboxCreatePatch; + }>; + try { + gpuCreateOutcome = await sandboxGpuCreateAttempt.executeSandboxGpuCreatePlan( + input.gpuRoutePlan, + { + runAttempt: attemptRunner.runAttempt, + captureNativeFailure: (failure) => { + const routeAdapter = adaptDockerGpuRouteForPatch(failure.route); + const diagnostics = collectDockerGpuPatchDiagnostics( + input.sandboxName, + { + error: failure.error, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }, + effectiveDeps, + ); + if (diagnostics) console.error(` Native GPU diagnostics saved: ${diagnostics.dir}`); + }, + cleanupNativeFailure: () => + sandboxGpuCreateAttempt.cleanupNativeGpuAttemptForFallback(input.sandboxName, { + runOpenshell: effectiveDeps.runOpenshell, + queryContainers: (sandboxName) => + queryOpenShellDockerSandboxContainers(sandboxName, effectiveDeps), + sleep: effectiveDeps.sleep, + }), + prepareCompatibilityAttempt: async () => { + if (!input.compatibilityPolicyPath) { + throw new Error("Compatibility retry policy was not materialized."); + } + const nativeRuntimeSnapshot = attemptRunner.state.nativeRuntimeSnapshot; + const imageId = resolveCompatibilityRetryImageId( + input.prebuild, + nativeRuntimeSnapshot?.imageId ?? null, + ); + if ( + !registryImageRef && + nativeRuntimeSnapshot?.bookkeepingImageRef && + !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) + ) { + registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; + } + const compatibilityArgs = renderCompatibilityFallbackCreateArgs( + input.prebuild.createArgs, + { + imageRef: imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + }, + ); + attemptRunner.state.compatibilityArgv = effectiveDeps.openshellArgv([ + "sandbox", + "create", + ...compatibilityArgs, + "--", + ...input.sandboxStartupCommand, + ]); + if (attemptRunner.state.compatibilityArgv.length === 0) { + throw new Error("Compatibility sandbox create executable is missing."); + } + }, + activateCompatibilityAttempt: async () => { + await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( + input.provider, + input.sandboxGpuConfig, + { + dockerDriverGateway: input.dockerDriverGateway, + selectedRoute: "compatibility", + gatewayPort: input.gatewayPort, + log: console.log, + reverifyBridgeReachability: retainedDockerRuntime + ? () => retainedDockerRuntime.reverifyBridgeReachability(input.gatewayPort) + : undefined, + }, + ); + input.sandboxGpuConfig.sandboxGpuProof = null; + }, + traceEvent: addTraceEvent, }, - activateCompatibilityAttempt: async () => { - await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( - input.provider, - input.sandboxGpuConfig, - { - dockerDriverGateway: input.dockerDriverGateway, - selectedRoute: "compatibility", - gatewayPort: input.gatewayPort, - log: console.log, - }, - ); - input.sandboxGpuConfig.sandboxGpuProof = null; - }, - traceEvent: addTraceEvent, - }, - ); + ); + } catch (error) { + abortUnusedRetainedImage(input); + throw error; + } if (!gpuCreateOutcome.ok) { console.error(""); console.error(" Operator-authorized GPU fallback stopped before compatibility retry."); @@ -164,13 +264,22 @@ export async function runSandboxGpuCreateFlow( ); } console.error(` Manual cleanup: openshell sandbox delete "${input.sandboxName}"`); + abortUnusedRetainedImage(input); process.exit(1); } + const retainedImageFinalization = + input.prebuild.preparedOpenClawLegacyImage?.finalizeAfterCreate() ?? null; + if (input.prebuild.preparedOpenClawLegacyImage && !retainedImageFinalization) { + abortUnusedRetainedImage(input); + throw new Error("Retained OpenClaw rebuild image could not be finalized after creation."); + } + return { ...gpuCreateOutcome.value, route: gpuCreateOutcome.route, firstCreateOutput: attemptRunner.state.firstCreateOutput, registryImageRef, + retainedImageFinalization, }; } diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 726758cc0ac..edf5ba6396a 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -44,6 +44,7 @@ const COMPATIBILITY_STABLE_READY_POLLS = 2; export function createSandboxGpuCreateAttemptRunner( input: SandboxGpuCreateFlowInput, deps: SandboxGpuCreateFlowDeps, + dockerDesktopWsl?: boolean, ) { const state: SandboxGpuCreateAttemptState = { firstCreateOutput: "", @@ -53,11 +54,12 @@ export function createSandboxGpuCreateAttemptRunner( }; const nativeFallbackBaseline = input.initialGpuRoute === "native" && input.gpuRoutePlan === "native-with-fallback" - ? queryOpenShellDockerSandboxContainers(input.sandboxName) + ? queryOpenShellDockerSandboxContainers(input.sandboxName, deps) : null; const nativeFallbackHasCleanBaseline = nativeFallbackBaseline?.ok === true && nativeFallbackBaseline.ids.length === 0; - const inspectNativeRuntime = () => queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName); + const inspectNativeRuntime = () => + queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName, deps); const runAttempt = async (route: SelectedDockerGpuRoute) => { const compatibility = route === "compatibility"; @@ -83,11 +85,16 @@ export function createSandboxGpuCreateAttemptRunner( requiredUlimits: input.requiredUlimits, timeoutSecs: input.sandboxReadyTimeoutSecs, backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", + dockerDesktopWsl, deps, }); const attemptArgv = state.compatibilityArgv ?? input.createArgv; const [createExecutable, ...createExecutableArgs] = attemptArgv; if (!createExecutable) throw new Error("Sandbox create executable is missing."); + const preparedImage = input.prebuild.preparedOpenClawLegacyImage; + if (preparedImage && !preparedImage.verifyForCreate()) { + throw new Error("Retained OpenClaw rebuild image changed before sandbox creation."); + } const createResult = await streamSandboxCreate( createExecutable, createExecutableArgs, @@ -253,6 +260,8 @@ export function createSandboxGpuCreateAttemptRunner( reportGpuProofFailure: !deferNativeProofFailure, selectedMode: dockerGpuCreatePatch.selectedMode, runCaptureOpenshell: deps.runCaptureOpenshell, + dockerCapture: deps.dockerCapture, + dockerLogs: deps.dockerLogs, log: console.log, }, ); diff --git a/src/lib/onboard/sandbox-prebuild.test.ts b/src/lib/onboard/sandbox-prebuild.test.ts index 961a891f90d..eaf79f9a5f4 100644 --- a/src/lib/onboard/sandbox-prebuild.test.ts +++ b/src/lib/onboard/sandbox-prebuild.test.ts @@ -17,6 +17,7 @@ vi.mock("../adapters/docker/exec", async (importOriginal) => ({ import { withStdoutRedirectedToStderr } from "../cli/stdout-guard"; import { SANDBOX_BUILD_CONTEXT_PREFIX } from "../sandbox/build-context"; +import type { PreparedOpenClawLegacyImage } from "./build-context-stage"; import { dockerBuildSubprocessEnv, prebuildSandboxImageIfEligible, @@ -28,6 +29,24 @@ const BUILD_ID = "1234567890"; const IMAGE_ID = `sha256:${"a".repeat(64)}`; const temporaryDirectories: string[] = []; +function createPreparedOpenClawLegacyImageFixture(): PreparedOpenClawLegacyImage { + return { + dockerEnv: { DOCKER_CONTEXT: "retained-context" }, + engineId: "retained-engine", + imageRef: "nemoclaw-sandbox-local:retained-openclaw", + imageId: IMAGE_ID, + verify: vi.fn(() => true), + retainForRecreate: vi.fn(() => true), + verifyForCreate: vi.fn(() => true), + finalizeAfterCreate: vi.fn(() => ({ + mutableTagVerified: true, + registryImageRef: null, + })), + abort: vi.fn(() => true), + dispose: vi.fn(() => true), + }; +} + function createBuildContext( parent = os.tmpdir(), prefix = SANDBOX_BUILD_CONTEXT_PREFIX, @@ -327,6 +346,59 @@ describe("sandbox BuildKit prebuild", () => { expect(log).toHaveBeenCalledWith(expect.stringContaining("could not be inspected")); }); + it("reuses a retained OpenClaw image by immutable ID without rebuilding", async () => { + const { buildCtx, createArgs } = createBuildContext(); + const preparedOpenClawLegacyImage = createPreparedOpenClawLegacyImageFixture(); + const buildImage = vi.fn(async () => 0); + + await expect( + prebuildSandboxImageIfEligible({ + buildCtx, + buildId: BUILD_ID, + origin: "generated", + createArgs, + sandboxName: "alpha", + dockerDriverGateway: true, + preparedOpenClawLegacyImage, + env: { NEMOCLAW_SANDBOX_PREBUILD: "0" }, + buildImage, + }), + ).resolves.toEqual({ + createArgs: ["--from", IMAGE_ID, "--name", "alpha"], + imageRef: preparedOpenClawLegacyImage.imageRef, + imageId: IMAGE_ID, + preparedOpenClawLegacyImage, + }); + expect(buildImage).not.toHaveBeenCalled(); + expect(preparedOpenClawLegacyImage.verifyForCreate).not.toHaveBeenCalled(); + expect(preparedOpenClawLegacyImage.finalizeAfterCreate).not.toHaveBeenCalled(); + }); + + it.each([ + ["a nonlocal gateway", { dockerDriverGateway: false }], + ["a custom Dockerfile", { origin: "custom" as const }], + ["changed create arguments", { createArgs: ["--from", "/tmp/changed/Dockerfile"] }], + ])("fails closed instead of rebuilding or falling back after %s", async (_label, override) => { + const { buildCtx, createArgs } = createBuildContext(); + const buildImage = vi.fn(async () => 0); + + await expect( + prebuildSandboxImageIfEligible({ + buildCtx, + buildId: BUILD_ID, + origin: "generated", + createArgs, + sandboxName: "alpha", + dockerDriverGateway: true, + preparedOpenClawLegacyImage: createPreparedOpenClawLegacyImageFixture(), + env: { NEMOCLAW_SANDBOX_PREBUILD: "0" }, + buildImage, + ...override, + }), + ).rejects.toThrow(/retained OpenClaw rebuild image/i); + expect(buildImage).not.toHaveBeenCalled(); + }); + it("uses the argv-based Docker helper and returns the local image on success", async () => { const { buildCtx, createArgs, dockerfile } = createBuildContext(); const buildImage = vi.fn(async () => 0); diff --git a/src/lib/onboard/sandbox-prebuild.ts b/src/lib/onboard/sandbox-prebuild.ts index 888b094c1fe..b4da1d34202 100644 --- a/src/lib/onboard/sandbox-prebuild.ts +++ b/src/lib/onboard/sandbox-prebuild.ts @@ -14,12 +14,14 @@ import { type SandboxBuildContextOrigin, } from "../sandbox/build-context"; import { buildSubprocessEnv } from "../subprocess-env"; +import type { PreparedOpenClawLegacyImage } from "./build-context-stage"; import { isImmutableDockerImageId } from "./openshell-docker-sandbox-containers"; const TRUTHY_FLAG_VALUES = new Set(["1", "true", "yes", "on"]); const FALSY_FLAG_VALUES = new Set(["0", "false", "no", "off"]); const LOCAL_IMAGE_REPO = LOCAL_SANDBOX_IMAGE_REPO; -const DOCKER_ENV_NAMES = [ +export const DOCKER_SELECTOR_ENV_NAMES = [ + "DOCKER_HOST", "DOCKER_API_VERSION", "DOCKER_CERT_PATH", "DOCKER_CONFIG", @@ -34,6 +36,7 @@ export interface SandboxPrebuildInput { sandboxName: string; dockerDriverGateway: boolean; origin: SandboxBuildContextOrigin; + preparedOpenClawLegacyImage?: PreparedOpenClawLegacyImage; env?: NodeJS.ProcessEnv; buildImage?: ( args: readonly string[], @@ -48,6 +51,7 @@ export interface SandboxPrebuildResult { imageRef: string | null; /** Immutable local image identity; mutable tags never authorize fallback. */ imageId: string | null; + preparedOpenClawLegacyImage?: PreparedOpenClawLegacyImage; } interface TrustedStagedBuildContext { @@ -96,7 +100,7 @@ function resolveTrustedStagedBuildContext(buildCtx: string): TrustedStagedBuildC /** Restrict the host Docker build to environment values used by Docker itself. */ export function dockerBuildSubprocessEnv(): Record { const env = buildSubprocessEnv(); - for (const key of DOCKER_ENV_NAMES) { + for (const key of DOCKER_SELECTOR_ENV_NAMES) { const value = process.env[key]; if (value !== undefined) env[key] = value; } @@ -153,6 +157,38 @@ export async function prebuildSandboxImageIfEligible( input: SandboxPrebuildInput, ): Promise { const createArgs = [...input.createArgs]; + const preparedImage = input.preparedOpenClawLegacyImage; + if (preparedImage) { + if (!input.dockerDriverGateway) { + throw new Error( + "A retained OpenClaw rebuild image cannot be used by a nonlocal OpenShell gateway.", + ); + } + if (input.origin !== "generated") { + throw new Error("A retained OpenClaw rebuild image cannot be used with a custom Dockerfile."); + } + const fromIndexes = createArgs.flatMap((arg, index) => (arg === "--from" ? [index] : [])); + const fromIndex = fromIndexes[0] ?? -1; + const fromDockerfile = createArgs[fromIndex + 1]; + if ( + fromIndexes.length !== 1 || + !fromDockerfile || + path.resolve(fromDockerfile) !== path.resolve(input.buildCtx, "Dockerfile") + ) { + throw new Error("Retained OpenClaw rebuild image arguments changed before sandbox creation."); + } + if (!isImmutableDockerImageId(preparedImage.imageId)) { + throw new Error("Retained OpenClaw rebuild image identity is invalid."); + } + const imageId = preparedImage.imageId.toLowerCase(); + createArgs[fromIndex + 1] = imageId; + return { + createArgs, + imageRef: preparedImage.imageRef, + imageId, + preparedOpenClawLegacyImage: preparedImage, + }; + } const env = input.env ?? process.env; const log = input.log ?? console.log; if (!resolveSandboxPrebuildEnabled(env, input.dockerDriverGateway)) { diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index 325d6c99087..6ea5462f270 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -39,6 +39,9 @@ const registryPersistence = requireDist("../../state/registry/persistence.js"); const sandboxState = requireDist("../../state/sandbox.js"); const sandboxSession = requireDist("../../state/sandbox-session.js"); const sandboxVersion = requireDist("../../sandbox/version.js"); +const mcpLifecycleLock = requireDist( + "../../state/mcp-lifecycle-lock.js", +) as typeof import("../../src/lib/state/mcp-lifecycle-lock"); const destroy = requireDist("./destroy.js"); const rebuildShields = requireDist("./rebuild-shields.js"); const nim = requireDist("../../inference/nim.js"); @@ -137,7 +140,13 @@ export type RebuildFlowOverrides = { openShieldsWindow?: () => { relocked: boolean; wasLocked: boolean } | null; preflightMessagingConflicts?: () => Promise | void; preflightAuthoritativeRebuildTarget?: (options: Record) => Promise | void; - rebuildImagePreflightResult?: { ok: false; detail: string } | { ok: true; imageTag: null }; + rebuildImagePreflightResult?: + | { ok: false; detail: string } + | { + ok: true; + imageTag: string | null; + prepared?: unknown; + }; mcpPreparation?: { entries: Array>; detachedProviderEntries: Array>; @@ -288,6 +297,9 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined); + vi.spyOn(mcpLifecycleLock, "withMcpLifecycleLock").mockImplementation( + async (_sandboxName: string, work: () => Promise | T): Promise => await work(), + ); const session = createRebuildFlowSession(onboardSession.MACHINE_SNAPSHOT_VERSION); const rebuildShieldsWindow = { relocked: false, wasLocked: false }; diff --git a/test/helpers/rebuild-flow-target-image-cases.ts b/test/helpers/rebuild-flow-target-image-cases.ts index 7249cc78c43..5d5f1c04428 100644 --- a/test/helpers/rebuild-flow-target-image-cases.ts +++ b/test/helpers/rebuild-flow-target-image-cases.ts @@ -236,7 +236,7 @@ export function registerRebuildFlowTargetImageTests(): void { try { await expect( harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).rejects.toThrow("Replacement sandbox image context changed before delete"); + ).rejects.toThrow("Replacement sandbox image inputs changed before delete"); expectNoSandboxDelete(harness.runOpenshellSpy); expect(harness.onboardSpy).not.toHaveBeenCalled(); @@ -296,7 +296,7 @@ export function registerRebuildFlowTargetImageTests(): void { try { await expect( harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).rejects.toThrow("Replacement sandbox image context changed before delete"); + ).rejects.toThrow("Replacement sandbox image inputs changed before delete"); expectNoSandboxDelete(harness.runOpenshellSpy); expect(harness.onboardSpy).not.toHaveBeenCalled(); expect(cleanupBuildCtx).toHaveBeenCalledOnce(); From ce5cf9d83c30c5694e15d211581b257bef4a9a99 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 18:23:38 -0400 Subject: [PATCH 09/16] refactor(onboard): keep rebuild orchestration net-neutral Signed-off-by: Julie Yaunches --- src/lib/onboard.ts | 55 ++++++++++------------------------------------ 1 file changed, 12 insertions(+), 43 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index be84219b901..fe7c52ca9ca 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2247,19 +2247,10 @@ async function createSandboxWithBaseImageResolution( : planRegisteredExtraProviders(GATEWAY_NAME, { runOpenshell }); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const baseResolvedCreateIntent = createIntent?.resolved ?? (await sandboxCreateIntentResolver.resolve({ sandboxName, inferenceProvider: provider, enabledChannels, webSearchConfig, agent, sandboxGpuConfig: effectiveSandboxGpuConfig, resourceProfile, hermesToolGateways, extraProviders: extraProviderPlan.extraProviders, staleExtraProviders: extraProviderPlan.staleExtraProviders, baselineExclusions: sandboxRegistration.baselineExclusionsForCreate(sandboxName), ...(createIntent?.reuseRegisteredCredentials ? { reuseRegisteredCredentials: true } : {}), ...(createIntent?.policyTier !== undefined ? { policyTier: createIntent.policyTier } : {}) })); - const retainedDockerRuntime = preparedBuildContext?.preparedOpenClawLegacyImage - ? sandboxGpuCreateFlow.createRetainedOpenClawDockerRuntime( - preparedBuildContext.preparedOpenClawLegacyImage, - ) - : null; - const resolvedCreateIntent = retainedDockerRuntime - ? sandboxGpuCreateFlow.bindRetainedOpenClawGpuRoute( - baseResolvedCreateIntent, - effectiveSandboxGpuConfig, - isLinuxDockerDriverGatewayEnabled(), - retainedDockerRuntime, - ) - : baseResolvedCreateIntent; + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const retainedDockerRuntime = preparedBuildContext?.preparedOpenClawLegacyImage ? sandboxGpuCreateFlow.createRetainedOpenClawDockerRuntime(preparedBuildContext.preparedOpenClawLegacyImage) : null; + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const resolvedCreateIntent = retainedDockerRuntime ? sandboxGpuCreateFlow.bindRetainedOpenClawGpuRoute(baseResolvedCreateIntent, effectiveSandboxGpuConfig, isLinuxDockerDriverGatewayEnabled(), retainedDockerRuntime) : baseResolvedCreateIntent; const messagingCapabilities = await sandboxCreateIntentResolver.rebind( { sandboxName, @@ -2721,25 +2712,14 @@ async function createSandboxWithBaseImageResolution( manageDashboard, openshellShellCommand, openshellArgv, - prebuild: { - buildCtx, - buildId, - dockerDriverGateway, - origin, - preparedOpenClawLegacyImage: preparedBuildContext?.preparedOpenClawLegacyImage, - }, + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + prebuild: { buildCtx, buildId, dockerDriverGateway, origin, preparedOpenClawLegacyImage: preparedBuildContext?.preparedOpenClawLegacyImage }, }); const restoreBackupPath = pendingStateRestore?.manifest?.backupPath ?? pendingStateRestoreBackupPath; recreateRuntime.advance("creating"); - const { - createResult, - dockerGpuCreatePatch, - route: selectedGpuRoute, - firstCreateOutput, - registryImageRef, - retainedImageFinalization, - } = await sandboxGpuCreateFlow.runSandboxGpuCreateFlow( + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const { createResult, dockerGpuCreatePatch, route: selectedGpuRoute, firstCreateOutput, registryImageRef, retainedImageFinalization } = await sandboxGpuCreateFlow.runSandboxGpuCreateFlow( { sandboxName, provider, @@ -2764,9 +2744,8 @@ async function createSandboxWithBaseImageResolution( sleep: sleepSeconds, openshellArgv, verifyDirectSandboxGpu, - ...(retainedDockerRuntime - ? { createRetainedDockerRuntime: () => retainedDockerRuntime } - : {}), + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + ...(retainedDockerRuntime ? { createRetainedDockerRuntime: () => retainedDockerRuntime } : {}), }, ); @@ -2825,18 +2804,8 @@ async function createSandboxWithBaseImageResolution( } // openshell tags images with seconds; buildId is ms. Parse actual tag from output. Fixes #2672. - const resolvedImageTag = sandboxGpuCreateFlow.resolveCreatedSandboxRegistryImageRef( - retainedImageFinalization, - [ - registryImageRef, - prebuild.imageRef, - buildContext.extractBuiltImageRef(`${firstCreateOutput}\n${createResult.output}`), - resolveSandboxImageTagFromCreateOutput( - `${firstCreateOutput}\n${createResult.output}`, - buildId, - ), - ], - ); + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const resolvedImageTag = sandboxGpuCreateFlow.resolveCreatedSandboxRegistryImageRef(retainedImageFinalization, [registryImageRef, prebuild.imageRef, buildContext.extractBuiltImageRef(`${firstCreateOutput}\n${createResult.output}`), resolveSandboxImageTagFromCreateOutput(`${firstCreateOutput}\n${createResult.output}`, buildId)]); const sandboxRuntimeFields = getSandboxRuntimeRegistryFields(effectiveSandboxGpuConfig); recreateRuntime.recordCreated(); finalizeCreatedSandbox( From f5e80b7d1fa9413a7a44e0b4f0f147a9dc0baffe Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 18:33:18 -0400 Subject: [PATCH 10/16] test(rebuild): keep guardrail scenarios linear Signed-off-by: Julie Yaunches --- .../rebuild-custom-image-preflight.test.ts | 19 +++--- .../sandbox/rebuild-destroy-phase.test.ts | 6 +- .../rebuild-openclaw-legacy-image.test.ts | 61 ++++++++++--------- ...ocker-gpu-sandbox-create-lifecycle.test.ts | 12 ++-- .../onboard/prepared-dcode-rebuild.test.ts | 45 ++++++++------ .../retained-openclaw-docker-runtime.test.ts | 14 +++-- 6 files changed, 92 insertions(+), 65 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts index 58550159585..e5ca76d42ad 100644 --- a/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts +++ b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts @@ -494,13 +494,18 @@ describe("preflightRebuildImage", () => { expect(result).toEqual({ ok: false, detail: diagnostic }); expect(buildImage).toHaveBeenCalledOnce(); - expect(captureLegacyDockerBinding).toHaveBeenCalledTimes(testCase.capturesBinding ? 1 : 0); - if (testCase.capturesBinding) { - expect(captureLegacyDockerBinding).toHaveBeenCalledWith({ - buildDockerEnv: expect.any(Function), - cwd: ROOT, - }); - } + expect(captureLegacyDockerBinding.mock.calls).toEqual( + testCase.capturesBinding + ? [ + [ + { + buildDockerEnv: expect.any(Function), + cwd: ROOT, + }, + ], + ] + : [], + ); expect(buildxAvailable).toHaveBeenCalledTimes(testCase.capturesBinding ? 1 : 0); expect(cleanupBuildCtx).toHaveBeenCalledOnce(); } finally { diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index a81fcf6fcda..f2035d0a7fc 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -381,9 +381,9 @@ describe("rebuild destroy phase", () => { return { ok: true as const }; }); const log = vi.fn((message: string) => { - if (message.startsWith("Running: openshell sandbox delete")) { - deleteEdgeEvents.push("log"); - } + deleteEdgeEvents.push( + ...(message.startsWith("Running: openshell sandbox delete") ? ["log"] : []), + ); }); mocks.runOpenshell.mockImplementation((args: string[]) => { deleteEdgeEvents.push(args[1] ?? "unknown"); diff --git a/src/lib/actions/sandbox/rebuild-openclaw-legacy-image.test.ts b/src/lib/actions/sandbox/rebuild-openclaw-legacy-image.test.ts index 0965b46e86b..f70b49b5299 100644 --- a/src/lib/actions/sandbox/rebuild-openclaw-legacy-image.test.ts +++ b/src/lib/actions/sandbox/rebuild-openclaw-legacy-image.test.ts @@ -54,26 +54,22 @@ function createDockerHarness(overrides: Partial = {}) { }; const exitListeners = new Set<() => void>(); const runDocker = vi.fn((args: readonly string[], _options: SpawnSyncOptions = {}) => { - if (args.join(" ") === "context show") { - return dockerResult(state.context, state.contextStatus); + switch (args.join(" ")) { + case "context show": + return dockerResult(state.context, state.contextStatus); + case "info --format {{.ID}}": + return dockerResult(state.engineId, state.infoStatus); + case `image inspect --format {{.Id}} ${IMAGE_REF}`: + return dockerResult(state.tagImageId); + case `image inspect --format {{.Id}} ${IMAGE_ID}`: + return dockerResult(state.directImageId); + case `image inspect --format {{.Id}} ${OTHER_IMAGE_ID}`: + return dockerResult(OTHER_IMAGE_ID); + case `rmi ${IMAGE_ID}`: + return dockerResult("", state.rmiStatus); + default: + throw new Error(`Unexpected Docker command: ${args.join(" ")}`); } - if (args.join(" ") === "info --format {{.ID}}") { - return dockerResult(state.engineId, state.infoStatus); - } - if ( - args[0] === "image" && - args[1] === "inspect" && - args[2] === "--format" && - args[3] === "{{.Id}}" - ) { - const selector = args[4]; - if (selector === IMAGE_REF) return dockerResult(state.tagImageId); - if (selector === IMAGE_ID) return dockerResult(state.directImageId); - if (selector === OTHER_IMAGE_ID) return dockerResult(OTHER_IMAGE_ID); - return dockerResult("", 1); - } - if (args[0] === "rmi") return dockerResult("", state.rmiStatus); - throw new Error(`Unexpected Docker command: ${args.join(" ")}`); }); const deps: OpenClawLegacyDockerBindingDeps = { cwd: CANONICAL_CWD, @@ -157,8 +153,7 @@ describe("OpenClaw legacy-image Docker binding", () => { it("keeps a relative Docker config on canonical engine A for identity and cleanup", () => { const harness = createDockerHarness(); - const runEngineA = harness.runDocker.getMockImplementation(); - if (!runEngineA) throw new Error("engine A Docker harness is missing"); + const runEngineA = harness.runDocker.getMockImplementation()!; const engineBCalls: string[][] = []; harness.deps.buildDockerEnv = () => ({ DOCKER_CONFIG: "relative/docker-config", @@ -166,13 +161,23 @@ describe("OpenClaw legacy-image Docker binding", () => { }); harness.runDocker.mockImplementation( (args: readonly string[], options: SpawnSyncOptions = {}) => { - if (options.cwd === CANONICAL_CWD) return runEngineA(args, options); - engineBCalls.push([...args]); - if (args.join(" ") === "context show") return dockerResult("ambient-context"); - if (args.join(" ") === "info --format {{.ID}}") return dockerResult("engine-b"); - if (args[0] === "image" && args[1] === "inspect") return dockerResult(OTHER_IMAGE_ID); - if (args[0] === "rmi") return dockerResult(""); - throw new Error(`Unexpected engine B Docker command: ${args.join(" ")}`); + const runEngineB = () => { + engineBCalls.push([...args]); + switch (args.join(" ")) { + case "context show": + return dockerResult("ambient-context"); + case "info --format {{.ID}}": + return dockerResult("engine-b"); + case `image inspect --format {{.Id}} ${IMAGE_REF}`: + case `image inspect --format {{.Id}} ${IMAGE_ID}`: + return dockerResult(OTHER_IMAGE_ID); + case `rmi ${IMAGE_ID}`: + return dockerResult(""); + default: + throw new Error(`Unexpected engine B Docker command: ${args.join(" ")}`); + } + }; + return options.cwd === CANONICAL_CWD ? runEngineA(args, options) : runEngineB(); }, ); diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index 936457078ec..d6c0aa4b335 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -280,11 +280,13 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { vi.spyOn(dockerAdapters, "dockerStart").mockImplementation(failOnEngineB), vi.spyOn(dockerAdapters, "dockerStop").mockImplementation(failOnEngineB), ]; - const dockerCapture = vi.fn((args: readonly string[]) => { - if (args[0] === "ps") return "old-container-id\n"; - if (args[0] === "inspect") return JSON.stringify([createDockerGpuInspectFixture()]); - return ""; - }); + const dockerCaptureOutput: Readonly> = { + ps: "old-container-id\n", + inspect: JSON.stringify([createDockerGpuInspectFixture()]), + }; + const dockerCapture = vi.fn( + (args: readonly string[]) => dockerCaptureOutput[args[0] ?? ""] ?? "", + ); const dockerRun = vi.fn(() => ({ status: 0, stdout: "engine-a-probe\n" })); const dockerRunDetached = vi.fn(() => ({ status: 1, diff --git a/src/lib/onboard/prepared-dcode-rebuild.test.ts b/src/lib/onboard/prepared-dcode-rebuild.test.ts index abfc4472dca..67beb3f9051 100644 --- a/src/lib/onboard/prepared-dcode-rebuild.test.ts +++ b/src/lib/onboard/prepared-dcode-rebuild.test.ts @@ -220,24 +220,35 @@ describe("prepared DCode rebuild adapter", () => { }); it.each([ - "custom origin", - "custom Dockerfile", - "Hermes agent", - ])("rejects a retained OpenClaw image for a %s", (unsupportedTarget) => { + [ + "custom origin", + (options: ReturnType) => { + options.preparedImageRebuild!.buildContext.origin = "custom"; + }, + ], + [ + "custom Dockerfile", + (options: ReturnType) => { + options.fromDockerfile = "/tmp/custom/Dockerfile"; + options.preparedImageRebuild!.buildContext.rebuildTarget = { + agentName: null, + fromDockerfile: "/tmp/custom/Dockerfile", + }; + }, + ], + [ + "Hermes agent", + (options: ReturnType) => { + options.agent = "hermes"; + options.preparedImageRebuild!.buildContext.rebuildTarget = { + agentName: "hermes", + fromDockerfile: null, + }; + }, + ], + ] as const)("rejects a retained OpenClaw image for a %s", (_unsupportedTarget, mutate) => { const options = createPreparedOpenClawImageOptions(); - const buildContext = options.preparedImageRebuild!.buildContext; - if (unsupportedTarget === "custom origin") { - buildContext.origin = "custom"; - } else if (unsupportedTarget === "custom Dockerfile") { - options.fromDockerfile = "/tmp/custom/Dockerfile"; - buildContext.rebuildTarget = { - agentName: null, - fromDockerfile: "/tmp/custom/Dockerfile", - }; - } else { - options.agent = "hermes"; - buildContext.rebuildTarget = { agentName: "hermes", fromDockerfile: null }; - } + mutate(options); expect(() => createPreparedDcodeRebuildRuntime(options, "nemoclaw")).toThrow( /retained legacy image can only be used for a generated OpenClaw rebuild/, diff --git a/src/lib/onboard/retained-openclaw-docker-runtime.test.ts b/src/lib/onboard/retained-openclaw-docker-runtime.test.ts index ef2f2dea72a..1792bffd5c2 100644 --- a/src/lib/onboard/retained-openclaw-docker-runtime.test.ts +++ b/src/lib/onboard/retained-openclaw-docker-runtime.test.ts @@ -84,12 +84,16 @@ describe("retained OpenClaw Docker runtime", () => { const calls: Array<{ args: string[]; options: SpawnSyncOptions }> = []; const runDocker = vi.fn((args: readonly string[], options: SpawnSyncOptions = {}) => { calls.push({ args: [...args], options }); - if (args.join(" ") === "info --format {{json .OperatingSystem}}") { - return dockerResult('"Docker Desktop"\n'); + switch (true) { + case args.join(" ") === "info --format {{json .OperatingSystem}}": + return dockerResult('"Docker Desktop"\n'); + case args[0] === "image" && args[1] === "inspect": + return dockerResult("[]\n"); + case args[0] === "logs": + return dockerResult("engine-a logs\n"); + default: + return dockerResult("engine-a-result\n"); } - if (args[0] === "image" && args[1] === "inspect") return dockerResult("[]\n"); - if (args[0] === "logs") return dockerResult("engine-a logs\n"); - return dockerResult("engine-a-result\n"); }); const runtime = createRetainedOpenClawDockerRuntime(image, { runDocker: runDocker as typeof dockerSpawnSync, From 901e4a599cdf1423a947173a55236f9194cfadc9 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 18:45:27 -0400 Subject: [PATCH 11/16] docs(rebuild): clarify custom Dockerfile exclusion Signed-off-by: Julie Yaunches --- docs/reference/commands.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index c64d2edb952..36411b745e3 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -2716,7 +2716,7 @@ If either identity changes during replacement creation, NemoClaw stops instead o The legacy-builder retry does not apply in the following cases: -- The sandbox was recorded with a custom `--from` Dockerfile. +- The sandbox was recorded with a custom Dockerfile. - The OpenShell gateway is not local. - The host-side local prebuild is disabled. - The build has an unrelated failure. From 5ca7deb2597ec2706f809866ba20c6ff0c9a7805 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 19:18:33 -0400 Subject: [PATCH 12/16] test(rebuild): bind Docker engine fixtures Signed-off-by: Julie Yaunches --- test/gateway-state-reconcile-2276.test.ts | 9 +++++++++ test/rebuild-credential-preflight.test.ts | 6 +++++- test/rebuild-shields-auto-unlock.test.ts | 4 ++++ test/rebuild-stale-recovery.test.ts | 4 ++++ test/repro-2201.test.ts | 4 ++++ 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/test/gateway-state-reconcile-2276.test.ts b/test/gateway-state-reconcile-2276.test.ts index 72adb6ce150..c3a53d1c397 100644 --- a/test/gateway-state-reconcile-2276.test.ts +++ b/test/gateway-state-reconcile-2276.test.ts @@ -164,6 +164,11 @@ if (args[0] === "sandbox" && args[1] === "list") { process.exit(0); } +if (args[0] === "sandbox" && args[1] === "create") { + process.stderr.write("injected sandbox create failure\\n"); + process.exit(1); +} + if (args[0] === "inference" && args[1] === "get") { process.stdout.write("Gateway inference:\\n Provider: nvidia-prod\\n Model: nvidia/nemotron-3-super-120b-a12b\\n"); process.exit(0); @@ -261,6 +266,10 @@ const { isOpenClawSecurityInventoryProbe } = require(${JSON.stringify( path.join(import.meta.dirname, "helpers", "onboard-script-mocks.cjs"), )}); if (a[0] === "info") { + if (a.includes("{{.ID}}")) { + process.stdout.write("engine-a\\n"); + process.exit(0); + } process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); process.exit(0); } diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index 7ba8c428b63..bc20058d6d1 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -271,7 +271,11 @@ const { isOpenClawSecurityInventoryProbe } = require(${JSON.stringify( )}); const provenancePath = ${JSON.stringify(path.join(tmpDir, "docker-base-provenance"))}; const readProvenance = () => fs.existsSync(provenancePath) ? JSON.parse(fs.readFileSync(provenancePath, "utf8")) : {}; -if (a[0] === "info") { process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); process.exit(0); } +if (a[0] === "info") { + if (a.includes("{{.ID}}")) { process.stdout.write("engine-a\\n"); process.exit(0); } + process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); + process.exit(0); +} if (a[0] === "build") { const labelIndex = a.indexOf("--label"); const tagIndex = a.indexOf("-t"); diff --git a/test/rebuild-shields-auto-unlock.test.ts b/test/rebuild-shields-auto-unlock.test.ts index c7504cecc4b..6ab6019103f 100644 --- a/test/rebuild-shields-auto-unlock.test.ts +++ b/test/rebuild-shields-auto-unlock.test.ts @@ -235,6 +235,10 @@ function writeLockState(state) { fs.writeFileSync(lockStatePath, state); } if (a[0]==="info") { + if (a.includes("{{.ID}}")) { + process.stdout.write("engine-a\\n"); + process.exit(0); + } process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); process.exit(0); } diff --git a/test/rebuild-stale-recovery.test.ts b/test/rebuild-stale-recovery.test.ts index a1683a22f3c..77a1a4e0362 100644 --- a/test/rebuild-stale-recovery.test.ts +++ b/test/rebuild-stale-recovery.test.ts @@ -173,6 +173,10 @@ const { isOpenClawSecurityInventoryProbe } = require(${JSON.stringify( path.join(REPO_ROOT, "test", "helpers", "onboard-script-mocks.cjs"), )}); if (a[0]==="info") { + if (a.includes("{{.ID}}")) { + process.stdout.write("engine-a\\n"); + process.exit(0); + } process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); process.exit(0); } diff --git a/test/repro-2201.test.ts b/test/repro-2201.test.ts index 1766f1d5973..34a3aff401a 100644 --- a/test/repro-2201.test.ts +++ b/test/repro-2201.test.ts @@ -277,6 +277,10 @@ const { isOpenClawSecurityInventoryProbe } = require(${JSON.stringify( const provenancePath = ${JSON.stringify(path.join(tmpDir, "docker-base-provenance"))}; const readProvenance = () => fs.existsSync(provenancePath) ? JSON.parse(fs.readFileSync(provenancePath, "utf8")) : {}; if (a[0]==="info") { + if (a.includes("{{.ID}}")) { + process.stdout.write("engine-a\\n"); + process.exit(0); + } process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); process.exit(0); } From 95124f8f24cacdb8ede4fab6bfa1c595b1e4e8f4 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 19:34:03 -0400 Subject: [PATCH 13/16] test(rebuild): stop interactive fixture after backup Signed-off-by: Julie Yaunches --- test/rebuild-credential-preflight.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index bc20058d6d1..c13a20670a5 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -397,12 +397,16 @@ describe("atomic rebuild process contracts (#2273)", () => { expect(registryHasSandbox(fixture)).toBe(true); }); - it("accepts trimmed case-insensitive yes input before continuing into backup", () => { + it("accepts trimmed case-insensitive yes input before continuing into backup", { + timeout: testTimeout(60_000), + }, () => { const fixture = createFixture({ savedCredential: { key: "NVIDIA_INFERENCE_API_KEY", value: "nvapi-test-key-for-rebuild", }, + // This contract ends at backup; stop before recreation. + sandboxDeleteExitCode: 1, }); const result = runRebuild(fixture, {}, { yes: false, input: " YES \n" }); From 0c559f783f974800376ecc36c8f301a5589893e6 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 19:42:10 -0400 Subject: [PATCH 14/16] test(rebuild): preserve sandbox after delete failure Signed-off-by: Julie Yaunches --- test/rebuild-credential-preflight.test.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index c13a20670a5..187e36d7c16 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -179,7 +179,11 @@ if (a[0] === "sandbox" && a[1] === "get") { process.stdout.write("Sandbox: ${sandboxName}\\nPhase: Ready\\n"); process.exit(0); } -if (a[0] === "sandbox" && a[1] === "delete") { fs.writeFileSync(${JSON.stringify(deleteMarker)}, "deleted\\n"); process.exit(${sandboxDeleteExitCode}); } +if (a[0] === "sandbox" && a[1] === "delete") { + const exitCode = ${sandboxDeleteExitCode}; + if (exitCode === 0) fs.writeFileSync(${JSON.stringify(deleteMarker)}, "deleted\\n"); + process.exit(exitCode); +} if (a[0] === "sandbox" && a[1] === "get") { if (fs.existsSync(${JSON.stringify(deleteMarker)})) { process.stderr.write("sandbox ${sandboxName} not found\\n"); process.exit(1); } process.stdout.write("${sandboxName} Ready\\n"); @@ -405,7 +409,7 @@ describe("atomic rebuild process contracts (#2273)", () => { key: "NVIDIA_INFERENCE_API_KEY", value: "nvapi-test-key-for-rebuild", }, - // This contract ends at backup; stop before recreation. + // This contract ends at backup; preserve the sandbox when delete fails. sandboxDeleteExitCode: 1, }); @@ -416,6 +420,8 @@ describe("atomic rebuild process contracts (#2273)", () => { expect(output).not.toContain("Cancelled."); expect(output).not.toContain("preflight failed"); expect(output).toContain("Backing up sandbox state"); + expect(output).not.toContain("Creating new sandbox with current image"); + expect(fs.existsSync(fixture.deleteMarker)).toBe(false); }); it("prints an active SSH session warning before interactive confirmation and cancel", () => { From 8828c522e89914266e6bedd1bb4588a001476fde Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 3 Aug 2026 14:01:20 -0700 Subject: [PATCH 15/16] fix(rebuild): refresh retained OpenClaw rebuild Signed-off-by: Carlos Villela --- .../sandbox/rebuild-destroy-phase.test.ts | 2 + src/lib/actions/sandbox/rebuild-pipeline.ts | 6 +- src/lib/onboard.ts | 17 +- src/lib/onboard/docker-gpu-local-inference.ts | 5 +- .../managed-bootstrap/runtime-create.ts | 6 +- .../retained-openclaw-docker-runtime.ts | 6 +- src/lib/onboard/sandbox-gpu-create-flow.ts | 192 +++++++++--------- 7 files changed, 129 insertions(+), 105 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index c06e23f6474..36c088070ab 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -384,6 +384,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -449,6 +450,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log, diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts index 7b3a22411c9..bbd1a923da2 100644 --- a/src/lib/actions/sandbox/rebuild-pipeline.ts +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -273,6 +273,8 @@ async function rebuildSandboxUnlocked( return; } + const preparedImageForAbort = preparedImage; + const mcpPreparation = await runRebuildDestroyPhase({ sandboxName, sandboxEntry, @@ -341,8 +343,8 @@ async function rebuildSandboxUnlocked( } return { ok: true }; }, - abortPreparedImageRecreate: preparedImage - ? () => abortPreparedImageRecreate(preparedImage) + abortPreparedImageRecreate: preparedImageForAbort + ? () => abortPreparedImageRecreate(preparedImageForAbort) : undefined, onDeleted: () => { sandboxStillExists = false; diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index fbf73426bb3..3ae8414daa4 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2802,11 +2802,18 @@ async function createSandboxWithBaseImageResolution( } // openshell tags images with seconds; buildId is ms. Parse actual tag from output. Fixes #2672. - const resolvedImageTag = - sandboxGpuCreateFlow.resolveCreatedSandboxRegistryImageRef(retainedImageFinalization, [registryImageRef, prebuild.imageRef, buildContext.extractBuiltImageRef(`${firstCreateOutput}\n${createResult.output}`), resolveSandboxImageTagFromCreateOutput(`${firstCreateOutput}\n${createResult.output}`, buildId)]) ?? - prebuild.imageRef ?? - buildContext.extractBuiltImageRef(`${firstCreateOutput}\n${createResult.output}`) ?? - resolveSandboxImageTagFromCreateOutput(`${firstCreateOutput}\n${createResult.output}`, buildId); + const resolvedImageTag = sandboxGpuCreateFlow.resolveCreatedSandboxRegistryImageRef( + retainedImageFinalization, + [ + registryImageRef, + prebuild.imageRef, + buildContext.extractBuiltImageRef(`${firstCreateOutput}\n${createResult.output}`), + resolveSandboxImageTagFromCreateOutput( + `${firstCreateOutput}\n${createResult.output}`, + buildId, + ), + ], + ); const sandboxRuntimeFields = getSandboxRuntimeRegistryFields(effectiveSandboxGpuConfig); recreateRuntime.recordCreated(); finalizeCreatedSandbox( diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index 42046274217..e274ad42ad5 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -8,7 +8,7 @@ import { getDockerGpuPatchNetworkMode, printDockerGpuProofFailure, } from "./docker-gpu-patch"; -import type { DockerGpuPatchMode } from "./docker-gpu-patch-types"; +import type { DockerGpuPatchDeps, DockerGpuPatchMode } from "./docker-gpu-patch-types"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; import type { ManagedBootstrapRuntimePatch } from "./managed-bootstrap/runtime-create"; @@ -390,6 +390,9 @@ export type GpuSandboxAfterReadyOptions = { reportGpuProofFailure?: boolean; selectedMode: ManagedBootstrapRuntimePatch["selectedMode"]; runCaptureOpenshell: (args: string[], opts?: Record) => string; + dockerCapture?: DockerGpuPatchDeps["dockerCapture"]; + dockerLogs?: DockerGpuPatchDeps["dockerLogs"]; + env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform; log?: (message: string) => void; diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.ts b/src/lib/onboard/managed-bootstrap/runtime-create.ts index 198d91710f0..740bca067f3 100644 --- a/src/lib/onboard/managed-bootstrap/runtime-create.ts +++ b/src/lib/onboard/managed-bootstrap/runtime-create.ts @@ -20,12 +20,12 @@ export interface ManagedBootstrapRuntimeCommandResult { } export interface ManagedBootstrapRuntimeDependencies { - readonly runCaptureOpenshell?: (args: string[], options?: Record) => string; - readonly runOpenshell?: ( + readonly runCaptureOpenshell: (args: string[], options?: Record) => string; + readonly runOpenshell: ( args: string[], options?: Record, ) => ManagedBootstrapRuntimeCommandResult; - readonly sleep?: (seconds: number) => void; + readonly sleep: (seconds: number) => void; } export type ManagedBootstrapRuntimeRoute = "none" | "native" | "compatibility"; diff --git a/src/lib/onboard/rebuild/retained-openclaw-docker-runtime.ts b/src/lib/onboard/rebuild/retained-openclaw-docker-runtime.ts index b4c8d75ab60..eb843d7eea8 100644 --- a/src/lib/onboard/rebuild/retained-openclaw-docker-runtime.ts +++ b/src/lib/onboard/rebuild/retained-openclaw-docker-runtime.ts @@ -2,10 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import type { SpawnSyncOptions } from "node:child_process"; +import path from "node:path"; + import { isIP } from "node:net"; import { dockerSpawnSync } from "../../adapters/docker/exec"; -import { ROOT } from "../../state/paths"; import type { PreparedOpenClawLegacyImage } from "../build-context-stage"; import type { DockerGpuPatchDeps } from "../docker-gpu-patch-types"; import { resolveDockerGpuSandboxCreatePlan } from "../docker-gpu-sandbox-create-plan"; @@ -29,6 +30,7 @@ import { detectWslDockerDesktopStatus } from "../wsl-docker-desktop-gpu"; const RETAINED_DOCKER_OPERATION_TIMEOUT_MS = 30_000; const RETAINED_DNS_PROBE_TIMEOUT_MS = 20_000; const RETAINED_DOCKER_PULL_TIMEOUT_MS = 60_000; +const REPOSITORY_ROOT = path.resolve(__dirname, "..", "..", "..", ".."); type DockerRunOptions = Record; type DockerRunResult = NonNullable>; @@ -123,7 +125,7 @@ export function createRetainedOpenClawDockerRuntime( ); } const spawnOptions: SpawnSyncOptions = { - cwd: ROOT, + cwd: REPOSITORY_ROOT, encoding: "utf-8", env: image.dockerEnv, shell: false, diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index a968844bdd0..943f979874f 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -11,6 +11,8 @@ import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { renderCompatibilityFallbackCreateArgs } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; import type { DockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +import type { PreparedOpenClawLegacyImageFinalization } from "./build-context-stage"; + import { isImmutableDockerImageId, queryOpenShellDockerSandboxContainers, @@ -19,6 +21,7 @@ import { bindRetainedOpenClawGpuRoute, createRetainedOpenClawDockerRuntime, } from "./rebuild/retained-openclaw-docker-runtime"; +export { bindRetainedOpenClawGpuRoute, createRetainedOpenClawDockerRuntime }; import type { ManagedBootstrapAdapter, ManagedBootstrapAgentIdentity, @@ -75,7 +78,8 @@ export interface SandboxGpuCreateFlowInput { requiredUlimits?: readonly DockerUlimit[] | null; } -export interface SandboxGpuCreateFlowDeps { +export interface SandboxGpuCreateFlowDeps + extends Pick { runOpenshell: RunOpenshell; runCaptureOpenshell: RunCaptureOpenshell; sleep: Sleep; @@ -126,6 +130,14 @@ function resolveCompatibilityRetryImageId( return retainedImageId; } +export function resolveCreatedSandboxRegistryImageRef( + finalization: PreparedOpenClawLegacyImageFinalization | null, + candidates: readonly (string | null | undefined)[], +): string | null { + if (finalization) return finalization.registryImageRef; + return candidates.find((candidate): candidate is string => Boolean(candidate)) ?? null; +} + function abortUnusedRetainedImage(input: SandboxGpuCreateFlowInput): void { const preparedImage = input.prebuild.preparedOpenClawLegacyImage; if (!preparedImage) return; @@ -167,11 +179,7 @@ export async function runSandboxGpuCreateFlow( return { retainedDockerRuntime, effectiveDeps, - attemptRunner: createSandboxGpuCreateAttemptRunner( - input, - effectiveDeps, - retainedDockerRuntime?.dockerDesktopWsl(), - ), + attemptRunner: createSandboxGpuCreateAttemptRunner(input, effectiveDeps), }; } catch (error) { abortUnusedRetainedImage(input); @@ -185,97 +193,97 @@ export async function runSandboxGpuCreateFlow( }>; try { gpuCreateOutcome = await sandboxGpuCreateAttempt.executeSandboxGpuCreatePlan( - input.gpuRoutePlan, - { - runAttempt: attemptRunner.runAttempt, - captureNativeFailure: (failure) => { - const routeAdapter = adaptDockerGpuRouteForPatch(failure.route); - const diagnostics = collectDockerGpuPatchDiagnostics( - input.sandboxName, - { - error: failure.error, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - }, - effectiveDeps, - ); - if (diagnostics) console.error(` Native GPU diagnostics saved: ${diagnostics.dir}`); - }, - cleanupNativeFailure: () => - sandboxGpuCreateAttempt.cleanupNativeGpuAttemptForFallback(input.sandboxName, { - runOpenshell: effectiveDeps.runOpenshell, - queryContainers: (sandboxName) => - queryOpenShellDockerSandboxContainers(sandboxName, effectiveDeps), - sleep: effectiveDeps.sleep, - }), - prepareCompatibilityAttempt: async () => { - if (!input.compatibilityPolicyPath) { - throw new Error("Compatibility retry policy was not materialized."); - } - const nativeRuntimeSnapshot = attemptRunner.state.nativeRuntimeSnapshot; - if (attemptRunner.managedRouting) { - const prepared = attemptRunner.managedRouting.prepareCompatibilityLaunch({ - createArgs: input.prebuild.createArgs, - currentRegistryImageRef: registryImageRef, - prebuildImageId: input.prebuild.imageId, - allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, - compatibilityPolicyPath: input.compatibilityPolicyPath, - startupCommand: input.sandboxStartupCommand, - runtimeSnapshot: nativeRuntimeSnapshot, - }); - attemptRunner.state.compatibilityArgv = [...prepared.createArgv]; - registryImageRef = prepared.registryImageRef; - } else { - const imageId = resolveCompatibilityRetryImageId( - input.prebuild, - nativeRuntimeSnapshot?.imageId ?? null, + input.gpuRoutePlan, + { + runAttempt: attemptRunner.runAttempt, + captureNativeFailure: (failure) => { + const routeAdapter = adaptDockerGpuRouteForPatch(failure.route); + const diagnostics = collectDockerGpuPatchDiagnostics( + input.sandboxName, + { + error: failure.error, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }, + effectiveDeps, ); - if ( - !registryImageRef && - nativeRuntimeSnapshot?.bookkeepingImageRef && - !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) - ) { - registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; + if (diagnostics) console.error(` Native GPU diagnostics saved: ${diagnostics.dir}`); + }, + cleanupNativeFailure: () => + sandboxGpuCreateAttempt.cleanupNativeGpuAttemptForFallback(input.sandboxName, { + runOpenshell: effectiveDeps.runOpenshell, + queryContainers: (sandboxName) => + queryOpenShellDockerSandboxContainers(sandboxName, effectiveDeps), + sleep: effectiveDeps.sleep, + }), + prepareCompatibilityAttempt: async () => { + if (!input.compatibilityPolicyPath) { + throw new Error("Compatibility retry policy was not materialized."); } - const compatibilityArgs = renderCompatibilityFallbackCreateArgs( - input.prebuild.createArgs, - { - imageRef: imageId, + const nativeRuntimeSnapshot = attemptRunner.state.nativeRuntimeSnapshot; + if (attemptRunner.managedRouting) { + const prepared = attemptRunner.managedRouting.prepareCompatibilityLaunch({ + createArgs: input.prebuild.createArgs, + currentRegistryImageRef: registryImageRef, + prebuildImageId: input.prebuild.imageId, allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, compatibilityPolicyPath: input.compatibilityPolicyPath, - }, - ); - attemptRunner.state.compatibilityArgv = effectiveDeps.openshellArgv([ - "sandbox", - "create", - ...compatibilityArgs, - "--", - ...input.sandboxStartupCommand, - ]); - } - if (attemptRunner.state.compatibilityArgv.length === 0) { - throw new Error("Compatibility sandbox create executable is missing."); - } - }, - activateCompatibilityAttempt: async () => { - if (!input.managedBootstrap) { - await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( - input.provider, - input.sandboxGpuConfig, - { - dockerDriverGateway: input.dockerDriverGateway, - selectedRoute: "compatibility", - gatewayPort: input.gatewayPort, - log: console.log, - reverifyBridgeReachability: retainedDockerRuntime - ? () => retainedDockerRuntime.reverifyBridgeReachability(input.gatewayPort) - : undefined, - }, - ); - } - input.sandboxGpuConfig.sandboxGpuProof = null; + startupCommand: input.sandboxStartupCommand, + runtimeSnapshot: nativeRuntimeSnapshot, + }); + attemptRunner.state.compatibilityArgv = [...prepared.createArgv]; + registryImageRef = prepared.registryImageRef; + } else { + const imageId = resolveCompatibilityRetryImageId( + input.prebuild, + nativeRuntimeSnapshot?.imageId ?? null, + ); + if ( + !registryImageRef && + nativeRuntimeSnapshot?.bookkeepingImageRef && + !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) + ) { + registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; + } + const compatibilityArgs = renderCompatibilityFallbackCreateArgs( + input.prebuild.createArgs, + { + imageRef: imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + }, + ); + attemptRunner.state.compatibilityArgv = effectiveDeps.openshellArgv([ + "sandbox", + "create", + ...compatibilityArgs, + "--", + ...input.sandboxStartupCommand, + ]); + } + if (attemptRunner.state.compatibilityArgv.length === 0) { + throw new Error("Compatibility sandbox create executable is missing."); + } + }, + activateCompatibilityAttempt: async () => { + if (!input.managedBootstrap) { + await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( + input.provider, + input.sandboxGpuConfig, + { + dockerDriverGateway: input.dockerDriverGateway, + selectedRoute: "compatibility", + gatewayPort: input.gatewayPort, + log: console.log, + reverifyBridgeReachability: retainedDockerRuntime + ? () => retainedDockerRuntime.reverifyBridgeReachability(input.gatewayPort) + : undefined, + }, + ); + } + input.sandboxGpuConfig.sandboxGpuProof = null; + }, + traceEvent: addTraceEvent, }, - traceEvent: addTraceEvent, - }, ); } catch (error) { abortUnusedRetainedImage(input); From 30938a59329c3a9e35a0d73a6a088155c0d23d3d Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 3 Aug 2026 14:06:37 -0700 Subject: [PATCH 16/16] fix(rebuild): keep onboard entrypoint net neutral Signed-off-by: Carlos Villela --- src/lib/onboard.ts | 22 +++------------------ src/lib/onboard/sandbox-gpu-create-flow.ts | 23 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 3ae8414daa4..26b30471f10 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2244,11 +2244,7 @@ async function createSandboxWithBaseImageResolution( ? { extraProviders: createIntent.extraProviders, staleExtraProviders: [] } : planRegisteredExtraProviders(GATEWAY_NAME, { runOpenshell }); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - const baseResolvedCreateIntent = createIntent?.resolved ?? (await sandboxCreateIntentResolver.resolve({ sandboxName, inferenceProvider: provider, enabledChannels, webSearchConfig, agent, sandboxGpuConfig: effectiveSandboxGpuConfig, resourceProfile, hermesToolGateways, extraProviders: extraProviderPlan.extraProviders, staleExtraProviders: extraProviderPlan.staleExtraProviders, baselineExclusions: sandboxRegistration.baselineExclusionsForCreate(sandboxName), ...(createIntent?.reuseRegisteredCredentials ? { reuseRegisteredCredentials: true } : {}), ...(createIntent?.policyTier !== undefined ? { policyTier: createIntent.policyTier } : {}) })); - // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - const retainedDockerRuntime = preparedBuildContext?.preparedOpenClawLegacyImage ? sandboxGpuCreateFlow.createRetainedOpenClawDockerRuntime(preparedBuildContext.preparedOpenClawLegacyImage) : null; - // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - const resolvedCreateIntent = retainedDockerRuntime ? sandboxGpuCreateFlow.bindRetainedOpenClawGpuRoute(baseResolvedCreateIntent, effectiveSandboxGpuConfig, isLinuxDockerDriverGatewayEnabled(), retainedDockerRuntime) : baseResolvedCreateIntent; + const resolvedCreateIntent = sandboxGpuCreateFlow.resolveRetainedOpenClawCreateIntent(createIntent?.resolved ?? (await sandboxCreateIntentResolver.resolve({ sandboxName, inferenceProvider: provider, enabledChannels, webSearchConfig, agent, sandboxGpuConfig: effectiveSandboxGpuConfig, resourceProfile, hermesToolGateways, extraProviders: extraProviderPlan.extraProviders, staleExtraProviders: extraProviderPlan.staleExtraProviders, baselineExclusions: sandboxRegistration.baselineExclusionsForCreate(sandboxName), ...(createIntent?.reuseRegisteredCredentials ? { reuseRegisteredCredentials: true } : {}), ...(createIntent?.policyTier !== undefined ? { policyTier: createIntent.policyTier } : {}) })), effectiveSandboxGpuConfig, isLinuxDockerDriverGatewayEnabled(), preparedBuildContext?.preparedOpenClawLegacyImage); const messagingCapabilities = await sandboxCreateIntentResolver.rebind( { sandboxName, @@ -2743,8 +2739,6 @@ async function createSandboxWithBaseImageResolution( sleep: sleepSeconds, openshellArgv, verifyDirectSandboxGpu, - // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - ...(retainedDockerRuntime ? { createRetainedDockerRuntime: () => retainedDockerRuntime } : {}), }, ); @@ -2802,18 +2796,8 @@ async function createSandboxWithBaseImageResolution( } // openshell tags images with seconds; buildId is ms. Parse actual tag from output. Fixes #2672. - const resolvedImageTag = sandboxGpuCreateFlow.resolveCreatedSandboxRegistryImageRef( - retainedImageFinalization, - [ - registryImageRef, - prebuild.imageRef, - buildContext.extractBuiltImageRef(`${firstCreateOutput}\n${createResult.output}`), - resolveSandboxImageTagFromCreateOutput( - `${firstCreateOutput}\n${createResult.output}`, - buildId, - ), - ], - ); + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const resolvedImageTag = sandboxGpuCreateFlow.resolveCreatedSandboxRegistryImageRef(retainedImageFinalization, [registryImageRef, prebuild.imageRef, buildContext.extractBuiltImageRef(`${firstCreateOutput}\n${createResult.output}`), resolveSandboxImageTagFromCreateOutput(`${firstCreateOutput}\n${createResult.output}`, buildId)]); const sandboxRuntimeFields = getSandboxRuntimeRegistryFields(effectiveSandboxGpuConfig); recreateRuntime.recordCreated(); finalizeCreatedSandbox( diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 943f979874f..2d1118a5e9b 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -138,6 +138,29 @@ export function resolveCreatedSandboxRegistryImageRef( return candidates.find((candidate): candidate is string => Boolean(candidate)) ?? null; } +type RetainedOpenClawCreateIntent = Parameters[0]; +type RetainedOpenClawGpuConfig = Parameters[1]; +type RetainedOpenClawImage = Parameters[0]; + +export function resolveRetainedOpenClawCreateIntent( + baseResolvedCreateIntent: RetainedOpenClawCreateIntent, + sandboxGpuConfig: RetainedOpenClawGpuConfig, + dockerDriverGateway: boolean, + preparedImage: RetainedOpenClawImage | null | undefined, +): RetainedOpenClawCreateIntent { + const retainedDockerRuntime = preparedImage + ? createRetainedOpenClawDockerRuntime(preparedImage) + : null; + return retainedDockerRuntime + ? bindRetainedOpenClawGpuRoute( + baseResolvedCreateIntent, + sandboxGpuConfig, + dockerDriverGateway, + retainedDockerRuntime, + ) + : baseResolvedCreateIntent; +} + function abortUnusedRetainedImage(input: SandboxGpuCreateFlowInput): void { const preparedImage = input.prebuild.preparedOpenClawLegacyImage; if (!preparedImage) return;