From e324c5cfdf2ada169583cc4d11619bd4b698f46f Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 04:22:18 -0400 Subject: [PATCH 01/51] fix(onboard): confirm readiness after runtime commit Signed-off-by: Julie Yaunches --- .../onboard/sandbox-gpu-create-flow.test.ts | 2 +- .../sandbox-gpu-create-identity-gate.test.ts | 116 ++++++++++++ .../onboard/sandbox-gpu-create-run-attempt.ts | 176 ++++++++++++++---- 3 files changed, 254 insertions(+), 40 deletions(-) diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index a039df5bad3..808c5eb0244 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -401,7 +401,7 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { mocks.waitForCreatedSandboxReadyWithTrace.mock.calls.map( ([options]) => options.stableReadyPolls, ), - ).toEqual([2, 2]); + ).toEqual([2, 2, 2]); vi.mocked(deps.runCaptureOpenshell).mockClear(); await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 41e0979a703..cfa489d14d7 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -111,6 +111,31 @@ function noGpuInput() { return input; } +function attachManagedBootstrap( + input: ReturnType, + patch: ReturnType, +): void { + input.managedBootstrap = { + bootstrapIdentity: "b".repeat(64), + stateRoot: "/tmp/nemoclaw-managed-bootstrap", + runtimeProvider: { + identity: { id: "mxc" }, + bootstrap: { + createOnboardRouting: () => ({ nativeFallbackHasCleanBaseline: false }), + createLifecycle: (options: { launchArgv: readonly string[] }) => ({ + launchArgv: options.launchArgv, + patch, + recoverUnfinished: async () => null, + prepareNetwork: async () => undefined, + runCreate: async () => { + throw new Error("resumed create must not launch"); + }, + }), + }, + }, + } as never; +} + function refuseEffectStartingWith(prefix: string): (operation: string) => void { return (operation) => { expect(operation, "checkpoint changed").not.toMatch(new RegExp(`^${prefix}`, "u")); @@ -121,6 +146,97 @@ beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); describe("created sandbox identity gate", () => { + it("reconfirms exact managed sandbox readiness after the runtime commit", async () => { + const sandboxId = "alpha-sandbox-id"; + const input = noGpuInput(); + input.resumeVerifiedCreate = { + route: "none", + liveIdentityFingerprint: fingerprintSandboxRecreateValue(sandboxId), + createAttemptNonce: "a".repeat(62), + }; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + attachManagedBootstrap(input, patch); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runOpenshell) + .mockReturnValueOnce({ + status: 0, + stdout: `Name: alpha\nId: ${sandboxId}\nState: Ready\n`, + stderr: "", + }) + .mockReturnValueOnce({ + status: 1, + stdout: "", + stderr: + "Error: × code: 'The system is not in a state required for the operation's\n" + + ' │ execution\', message: "sandbox is not ready"\n', + }) + .mockReturnValueOnce({ + status: 0, + stdout: `Name: alpha\nId: ${sandboxId}\nState: Ready\n`, + stderr: "", + }) + .mockReturnValue({ status: 0, stdout: "", stderr: "" }); + mocks.waitForCreatedSandboxReadyWithTrace + .mockReturnValueOnce({ ready: true, reason: "ready", failurePhase: null }) + .mockImplementationOnce((options) => { + expect(patch.commitAfterReady).toHaveBeenCalledOnce(); + expect(options.checkReadyIdentity?.()).toBe("not_ready"); + expect(options.checkReadyIdentity?.()).toBe("ready"); + return { ready: true, reason: "ready", failurePhase: null }; + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ + origin: "resumed", + route: "none", + }); + + expect(mocks.waitForCreatedSandboxReadyWithTrace).toHaveBeenCalledTimes(2); + expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + }); + + it("retains exact recovery when committed managed readiness does not return", async () => { + const sandboxId = "alpha-sandbox-id"; + const sandboxIdentityFingerprint = fingerprintSandboxRecreateValue(sandboxId); + const input = noGpuInput(); + input.resumeVerifiedCreate = { + route: "none", + liveIdentityFingerprint: sandboxIdentityFingerprint, + createAttemptNonce: "a".repeat(62), + }; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + attachManagedBootstrap(input, patch); + const deps = createGpuFlowDeps(sandboxId); + mocks.waitForCreatedSandboxReadyWithTrace + .mockReturnValueOnce({ ready: true, reason: "ready", failurePhase: null }) + .mockReturnValue({ ready: false, reason: "timeout", failurePhase: null }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "did not return to Ready after its managed runtime commit", + ); + + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining("did not return to executable Ready state"), + sandboxIdentityFingerprint, + "a".repeat(62), + ); + expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { + backupPath: null, + }); + }); + it("resumes the exact verified sandbox without issuing another create (#9833)", async () => { const events: string[] = []; const sandboxId = "alpha-sandbox-id"; diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 674a56d4116..75eca142e31 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -250,6 +250,120 @@ async function verifyCreatedSandboxBeforeEffects( }); } +function persistCreateAttemptRecovery(options: { + readonly input: SandboxGpuCreateFlowInput; + readonly createAttemptNonce: string | null; + readonly detail: string; + readonly sandboxIdentityFingerprint?: string; +}): void { + const { input, createAttemptNonce, detail, sandboxIdentityFingerprint } = options; + if (!createAttemptNonce) { + throw new Error("Sandbox create-attempt identity was not generated."); + } + const persist = input.persistRetainedSandboxRecovery; + if (!persist) { + throw new Error("Verified sandbox creation has no durable recovery evidence owner."); + } + const message = + `Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${createAttemptNonce}. ` + + (sandboxIdentityFingerprint + ? `Durable sandbox identity fingerprint: ${sandboxIdentityFingerprint}. ` + : "") + + detail; + let persisted = false; + try { + persisted = persist(message, sandboxIdentityFingerprint, createAttemptNonce); + } catch { + persisted = false; + } + console.error(` ${message}`); + if (!persisted) { + console.error( + " NemoClaw could not save this create-attempt evidence. Preserve the terminal output for an OpenShell administrator.", + ); + } +} + +function persistIdentitySettlementRecoveryEvidence(options: { + readonly input: SandboxGpuCreateFlowInput; + readonly createAttemptNonce: string | null; + readonly sandboxIdentityFingerprint: string | null; +}): void { + const { input, createAttemptNonce, sandboxIdentityFingerprint } = options; + const identityEvidence = sandboxIdentityFingerprint + ? `Sandbox '${input.sandboxName}' did not remain visible through owning gateway '${input.gatewayName}' before identity verification completed. ` + : `Sandbox '${input.sandboxName}' reached Ready before OpenShell returned one exact durable create identity. Gateway '${input.gatewayName}'. OpenShell did not return one exact durable sandbox identity for this create attempt. `; + persistCreateAttemptRecovery({ + input, + createAttemptNonce, + detail: + identityEvidence + + "Do not delete a sandbox by mutable name; preserve it until an OpenShell administrator resolves the create-attempt label to one sandbox.", + sandboxIdentityFingerprint: sandboxIdentityFingerprint ?? undefined, + }); +} + +function confirmManagedRuntimeCommitReadiness(options: { + readonly input: SandboxGpuCreateFlowInput; + readonly deps: SandboxGpuCreateFlowDeps; + readonly sandboxId: string | null; + readonly createAttemptNonce: string | null; +}): void { + const { input, deps, sandboxId } = options; + if (!sandboxId) return; + input.revalidateVerifiedSandboxBeforeEffect?.( + `confirm committed runtime readiness for sandbox '${input.sandboxName}'`, + ); + const committedReadiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ + sandboxName: input.sandboxName, + timeoutSecs: input.sandboxReadyTimeoutSecs, + runCaptureOpenshell: deps.runCaptureOpenshell, + isSandboxReady, + getSandboxFailurePhase, + stableReadyPolls: REPLACEMENT_STABLE_READY_POLLS, + checkReadyIdentity: (getRemainingMs = () => SANDBOX_RECREATE_PROBE_TIMEOUT_MS) => + checkRecreatedSandboxReadyIdentity( + input.sandboxName, + sandboxId, + deps, + getRemainingMs, + ), + sleep: deps.sleep, + }); + if (committedReadiness.ready) return; + console.error(""); + sandboxReadinessTracing.printReadinessFailure( + committedReadiness, + input.sandboxName, + input.sandboxReadyTimeoutSecs, + ); + const sandboxIdentityFingerprint = fingerprintSandboxRecreateValue(sandboxId); + persistCreateAttemptRecovery({ + input, + createAttemptNonce: options.createAttemptNonce, + sandboxIdentityFingerprint, + detail: + `Managed runtime commit completed for sandbox '${input.sandboxName}', but the same sandbox did not return to executable Ready state through owning gateway '${input.gatewayName}'. ` + + "Do not delete a sandbox by mutable name; preserve it for identity-bound recovery.", + }); + (deps.printCreateFailureDiagnostics ?? printSandboxCreateFailureDiagnostics)(input.sandboxName, { + backupPath: input.restoreBackupPath, + }); + console.error( + " NemoClaw did not start dashboard forwarding. NemoClaw left the sandbox in place for identity-bound recovery.", + ); + throw new Error( + `Sandbox '${input.sandboxName}' did not return to Ready after its managed runtime commit.`, + ); +} + +function selectManagedRuntimeCommitSandboxId( + managedBootstrap: SandboxGpuCreateFlowInput["managedBootstrap"], + verifiedCreatedSandboxId: string | null, +): string | null { + return managedBootstrap ? verifiedCreatedSandboxId : null; +} + function resolveCreateAttemptNonce( input: SandboxGpuCreateFlowInput, deferPostCreateEffects: boolean, @@ -446,34 +560,11 @@ export function createSandboxGpuCreateAttemptRunner( const persistIdentitySettlementRecovery = ( sandboxIdentityFingerprint: string | null = null, ): void => { - if (!createAttemptNonce) { - throw new Error("Sandbox create-attempt identity was not generated."); - } - const persist = input.persistRetainedSandboxRecovery; - if (!persist) { - throw new Error("Verified sandbox creation has no durable recovery evidence owner."); - } - const identityEvidence = sandboxIdentityFingerprint - ? `Durable sandbox identity fingerprint: ${sandboxIdentityFingerprint}. Sandbox '${input.sandboxName}' did not remain visible through owning gateway '${input.gatewayName}' before identity verification completed. ` - : `Sandbox '${input.sandboxName}' reached Ready before OpenShell returned one exact durable create identity. Gateway '${input.gatewayName}'. OpenShell did not return one exact durable sandbox identity for this create attempt. `; - const message = - `Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${createAttemptNonce}. ` + - identityEvidence + - "Do not delete a sandbox by mutable name; preserve it until an OpenShell administrator resolves the create-attempt label to one sandbox."; - let persisted = false; - try { - persisted = sandboxIdentityFingerprint - ? persist(message, sandboxIdentityFingerprint, createAttemptNonce) - : persist(message, undefined, createAttemptNonce); - } catch { - persisted = false; - } - console.error(` ${message}`); - if (!persisted) { - console.error( - " NemoClaw could not save this create-attempt evidence. Preserve the terminal output for an OpenShell administrator.", - ); - } + persistIdentitySettlementRecoveryEvidence({ + input, + createAttemptNonce, + sandboxIdentityFingerprint, + }); }; const waitForCreatedSandboxPublication = (sandboxId: string): void => { try { @@ -684,6 +775,12 @@ export function createSandboxGpuCreateAttemptRunner( let resumedSandboxId: string | null = null; let managedIncompleteCreateRecovered = false; let createdSandboxVerified = false; + let verifiedCreatedSandboxId: string | null = null; + const verifyAndRecordCreatedSandboxBeforeEffects = async (sandboxId: string): Promise => { + await verifyCreatedSandboxBeforeEffects(sandboxId, createAttemptNonce!, route, input); + verifiedCreatedSandboxId = sandboxId; + createdSandboxVerified = true; + }; const failAfterCreatedSandboxVerification = (message: string, status: number): never => { if (createdSandboxVerified) throw new Error(message); return process.exit(status); @@ -705,13 +802,7 @@ export function createSandboxGpuCreateAttemptRunner( ); } resumedSandboxId = identity.sandboxId; - await verifyCreatedSandboxBeforeEffects( - identity.sandboxId, - createAttemptNonce!, - route, - input, - ); - createdSandboxVerified = true; + await verifyAndRecordCreatedSandboxBeforeEffects(identity.sandboxId); if (deferPostCreateEffects) { revalidatePostCreateEffect(`activate managed sandbox network for '${input.sandboxName}'`); await managedLifecycle?.prepareNetwork(); @@ -786,8 +877,7 @@ export function createSandboxGpuCreateAttemptRunner( ); } waitForCreatedSandboxPublication(sandboxId); - await verifyCreatedSandboxBeforeEffects(sandboxId, createAttemptNonce!, route, input); - createdSandboxVerified = true; + await verifyAndRecordCreatedSandboxBeforeEffects(sandboxId); if (deferPostCreateEffects) { revalidatePostCreateEffect( `activate managed sandbox network for '${input.sandboxName}'`, @@ -919,8 +1009,7 @@ export function createSandboxGpuCreateAttemptRunner( ); } waitForCreatedSandboxPublication(sandboxId); - await verifyCreatedSandboxBeforeEffects(sandboxId, createAttemptNonce!, route, input); - createdSandboxVerified = true; + await verifyAndRecordCreatedSandboxBeforeEffects(sandboxId); } if (deferPostCreateEffects) { revalidatePostCreateEffect(`validate runtime patch for sandbox '${input.sandboxName}'`); @@ -1120,6 +1209,15 @@ export function createSandboxGpuCreateAttemptRunner( if (!input.sandboxGpuConfig.sandboxGpuEnabled) { revalidatePostCreateEffect(`commit runtime readiness for sandbox '${input.sandboxName}'`); await runtimePatch.commitAfterReady(); + confirmManagedRuntimeCommitReadiness({ + input, + deps, + sandboxId: selectManagedRuntimeCommitSandboxId( + managedBootstrap, + verifiedCreatedSandboxId, + ), + createAttemptNonce, + }); } return { ok: true, From dd0a9ff74e695b1ce02ac6fb88b6fb2b770675cb Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 05:15:18 -0400 Subject: [PATCH 02/51] fix(onboard): confirm GPU runtime readiness Signed-off-by: Julie Yaunches --- .../created-sandbox-finalization.test.ts | 94 +++++++++++++++++++ .../onboard/created-sandbox-finalization.ts | 1 + src/lib/onboard/sandbox-gpu-create-flow.ts | 4 + .../sandbox-gpu-create-identity-gate.test.ts | 4 + .../onboard/sandbox-gpu-create-run-attempt.ts | 71 +++++++------- src/lib/onboard/sandbox-readiness-tracing.ts | 22 +++++ 6 files changed, 158 insertions(+), 38 deletions(-) diff --git a/src/lib/onboard/created-sandbox-finalization.test.ts b/src/lib/onboard/created-sandbox-finalization.test.ts index 28ec63eb85c..def9ddc1634 100644 --- a/src/lib/onboard/created-sandbox-finalization.test.ts +++ b/src/lib/onboard/created-sandbox-finalization.test.ts @@ -19,6 +19,7 @@ import { finalizeCreatedSandbox, } from "./created-sandbox-finalization"; import { getDcodeSelectionDrift } from "./dcode-selection-drift"; +import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import type { HermesPortableConfiguredReceipt } from "./experimental/hermes-portable-receipt"; import { pendingSandboxCreateIdentityForBoundary } from "./sandbox-create/identity-boundary"; import type { SandboxGpuCreateFlowResult } from "./sandbox-gpu-create-flow"; @@ -984,6 +985,99 @@ describe("created OpenClaw sandbox finalization", () => { }); describe("created sandbox completion actions", () => { + it("stops GPU dashboard and registry effects when committed readiness does not return", async () => { + vi.spyOn( + dockerGpuLocalInference, + "verifyGpuSandboxLocalInferenceAndCommitAfterReady", + ).mockResolvedValue(); + const ensureForward = vi.fn(); + const registerCreatedSandbox = vi.fn(); + const lifecycleRegistration = { + lifecycleGeneration: "generation-1", + lifecycleLiveIdentityFingerprint: "a".repeat(64), + }; + const verifiedCreateBoundary = { + sandboxName: "alpha", + gatewayName: "nemoclaw", + gatewayPort: 8080, + ...lifecycleRegistration, + route: "native" as const, + }; + const completion = createCreatedSandboxCompletionActions( + { + finalization: { sandboxName: "alpha" }, + registration: { gatewayName: "nemoclaw", gatewayPort: 8080 }, + policy: { + initialPolicyPath: "/private/initial-policy.yaml", + compatibilityPolicyPath: null, + getVerifiedCreateBoundary: () => verifiedCreateBoundary, + getVerifiedCreateRegistrationAuthority: vi.fn(), + }, + gpu: { + config: {} as SandboxGpuConfig, + provider: "nvidia-prod", + dockerDriverGateway: true, + verifyDirectSandboxGpu: vi.fn(), + runCaptureOpenshell: vi.fn(), + }, + dashboard: { + chatUiUrl: "http://127.0.0.1:8643", + initialHermesState: { config: null, enabled: false }, + releasePort: vi.fn(), + ensureForward, + getForwardPort: vi.fn(), + resolveHermesState: vi.fn(), + ensureHermesForward: vi.fn(), + }, + workload: {}, + } as never, + { + revalidateSandboxIdentity: vi.fn(), + discoverFreshOpenClawImagePluginInstalls: vi.fn(), + restoreRecreatedSandboxState: vi.fn(), + getDcodeSelectionDrift: vi.fn(), + note: vi.fn(), + error: vi.fn(), + exitProcess: vi.fn() as never, + registerCreatedSandbox, + }, + ); + const confirmManagedRuntimeCommitReadiness = vi.fn(() => { + throw new Error("managed runtime did not return to Ready"); + }); + const created = { + origin: "created", + createResult: { status: 0, output: "", sawProgress: true }, + route: "native", + firstCreateOutput: "", + registryImageRef: null, + lifecycleRegistrationFields: { lifecycleGeneration: "generation-1" }, + runtimePatch: {}, + confirmManagedRuntimeCommitReadiness, + } as unknown as SandboxGpuCreateFlowResult; + const lifecycle = { + generation: "generation-1", + recordExactIdentity: vi.fn(), + capture: vi.fn(() => lifecycleRegistration), + revalidate: vi.fn(() => lifecycleRegistration), + }; + + await expect( + completion.complete( + created, + null, + "created", + true, + () => ({ lifecycleGeneration: "generation-1" }), + lifecycle, + ), + ).rejects.toThrow("managed runtime did not return to Ready"); + + expect(confirmManagedRuntimeCommitReadiness).toHaveBeenCalledOnce(); + expect(ensureForward).not.toHaveBeenCalled(); + expect(registerCreatedSandbox).not.toHaveBeenCalled(); + }); + it.each([ ["ordinary", true, false], ["schema-5", false, true], diff --git a/src/lib/onboard/created-sandbox-finalization.ts b/src/lib/onboard/created-sandbox-finalization.ts index bc023627988..155293a3882 100644 --- a/src/lib/onboard/created-sandbox-finalization.ts +++ b/src/lib/onboard/created-sandbox-finalization.ts @@ -372,6 +372,7 @@ export function createCreatedSandboxCompletionActions( `committing GPU capability for sandbox '${options.finalization.sandboxName}'`, ), ); + created.confirmManagedRuntimeCommitReadiness(); } function recordHermesGpuProof(): void { options.gpu.config.sandboxGpuProof = options.gpu.verifyDirectSandboxGpu( diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 7fc48e8bd39..028ad81f547 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -307,6 +307,8 @@ export interface SandboxGpuCreateFlowDeps { interface SandboxGpuCreateFlowResultCommon { runtimePatch: ManagedBootstrapRuntimePatch; + /** Confirm executable Ready state after the managed runtime commit. */ + confirmManagedRuntimeCommitReadiness(): void; route: SelectedDockerGpuRoute; /** Mutable tag/reference retained only for registry and image-GC bookkeeping. */ registryImageRef: string | null; @@ -626,6 +628,8 @@ export async function runSandboxGpuCreateFlow( const common = { runtimePatch: gpuCreateOutcome.value.runtimePatch, + confirmManagedRuntimeCommitReadiness: + gpuCreateOutcome.value.confirmManagedRuntimeCommitReadiness, route: gpuCreateOutcome.route, registryImageRef, lifecycleRegistrationFields: { diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index cfa489d14d7..0cf84bb7a28 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -201,6 +201,7 @@ describe("created sandbox identity gate", () => { }); it("retains exact recovery when committed managed readiness does not return", async () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); const sandboxId = "alpha-sandbox-id"; const sandboxIdentityFingerprint = fingerprintSandboxRecreateValue(sandboxId); const input = noGpuInput(); @@ -235,6 +236,9 @@ describe("created sandbox identity gate", () => { expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { backupPath: null, }); + expect(error.mock.calls.flat().join("\n")).toContain( + "Run `nemoclaw destroy` to attempt identity-bound recovery.", + ); }); it("resumes the exact verified sandbox without issuing another create (#9833)", async () => { diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 75eca142e31..6b4c221cb11 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -352,18 +352,12 @@ function confirmManagedRuntimeCommitReadiness(options: { console.error( " NemoClaw did not start dashboard forwarding. NemoClaw left the sandbox in place for identity-bound recovery.", ); + console.error(" Run `nemoclaw destroy` to attempt identity-bound recovery."); throw new Error( `Sandbox '${input.sandboxName}' did not return to Ready after its managed runtime commit.`, ); } -function selectManagedRuntimeCommitSandboxId( - managedBootstrap: SandboxGpuCreateFlowInput["managedBootstrap"], - verifiedCreatedSandboxId: string | null, -): string | null { - return managedBootstrap ? verifiedCreatedSandboxId : null; -} - function resolveCreateAttemptNonce( input: SandboxGpuCreateFlowInput, deferPostCreateEffects: boolean, @@ -392,21 +386,20 @@ function waitForCreatedOpenShellSandboxPublication( deps: SandboxGpuCreateFlowDeps, ): void { const timeoutMs = Math.max(1, Math.round(input.sandboxReadyTimeoutSecs * 1_000)); - const deadlineMs = Date.now() + timeoutMs; - const maxPolls = - Math.ceil(timeoutMs / (CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_SECONDS * 1_000)) + 1; - for (let poll = 0; poll < maxPolls; poll += 1) { - const remainingMs = Math.max(1, deadlineMs - Date.now()); - const result = deps.runOpenshell( - ["sandbox", "get", "-g", input.gatewayName, input.sandboxName], - { - ignoreError: true, - suppressOutput: true, - timeout: Math.min(SANDBOX_READY_PROBE_TIMEOUT_MS, remainingMs), - killSignal: "SIGKILL", - }, - ); - if (result.status === 0 && !result.error) { + const published = sandboxReadinessTracing.waitForCreatedSandboxPublication({ + timeoutMs, + pollIntervalMs: CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_SECONDS * 1_000, + probe: (getRemainingMs) => { + const result = deps.runOpenshell( + ["sandbox", "get", "-g", input.gatewayName, input.sandboxName], + { + ignoreError: true, + suppressOutput: true, + timeout: Math.min(SANDBOX_READY_PROBE_TIMEOUT_MS, getRemainingMs()), + killSignal: "SIGKILL", + }, + ); + if (result.status !== 0 || result.error) return false; const publishedSandboxId = parseOpenShellSandboxId(String(result.stdout ?? "")); if (!publishedSandboxId) { throw new Error( @@ -418,19 +411,15 @@ function waitForCreatedOpenShellSandboxPublication( `Created sandbox '${input.sandboxName}' changed identity before identity verification completed.`, ); } - return; - } - if (poll + 1 >= maxPolls || Date.now() >= deadlineMs) break; - deps.sleep( - Math.min( - CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_SECONDS, - Math.max(0, (deadlineMs - Date.now()) / 1_000), - ), + return true; + }, + sleep: deps.sleep, + }); + if (!published) { + throw new Error( + `Created sandbox '${input.sandboxName}' did not become visible through its owning gateway before identity verification completed.`, ); } - throw new Error( - `Created sandbox '${input.sandboxName}' did not become visible through its owning gateway before identity verification completed.`, - ); } function checkRecreatedSandboxReadyIdentity( @@ -1209,20 +1198,26 @@ export function createSandboxGpuCreateAttemptRunner( if (!input.sandboxGpuConfig.sandboxGpuEnabled) { revalidatePostCreateEffect(`commit runtime readiness for sandbox '${input.sandboxName}'`); await runtimePatch.commitAfterReady(); + confirmCommittedRuntimeReadiness(); + } + function confirmCommittedRuntimeReadiness(): void { confirmManagedRuntimeCommitReadiness({ input, deps, - sandboxId: selectManagedRuntimeCommitSandboxId( - managedBootstrap, - verifiedCreatedSandboxId, - ), + sandboxId: managedBootstrap ? verifiedCreatedSandboxId : null, createAttemptNonce, }); } return { ok: true, route, - value: createResult ? { createResult, runtimePatch } : { runtimePatch }, + value: createResult + ? { + createResult, + runtimePatch, + confirmManagedRuntimeCommitReadiness: confirmCommittedRuntimeReadiness, + } + : { runtimePatch, confirmManagedRuntimeCommitReadiness: confirmCommittedRuntimeReadiness }, } as const; }; diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index 200dd09448b..1cc1e50d94a 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -103,6 +103,28 @@ export interface SandboxReadyWaitOptions extends SandboxReadyWaitDeps { delaySeconds: number; } +/** Wait for one owner-scoped created-sandbox publication within a fixed deadline. */ +export function waitForCreatedSandboxPublication(options: { + readonly timeoutMs: number; + readonly pollIntervalMs: number; + readonly probe: (getRemainingMs: () => number) => boolean; + readonly sleep: (seconds: number) => void; + readonly now?: () => number; +}): boolean { + const now = options.now ?? Date.now; + const timeoutMs = Math.max(1, Math.round(options.timeoutMs)); + const pollIntervalMs = Math.max(0, options.pollIntervalMs); + const deadlineMs = now() + timeoutMs; + return waitUntil(() => options.probe(() => Math.max(1, deadlineMs - now())), { + deadlineMs, + initialIntervalMs: pollIntervalMs, + maxIntervalMs: pollIntervalMs, + maxAttempts: Math.ceil(timeoutMs / Math.max(1, pollIntervalMs)) + 1, + now, + sleep: (ms) => options.sleep(ms / 1_000), + }); +} + function pollSandboxReady( options: SandboxReadyWaitOptions & { trace?: (event: string, attributes: Record) => void; From dd0a66935067ee342d13b8969102dde962d877e4 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 05:31:45 -0400 Subject: [PATCH 03/51] fix(onboard): scope readiness probes to gateway Signed-off-by: Julie Yaunches --- .../onboard/sandbox-fresh-readiness.test.ts | 36 ++++++++++------ .../onboard/sandbox-gpu-create-flow.test.ts | 12 +++--- .../sandbox-gpu-create-identity-gate.test.ts | 13 +++++- .../onboard/sandbox-gpu-create-run-attempt.ts | 42 ++++++++++++------- 4 files changed, 69 insertions(+), 34 deletions(-) diff --git a/src/lib/onboard/sandbox-fresh-readiness.test.ts b/src/lib/onboard/sandbox-fresh-readiness.test.ts index 6380999db8b..1d53f18f41a 100644 --- a/src/lib/onboard/sandbox-fresh-readiness.test.ts +++ b/src/lib/onboard/sandbox-fresh-readiness.test.ts @@ -100,9 +100,9 @@ describe("fresh sandbox executable readiness", () => { const deps = createDeps(); vi.mocked(deps.runOpenshell).mockImplementation( createSequencedOpenShellRunner([ - ["sandbox get alpha", [readySandboxGetResult(), readySandboxGetResult()]], + ["sandbox get -g nemoclaw alpha", [readySandboxGetResult(), readySandboxGetResult()]], [ - "sandbox exec --name alpha -- true", + "sandbox exec -g nemoclaw --name alpha -- true", [ { status: 1, @@ -122,7 +122,9 @@ describe("fresh sandbox executable readiness", () => { expect( vi .mocked(deps.runOpenshell) - .mock.calls.filter(([args]) => args.join(" ") === "sandbox exec --name alpha -- true"), + .mock.calls.filter( + ([args]) => args.join(" ") === "sandbox exec -g nemoclaw --name alpha -- true", + ), ).toHaveLength(2); expect(deps.runOpenshell).not.toHaveBeenCalledWith( ["sandbox", "delete", "alpha"], @@ -134,9 +136,9 @@ describe("fresh sandbox executable readiness", () => { const deps = createDeps(); vi.mocked(deps.runOpenshell).mockImplementation( createSequencedOpenShellRunner([ - ["sandbox get alpha", [readySandboxGetResult()]], + ["sandbox get -g nemoclaw alpha", [readySandboxGetResult()]], [ - "sandbox exec --name alpha -- true", + "sandbox exec -g nemoclaw --name alpha -- true", [{ status: 1, stdout: "", stderr: "permission denied" }], ], ]), @@ -158,7 +160,10 @@ describe("fresh sandbox executable readiness", () => { const deps = createDeps(); vi.mocked(deps.runOpenshell).mockImplementation( createSequencedOpenShellRunner([ - ["sandbox get alpha", [{ status: 0, stdout: "Name: alpha\nState: Ready\n", stderr: "" }]], + [ + "sandbox get -g nemoclaw alpha", + [{ status: 0, stdout: "Name: alpha\nState: Ready\n", stderr: "" }], + ], ]), ); mockExit(); @@ -166,7 +171,7 @@ describe("fresh sandbox executable readiness", () => { await expect(runSandboxGpuCreateFlow(createInput(), deps)).rejects.toThrow("process.exit:1"); expect(deps.runOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "exec", "--name", "alpha", "--", "true"], + ["sandbox", "exec", "-g", "nemoclaw", "--name", "alpha", "--", "true"], expect.anything(), ); expect(deps.runOpenshell).not.toHaveBeenCalledWith( @@ -185,7 +190,10 @@ describe("fresh sandbox executable readiness", () => { const deps = createDeps(); vi.mocked(deps.runOpenshell).mockImplementation( createSequencedOpenShellRunner([ - ["sandbox get alpha", [timedOutOpenShellResult(SANDBOX_NOT_READY_OUTPUT)]], + [ + "sandbox get -g nemoclaw alpha", + [timedOutOpenShellResult(SANDBOX_NOT_READY_OUTPUT)], + ], ]), ); mockExit(); @@ -193,7 +201,7 @@ describe("fresh sandbox executable readiness", () => { await expect(runSandboxGpuCreateFlow(createInput(), deps)).rejects.toThrow("process.exit:1"); expect(deps.runOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "exec", "--name", "alpha", "--", "true"], + ["sandbox", "exec", "-g", "nemoclaw", "--name", "alpha", "--", "true"], expect.anything(), ); expect(deps.runOpenshell).not.toHaveBeenCalledWith( @@ -208,9 +216,9 @@ describe("fresh sandbox executable readiness", () => { input.sandboxReadyTimeoutSecs = 0.5; vi.mocked(deps.runOpenshell).mockImplementation( createSequencedOpenShellRunner([ - ["sandbox get alpha", [readySandboxGetResult()]], + ["sandbox get -g nemoclaw alpha", [readySandboxGetResult()]], [ - ["sandbox", "exec", "--name", "alpha", "--", "true"].join(" "), + ["sandbox", "exec", "-g", "nemoclaw", "--name", "alpha", "--", "true"].join(" "), [timedOutOpenShellResult(SANDBOX_NOT_READY_OUTPUT)], ], ]), @@ -221,10 +229,12 @@ describe("fresh sandbox executable readiness", () => { const identityOptions = vi .mocked(deps.runOpenshell) - .mock.calls.find(([args]) => args.join(" ") === "sandbox get alpha")?.[1]; + .mock.calls.find(([args]) => args.join(" ") === "sandbox get -g nemoclaw alpha")?.[1]; const executableOptions = vi .mocked(deps.runOpenshell) - .mock.calls.find(([args]) => args.join(" ") === "sandbox exec --name alpha -- true")?.[1]; + .mock.calls.find( + ([args]) => args.join(" ") === "sandbox exec -g nemoclaw --name alpha -- true", + )?.[1]; expect(identityOptions).toMatchObject({ killSignal: "SIGKILL" }); expect(executableOptions).toMatchObject({ killSignal: "SIGKILL" }); expect(identityOptions?.timeout).toBeGreaterThan(0); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index 808c5eb0244..df48d812545 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -707,9 +707,9 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { const deps = createDeps(); vi.mocked(deps.runOpenshell).mockImplementation( createSequencedOpenShellRunner([ - ["sandbox get alpha", [readySandboxGetResult(), readySandboxGetResult()]], + ["sandbox get -g nemoclaw alpha", [readySandboxGetResult(), readySandboxGetResult()]], [ - "sandbox exec --name alpha -- true", + "sandbox exec -g nemoclaw --name alpha -- true", [{ status: 1, stdout: "", stderr: "permission denied" }], ], ]), @@ -760,11 +760,11 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { vi.mocked(deps.runOpenshell).mockImplementation( createSequencedOpenShellRunner([ [ - "sandbox get alpha", + "sandbox get -g nemoclaw alpha", [readySandboxGetResult(), readySandboxGetResult(), readySandboxGetResult()], ], [ - "sandbox exec --name alpha -- true", + "sandbox exec -g nemoclaw --name alpha -- true", [ { status: 1, @@ -791,7 +791,9 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { expect( vi .mocked(deps.runOpenshell) - .mock.calls.filter(([args]) => args.join(" ") === "sandbox exec --name alpha -- true"), + .mock.calls.filter( + ([args]) => args.join(" ") === "sandbox exec -g nemoclaw --name alpha -- true", + ), ).toHaveLength(2); expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); expect(deps.runOpenshell).not.toHaveBeenCalledWith( diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 0cf84bb7a28..d02c899c4bd 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -149,6 +149,7 @@ describe("created sandbox identity gate", () => { it("reconfirms exact managed sandbox readiness after the runtime commit", async () => { const sandboxId = "alpha-sandbox-id"; const input = noGpuInput(); + input.gatewayName = "owner-gateway"; input.resumeVerifiedCreate = { route: "none", liveIdentityFingerprint: fingerprintSandboxRecreateValue(sandboxId), @@ -193,6 +194,14 @@ describe("created sandbox identity gate", () => { }); expect(mocks.waitForCreatedSandboxReadyWithTrace).toHaveBeenCalledTimes(2); + expect(deps.runOpenshell).toHaveBeenCalledWith( + ["sandbox", "get", "-g", "owner-gateway", "alpha"], + expect.objectContaining({ suppressOutput: true }), + ); + expect(deps.runOpenshell).toHaveBeenCalledWith( + ["sandbox", "exec", "-g", "owner-gateway", "--name", "alpha", "--", "true"], + expect.objectContaining({ suppressOutput: true }), + ); expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); expect(deps.runOpenshell).not.toHaveBeenCalledWith( ["sandbox", "delete", "alpha"], @@ -274,7 +283,7 @@ describe("created sandbox identity gate", () => { }); const deps = createGpuFlowDeps(); vi.mocked(deps.runOpenshell).mockImplementation((args) => - args.join(" ") === "sandbox get alpha" + args.join(" ") === "sandbox get -g nemoclaw alpha" ? { status: 0, stdout: `Name: alpha\nId: ${sandboxId}\nState: Ready\n`, stderr: "" } : { status: 0, stdout: "", stderr: "" }, ); @@ -319,7 +328,7 @@ describe("created sandbox identity gate", () => { mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); const deps = createGpuFlowDeps(); vi.mocked(deps.runOpenshell).mockImplementation((args) => - args.join(" ") === "sandbox get alpha" + args.join(" ") === "sandbox get -g nemoclaw alpha" ? { status: 0, stdout: "Name: alpha\nId: replacement-id\nState: Ready\n", stderr: "" } : { status: 0, stdout: "", stderr: "" }, ); diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 6b4c221cb11..ee00fcc40dc 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -212,12 +212,13 @@ function remainingReadinessProbeTimeout(getRemainingMs: () => number): number | function probeExactOpenShellSandboxId( sandboxName: string, + gatewayName: string, deps: SandboxGpuCreateFlowDeps, getRemainingMs: () => number = () => SANDBOX_RECREATE_PROBE_TIMEOUT_MS, ): OpenShellSandboxIdentityProbe { const timeout = remainingReadinessProbeTimeout(getRemainingMs); if (timeout === null) return { state: "not_ready" }; - const result = deps.runOpenshell(["sandbox", "get", sandboxName], { + const result = deps.runOpenshell(["sandbox", "get", "-g", gatewayName, sandboxName], { ignoreError: true, suppressOutput: true, timeout, @@ -324,6 +325,7 @@ function confirmManagedRuntimeCommitReadiness(options: { checkReadyIdentity: (getRemainingMs = () => SANDBOX_RECREATE_PROBE_TIMEOUT_MS) => checkRecreatedSandboxReadyIdentity( input.sandboxName, + input.gatewayName, sandboxId, deps, getRemainingMs, @@ -424,41 +426,47 @@ function waitForCreatedOpenShellSandboxPublication( function checkRecreatedSandboxReadyIdentity( sandboxName: string, + gatewayName: string, expectedSandboxId: string, deps: SandboxGpuCreateFlowDeps, getRemainingMs: () => number, ): ReturnType { - const identity = probeExactOpenShellSandboxId(sandboxName, deps, getRemainingMs); + const identity = probeExactOpenShellSandboxId(sandboxName, gatewayName, deps, getRemainingMs); if (identity.state === "not_ready") return "not_ready"; if (identity.state === "failed") return "probe_failed"; if (identity.sandboxId !== expectedSandboxId) return "identity_changed"; - return checkSandboxExecutableReadiness(sandboxName, deps, getRemainingMs); + return checkSandboxExecutableReadiness(sandboxName, gatewayName, deps, getRemainingMs); } function checkCreatedSandboxReadyIdentity( sandboxName: string, + gatewayName: string, deps: SandboxGpuCreateFlowDeps, getRemainingMs: () => number, ): ReturnType { - const identity = probeExactOpenShellSandboxId(sandboxName, deps, getRemainingMs); + const identity = probeExactOpenShellSandboxId(sandboxName, gatewayName, deps, getRemainingMs); if (identity.state === "not_ready") return "not_ready"; if (identity.state === "failed") return "probe_failed"; - return checkSandboxExecutableReadiness(sandboxName, deps, getRemainingMs); + return checkSandboxExecutableReadiness(sandboxName, gatewayName, deps, getRemainingMs); } function checkSandboxExecutableReadiness( sandboxName: string, + gatewayName: string, deps: SandboxGpuCreateFlowDeps, getRemainingMs: () => number, ): ReturnType { const timeout = remainingReadinessProbeTimeout(getRemainingMs); if (timeout === null) return "not_ready"; - const result = deps.runOpenshell(["sandbox", "exec", "--name", sandboxName, "--", "true"], { - ignoreError: true, - suppressOutput: true, - timeout, - killSignal: "SIGKILL", - }); + const result = deps.runOpenshell( + ["sandbox", "exec", "-g", gatewayName, "--name", sandboxName, "--", "true"], + { + ignoreError: true, + suppressOutput: true, + timeout, + killSignal: "SIGKILL", + }, + ); if (result.status === 0 && !result.error) return "ready"; if (result.error || result.status === null || ("signal" in result && result.signal)) { return "probe_failed"; @@ -778,7 +786,7 @@ export function createSandboxGpuCreateAttemptRunner( if (route !== input.resumeVerifiedCreate.route) { throw new Error("Verified sandbox recovery route changed before continuation."); } - const identity = probeExactOpenShellSandboxId(input.sandboxName, deps); + const identity = probeExactOpenShellSandboxId(input.sandboxName, input.gatewayName, deps); if (identity.state !== "identified") { throw new Error( `Cannot resume sandbox '${input.sandboxName}': its exact live identity is unavailable.`, @@ -1006,7 +1014,7 @@ export function createSandboxGpuCreateAttemptRunner( } const preRecreateIdentity = deferRestartSafeCutover && !resumedSandboxId - ? probeExactOpenShellSandboxId(input.sandboxName, deps) + ? probeExactOpenShellSandboxId(input.sandboxName, input.gatewayName, deps) : null; const expectedRecreatedSandboxId = resumedSandboxId ?? @@ -1045,6 +1053,7 @@ export function createSandboxGpuCreateAttemptRunner( ? (getRemainingMs = () => SANDBOX_RECREATE_PROBE_TIMEOUT_MS) => checkRecreatedSandboxReadyIdentity( input.sandboxName, + input.gatewayName, expectedRecreatedSandboxId, deps, getRemainingMs, @@ -1052,7 +1061,12 @@ export function createSandboxGpuCreateAttemptRunner( : input.terminalAgent ? undefined : (getRemainingMs = () => SANDBOX_RECREATE_PROBE_TIMEOUT_MS) => - checkCreatedSandboxReadyIdentity(input.sandboxName, deps, getRemainingMs), + checkCreatedSandboxReadyIdentity( + input.sandboxName, + input.gatewayName, + deps, + getRemainingMs, + ), sleep: deps.sleep, }); if (!readiness.ready) { From 661786d37442611cd34408704da343ef31d7a576 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 05:56:22 -0400 Subject: [PATCH 04/51] fix(onboard): scope readiness lists to gateway Signed-off-by: Julie Yaunches --- .../checks/run-managed-image-openshell-e2e.ts | 1 + .../sandbox-gpu-create-flow.ts | 1 + .../hermes-portable-onboarding.ts | 69 ++++++++++++++----- .../onboard/sandbox-create/orchestration.ts | 1 + .../onboard/sandbox-gpu-create-flow.test.ts | 10 ++- src/lib/onboard/sandbox-gpu-create-flow.ts | 2 + .../sandbox-gpu-create-identity-gate.test.ts | 7 +- .../onboard/sandbox-gpu-create-run-attempt.ts | 11 ++- .../sandbox-readiness-stability.test.ts | 29 ++++++++ .../onboard/sandbox-readiness-tracing.test.ts | 14 ++++ src/lib/onboard/sandbox-readiness-tracing.ts | 6 +- 11 files changed, 126 insertions(+), 25 deletions(-) diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 917cc682d16..8b970a9ad22 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -1017,6 +1017,7 @@ async function run { vi.mocked(deps.runCaptureOpenshell).mockClear(); await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); - expect(deps.runCaptureOpenshell).toHaveBeenCalledWith(["sandbox", "list"], READY_CHECK_OPTIONS); + expect(deps.runCaptureOpenshell).toHaveBeenCalledWith( + ["sandbox", "list", "-g", "nemoclaw"], + READY_CHECK_OPTIONS, + ); expect(vi.mocked(console.warn).mock.calls.flat().join("\n")).toContain( "unrelated sandbox 'bravo'", @@ -620,7 +623,10 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { const result = await runSandboxGpuCreateFlow(createInput(), deps); expect(result).toMatchObject({ route: "native" }); - expect(deps.runCaptureOpenshell).toHaveBeenCalledWith(["sandbox", "list"], READY_CHECK_OPTIONS); + expect(deps.runCaptureOpenshell).toHaveBeenCalledWith( + ["sandbox", "list", "-g", "nemoclaw"], + READY_CHECK_OPTIONS, + ); }); it("defers restart-safe no-GPU recreation until the create process exits (#8720)", async () => { diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 028ad81f547..b36c7a75e9d 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -210,6 +210,8 @@ type LifecycleRegistrationFields = Pick; export interface SandboxGpuCreateFlowInput { sandboxName: string; + /** Active CLI spelling used in identity-bound recovery guidance. */ + cliName: string; /** Resume the exact sandbox retained after its verified-create checkpoint was persisted. */ resumeVerifiedCreate?: { readonly route: SelectedDockerGpuRoute; diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index d02c899c4bd..4a1fe444d4b 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -214,6 +214,7 @@ describe("created sandbox identity gate", () => { const sandboxId = "alpha-sandbox-id"; const sandboxIdentityFingerprint = fingerprintSandboxRecreateValue(sandboxId); const input = noGpuInput(); + input.cliName = "nemohermes"; input.resumeVerifiedCreate = { route: "none", liveIdentityFingerprint: sandboxIdentityFingerprint, @@ -245,9 +246,9 @@ describe("created sandbox identity gate", () => { expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { backupPath: null, }); - expect(error.mock.calls.flat().join("\n")).toContain( - "Run `nemoclaw destroy` to attempt identity-bound recovery.", - ); + const recoveryOutput = error.mock.calls.flat().join("\n"); + expect(recoveryOutput).toContain("Run `nemohermes alpha destroy` to recover"); + expect(recoveryOutput).toContain("Stop if the command cannot verify its retained identity."); }); it("resumes the exact verified sandbox without issuing another create (#9833)", async () => { diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index ee00fcc40dc..0aec4431ffb 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -317,6 +317,7 @@ function confirmManagedRuntimeCommitReadiness(options: { ); const committedReadiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ sandboxName: input.sandboxName, + gatewayName: input.gatewayName, timeoutSecs: input.sandboxReadyTimeoutSecs, runCaptureOpenshell: deps.runCaptureOpenshell, isSandboxReady, @@ -354,7 +355,9 @@ function confirmManagedRuntimeCommitReadiness(options: { console.error( " NemoClaw did not start dashboard forwarding. NemoClaw left the sandbox in place for identity-bound recovery.", ); - console.error(" Run `nemoclaw destroy` to attempt identity-bound recovery."); + console.error( + ` Run \`${input.cliName} ${input.sandboxName} destroy\` to recover the retained sandbox. Stop if the command cannot verify its retained identity.`, + ); throw new Error( `Sandbox '${input.sandboxName}' did not return to Ready after its managed runtime commit.`, ); @@ -715,7 +718,7 @@ export function createSandboxGpuCreateAttemptRunner( streamSandboxCreate(createExecutable, createExecutableArgs, createEnv, { ...(input.createWorkingDirectory ? { cwd: input.createWorkingDirectory } : {}), readyCheck: () => { - const list = deps.runCaptureOpenshell(["sandbox", "list"], { + const list = deps.runCaptureOpenshell(["sandbox", "list", "-g", input.gatewayName], { ignoreError: true, timeout: SANDBOX_READY_PROBE_TIMEOUT_MS, }); @@ -826,6 +829,7 @@ export function createSandboxGpuCreateAttemptRunner( if (createFailure?.kind === "sandbox_create_incomplete") { const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ sandboxName: input.sandboxName, + gatewayName: input.gatewayName, timeoutSecs: input.sandboxReadyTimeoutSecs, runCaptureOpenshell: deps.runCaptureOpenshell, isSandboxReady, @@ -846,7 +850,7 @@ export function createSandboxGpuCreateAttemptRunner( ); } } else { - const list = deps.runCaptureOpenshell(["sandbox", "list"], { + const list = deps.runCaptureOpenshell(["sandbox", "list", "-g", input.gatewayName], { ignoreError: true, timeout: SANDBOX_READY_PROBE_TIMEOUT_MS, }); @@ -1041,6 +1045,7 @@ export function createSandboxGpuCreateAttemptRunner( console.log(" Waiting for sandbox to become ready..."); const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ sandboxName: input.sandboxName, + gatewayName: input.gatewayName, timeoutSecs: input.sandboxReadyTimeoutSecs, runCaptureOpenshell: deps.runCaptureOpenshell, isSandboxReady, diff --git a/src/lib/onboard/sandbox-readiness-stability.test.ts b/src/lib/onboard/sandbox-readiness-stability.test.ts index 4fba29d0ffc..91d18a20754 100644 --- a/src/lib/onboard/sandbox-readiness-stability.test.ts +++ b/src/lib/onboard/sandbox-readiness-stability.test.ts @@ -15,11 +15,39 @@ function replay(outputs: readonly string[]) { } describe("created sandbox Ready stability", () => { + it("reads same-name readiness only from the owning gateway", () => { + let nowMs = 0; + const runCaptureOpenshell = vi.fn((args: string[]) => + args.join(" ") === "sandbox list -g owner-gateway" ? `${NAME} Pending` : `${NAME} Ready`, + ); + const sleep = vi.fn((seconds: number) => { + nowMs += seconds * 1_000; + }); + + const ready = waitForCreatedSandboxReadyWithTrace({ + sandboxName: NAME, + gatewayName: "owner-gateway", + timeoutSecs: 1, + runCaptureOpenshell, + isSandboxReady, + getSandboxFailurePhase, + sleep, + now: () => nowMs, + }); + + expect(ready).toEqual({ ready: false, reason: "timeout", failurePhase: null }); + expect(runCaptureOpenshell).toHaveBeenCalledWith(["sandbox", "list", "-g", "owner-gateway"], { + ignoreError: true, + }); + expect(runCaptureOpenshell).not.toHaveBeenCalledWith(["sandbox", "list"], expect.anything()); + }); + it("preserves single-poll Ready acceptance by default", () => { const { runCaptureOpenshell, sleep } = replay([`${NAME} Ready 1s ago`]); const ready = waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", timeoutSecs: 600, runCaptureOpenshell, isSandboxReady, @@ -45,6 +73,7 @@ describe("created sandbox Ready stability", () => { const ready = waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", timeoutSecs: 600, runCaptureOpenshell, isSandboxReady, diff --git a/src/lib/onboard/sandbox-readiness-tracing.test.ts b/src/lib/onboard/sandbox-readiness-tracing.test.ts index 52b9459fc46..df706b1eecb 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.test.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.test.ts @@ -93,6 +93,7 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { expect( waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", timeoutSecs: 30, runCaptureOpenshell, isSandboxReady, @@ -112,6 +113,7 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { expect( waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", timeoutSecs: 30, runCaptureOpenshell, isSandboxReady, @@ -128,6 +130,7 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { const readiness = waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", timeoutSecs: 30, runCaptureOpenshell, isSandboxReady, @@ -152,6 +155,7 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { expect( waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", timeoutSecs: 0, runCaptureOpenshell, isSandboxReady, @@ -169,6 +173,7 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { const ready = waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", // 600 / 2 = 300 readyAttempts. With the K=1 (no-debounce) opt-out we bail // out after the 2nd poll, preserving the original fast-fail intent. timeoutSecs: 600, @@ -202,6 +207,7 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { const ready = waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", timeoutSecs: 600, runCaptureOpenshell, isSandboxReady, @@ -224,6 +230,7 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { const ready = waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", timeoutSecs: 600, runCaptureOpenshell, isSandboxReady, @@ -241,6 +248,7 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { const ready = waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", timeoutSecs: 600, runCaptureOpenshell, isSandboxReady, @@ -268,6 +276,7 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { const ready = waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", timeoutSecs: 2, // -> readyAttempts = 1, far below the default 30-poll debounce runCaptureOpenshell, isSandboxReady, @@ -293,6 +302,7 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { const ready = waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", timeoutSecs: 600, runCaptureOpenshell, isSandboxReady, @@ -313,6 +323,7 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { const ready = waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", timeoutSecs: 600, runCaptureOpenshell, isSandboxReady, @@ -344,6 +355,7 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { const ready = waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", timeoutSecs: 600, runCaptureOpenshell, isSandboxReady, @@ -507,6 +519,7 @@ describe("DGX Spark fresh-onboard readiness replay (#6043)", () => { const { runCaptureOpenshell, sleep } = replay(reporterSequence); const ready = waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", timeoutSecs: 1500, runCaptureOpenshell, isSandboxReady, @@ -535,6 +548,7 @@ describe("DGX Spark fresh-onboard readiness replay (#6043)", () => { const { runCaptureOpenshell, sleep } = replay(reporterSequence); const ready = waitForCreatedSandboxReadyWithTrace({ sandboxName: NAME, + gatewayName: "owner-gateway", timeoutSecs: 1500, runCaptureOpenshell, isSandboxReady, diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index 1cc1e50d94a..da99e4dea34 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -213,6 +213,7 @@ export function createSandboxReadyWaiter( export function waitForCreatedSandboxReadyWithTrace(options: { sandboxName: string; + gatewayName: string; timeoutSecs: number; runCaptureOpenshell: RunCaptureOpenshell; isSandboxReady: (output: string, sandboxName: string) => boolean; @@ -265,6 +266,7 @@ export function waitForCreatedSandboxReadyWithTrace(options: { }): CreatedSandboxReadinessResult { const { sandboxName, + gatewayName, timeoutSecs, runCaptureOpenshell, isSandboxReady, @@ -307,7 +309,9 @@ export function waitForCreatedSandboxReadyWithTrace(options: { let result: CreatedSandboxReadinessResult | null = null; waitUntil(() => { attempt += 1; - const list = runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); + const list = runCaptureOpenshell(["sandbox", "list", "-g", gatewayName], { + ignoreError: true, + }); if (isSandboxReady(list, sandboxName)) { const identity = options.checkReadyIdentity?.(getRemainingMs) ?? "ready"; if (identity === "identity_changed") { From 9f3629ddd65c9b717bf528517390ed9a053f08ad Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 06:32:45 -0400 Subject: [PATCH 05/51] test(onboard): isolate scoped Hermes readiness Signed-off-by: Julie Yaunches --- .../hermes-portable-onboarding.test.ts | 21 ++++++++++++ .../hermes-portable-onboarding.ts | 33 +++++++++---------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts b/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts index 9eb38f7c1e3..55244d69cbf 100644 --- a/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts +++ b/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts @@ -1359,6 +1359,27 @@ network_policies: ); }); + it("accepts readiness commands already scoped to the receipt gateway", () => { + const capture = vi.fn(() => ({ + status: 0, + stdout: Buffer.from("ready"), + stderr: Buffer.alloc(0), + })); + const run = createHermesPortableReadyRunner("alpha", "nemoclaw", capture); + + expect(run(["sandbox", "list", "-g", "nemoclaw"]).status).toBe(0); + expect( + run(["sandbox", "exec", "-g", "nemoclaw", "--name", "alpha", "--", "true"]).status, + ).toBe(0); + expect(capture.mock.calls).toEqual([ + [["sandbox", "list", "-g", "nemoclaw"]], + [["sandbox", "exec", "-g", "nemoclaw", "--name", "alpha", "--", "true"]], + ]); + expect(() => run(["sandbox", "list", "-g", "other-gateway"])).toThrow( + "unsupported OpenShell command", + ); + }); + it("rejects exact-gateway identity that has not reached Ready (#9203)", () => { const capture = vi .fn() diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding.ts b/src/lib/onboard/experimental/hermes-portable-onboarding.ts index bcc9e8bc51c..6d05025d68a 100644 --- a/src/lib/onboard/experimental/hermes-portable-onboarding.ts +++ b/src/lib/onboard/experimental/hermes-portable-onboarding.ts @@ -316,9 +316,6 @@ function scopeHermesPortableReadyGetArgs( } function scopeHermesPortableReadyListArgs(args: string[], gatewayName: string): string[] | null { - if (args.length === 2 && args[0] === "sandbox" && args[1] === "list") { - return ["sandbox", "list", "-g", gatewayName]; - } if ( args.length === 4 && args[0] === "sandbox" && @@ -336,17 +333,6 @@ function scopeHermesPortableReadyExecArgs( sandboxName: string, gatewayName: string, ): string[] | null { - if ( - args.length === 6 && - args[0] === "sandbox" && - args[1] === "exec" && - args[2] === "--name" && - args[3] === sandboxName && - args[4] === "--" && - args[5] === "true" - ) { - return ["sandbox", "exec", "-g", gatewayName, "--name", sandboxName, "--", "true"]; - } if ( args.length === 8 && args[0] === "sandbox" && @@ -375,9 +361,22 @@ export function createHermesPortableReadyRunner( scopeHermesPortableReadyGetArgs(args, sandboxName, gatewayName) ?? scopeHermesPortableReadyListArgs(args, gatewayName) ?? scopeHermesPortableReadyExecArgs(args, sandboxName, gatewayName) ?? - (args[0] === "sandbox" && args[1] === "delete" && args.length === 3 && args[2] === sandboxName - ? ["sandbox", "delete", "-g", gatewayName, args[2]!] - : null); + (args[0] === "sandbox" && args[1] === "list" && args.length === 2 + ? ["sandbox", "list", "-g", gatewayName] + : args[0] === "sandbox" && + args[1] === "delete" && + args.length === 3 && + args[2] === sandboxName + ? ["sandbox", "delete", "-g", gatewayName, args[2]!] + : args.length === 6 && + args[0] === "sandbox" && + args[1] === "exec" && + args[2] === "--name" && + args[3] === sandboxName && + args[4] === "--" && + args[5] === "true" + ? ["sandbox", "exec", "-g", gatewayName, "--name", args[3]!, "--", "true"] + : null); if (!scoped) fail("create lifecycle attempted an unsupported OpenShell command"); return capture(scoped); }; From 8507e63799296cab9880be2009f2d6b4363d99c3 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 06:42:43 -0400 Subject: [PATCH 06/51] fix(onboard): clarify retained sandbox recovery Signed-off-by: Julie Yaunches --- src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts | 7 +++++-- src/lib/onboard/sandbox-gpu-create-run-attempt.ts | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 4a1fe444d4b..8bbc2aa848c 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -247,8 +247,11 @@ describe("created sandbox identity gate", () => { backupPath: null, }); const recoveryOutput = error.mock.calls.flat().join("\n"); - expect(recoveryOutput).toContain("Run `nemohermes alpha destroy` to recover"); - expect(recoveryOutput).toContain("Stop if the command cannot verify its retained identity."); + expect(recoveryOutput).toContain("Do not delete sandbox 'alpha' by name."); + expect(recoveryOutput).toContain( + "Give the create-attempt label above to an OpenShell administrator", + ); + expect(recoveryOutput).toContain("run `nemohermes alpha destroy --yes`"); }); it("resumes the exact verified sandbox without issuing another create (#9833)", async () => { diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 0aec4431ffb..c44f3415853 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -356,7 +356,7 @@ function confirmManagedRuntimeCommitReadiness(options: { " NemoClaw did not start dashboard forwarding. NemoClaw left the sandbox in place for identity-bound recovery.", ); console.error( - ` Run \`${input.cliName} ${input.sandboxName} destroy\` to recover the retained sandbox. Stop if the command cannot verify its retained identity.`, + ` Do not delete sandbox '${input.sandboxName}' by name. Give the create-attempt label above to an OpenShell administrator and ask them to remove that exact sandbox through an identity-bound procedure. After OpenShell confirms the sandbox is absent, run \`${input.cliName} ${input.sandboxName} destroy --yes\` to reconcile the retained local state.`, ); throw new Error( `Sandbox '${input.sandboxName}' did not return to Ready after its managed runtime commit.`, From e990fcaf713488316221d6149cc9df8b90669b37 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 06:52:50 -0400 Subject: [PATCH 07/51] fix(onboard): reject mutable-name portable cleanup Signed-off-by: Julie Yaunches --- .../hermes-portable-onboarding.test.ts | 3 +- .../hermes-portable-onboarding.ts | 25 ++++----- src/lib/onboard/sandbox-readiness-tracing.ts | 56 ++----------------- 3 files changed, 16 insertions(+), 68 deletions(-) diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts b/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts index 55244d69cbf..5c8d2f40a67 100644 --- a/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts +++ b/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts @@ -1346,13 +1346,12 @@ network_policies: const run = createHermesPortableReadyRunner("alpha", "nemoclaw", capture); expect(run(["sandbox", "get", "alpha"]).status).toBe(0); - expect(run(["sandbox", "delete", "alpha"]).status).toBe(0); expect(run(["sandbox", "exec", "--name", "alpha", "--", "true"]).status).toBe(0); expect(capture.mock.calls).toEqual([ [["sandbox", "get", "-g", "nemoclaw", "alpha"]], - [["sandbox", "delete", "-g", "nemoclaw", "alpha"]], [["sandbox", "exec", "-g", "nemoclaw", "--name", "alpha", "--", "true"]], ]); + expect(() => run(["sandbox", "delete", "alpha"])).toThrow("unsupported OpenShell command"); expect(() => run(["sandbox", "get", "beta"])).toThrow("unsupported OpenShell command"); expect(() => run(["sandbox", "exec", "--name", "beta", "--", "true"])).toThrow( "unsupported OpenShell command", diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding.ts b/src/lib/onboard/experimental/hermes-portable-onboarding.ts index 6d05025d68a..ca54e1b1f90 100644 --- a/src/lib/onboard/experimental/hermes-portable-onboarding.ts +++ b/src/lib/onboard/experimental/hermes-portable-onboarding.ts @@ -349,7 +349,7 @@ function scopeHermesPortableReadyExecArgs( return null; } -/** Route create readiness and failed-create cleanup through exact schema-7 authority. */ +/** Route create readiness through exact schema-7 authority. */ export function createHermesPortableReadyRunner( sandboxName: string, gatewayName: string, @@ -363,20 +363,15 @@ export function createHermesPortableReadyRunner( scopeHermesPortableReadyExecArgs(args, sandboxName, gatewayName) ?? (args[0] === "sandbox" && args[1] === "list" && args.length === 2 ? ["sandbox", "list", "-g", gatewayName] - : args[0] === "sandbox" && - args[1] === "delete" && - args.length === 3 && - args[2] === sandboxName - ? ["sandbox", "delete", "-g", gatewayName, args[2]!] - : args.length === 6 && - args[0] === "sandbox" && - args[1] === "exec" && - args[2] === "--name" && - args[3] === sandboxName && - args[4] === "--" && - args[5] === "true" - ? ["sandbox", "exec", "-g", gatewayName, "--name", args[3]!, "--", "true"] - : null); + : args.length === 6 && + args[0] === "sandbox" && + args[1] === "exec" && + args[2] === "--name" && + args[3] === sandboxName && + args[4] === "--" && + args[5] === "true" + ? ["sandbox", "exec", "-g", gatewayName, "--name", args[3]!, "--", "true"] + : null); if (!scoped) fail("create lifecycle attempted an unsupported OpenShell command"); return capture(scoped); }; diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index da99e4dea34..49e74a99ac3 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -15,57 +15,11 @@ type RunCaptureOpenshell = (args: string[], options?: { ignoreError?: boolean }) export const SANDBOX_READY_ERROR_DEBOUNCE_ENV = "NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE"; /* - * Create/readiness Error-phase debounce. - * - * Invalid state - * ------------- - * On a fresh onboard the OpenShell gateway may (re)start its supervisor - * session and re-register the just-created sandbox. During that window - * `openshell sandbox list` briefly reports the sandbox in the transient - * "Error" phase before it flips to Ready. Observed on DGX Spark, where the - * dashboard port fallback (18789 -> 18794) and supervisor restart race the - * sandbox bootstrap (#6043). Fast-failing on the first Error poll turns that - * recoverable transient into a terminal onboard failure. - * - * Source-of-truth boundary - * ------------------------ - * The transient lives in the OpenShell gateway's `sandbox list` cache: the - * preferred fix is upstream — `sandbox list` should not report a terminal - * phase for a sandbox the gateway is still registering. Until that ships, - * NemoClaw tolerates the transient at this layer via a consecutive-Error-poll - * debounce, mirroring the Docker GPU supervisor-reconnect path - * (docker-gpu-supervisor-reconnect.ts), which tolerates the same class of - * transient while a recreated GPU container reconnects. - * - * Scope - * ----- - * Only the "Error" phase is debounced. "Failed" and "CrashLoopBackOff" are - * genuinely terminal and still fast-fail immediately. A sandbox that stays in - * Error also fast-fails after the bounded debounce window (well before the - * full readiness timeout), and the caller still captures full failure - * diagnostics — this does NOT hide terminal failures. - * - * Regression evidence / removal condition - * --------------------------------------- - * Delete this debounce once OpenShell guarantees `sandbox list` skips the - * brief Error transition during a known registration. The runtime evidence - * required is a fresh-onboard reproduction (DGX Spark, or the deterministic - * `sandbox list` replay in sandbox-readiness-tracing.test.ts) showing a - * transient create-time Error that recovers to Ready. - * - * Tracking mechanism: removal is tracked on NemoClaw #6043 - * (https://github.com/NVIDIA/NemoClaw/issues/6043), which owns the pending - * OpenShell `sandbox list` fix. The maintainer-enabled removal-signal - * test `upstream_openshell_sandbox_list_error_transient_fixed` - * (sandbox-readiness-tracing.test.ts, currently `it.skip`) is the executable - * checkpoint — point it at a captured `sandbox list` trace from a fixed - * OpenShell and, once it passes (no transient Error), this debounce can be - * removed. Escalate to a dedicated OpenShell-fix tracking issue (referenced - * here and in the test) if the workaround outlives a release cycle. - * - * The readiness loop starts at 250ms and backs off to 2 seconds. The default - * of 30 therefore tolerates a substantial transient window while the overall - * sandbox readiness deadline remains authoritative. + * OpenShell can briefly report Error while it registers a new sandbox. + * Debounce only Error; Failed and CrashLoopBackOff remain terminal. Remove + * this workaround only after a fresh-onboard trace from fixed OpenShell + * contains no transient Error phase. The skipped removal-signal test in + * sandbox-readiness-tracing.test.ts records that checkpoint (#6043). */ const SANDBOX_READY_ERROR_PHASE_DEFAULT_DEBOUNCE_POLLS = 30; From 347f25212c0266571bbbdf766582c8ea0172aadc Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 07:07:16 -0400 Subject: [PATCH 08/51] test(e2e): expect gateway-scoped readiness probes Signed-off-by: Julie Yaunches --- test/helpers/managed-image-buildless-e2e.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/helpers/managed-image-buildless-e2e.ts b/test/helpers/managed-image-buildless-e2e.ts index 6a77cb6b912..b9e73149d87 100644 --- a/test/helpers/managed-image-buildless-e2e.ts +++ b/test/helpers/managed-image-buildless-e2e.ts @@ -839,12 +839,14 @@ function assertManagedLaunch( } else { expect( result.payload.runnerCommands.some((command) => - command.includes(`sandbox get ${bootstrapRequest?.sandboxName}`), + command.includes(`sandbox get -g nemoclaw ${bootstrapRequest?.sandboxName}`), ), ).toBe(true); expect( result.payload.runnerCommands.some((command) => - command.includes(`sandbox exec --name ${bootstrapRequest?.sandboxName} -- true`), + command.includes( + `sandbox exec -g nemoclaw --name ${bootstrapRequest?.sandboxName} -- true`, + ), ), ).toBe(true); } From 89b851aac1c1af2e1145f2f919e2f720aa016cfd Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 07:21:11 -0400 Subject: [PATCH 09/51] test(security): expect scoped readiness commands Signed-off-by: Julie Yaunches --- test/security/shellquote-sandbox.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/security/shellquote-sandbox.test.ts b/test/security/shellquote-sandbox.test.ts index af27470fc5d..2f2d41ed642 100644 --- a/test/security/shellquote-sandbox.test.ts +++ b/test/security/shellquote-sandbox.test.ts @@ -222,12 +222,12 @@ try { expect(dnsCommand.command).not.toContain("bash -c"); expect( payload.commands.some((entry: { command: string }) => - entry.command.includes("sandbox get my-assistant"), + entry.command.includes("sandbox get -g nemoclaw my-assistant"), ), ).toBe(true); expect( payload.commands.some((entry: { command: string }) => - entry.command.includes("sandbox exec --name my-assistant -- true"), + entry.command.includes("sandbox exec -g nemoclaw --name my-assistant -- true"), ), ).toBe(true); } finally { From 474b30a9780d18148ecdfb522adb7b9b49444cc3 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 07:28:31 -0400 Subject: [PATCH 10/51] docs(onboard): describe readiness phase handling Signed-off-by: Julie Yaunches --- src/lib/onboard/sandbox-readiness-tracing.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index 49e74a99ac3..971bee862f1 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -172,10 +172,9 @@ export function waitForCreatedSandboxReadyWithTrace(options: { runCaptureOpenshell: RunCaptureOpenshell; isSandboxReady: (output: string, sandboxName: string) => boolean; /** - * Optional terminal-failure-phase classifier. When provided, the waiter - * short-circuits as soon as the sandbox enters a terminal failure phase - * (e.g. Error / Failed / CrashLoopBackOff) rather than burning the full - * timeout window before reporting "did not become ready" (#4316). + * Optional terminal-failure-phase classifier. Failed and CrashLoopBackOff + * stop the wait immediately. Error stops it after the configured number of + * consecutive Error observations (#4316). */ getSandboxFailurePhase?: (output: string, sandboxName: string) => string | null; /** From 008e834db2c184b47c7175bfbadaf83c0b2a0e68 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 07:34:34 -0400 Subject: [PATCH 11/51] docs(onboard): make debounce removal evidence explicit Signed-off-by: Julie Yaunches --- src/lib/onboard/sandbox-readiness-tracing.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index 971bee862f1..5f29ca6edf5 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -17,9 +17,11 @@ export const SANDBOX_READY_ERROR_DEBOUNCE_ENV = "NEMOCLAW_SANDBOX_READY_ERROR_DE /* * OpenShell can briefly report Error while it registers a new sandbox. * Debounce only Error; Failed and CrashLoopBackOff remain terminal. Remove - * this workaround only after a fresh-onboard trace from fixed OpenShell - * contains no transient Error phase. The skipped removal-signal test in - * sandbox-readiness-tracing.test.ts records that checkpoint (#6043). + * this workaround only after an OpenShell release guarantees that sandbox + * list does not report Error while registering a newly created sandbox. + * Capture a fresh-onboarding trace from that release and enable the skipped + * removal-signal test in sandbox-readiness-tracing.test.ts before removal + * (#6043). */ const SANDBOX_READY_ERROR_PHASE_DEFAULT_DEBOUNCE_POLLS = 30; From 6f274756424969e3a9d35ea264260cf98081c71b Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 07:54:14 -0400 Subject: [PATCH 12/51] refactor(onboard): consolidate readiness probes Signed-off-by: Julie Yaunches --- .../onboard/sandbox-gpu-create-run-attempt.ts | 63 +++++++++---------- src/lib/onboard/sandbox-readiness-tracing.ts | 22 ------- 2 files changed, 29 insertions(+), 56 deletions(-) diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index c44f3415853..20f30075ce2 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -324,7 +324,7 @@ function confirmManagedRuntimeCommitReadiness(options: { getSandboxFailurePhase, stableReadyPolls: REPLACEMENT_STABLE_READY_POLLS, checkReadyIdentity: (getRemainingMs = () => SANDBOX_RECREATE_PROBE_TIMEOUT_MS) => - checkRecreatedSandboxReadyIdentity( + checkCreatedSandboxReadyIdentity( input.sandboxName, input.gatewayName, sandboxId, @@ -391,20 +391,22 @@ function waitForCreatedOpenShellSandboxPublication( deps: SandboxGpuCreateFlowDeps, ): void { const timeoutMs = Math.max(1, Math.round(input.sandboxReadyTimeoutSecs * 1_000)); - const published = sandboxReadinessTracing.waitForCreatedSandboxPublication({ - timeoutMs, - pollIntervalMs: CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_SECONDS * 1_000, - probe: (getRemainingMs) => { - const result = deps.runOpenshell( - ["sandbox", "get", "-g", input.gatewayName, input.sandboxName], - { - ignoreError: true, - suppressOutput: true, - timeout: Math.min(SANDBOX_READY_PROBE_TIMEOUT_MS, getRemainingMs()), - killSignal: "SIGKILL", - }, - ); - if (result.status !== 0 || result.error) return false; + const pollIntervalMs = CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_SECONDS * 1_000; + const deadlineMs = Date.now() + timeoutMs; + const maxAttempts = Math.ceil(timeoutMs / Math.max(1, pollIntervalMs)) + 1; + let published = false; + for (let attempt = 0; attempt < maxAttempts && Date.now() < deadlineMs; attempt += 1) { + const getRemainingMs = (): number => Math.max(1, deadlineMs - Date.now()); + const result = deps.runOpenshell( + ["sandbox", "get", "-g", input.gatewayName, input.sandboxName], + { + ignoreError: true, + suppressOutput: true, + timeout: Math.min(SANDBOX_READY_PROBE_TIMEOUT_MS, getRemainingMs()), + killSignal: "SIGKILL", + }, + ); + if (result.status === 0 && !result.error) { const publishedSandboxId = parseOpenShellSandboxId(String(result.stdout ?? "")); if (!publishedSandboxId) { throw new Error( @@ -416,10 +418,14 @@ function waitForCreatedOpenShellSandboxPublication( `Created sandbox '${input.sandboxName}' changed identity before identity verification completed.`, ); } - return true; - }, - sleep: deps.sleep, - }); + published = true; + break; + } + const remainingMs = deadlineMs - Date.now(); + if (attempt + 1 < maxAttempts && remainingMs > 0) { + deps.sleep(Math.min(pollIntervalMs, remainingMs) / 1_000); + } + } if (!published) { throw new Error( `Created sandbox '${input.sandboxName}' did not become visible through its owning gateway before identity verification completed.`, @@ -427,29 +433,17 @@ function waitForCreatedOpenShellSandboxPublication( } } -function checkRecreatedSandboxReadyIdentity( - sandboxName: string, - gatewayName: string, - expectedSandboxId: string, - deps: SandboxGpuCreateFlowDeps, - getRemainingMs: () => number, -): ReturnType { - const identity = probeExactOpenShellSandboxId(sandboxName, gatewayName, deps, getRemainingMs); - if (identity.state === "not_ready") return "not_ready"; - if (identity.state === "failed") return "probe_failed"; - if (identity.sandboxId !== expectedSandboxId) return "identity_changed"; - return checkSandboxExecutableReadiness(sandboxName, gatewayName, deps, getRemainingMs); -} - function checkCreatedSandboxReadyIdentity( sandboxName: string, gatewayName: string, + expectedSandboxId: string | null, deps: SandboxGpuCreateFlowDeps, getRemainingMs: () => number, ): ReturnType { const identity = probeExactOpenShellSandboxId(sandboxName, gatewayName, deps, getRemainingMs); if (identity.state === "not_ready") return "not_ready"; if (identity.state === "failed") return "probe_failed"; + if (expectedSandboxId && identity.sandboxId !== expectedSandboxId) return "identity_changed"; return checkSandboxExecutableReadiness(sandboxName, gatewayName, deps, getRemainingMs); } @@ -1056,7 +1050,7 @@ export function createSandboxGpuCreateAttemptRunner( : 1, checkReadyIdentity: expectedRecreatedSandboxId ? (getRemainingMs = () => SANDBOX_RECREATE_PROBE_TIMEOUT_MS) => - checkRecreatedSandboxReadyIdentity( + checkCreatedSandboxReadyIdentity( input.sandboxName, input.gatewayName, expectedRecreatedSandboxId, @@ -1069,6 +1063,7 @@ export function createSandboxGpuCreateAttemptRunner( checkCreatedSandboxReadyIdentity( input.sandboxName, input.gatewayName, + null, deps, getRemainingMs, ), diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index 5f29ca6edf5..7dd1746f2c3 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -59,28 +59,6 @@ export interface SandboxReadyWaitOptions extends SandboxReadyWaitDeps { delaySeconds: number; } -/** Wait for one owner-scoped created-sandbox publication within a fixed deadline. */ -export function waitForCreatedSandboxPublication(options: { - readonly timeoutMs: number; - readonly pollIntervalMs: number; - readonly probe: (getRemainingMs: () => number) => boolean; - readonly sleep: (seconds: number) => void; - readonly now?: () => number; -}): boolean { - const now = options.now ?? Date.now; - const timeoutMs = Math.max(1, Math.round(options.timeoutMs)); - const pollIntervalMs = Math.max(0, options.pollIntervalMs); - const deadlineMs = now() + timeoutMs; - return waitUntil(() => options.probe(() => Math.max(1, deadlineMs - now())), { - deadlineMs, - initialIntervalMs: pollIntervalMs, - maxIntervalMs: pollIntervalMs, - maxAttempts: Math.ceil(timeoutMs / Math.max(1, pollIntervalMs)) + 1, - now, - sleep: (ms) => options.sleep(ms / 1_000), - }); -} - function pollSandboxReady( options: SandboxReadyWaitOptions & { trace?: (event: string, attributes: Record) => void; From 4b04b867e54b8702d500305c7b7428e6934f4df3 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 08:56:03 -0400 Subject: [PATCH 13/51] fix(onboard): close readiness recovery gaps Signed-off-by: Julie Yaunches --- .../sandbox-gpu-create-identity-gate.test.ts | 3 +- .../onboard/sandbox-gpu-create-run-attempt.ts | 2 +- .../onboard/sandbox-readiness-tracing.test.ts | 214 +++--------------- src/lib/onboard/sandbox-readiness-tracing.ts | 52 +---- 4 files changed, 42 insertions(+), 229 deletions(-) diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 8bbc2aa848c..58f2518075d 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -251,7 +251,8 @@ describe("created sandbox identity gate", () => { expect(recoveryOutput).toContain( "Give the create-attempt label above to an OpenShell administrator", ); - expect(recoveryOutput).toContain("run `nemohermes alpha destroy --yes`"); + expect(recoveryOutput).not.toContain("destroy --yes"); + expect(recoveryOutput).not.toContain("After OpenShell confirms the sandbox is absent"); }); it("resumes the exact verified sandbox without issuing another create (#9833)", async () => { diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 20f30075ce2..6797ff4049e 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -356,7 +356,7 @@ function confirmManagedRuntimeCommitReadiness(options: { " NemoClaw did not start dashboard forwarding. NemoClaw left the sandbox in place for identity-bound recovery.", ); console.error( - ` Do not delete sandbox '${input.sandboxName}' by name. Give the create-attempt label above to an OpenShell administrator and ask them to remove that exact sandbox through an identity-bound procedure. After OpenShell confirms the sandbox is absent, run \`${input.cliName} ${input.sandboxName} destroy --yes\` to reconcile the retained local state.`, + ` Do not delete sandbox '${input.sandboxName}' by name. Give the create-attempt label above to an OpenShell administrator and ask them to remove that exact sandbox and reconcile its retained recovery state through an identity-bound procedure.`, ); throw new Error( `Sandbox '${input.sandboxName}' did not return to Ready after its managed runtime commit.`, diff --git a/src/lib/onboard/sandbox-readiness-tracing.test.ts b/src/lib/onboard/sandbox-readiness-tracing.test.ts index df706b1eecb..48fa8989918 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.test.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.test.ts @@ -7,8 +7,6 @@ import { getSandboxFailurePhase, isSandboxReady } from "../state/gateway"; import { createSandboxReadyWaiter, formatCreatedSandboxReadinessFailureMessage, - getSandboxReadyErrorDebouncePolls, - SANDBOX_READY_ERROR_DEBOUNCE_ENV, waitForCreatedSandboxReadyWithTrace, waitForDashboardReadyWithTrace, waitForSandboxReadyWithTrace, @@ -165,35 +163,6 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { expect(runCaptureOpenshell).not.toHaveBeenCalled(); }); - it("fast-fails on the first Error poll when the debounce is opted out (K=1)", () => { - const { runCaptureOpenshell, sleep } = replay([ - `${NAME} Provisioning 1s ago`, - `${NAME} Error 3s ago`, - ]); - - const ready = waitForCreatedSandboxReadyWithTrace({ - sandboxName: NAME, - gatewayName: "owner-gateway", - // 600 / 2 = 300 readyAttempts. With the K=1 (no-debounce) opt-out we bail - // out after the 2nd poll, preserving the original fast-fail intent. - timeoutSecs: 600, - runCaptureOpenshell, - isSandboxReady, - getSandboxFailurePhase, - errorPhaseDebouncePolls: 1, - sleep, - }); - - expect(ready).toEqual({ - ready: false, - reason: "terminal_failure_phase", - failurePhase: "Error", - }); - expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); - // Should not sleep after detecting the terminal phase. - expect(sleep).toHaveBeenCalledTimes(1); - }); - it("recovers when a transient Error flips to Ready within the debounce window (#6043)", () => { // DGX Spark repro: the gateway re-registers the just-created sandbox and // `sandbox list` briefly reports Error before flipping to Ready. The @@ -220,11 +189,12 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { }); it("resets the debounce counter when a non-Error poll interrupts the Error streak", () => { - // Flapping Error must not accumulate toward the terminal threshold. + const firstErrorStreak = Array.from({ length: 29 }, () => `${NAME} Error 1s ago`); + const secondErrorStreak = Array.from({ length: 29 }, () => `${NAME} Error 5s ago`); const { runCaptureOpenshell, sleep } = replay([ - `${NAME} Error 1s ago`, + ...firstErrorStreak, `${NAME} Provisioning 3s ago`, - `${NAME} Error 5s ago`, + ...secondErrorStreak, `${NAME} Ready 7s ago`, ]); @@ -235,11 +205,9 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { runCaptureOpenshell, isSandboxReady, getSandboxFailurePhase, - errorPhaseDebouncePolls: 2, sleep, }); - // Never two consecutive Error polls, so it never crosses the threshold. expect(ready).toEqual({ ready: true, reason: "ready", failurePhase: null }); }); @@ -253,7 +221,6 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { runCaptureOpenshell, isSandboxReady, getSandboxFailurePhase, - errorPhaseDebouncePolls: 3, sleep, }); @@ -262,10 +229,8 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { reason: "terminal_failure_phase", failurePhase: "Error", }); - // 3 consecutive Error polls trigger the terminal failure; the wait sleeps - // twice between the first three polls and stops before the full timeout. - expect(runCaptureOpenshell).toHaveBeenCalledTimes(3); - expect(sleep).toHaveBeenCalledTimes(2); + expect(runCaptureOpenshell).toHaveBeenCalledTimes(30); + expect(sleep).toHaveBeenCalledTimes(29); }); it("reports the Error phase (not a generic timeout) when the debounce outlasts the timeout", () => { @@ -291,81 +256,33 @@ describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { }); }); - it.each([ - "Failed", - "CrashLoopBackOff", - ])("fast-fails immediately on genuinely terminal phase %s even with a large debounce", (phase) => { - const { runCaptureOpenshell, sleep } = replay([ - `${NAME} Provisioning 1s ago`, - `${NAME} ${phase} 3s ago`, - ]); - - const ready = waitForCreatedSandboxReadyWithTrace({ - sandboxName: NAME, - gatewayName: "owner-gateway", - timeoutSecs: 600, - runCaptureOpenshell, - isSandboxReady, - getSandboxFailurePhase, - // Even with a very large debounce, non-Error terminal phases must not - // be debounced (#6043 CodeRabbit/advisor: debounce is Error-only). - errorPhaseDebouncePolls: 999, - sleep, - }); - - expect(ready).toEqual({ ready: false, reason: "terminal_failure_phase", failurePhase: phase }); - expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); - expect(sleep).toHaveBeenCalledTimes(1); - }); - - it("rounds a fractional debounce override (2.6 -> 3), matching envInt semantics", () => { - const { runCaptureOpenshell, sleep } = replay([`${NAME} Error 3s ago`]); - - const ready = waitForCreatedSandboxReadyWithTrace({ - sandboxName: NAME, - gatewayName: "owner-gateway", - timeoutSecs: 600, - runCaptureOpenshell, - isSandboxReady, - getSandboxFailurePhase, - errorPhaseDebouncePolls: 2.6, - sleep, - }); - - expect(ready).toEqual({ - ready: false, - reason: "terminal_failure_phase", - failurePhase: "Error", - }); - // round(2.6) === 3 (truncation would give 2), so the 3rd consecutive Error - // poll is terminal — the same rounding rule as the - // NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE env path. - expect(runCaptureOpenshell).toHaveBeenCalledTimes(3); - }); - - it("ignores a non-finite debounce override and falls back to the env/default", () => { - // NaN is not finite, so the override is dropped and the default (30) is - // used: a 4-poll transient Error still recovers to Ready. - const { runCaptureOpenshell } = replay([ - `${NAME} Error 1s ago`, - `${NAME} Error 3s ago`, - `${NAME} Error 5s ago`, - `${NAME} Ready 7s ago`, - ]); + it.each(["Failed", "CrashLoopBackOff"])( + "fast-fails immediately on genuinely terminal phase %s", + (phase) => { + const { runCaptureOpenshell, sleep } = replay([ + `${NAME} Provisioning 1s ago`, + `${NAME} ${phase} 3s ago`, + ]); - const ready = waitForCreatedSandboxReadyWithTrace({ - sandboxName: NAME, - gatewayName: "owner-gateway", - timeoutSecs: 600, - runCaptureOpenshell, - isSandboxReady, - getSandboxFailurePhase, - errorPhaseDebouncePolls: Number.NaN, - sleep: () => {}, - }); - - expect(ready).toEqual({ ready: true, reason: "ready", failurePhase: null }); - }); + const ready = waitForCreatedSandboxReadyWithTrace({ + sandboxName: NAME, + gatewayName: "owner-gateway", + timeoutSecs: 600, + runCaptureOpenshell, + isSandboxReady, + getSandboxFailurePhase, + sleep, + }); + + expect(ready).toEqual({ + ready: false, + reason: "terminal_failure_phase", + failurePhase: phase, + }); + expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); + expect(sleep).toHaveBeenCalledTimes(1); + }, + ); }); describe("waitForDashboardReadyWithTrace", () => { @@ -450,54 +367,6 @@ describe("waitForDashboardReadyWithTrace", () => { }); }); -describe("getSandboxReadyErrorDebouncePolls env contract", () => { - it("defaults to 30 when the env var is unset", () => { - expect(getSandboxReadyErrorDebouncePolls({})).toBe(30); - }); - - it("honors a valid override", () => { - expect(getSandboxReadyErrorDebouncePolls({ [SANDBOX_READY_ERROR_DEBOUNCE_ENV]: "12" })).toBe( - 12, - ); - }); - - it("falls back to the default for empty or non-numeric values", () => { - expect(getSandboxReadyErrorDebouncePolls({ [SANDBOX_READY_ERROR_DEBOUNCE_ENV]: "" })).toBe(30); - expect(getSandboxReadyErrorDebouncePolls({ [SANDBOX_READY_ERROR_DEBOUNCE_ENV]: "abc" })).toBe( - 30, - ); - expect( - getSandboxReadyErrorDebouncePolls({ [SANDBOX_READY_ERROR_DEBOUNCE_ENV]: "Infinity" }), - ).toBe(30); - expect(getSandboxReadyErrorDebouncePolls({ [SANDBOX_READY_ERROR_DEBOUNCE_ENV]: "NaN" })).toBe( - 30, - ); - }); - - it("clamps to a minimum of 1 poll", () => { - expect(getSandboxReadyErrorDebouncePolls({ [SANDBOX_READY_ERROR_DEBOUNCE_ENV]: "0" })).toBe(1); - // envInt rounds 0.4 -> 0, then the clamp lifts it to 1. - expect(getSandboxReadyErrorDebouncePolls({ [SANDBOX_READY_ERROR_DEBOUNCE_ENV]: "0.4" })).toBe( - 1, - ); - }); - - it("falls back for a negative override instead of clamping it to the minimum", () => { - // A negative is invalid input, so it reaches the documented default the - // same way "abc" does above, rather than silently becoming the smallest - // legal debounce (#7881). - expect(getSandboxReadyErrorDebouncePolls({ [SANDBOX_READY_ERROR_DEBOUNCE_ENV]: "-5" })).toBe( - 30, - ); - }); - - it("rounds fractional env values (envInt semantics)", () => { - expect(getSandboxReadyErrorDebouncePolls({ [SANDBOX_READY_ERROR_DEBOUNCE_ENV]: "2.6" })).toBe( - 3, - ); - }); -}); - // PRA-5 acceptance: deterministic replay of the reporter's DGX Spark // gateway/port-fallback create sequence through the real readiness waiter. DGX // Spark hardware is unavailable, so this checked-in replay is the acceptance @@ -515,25 +384,6 @@ describe("DGX Spark fresh-onboard readiness replay (#6043)", () => { `${NAME} Ready 14s ago`, ] as const; - it("regressed pre-fix: fast-fail (K=1) surfaces the exact reporter failure line", () => { - const { runCaptureOpenshell, sleep } = replay(reporterSequence); - const ready = waitForCreatedSandboxReadyWithTrace({ - sandboxName: NAME, - gatewayName: "owner-gateway", - timeoutSecs: 1500, - runCaptureOpenshell, - isSandboxReady, - getSandboxFailurePhase, - errorPhaseDebouncePolls: 1, - sleep, - }); - - expect(ready.ready).toBe(false); - expect(formatCreatedSandboxReadinessFailureMessage(NAME, ready, 1500)).toContain( - "entered Error phase before it became ready (waited up to 1500s)", - ); - }); - it("retains the terminal phase in managed-bootstrap readiness diagnostics (#9819)", () => { expect( formatCreatedSandboxReadinessFailureMessage( diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index 7dd1746f2c3..5efe35ba740 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { waitUntil } from "../core/wait"; -import { envInt } from "./env"; import { createReadinessWaitOptions, formatReadinessDeadline, @@ -12,8 +11,6 @@ import { addTraceEvent, withDashboardReadinessTrace, withSandboxReadinessTrace } type RunCaptureOpenshell = (args: string[], options?: { ignoreError?: boolean }) => string; -export const SANDBOX_READY_ERROR_DEBOUNCE_ENV = "NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE"; - /* * OpenShell can briefly report Error while it registers a new sandbox. * Debounce only Error; Failed and CrashLoopBackOff remain terminal. Remove @@ -25,15 +22,6 @@ export const SANDBOX_READY_ERROR_DEBOUNCE_ENV = "NEMOCLAW_SANDBOX_READY_ERROR_DE */ const SANDBOX_READY_ERROR_PHASE_DEFAULT_DEBOUNCE_POLLS = 30; -export function getSandboxReadyErrorDebouncePolls( - env: Record = process.env, -): number { - return Math.max( - 1, - envInt(SANDBOX_READY_ERROR_DEBOUNCE_ENV, SANDBOX_READY_ERROR_PHASE_DEFAULT_DEBOUNCE_POLLS, env), - ); -} - export type CreatedSandboxReadinessResult = | { ready: true; reason: "ready"; failurePhase: null } | { ready: false; reason: "terminal_failure_phase"; failurePhase: string | null } @@ -153,8 +141,8 @@ export function waitForCreatedSandboxReadyWithTrace(options: { isSandboxReady: (output: string, sandboxName: string) => boolean; /** * Optional terminal-failure-phase classifier. Failed and CrashLoopBackOff - * stop the wait immediately. Error stops it after the configured number of - * consecutive Error observations (#4316). + * stop the wait immediately. Error stops it after the fixed bounded number + * of consecutive Error observations (#4316). */ getSandboxFailurePhase?: (output: string, sandboxName: string) => string | null; /** @@ -174,26 +162,6 @@ export function waitForCreatedSandboxReadyWithTrace(options: { * terminal. */ checkReadyIdentity?: CreatedSandboxReadyIdentityCheck; - /** - * Consecutive Error-phase polls required before the wait treats the phase as - * terminal. Defaults to {@link getSandboxReadyErrorDebouncePolls} (30 polls). - * - * Trade-off: on a fresh create — the path this waiter guards — a healthy - * sandbox that briefly transits Error costs nothing (it flips to Ready and - * the wait returns on that poll), while a genuinely stuck Error is reported - * after the configured number of observations. The default is deliberately - * conservative rather than tuned to the shortest observed transient: the - * re-registration window scales with host/gateway speed (slower on - * ARM64/DGX-class hosts), so a too-low default risks re-introducing #6043. - * The readiness deadline still bounds the wait; operators who want fewer - * observations set NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE. - * - * Fractional values are rounded (Math.round), matching the env-var path's - * envInt rounding for one consistent rule across both entry points. Pass 1 to - * restore the original fast-fail-on-first-Error behavior (used by callers - * that have already ruled out the transient supervisor-reconnect race). - */ - errorPhaseDebouncePolls?: number; sleep: (seconds: number) => void; now?: () => number; }): CreatedSandboxReadinessResult { @@ -206,12 +174,6 @@ export function waitForCreatedSandboxReadyWithTrace(options: { getSandboxFailurePhase, sleep, } = options; - const errorPhaseDebouncePolls = - options.errorPhaseDebouncePolls == null || !Number.isFinite(options.errorPhaseDebouncePolls) - ? getSandboxReadyErrorDebouncePolls() - : // Round (not truncate) so a fractional override matches the env-var - // path's envInt rounding — one consistent rule for both entry points. - Math.max(1, Math.round(options.errorPhaseDebouncePolls)); const stableReadyPolls = options.stableReadyPolls == null || !Number.isFinite(options.stableReadyPolls) ? 1 @@ -306,7 +268,7 @@ export function waitForCreatedSandboxReadyWithTrace(options: { lastFailurePhase = failurePhase; // Sustained Error is terminal; a transient Error while the gateway // re-registers the sandbox recovers on a later poll (#6043). - if (consecutiveFailurePolls >= errorPhaseDebouncePolls) { + if (consecutiveFailurePolls >= SANDBOX_READY_ERROR_PHASE_DEFAULT_DEBOUNCE_POLLS) { addTraceEvent("terminal_failure_phase", { attempt, failure_phase: failurePhase, @@ -319,7 +281,7 @@ export function waitForCreatedSandboxReadyWithTrace(options: { attempt, failure_phase: failurePhase, consecutive_polls: consecutiveFailurePolls, - debounce_polls: errorPhaseDebouncePolls, + debounce_polls: SANDBOX_READY_ERROR_PHASE_DEFAULT_DEBOUNCE_POLLS, }); } else { consecutiveFailurePolls = 0; @@ -328,8 +290,8 @@ export function waitForCreatedSandboxReadyWithTrace(options: { }, waitOptions); if (result) return result; // If the sandbox is still in Error on the final poll, surface the terminal - // phase instead of a generic timeout. This happens when the configured - // debounce window is larger than the readiness timeout allows (e.g. a low + // phase instead of a generic timeout. This happens when the fixed debounce + // window is larger than the readiness timeout allows (e.g. a low // NEMOCLAW_SANDBOX_READY_TIMEOUT with the default 30-poll debounce), so a // genuinely stuck Error would otherwise be misreported as "did not become // ready" and drop the phase (#6043 review). @@ -338,7 +300,7 @@ export function waitForCreatedSandboxReadyWithTrace(options: { attempts: attempt, failure_phase: lastFailurePhase, consecutive_polls: consecutiveFailurePolls, - debounce_polls: errorPhaseDebouncePolls, + debounce_polls: SANDBOX_READY_ERROR_PHASE_DEFAULT_DEBOUNCE_POLLS, note: "debounce_window_exceeded_timeout", }); return { ready: false, reason: "terminal_failure_phase", failurePhase: lastFailurePhase }; From 383cfe8b291fd57025d1740cb7b4f1c462eef4d3 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 09:10:28 -0400 Subject: [PATCH 14/51] fix(onboard): block on recovery persistence Signed-off-by: Julie Yaunches --- .../sandbox-gpu-create-identity-gate.test.ts | 48 +++++++++++++++++++ .../onboard/sandbox-gpu-create-run-attempt.ts | 20 +++++--- 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 58f2518075d..aa339ec9dd6 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -142,6 +142,42 @@ function refuseEffectStartingWith(prefix: string): (operation: string) => void { }; } +async function expectCommittedReadinessPersistenceFailure( + persist: NonNullable["persistRetainedSandboxRecovery"]>, +): Promise { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + const input = noGpuInput(); + input.resumeVerifiedCreate = { + route: "none", + liveIdentityFingerprint: fingerprintSandboxRecreateValue("alpha-sandbox-id"), + createAttemptNonce: "a".repeat(62), + }; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + input.persistRetainedSandboxRecovery = persist; + const patch = createGpuPatchFixture(); + attachManagedBootstrap(input, patch); + const deps = createGpuFlowDeps("alpha-sandbox-id"); + mocks.waitForCreatedSandboxReadyWithTrace + .mockReturnValueOnce({ ready: true, reason: "ready", failurePhase: null }) + .mockReturnValue({ ready: false, reason: "timeout", failurePhase: null }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "the recovery-only session remains blocked", + ); + + expect(persist).toHaveBeenCalledOnce(); + expect(error.mock.calls.flat().join("\n")).toContain( + "The recovery-only session remains blocked until its durable recovery record can be saved.", + ); + expect(error.mock.calls.flat().join("\n")).not.toContain("Preserve the terminal output"); + expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); +} + beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); @@ -255,6 +291,18 @@ describe("created sandbox identity gate", () => { expect(recoveryOutput).not.toContain("After OpenShell confirms the sandbox is absent"); }); + it("blocks committed-readiness recovery when durable persistence returns false", async () => { + await expectCommittedReadinessPersistenceFailure(vi.fn(() => false)); + }); + + it("blocks committed-readiness recovery when durable persistence throws", async () => { + await expectCommittedReadinessPersistenceFailure( + vi.fn(() => { + throw new Error("durable writer failed"); + }), + ); + }); + it("resumes the exact verified sandbox without issuing another create (#9833)", async () => { const events: string[] = []; const sandboxId = "alpha-sandbox-id"; diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 6797ff4049e..353ce217a35 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -271,16 +271,24 @@ function persistCreateAttemptRecovery(options: { ? `Durable sandbox identity fingerprint: ${sandboxIdentityFingerprint}. ` : "") + detail; - let persisted = false; + let persistenceFailure: unknown = null; try { - persisted = persist(message, sandboxIdentityFingerprint, createAttemptNonce); - } catch { - persisted = false; + if (!persist(message, sandboxIdentityFingerprint, createAttemptNonce)) { + persistenceFailure = new Error( + "The retained sandbox recovery writer did not confirm durable persistence.", + ); + } + } catch (error) { + persistenceFailure = error; } console.error(` ${message}`); - if (!persisted) { + if (persistenceFailure) { console.error( - " NemoClaw could not save this create-attempt evidence. Preserve the terminal output for an OpenShell administrator.", + " NemoClaw could not save this create-attempt evidence. The recovery-only session remains blocked until its durable recovery record can be saved.", + ); + throw new Error( + "NemoClaw could not save the retained sandbox recovery record; the recovery-only session remains blocked.", + { cause: persistenceFailure }, ); } } From 73b3f38ffa81422d3b6c47c3cc64b8bf4a429cb2 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 10:15:49 -0400 Subject: [PATCH 15/51] test(onboard): reject unscoped readiness probes Signed-off-by: Julie Yaunches --- .../sandbox-gpu-create-flow.ts | 1 - src/lib/onboard/sandbox-gpu-create-flow.ts | 2 -- test/helpers/managed-image-buildless-e2e.ts | 21 ++++++++++++++++ test/security/shellquote-sandbox.test.ts | 24 +++++++++++++++++++ 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index 8421daf6d9a..444781b3b05 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -28,7 +28,6 @@ export const GPU_IMAGE_ID = `sha256:${"a".repeat(64)}`; export function createGpuFlowInput(): SandboxGpuCreateFlowInput { return { sandboxName: "alpha", - cliName: "nemoclaw", provider: "nim", sandboxGpuConfig: { mode: "1", diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index b36c7a75e9d..028ad81f547 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -210,8 +210,6 @@ type LifecycleRegistrationFields = Pick; export interface SandboxGpuCreateFlowInput { sandboxName: string; - /** Active CLI spelling used in identity-bound recovery guidance. */ - cliName: string; /** Resume the exact sandbox retained after its verified-create checkpoint was persisted. */ resumeVerifiedCreate?: { readonly route: SelectedDockerGpuRoute; diff --git a/test/helpers/managed-image-buildless-e2e.ts b/test/helpers/managed-image-buildless-e2e.ts index b9e73149d87..1d13a2ea000 100644 --- a/test/helpers/managed-image-buildless-e2e.ts +++ b/test/helpers/managed-image-buildless-e2e.ts @@ -849,6 +849,27 @@ function assertManagedLaunch( ), ), ).toBe(true); + const sandboxGetCommands = result.payload.runnerCommands.filter( + (command) => + command.includes("sandbox get") && command.includes(bootstrapRequest?.sandboxName ?? ""), + ); + expect( + sandboxGetCommands.filter( + (command) => + !command.includes("sandbox get -g nemoclaw") && + !command.includes("sandbox get --gateway nemoclaw"), + ), + ).toEqual([]); + const sandboxExecCommands = result.payload.runnerCommands.filter( + (command) => + command.includes("sandbox exec") && command.includes(bootstrapRequest?.sandboxName ?? ""), + ); + expect( + sandboxExecCommands.filter( + (command) => + !command.includes("sandbox exec -g nemoclaw") && !command.includes("--gateway nemoclaw"), + ), + ).toEqual([]); } expect(createArgs.filter((arg) => arg.startsWith("NEMOCLAW_CORPORATE_CA_B64="))).toEqual([]); expect(profile.proxy).toMatchObject({ diff --git a/test/security/shellquote-sandbox.test.ts b/test/security/shellquote-sandbox.test.ts index 2f2d41ed642..f548e4c6ed5 100644 --- a/test/security/shellquote-sandbox.test.ts +++ b/test/security/shellquote-sandbox.test.ts @@ -230,6 +230,30 @@ try { entry.command.includes("sandbox exec -g nemoclaw --name my-assistant -- true"), ), ).toBe(true); + expect( + payload.commands + .filter( + (entry: { command: string }) => + entry.command.includes("sandbox get") && entry.command.includes("my-assistant"), + ) + .every( + (entry: { command: string }) => + entry.command.includes("sandbox get -g nemoclaw") || + entry.command.includes("sandbox get --gateway nemoclaw"), + ), + ).toBe(true); + expect( + payload.commands + .filter( + (entry: { command: string }) => + entry.command.includes("sandbox exec") && entry.command.includes("my-assistant"), + ) + .every( + (entry: { command: string }) => + entry.command.includes("sandbox exec -g nemoclaw") || + entry.command.includes("--gateway nemoclaw"), + ), + ).toBe(true); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } From 7d26771786886a9d8c2bc67c2e6226ce143abb39 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 10:22:32 -0400 Subject: [PATCH 16/51] test(onboard): remove unused CLI recovery input Signed-off-by: Julie Yaunches --- scripts/checks/run-managed-image-openshell-e2e.ts | 1 - src/lib/onboard/sandbox-create/orchestration.ts | 1 - src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts | 1 - 3 files changed, 3 deletions(-) diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 8b970a9ad22..917cc682d16 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -1017,7 +1017,6 @@ async function run { const sandboxId = "alpha-sandbox-id"; const sandboxIdentityFingerprint = fingerprintSandboxRecreateValue(sandboxId); const input = noGpuInput(); - input.cliName = "nemohermes"; input.resumeVerifiedCreate = { route: "none", liveIdentityFingerprint: sandboxIdentityFingerprint, From d1a395a2fa8f5b916ab516a9e78c9c3447bd982a Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 13:31:31 -0400 Subject: [PATCH 17/51] refactor(onboard): reuse readiness waiter Signed-off-by: Julie Yaunches --- src/lib/onboard/sandbox-gpu-create-flow.ts | 2 + .../sandbox-gpu-create-identity-gate.test.ts | 49 ++++++++++++++ .../onboard/sandbox-gpu-create-run-attempt.ts | 67 +++++++++---------- src/lib/onboard/sandbox-readiness-tracing.ts | 24 +++++++ 4 files changed, 107 insertions(+), 35 deletions(-) diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 028ad81f547..280f2c1ca39 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -293,6 +293,8 @@ export interface SandboxGpuCreateFlowDeps { runOpenshell: RunOpenshell; runCaptureOpenshell: RunCaptureOpenshell; sleep: Sleep; + /** Production callers use the system clock; tests may inject publication-wait time. */ + now?: () => number; openshellArgv(args: string[]): string[]; verifyDirectSandboxGpu(sandboxName: string): SandboxGpuProofResult; printCreateFailureDiagnostics?: ( diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 304b6496400..d270e96d785 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -749,6 +749,11 @@ describe("created sandbox identity gate", () => { return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; }); const deps = createGpuFlowDeps(); + let nowMs = 0; + deps.now = () => nowMs; + vi.mocked(deps.sleep).mockImplementation((seconds) => { + nowMs += seconds * 1_000; + }); vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), ); @@ -814,6 +819,44 @@ describe("created sandbox identity gate", () => { expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); }); + it("rejects malformed owner-scoped publication before post-create effects (#9833)", async () => { + let nonce = ""; + const input = noGpuInput(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + vi.mocked(deps.runOpenshell).mockReturnValue({ + status: 0, + stdout: "Name: alpha\nState: Ready\n", + stderr: "", + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "returned no exact durable ID for created sandbox", + ); + + expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining( + `Durable sandbox identity fingerprint: ${fingerprintSandboxRecreateValue("alpha-sandbox-id")}`, + ), + fingerprintSandboxRecreateValue("alpha-sandbox-id"), + nonce, + ); + expect(patch.exitOnPatchError).not.toHaveBeenCalled(); + expect(patch.ensureApplied).not.toHaveBeenCalled(); + expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); + }); + it("stops when owner-scoped sandbox publication exceeds the deadline (#9833)", async () => { let nonce = ""; const input = noGpuInput(); @@ -827,6 +870,11 @@ describe("created sandbox identity gate", () => { return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; }); const deps = createGpuFlowDeps(); + let nowMs = 0; + deps.now = () => nowMs; + vi.mocked(deps.sleep).mockImplementation((seconds) => { + nowMs += seconds * 1_000; + }); vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), ); @@ -852,6 +900,7 @@ describe("created sandbox identity gate", () => { expect(patch.exitOnPatchError).not.toHaveBeenCalled(); expect(patch.ensureApplied).not.toHaveBeenCalled(); expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); + expect(deps.sleep).toHaveBeenCalledExactlyOnceWith(0.001); }); it("rejects a same-name replacement before post-create effects (#9833)", async () => { diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 353ce217a35..36b887b6d5b 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -69,7 +69,7 @@ export type SandboxGpuCreateAttemptState = { // to live validation or the GPU proof. const REPLACEMENT_STABLE_READY_POLLS = 2; const SANDBOX_READY_PROBE_TIMEOUT_MS = 5_000; -const CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_SECONDS = 1; +const CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_MS = 1_000; async function streamSandboxCreateWithPublicImageCredentialIsolation( isolate: boolean, @@ -399,41 +399,38 @@ function waitForCreatedOpenShellSandboxPublication( deps: SandboxGpuCreateFlowDeps, ): void { const timeoutMs = Math.max(1, Math.round(input.sandboxReadyTimeoutSecs * 1_000)); - const pollIntervalMs = CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_SECONDS * 1_000; - const deadlineMs = Date.now() + timeoutMs; - const maxAttempts = Math.ceil(timeoutMs / Math.max(1, pollIntervalMs)) + 1; - let published = false; - for (let attempt = 0; attempt < maxAttempts && Date.now() < deadlineMs; attempt += 1) { - const getRemainingMs = (): number => Math.max(1, deadlineMs - Date.now()); - const result = deps.runOpenshell( - ["sandbox", "get", "-g", input.gatewayName, input.sandboxName], - { - ignoreError: true, - suppressOutput: true, - timeout: Math.min(SANDBOX_READY_PROBE_TIMEOUT_MS, getRemainingMs()), - killSignal: "SIGKILL", - }, - ); - if (result.status === 0 && !result.error) { - const publishedSandboxId = parseOpenShellSandboxId(String(result.stdout ?? "")); - if (!publishedSandboxId) { - throw new Error( - `OpenShell returned no exact durable ID for created sandbox '${input.sandboxName}'.`, - ); - } - if (publishedSandboxId !== sandboxId) { - throw new Error( - `Created sandbox '${input.sandboxName}' changed identity before identity verification completed.`, - ); + const published = sandboxReadinessTracing.waitForCreatedSandboxPublication({ + budgetMs: timeoutMs, + pollIntervalMs: CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_MS, + now: deps.now, + sleep: deps.sleep, + probe: (getRemainingMs) => { + const result = deps.runOpenshell( + ["sandbox", "get", "-g", input.gatewayName, input.sandboxName], + { + ignoreError: true, + suppressOutput: true, + timeout: Math.min(SANDBOX_READY_PROBE_TIMEOUT_MS, getRemainingMs()), + killSignal: "SIGKILL", + }, + ); + if (result.status === 0 && !result.error) { + const publishedSandboxId = parseOpenShellSandboxId(String(result.stdout ?? "")); + if (!publishedSandboxId) { + throw new Error( + `OpenShell returned no exact durable ID for created sandbox '${input.sandboxName}'.`, + ); + } + if (publishedSandboxId !== sandboxId) { + throw new Error( + `Created sandbox '${input.sandboxName}' changed identity before identity verification completed.`, + ); + } + return true; } - published = true; - break; - } - const remainingMs = deadlineMs - Date.now(); - if (attempt + 1 < maxAttempts && remainingMs > 0) { - deps.sleep(Math.min(pollIntervalMs, remainingMs) / 1_000); - } - } + return false; + }, + }); if (!published) { throw new Error( `Created sandbox '${input.sandboxName}' did not become visible through its owning gateway before identity verification completed.`, diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index 5efe35ba740..1ba1140d094 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -47,6 +47,30 @@ export interface SandboxReadyWaitOptions extends SandboxReadyWaitDeps { delaySeconds: number; } +/** Wait for one created-sandbox publication condition inside a shared bounded deadline. */ +export function waitForCreatedSandboxPublication(options: { + budgetMs: number; + pollIntervalMs: number; + probe: (getRemainingMs: () => number) => boolean; + sleep: (seconds: number) => void; + now?: () => number; +}): boolean { + const waitOptions = createReadinessWaitOptions({ + budgetMs: options.budgetMs, + initialIntervalMs: options.pollIntervalMs, + maxIntervalMs: options.pollIntervalMs, + now: options.now, + sleep: (milliseconds) => options.sleep(milliseconds / 1_000), + }); + const deadlineMs = waitOptions?.deadlineMs; + const now = waitOptions?.now; + if (!waitOptions || deadlineMs === undefined || !now) return false; + return waitUntil( + () => options.probe(() => Math.max(1, deadlineMs - now())), + waitOptions, + ); +} + function pollSandboxReady( options: SandboxReadyWaitOptions & { trace?: (event: string, attributes: Record) => void; From 65fe3179d4e2b3f35414b4b15c244c3076af3391 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 13:37:38 -0400 Subject: [PATCH 18/51] fix(onboard): isolate publication wait clock Signed-off-by: Julie Yaunches --- src/lib/onboard/sandbox-gpu-create-flow.ts | 2 +- src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts | 4 ++-- src/lib/onboard/sandbox-gpu-create-run-attempt.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 280f2c1ca39..76a471e661c 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -294,7 +294,7 @@ export interface SandboxGpuCreateFlowDeps { runCaptureOpenshell: RunCaptureOpenshell; sleep: Sleep; /** Production callers use the system clock; tests may inject publication-wait time. */ - now?: () => number; + publicationNow?: () => number; openshellArgv(args: string[]): string[]; verifyDirectSandboxGpu(sandboxName: string): SandboxGpuProofResult; printCreateFailureDiagnostics?: ( diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index d270e96d785..c190fe0daec 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -750,7 +750,7 @@ describe("created sandbox identity gate", () => { }); const deps = createGpuFlowDeps(); let nowMs = 0; - deps.now = () => nowMs; + deps.publicationNow = () => nowMs; vi.mocked(deps.sleep).mockImplementation((seconds) => { nowMs += seconds * 1_000; }); @@ -871,7 +871,7 @@ describe("created sandbox identity gate", () => { }); const deps = createGpuFlowDeps(); let nowMs = 0; - deps.now = () => nowMs; + deps.publicationNow = () => nowMs; vi.mocked(deps.sleep).mockImplementation((seconds) => { nowMs += seconds * 1_000; }); diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 36b887b6d5b..b30b57dc05b 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -402,7 +402,7 @@ function waitForCreatedOpenShellSandboxPublication( const published = sandboxReadinessTracing.waitForCreatedSandboxPublication({ budgetMs: timeoutMs, pollIntervalMs: CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_MS, - now: deps.now, + now: deps.publicationNow, sleep: deps.sleep, probe: (getRemainingMs) => { const result = deps.runOpenshell( From 2308f78d35c6e903643caed37fd93a47e7b733d9 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 13:52:14 -0400 Subject: [PATCH 19/51] docs(onboard): remove stale readiness override Signed-off-by: Julie Yaunches --- docs/reference/commands.mdx | 1 - docs/reference/troubleshooting.mdx | 7 ------- 2 files changed, 8 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 6ce7620c2ea..8fdd77118ed 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3871,7 +3871,6 @@ The following environment variables tune onboard-time and recovery wall-clock li | `NEMOCLAW_OLLAMA_PULL_TIMEOUT` | `1800` (30 minutes) | Wall-clock timeout for `ollama pull` during onboard, in seconds. Accepts integer or float values. Already-downloaded layers are kept; re-running the pull resumes them. | | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | `180` | Wall-clock timeout for the inference-server validation probe during onboard, in seconds. Raise on slow networks or for very large prompts. | | `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Wall-clock timeout for post-create readiness, in seconds. Raise the timeout when the managed-image pull, explicit custom image build, gateway upload, or in-sandbox boot exceeds the default (typical on 70B+ models, first-time gateway uploads over slow links, or DGX Station / remote-VM first runs). Ordinary onboarding deletes the partially created sandbox when the deadline expires and prints the retry hint. Portable OpenClaw onboarding instead preserves the sandbox when NemoClaw cannot verify its runtime identity. | -| `NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE` | `30` | Consecutive `Error`-phase polls the post-create readiness wait tolerates before treating `Error` as terminal. Polling starts at 250ms and backs off to a 2-second cap, while `NEMOCLAW_SANDBOX_READY_TIMEOUT` remains the overall deadline. The gateway can briefly report a just-created sandbox in `Error` while it re-registers the sandbox (seen on DGX Spark); the debounce lets that transient recover to `Ready`. `Failed` and `CrashLoopBackOff` always fail immediately. Set to `1` to restore fast-fail on the first `Error` poll. | | `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` | `30`, `90`, or `120`, depending on the recovery phase | Wall-clock timeout for OpenShell command re-registration after policy application, plus gateway health and re-registration during managed OpenClaw or Hermes recovery. A valid finite, nonnegative value overrides the internal budget for the current recovery phase. | diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index cd2657a40d7..036cd2985ed 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2050,13 +2050,6 @@ On a fresh onboard the OpenShell gateway can (re)start its supervisor session an NemoClaw polls immediately, starts retrying after 250ms, and backs off to a 2-second cap. It tolerates 30 consecutive `Error` observations by default so this transient recovers on its own. Only `Error` that persists through the debounce count is terminal, unless the overall `NEMOCLAW_SANDBOX_READY_TIMEOUT` deadline expires first. `Failed` and `CrashLoopBackOff` are always terminal and fail immediately. -If your host needs more observations for slower re-registration, raise the debounce. Raise `NEMOCLAW_SANDBOX_READY_TIMEOUT` too if the overall deadline is too short. To fail fast on the first `Error` poll, set the debounce to `1`: - -```bash -export NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE=1 -$$nemoclaw onboard -``` - If the failure persists after the debounce, the sandbox is stuck. Inspect the retained diagnostics and gateway state: ```bash From b3bc5731271750f63f7db9be2206be11633ea420 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 18:20:22 -0400 Subject: [PATCH 20/51] fix(onboard): preserve committed readiness recovery Signed-off-by: Julie Yaunches --- docs/reference/troubleshooting.mdx | 2 +- .../sandbox-gpu-create-identity-gate.test.ts | 182 ++++++++++ .../onboard/sandbox-gpu-create-run-attempt.ts | 316 +++++++++++------- src/lib/onboard/sandbox-readiness-tracing.ts | 21 ++ 4 files changed, 408 insertions(+), 113 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 404ef3e075d..9e587724aa1 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2048,7 +2048,7 @@ Onboarding ends with: On a fresh onboard the OpenShell gateway can (re)start its supervisor session and re-register the just-created sandbox. During that window `openshell sandbox list` briefly reports the sandbox in the transient `Error` phase before it flips to `Ready`, as seen on DGX Spark when supervisor restart races the sandbox bootstrap. -NemoClaw polls immediately, starts retrying after 250ms, and backs off to a 2-second cap. +NemoClaw polls immediately. A single-observation wait retries after 250 ms and backs off to a two-second cap. A stable-readiness wait that requires two consecutive `Ready` observations checks every two seconds. It tolerates 30 consecutive `Error` observations by default so this transient recovers on its own. Only `Error` that persists through the debounce count is terminal, unless the overall `NEMOCLAW_SANDBOX_READY_TIMEOUT` deadline expires first. Every terminal observation outside the `Error` phase, including one with no reported phase, fails immediately. diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 6e808ef6993..a1fbcf1434b 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -111,12 +111,73 @@ function noGpuInput() { return input; } +function attachManagedBootstrap( + input: ReturnType, + patch: ReturnType, +): void { + input.managedBootstrap = { + bootstrapIdentity: "b".repeat(64), + stateRoot: "/tmp/nemoclaw-managed-bootstrap", + runtimeProvider: { + identity: { id: "mxc" }, + bootstrap: { + createOnboardRouting: () => ({ nativeFallbackHasCleanBaseline: false }), + createLifecycle: (options: { launchArgv: readonly string[] }) => ({ + launchArgv: options.launchArgv, + patch, + recoverUnfinished: async () => null, + prepareNetwork: async () => undefined, + runCreate: async () => { + throw new Error("resumed create must not launch"); + }, + }), + }, + }, + } as never; +} + function refuseEffectStartingWith(prefix: string): (operation: string) => void { return (operation) => { expect(operation, "checkpoint changed").not.toMatch(new RegExp(`^${prefix}`, "u")); }; } +async function expectCommittedReadinessPersistenceFailure( + persist: NonNullable["persistRetainedSandboxRecovery"]>, +): Promise { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + const input = noGpuInput(); + input.resumeVerifiedCreate = { + route: "none", + liveIdentityFingerprint: fingerprintSandboxRecreateValue("alpha-sandbox-id"), + createAttemptNonce: "a".repeat(62), + }; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + input.persistRetainedSandboxRecovery = persist; + const patch = createGpuPatchFixture(); + attachManagedBootstrap(input, patch); + const deps = createGpuFlowDeps("alpha-sandbox-id"); + mocks.waitForCreatedSandboxReadyWithTrace + .mockResolvedValueOnce({ ready: true, reason: "ready", failurePhase: null }) + .mockResolvedValue({ ready: false, reason: "timeout", failurePhase: null }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "the recovery-only session remains blocked", + ); + + expect(persist).toHaveBeenCalledOnce(); + expect(error.mock.calls.flat().join("\n")).toContain( + "The recovery-only session remains blocked until its durable recovery record can be saved.", + ); + expect(error.mock.calls.flat().join("\n")).not.toContain("Preserve the terminal output"); + expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); +} + beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); @@ -157,6 +218,127 @@ describe("created sandbox identity gate", () => { ); }); + it("reconfirms exact managed sandbox readiness after the runtime commit (#9211)", async () => { + const sandboxId = "alpha-sandbox-id"; + const input = noGpuInput(); + input.gatewayName = "owner-gateway"; + input.resumeVerifiedCreate = { + route: "none", + liveIdentityFingerprint: fingerprintSandboxRecreateValue(sandboxId), + createAttemptNonce: "a".repeat(62), + }; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + attachManagedBootstrap(input, patch); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runOpenshell) + .mockReturnValueOnce({ + status: 0, + stdout: `Name: alpha\nId: ${sandboxId}\nState: Ready\n`, + stderr: "", + }) + .mockReturnValueOnce({ + status: 1, + stdout: "", + stderr: + "Error: × code: 'The system is not in a state required for the operation's\n" + + ' │ execution\', message: "sandbox is not ready"\n', + }) + .mockReturnValueOnce({ + status: 0, + stdout: `Name: alpha\nId: ${sandboxId}\nState: Ready\n`, + stderr: "", + }) + .mockReturnValue({ status: 0, stdout: "", stderr: "" }); + mocks.waitForCreatedSandboxReadyWithTrace + .mockResolvedValueOnce({ ready: true, reason: "ready", failurePhase: null }) + .mockImplementationOnce(async (options) => { + expect(patch.commitAfterReady).toHaveBeenCalledOnce(); + expect(options.target).toEqual({ kind: "named", gatewayName: "owner-gateway" }); + expect(options.checkReadyIdentity?.()).toBe("not_ready"); + expect(options.checkReadyIdentity?.()).toBe("ready"); + return { ready: true, reason: "ready", failurePhase: null }; + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ + origin: "resumed", + route: "none", + }); + + expect(mocks.waitForCreatedSandboxReadyWithTrace).toHaveBeenCalledTimes(2); + expect(deps.runOpenshell).toHaveBeenCalledWith( + ["sandbox", "get", "-g", "owner-gateway", "alpha"], + expect.objectContaining({ suppressOutput: true }), + ); + expect(deps.runOpenshell).toHaveBeenCalledWith( + ["sandbox", "exec", "-g", "owner-gateway", "--name", "alpha", "--", "true"], + expect.objectContaining({ suppressOutput: true }), + ); + expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + }); + + it("retains exact recovery when committed managed readiness does not return (#9211)", async () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + const sandboxId = "alpha-sandbox-id"; + const sandboxIdentityFingerprint = fingerprintSandboxRecreateValue(sandboxId); + const input = noGpuInput(); + input.resumeVerifiedCreate = { + route: "none", + liveIdentityFingerprint: sandboxIdentityFingerprint, + createAttemptNonce: "a".repeat(62), + }; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + attachManagedBootstrap(input, patch); + const deps = createGpuFlowDeps(sandboxId); + mocks.waitForCreatedSandboxReadyWithTrace + .mockResolvedValueOnce({ ready: true, reason: "ready", failurePhase: null }) + .mockResolvedValue({ ready: false, reason: "timeout", failurePhase: null }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "did not return to Ready after its managed runtime commit", + ); + + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining("did not return to executable Ready state"), + sandboxIdentityFingerprint, + "a".repeat(62), + ); + expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { + backupPath: null, + }); + const recoveryOutput = error.mock.calls.flat().join("\n"); + expect(recoveryOutput).toContain("Do not delete sandbox 'alpha' by name."); + expect(recoveryOutput).toContain( + "Give the create-attempt label above to an OpenShell administrator", + ); + expect(recoveryOutput).not.toContain("destroy --yes"); + expect(recoveryOutput).not.toContain("After OpenShell confirms the sandbox is absent"); + }); + + it("blocks committed-readiness recovery when durable persistence returns false (#9211)", async () => { + await expectCommittedReadinessPersistenceFailure(vi.fn(() => false)); + }); + + it("blocks committed-readiness recovery when durable persistence throws (#9211)", async () => { + await expectCommittedReadinessPersistenceFailure( + vi.fn(() => { + throw new Error("durable writer failed"); + }), + ); + }); + it("resumes the exact verified sandbox without issuing another create (#9833)", async () => { const events: string[] = []; const sandboxId = "alpha-sandbox-id"; diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 1da21b4c6d0..11bdd463de8 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -156,6 +156,19 @@ type NativeFallbackCleanupEvidence = Readonly<{ nativeCleanupHandoff?: ManagedBootstrapNativeGpuFallbackOwnerCleanupHandoff; }>; +function warnForAuthorizedCompatibilityRetry( + route: SelectedDockerGpuRoute, + initialGpuRoute: SelectedDockerGpuRoute, +): void { + if (route !== "compatibility" || initialGpuRoute !== "native") return; + console.warn( + " Native OpenShell GPU onboarding did not complete; retrying once by recreating the OpenShell-managed Docker container with the legacy GPU compatibility envelope.", + ); + console.warn( + " This compatibility container swap may relax container confinement compared with native injection. The retry is running only because NEMOCLAW_DOCKER_GPU_PATCH=fallback explicitly authorized it.", + ); +} + async function rollbackNativeGpuFailureForFallback( managedLifecycle: ManagedBootstrapRuntimeCreateLifecycle | null, runtimePatch: ManagedBootstrapRuntimePatch, @@ -251,6 +264,148 @@ async function verifyCreatedSandboxBeforeEffects( }); } +function persistCreateAttemptRecovery(options: { + readonly input: SandboxGpuCreateFlowInput; + readonly createAttemptNonce: string | null; + readonly detail: string; + readonly sandboxIdentityFingerprint?: string; +}): void { + const { input, createAttemptNonce, detail, sandboxIdentityFingerprint } = options; + if (!createAttemptNonce) { + throw new Error("Sandbox create-attempt identity was not generated."); + } + const persist = input.persistRetainedSandboxRecovery; + if (!persist) { + throw new Error("Verified sandbox creation has no durable recovery evidence owner."); + } + const message = + `Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${createAttemptNonce}. ` + + (sandboxIdentityFingerprint + ? `Durable sandbox identity fingerprint: ${sandboxIdentityFingerprint}. ` + : "") + + detail; + let persistenceFailure: unknown = null; + try { + if (!persist(message, sandboxIdentityFingerprint, createAttemptNonce)) { + persistenceFailure = new Error( + "The retained sandbox recovery writer did not confirm durable persistence.", + ); + } + } catch (error) { + persistenceFailure = error; + } + console.error(` ${message}`); + if (persistenceFailure) { + console.error( + " NemoClaw could not save this create-attempt evidence. The recovery-only session remains blocked until its durable recovery record can be saved.", + ); + throw new Error( + "NemoClaw could not save the retained sandbox recovery record; the recovery-only session remains blocked.", + { cause: persistenceFailure }, + ); + } +} + +function persistIdentitySettlementRecoveryEvidence(options: { + readonly input: SandboxGpuCreateFlowInput; + readonly createAttemptNonce: string | null; + readonly sandboxIdentityFingerprint: string | null; +}): void { + const { input, createAttemptNonce, sandboxIdentityFingerprint } = options; + const identityEvidence = sandboxIdentityFingerprint + ? `Sandbox '${input.sandboxName}' did not remain visible through owning gateway '${input.gatewayName}' before identity verification completed. ` + : `Sandbox '${input.sandboxName}' reached Ready before OpenShell returned one exact durable create identity. Gateway '${input.gatewayName}'. OpenShell did not return one exact durable sandbox identity for this create attempt. `; + persistCreateAttemptRecovery({ + input, + createAttemptNonce, + detail: + identityEvidence + + "Do not delete a sandbox by mutable name; preserve it until an OpenShell administrator resolves the create-attempt label to one sandbox.", + sandboxIdentityFingerprint: sandboxIdentityFingerprint ?? undefined, + }); +} + +async function confirmManagedRuntimeCommitReadiness(options: { + readonly input: SandboxGpuCreateFlowInput; + readonly deps: SandboxGpuCreateFlowDeps; + readonly sandboxId: string | null; + readonly createAttemptNonce: string | null; +}): Promise { + const { input, deps, sandboxId } = options; + if (!sandboxId) return; + input.revalidateVerifiedSandboxBeforeEffect?.( + `confirm committed runtime readiness for sandbox '${input.sandboxName}'`, + ); + const committedReadiness = await sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ + sandboxName: input.sandboxName, + timeoutSecs: input.sandboxReadyTimeoutSecs, + observer: deps.sandboxObserver, + target: { kind: "named", gatewayName: input.gatewayName }, + stableReadyPolls: REPLACEMENT_STABLE_READY_POLLS, + checkReadyIdentity: (getRemainingMs = () => SANDBOX_RECREATE_PROBE_TIMEOUT_MS) => + checkRecreatedSandboxReadyIdentity( + input.sandboxName, + input.gatewayName, + sandboxId, + deps, + getRemainingMs, + ), + sleep: deps.sleep, + }); + if (committedReadiness.ready) return; + console.error(""); + sandboxReadinessTracing.printReadinessFailure( + committedReadiness, + input.sandboxName, + input.sandboxReadyTimeoutSecs, + ); + const sandboxIdentityFingerprint = fingerprintSandboxRecreateValue(sandboxId); + persistCreateAttemptRecovery({ + input, + createAttemptNonce: options.createAttemptNonce, + sandboxIdentityFingerprint, + detail: + `Managed runtime commit completed for sandbox '${input.sandboxName}', but the same sandbox did not return to executable Ready state through owning gateway '${input.gatewayName}'. ` + + "Do not delete a sandbox by mutable name; preserve it for identity-bound recovery.", + }); + (deps.printCreateFailureDiagnostics ?? printSandboxCreateFailureDiagnostics)(input.sandboxName, { + backupPath: input.restoreBackupPath, + }); + console.error( + " NemoClaw did not start dashboard forwarding. NemoClaw left the sandbox in place for identity-bound recovery.", + ); + console.error( + ` Do not delete sandbox '${input.sandboxName}' by name. Give the create-attempt label above to an OpenShell administrator and ask them to remove that exact sandbox and reconcile its retained recovery state through an identity-bound procedure.`, + ); + throw new Error( + `Sandbox '${input.sandboxName}' did not return to Ready after its managed runtime commit.`, + ); +} + +async function requireManagedBootstrapCreatedSandboxReady(options: { + readonly input: SandboxGpuCreateFlowInput; + readonly deps: SandboxGpuCreateFlowDeps; + readonly createAttemptNonce: string | null; + readonly persistIdentitySettlementRecovery: () => void; +}): Promise { + const observation = await sandboxReadinessTracing.observeOpenShellSandbox( + options.deps.sandboxObserver, + { kind: "named", gatewayName: options.input.gatewayName }, + options.input.sandboxName, + SANDBOX_READY_PROBE_TIMEOUT_MS, + ); + if (!observation.ok) { + if (options.createAttemptNonce) options.persistIdentitySettlementRecovery(); + throw new Error( + `Managed bootstrap create completed, but NemoClaw could not observe the sandbox. ${observation.error.message}`, + ); + } + if (observation.value.state !== "present" || observation.value.sandbox.readiness !== "ready") { + if (options.createAttemptNonce) options.persistIdentitySettlementRecovery(); + throw new Error("Managed bootstrap create completed without an authoritative Ready sandbox."); + } +} + function resolveCreateAttemptNonce( input: SandboxGpuCreateFlowInput, deferPostCreateEffects: boolean, @@ -318,6 +473,20 @@ function waitForCreatedOpenShellSandboxPublication( } } +function waitForCreatedSandboxPublicationOrPersist( + sandboxId: string, + input: SandboxGpuCreateFlowInput, + deps: SandboxGpuCreateFlowDeps, + persistIdentitySettlementRecovery: (sandboxIdentityFingerprint: string) => void, +): void { + try { + waitForCreatedOpenShellSandboxPublication(sandboxId, input, deps); + } catch (error) { + persistIdentitySettlementRecovery(fingerprintSandboxRecreateValue(sandboxId)); + throw error; + } +} + function checkRecreatedSandboxReadyIdentity( sandboxName: string, gatewayName: string, @@ -435,14 +604,7 @@ export function createSandboxGpuCreateAttemptRunner( const runAttempt = async (route: SelectedDockerGpuRoute) => { const deferPostCreateEffects = input.verifyCreatedSandboxBeforeEffects !== undefined; const compatibility = route === "compatibility"; - if (compatibility && input.initialGpuRoute === "native") { - console.warn( - " Native OpenShell GPU onboarding did not complete; retrying once by recreating the OpenShell-managed Docker container with the legacy GPU compatibility envelope.", - ); - console.warn( - " This compatibility container swap may relax container confinement compared with native injection. The retry is running only because NEMOCLAW_DOCKER_GPU_PATCH=fallback explicitly authorized it.", - ); - } + warnForAuthorizedCompatibilityRetry(route, input.initialGpuRoute); const hasRequiredUlimits = (input.requiredUlimits?.length ?? 0) > 0; const managedBootstrap = input.managedBootstrap ?? null; const unboundAttemptArgv = state.compatibilityArgv ?? input.createArgv; @@ -451,49 +613,19 @@ export function createSandboxGpuCreateAttemptRunner( const persistIdentitySettlementRecovery = ( sandboxIdentityFingerprint: string | null = null, ): void => { - if (!createAttemptNonce) { - throw new Error("Sandbox create-attempt identity was not generated."); - } - const persist = input.persistRetainedSandboxRecovery; - if (!persist) { - throw new Error("Verified sandbox creation has no durable recovery evidence owner."); - } - const identityEvidence = sandboxIdentityFingerprint - ? `Durable sandbox identity fingerprint: ${sandboxIdentityFingerprint}. Sandbox '${input.sandboxName}' did not remain visible through owning gateway '${input.gatewayName}' before identity verification completed. ` - : `Sandbox '${input.sandboxName}' reached Ready before OpenShell returned one exact durable create identity. Gateway '${input.gatewayName}'. OpenShell did not return one exact durable sandbox identity for this create attempt. `; - const message = - `Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${createAttemptNonce}. ` + - identityEvidence + - "Do not delete a sandbox by mutable name; preserve it until an OpenShell administrator resolves the create-attempt label to one sandbox."; - let persistenceFailure: unknown = null; - try { - if (!persist(message, sandboxIdentityFingerprint ?? undefined, createAttemptNonce)) { - persistenceFailure = new Error( - "The retained sandbox recovery writer did not confirm durable persistence.", - ); - } - } catch (error) { - persistenceFailure = error; - } - console.error(` ${message}`); - if (persistenceFailure) { - console.error( - " NemoClaw could not save this create-attempt evidence. The recovery-only session remains blocked until its durable recovery record can be saved.", - ); - throw new Error( - "NemoClaw could not save the retained sandbox recovery record; the recovery-only session remains blocked.", - { cause: persistenceFailure }, - ); - } - }; - const waitForCreatedSandboxPublication = (sandboxId: string): void => { - try { - waitForCreatedOpenShellSandboxPublication(sandboxId, input, deps); - } catch (error) { - persistIdentitySettlementRecovery(fingerprintSandboxRecreateValue(sandboxId)); - throw error; - } + persistIdentitySettlementRecoveryEvidence({ + input, + createAttemptNonce, + sandboxIdentityFingerprint, + }); }; + const waitForCreatedSandboxPublication = (sandboxId: string): void => + waitForCreatedSandboxPublicationOrPersist( + sandboxId, + input, + deps, + persistIdentitySettlementRecovery, + ); const captureRetainedSandboxRecovery = () => { if (!input.requirePolicylessCreate || !createAttemptNonce) return {}; let liveIdentityFingerprint: string | null = null; @@ -770,27 +902,12 @@ export function createSandboxGpuCreateAttemptRunner( ); } } else { - const observation = await sandboxReadinessTracing.observeOpenShellSandbox( - deps.sandboxObserver, - { kind: "named", gatewayName: input.gatewayName }, - input.sandboxName, - SANDBOX_READY_PROBE_TIMEOUT_MS, - ); - if (!observation.ok) { - if (createAttemptNonce) persistIdentitySettlementRecovery(); - throw new Error( - `Managed bootstrap create completed, but NemoClaw could not observe the sandbox. ${observation.error.message}`, - ); - } - if ( - observation.value.state !== "present" || - observation.value.sandbox.readiness !== "ready" - ) { - if (createAttemptNonce) persistIdentitySettlementRecovery(); - throw new Error( - "Managed bootstrap create completed without an authoritative Ready sandbox.", - ); - } + await requireManagedBootstrapCreatedSandboxReady({ + input, + deps, + createAttemptNonce, + persistIdentitySettlementRecovery, + }); } let sandboxId: string; try { @@ -1155,54 +1272,29 @@ export function createSandboxGpuCreateAttemptRunner( revalidatePostCreateEffect(`commit runtime readiness for sandbox '${input.sandboxName}'`); await runtimePatch.commitAfterReady(); } - const confirmManagedRuntimeCommitReadiness = async (): Promise => { - if (!managedBootstrap || !verifiedCreatedSandboxId) return; - input.revalidateVerifiedSandboxBeforeEffect?.( - `confirm committed runtime readiness for sandbox '${input.sandboxName}'`, - ); - const committedReadiness = await sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ - sandboxName: input.sandboxName, - timeoutSecs: input.sandboxReadyTimeoutSecs, - observer: deps.sandboxObserver, - target: { kind: "named", gatewayName: input.gatewayName }, - stableReadyPolls: REPLACEMENT_STABLE_READY_POLLS, - checkReadyIdentity: (getRemainingMs = () => SANDBOX_RECREATE_PROBE_TIMEOUT_MS) => - checkRecreatedSandboxReadyIdentity( - input.sandboxName, - input.gatewayName, - verifiedCreatedSandboxId, - deps, - getRemainingMs, - ), - sleep: deps.sleep, - }); - if (committedReadiness.ready) return; - console.error(""); - sandboxReadinessTracing.printReadinessFailure( - committedReadiness, - input.sandboxName, - input.sandboxReadyTimeoutSecs, - ); - persistIdentitySettlementRecovery(fingerprintSandboxRecreateValue(verifiedCreatedSandboxId)); - (deps.printCreateFailureDiagnostics ?? printSandboxCreateFailureDiagnostics)(input.sandboxName, { - backupPath: input.restoreBackupPath, + const confirmCommittedRuntimeReadiness = () => + confirmManagedRuntimeCommitReadiness({ + input, + deps, + sandboxId: managedBootstrap ? verifiedCreatedSandboxId : null, + createAttemptNonce, }); - console.error( - " NemoClaw did not start dashboard forwarding. NemoClaw left the sandbox in place for identity-bound recovery.", - ); - throw new Error( - `Sandbox '${input.sandboxName}' did not return to Ready after its managed runtime commit.`, - ); - }; if (!input.sandboxGpuConfig.sandboxGpuEnabled) { - await confirmManagedRuntimeCommitReadiness(); + await confirmCommittedRuntimeReadiness(); } return { ok: true, route, value: createResult - ? { createResult, runtimePatch, confirmManagedRuntimeCommitReadiness } - : { runtimePatch, confirmManagedRuntimeCommitReadiness }, + ? { + createResult, + runtimePatch, + confirmManagedRuntimeCommitReadiness: confirmCommittedRuntimeReadiness, + } + : { + runtimePatch, + confirmManagedRuntimeCommitReadiness: confirmCommittedRuntimeReadiness, + }, } as const; }; diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index 6225eca9a82..84463d889f8 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -130,6 +130,27 @@ export interface SandboxReadyWaitOptions extends SandboxReadyWaitDeps { delaySeconds: number; } +/** Wait for one created-sandbox publication condition inside a shared bounded deadline. */ +export function waitForCreatedSandboxPublication(options: { + budgetMs: number; + pollIntervalMs: number; + probe: (getRemainingMs: () => number) => boolean; + sleep: (seconds: number) => void; + now?: () => number; +}): boolean { + const waitOptions = createReadinessWaitOptions({ + budgetMs: options.budgetMs, + initialIntervalMs: options.pollIntervalMs, + maxIntervalMs: options.pollIntervalMs, + now: options.now, + sleep: (milliseconds) => options.sleep(milliseconds / 1_000), + }); + const deadlineMs = waitOptions?.deadlineMs; + const now = waitOptions?.now; + if (!waitOptions || deadlineMs === undefined || !now) return false; + return waitUntil(() => options.probe(() => Math.max(1, deadlineMs - now())), waitOptions); +} + export async function observeOpenShellSandbox( observer: OpenShellSandboxObserver, target: OpenShellGatewayTarget, From c5b71b3625baf145a7f02a20b416a07dd835fb1c Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 22:02:58 -0400 Subject: [PATCH 21/51] fix(onboard): align readiness authority checks Signed-off-by: Julie Yaunches --- src/lib/onboard/experimental/hermes-portable-onboarding.ts | 7 ++----- src/lib/onboard/sandbox-gpu-create-flow.test.ts | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding.ts b/src/lib/onboard/experimental/hermes-portable-onboarding.ts index b3b3662024a..091d34bec82 100644 --- a/src/lib/onboard/experimental/hermes-portable-onboarding.ts +++ b/src/lib/onboard/experimental/hermes-portable-onboarding.ts @@ -354,7 +354,7 @@ function scopeHermesPortableReadyExecArgs( return null; } -/** Route create readiness and failed-create cleanup through exact schema-7 authority. */ +/** Route create readiness through exact schema-7 authority. */ export function createHermesPortableReadyRunner( sandboxName: string, gatewayName: string, @@ -365,10 +365,7 @@ export function createHermesPortableReadyRunner( scopeHermesPortableCreatedIdentityArgs(args, gatewayName) ?? scopeHermesPortableReadyGetArgs(args, sandboxName, gatewayName) ?? scopeHermesPortableReadyListArgs(args, gatewayName) ?? - scopeHermesPortableReadyExecArgs(args, sandboxName, gatewayName) ?? - (args[0] === "sandbox" && args[1] === "delete" && args.length === 3 && args[2] === sandboxName - ? ["sandbox", "delete", "-g", gatewayName, args[2]!] - : null); + scopeHermesPortableReadyExecArgs(args, sandboxName, gatewayName); if (!scoped) fail("create lifecycle attempted an unsupported OpenShell command"); return capture(scoped); }; diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index fbfe616dde5..6d6d90066eb 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -401,7 +401,7 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { mocks.waitForCreatedSandboxReadyWithTrace.mock.calls.map( ([options]) => options.stableReadyPolls, ), - ).toEqual([2, 2]); + ).toEqual([2, 2, 2]); vi.mocked(deps.runCaptureOpenshell).mockClear(); await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); expect(deps.runCaptureOpenshell).toHaveBeenCalledWith( From aa28948f7cd45107ce9bc851b4157c582b03b73c Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 22:16:23 -0400 Subject: [PATCH 22/51] test(onboard): expose persistence failure assertions Signed-off-by: Julie Yaunches --- .../sandbox-gpu-create-identity-gate.test.ts | 63 ++++++++++++------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index a1fbcf1434b..5bcf84cc087 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -142,9 +142,7 @@ function refuseEffectStartingWith(prefix: string): (operation: string) => void { }; } -async function expectCommittedReadinessPersistenceFailure( - persist: NonNullable["persistRetainedSandboxRecovery"]>, -): Promise { +function createCommittedReadinessPersistenceFixture() { const error = vi.spyOn(console, "error").mockImplementation(() => undefined); const input = noGpuInput(); input.resumeVerifiedCreate = { @@ -154,28 +152,13 @@ async function expectCommittedReadinessPersistenceFailure( }; input.verifyCreatedSandboxBeforeEffects = vi.fn(); input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - input.persistRetainedSandboxRecovery = persist; const patch = createGpuPatchFixture(); attachManagedBootstrap(input, patch); const deps = createGpuFlowDeps("alpha-sandbox-id"); mocks.waitForCreatedSandboxReadyWithTrace .mockResolvedValueOnce({ ready: true, reason: "ready", failurePhase: null }) .mockResolvedValue({ ready: false, reason: "timeout", failurePhase: null }); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "the recovery-only session remains blocked", - ); - - expect(persist).toHaveBeenCalledOnce(); - expect(error.mock.calls.flat().join("\n")).toContain( - "The recovery-only session remains blocked until its durable recovery record can be saved.", - ); - expect(error.mock.calls.flat().join("\n")).not.toContain("Preserve the terminal output"); - expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); - expect(deps.runOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), - ); + return { deps, error, input, patch }; } beforeEach(() => setupGpuFlowMocks(mocks)); @@ -328,14 +311,46 @@ describe("created sandbox identity gate", () => { }); it("blocks committed-readiness recovery when durable persistence returns false (#9211)", async () => { - await expectCommittedReadinessPersistenceFailure(vi.fn(() => false)); + const { deps, error, input, patch } = createCommittedReadinessPersistenceFixture(); + const persist = vi.fn(() => false); + input.persistRetainedSandboxRecovery = persist; + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "the recovery-only session remains blocked", + ); + + expect(persist).toHaveBeenCalledOnce(); + expect(error.mock.calls.flat().join("\n")).toContain( + "The recovery-only session remains blocked until its durable recovery record can be saved.", + ); + expect(error.mock.calls.flat().join("\n")).not.toContain("Preserve the terminal output"); + expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); }); it("blocks committed-readiness recovery when durable persistence throws (#9211)", async () => { - await expectCommittedReadinessPersistenceFailure( - vi.fn(() => { - throw new Error("durable writer failed"); - }), + const { deps, error, input, patch } = createCommittedReadinessPersistenceFixture(); + const persist = vi.fn(() => { + throw new Error("durable writer failed"); + }); + input.persistRetainedSandboxRecovery = persist; + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "the recovery-only session remains blocked", + ); + + expect(persist).toHaveBeenCalledOnce(); + expect(error.mock.calls.flat().join("\n")).toContain( + "The recovery-only session remains blocked until its durable recovery record can be saved.", + ); + expect(error.mock.calls.flat().join("\n")).not.toContain("Preserve the terminal output"); + expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), ); }); From 70ab052d084f321150d381da6a6bedd86df83db2 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 31 Aug 2026 22:47:45 -0400 Subject: [PATCH 23/51] docs: explain retained sandbox recovery Signed-off-by: Julie Yaunches --- docs/reference/troubleshooting.mdx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 9e587724aa1..0b1b5421a53 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2004,6 +2004,11 @@ This is a separate budget from `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT`. It covers the For a newly created OpenClaw or Hermes sandbox, `Ready` is not the final acceptance signal. Within this same budget, NemoClaw also requires OpenShell to return a durable sandbox ID and accept `openshell sandbox exec --name -- true`. NemoClaw keeps waiting only when OpenShell returns its exact `sandbox is not ready` response. A missing or malformed ID, or another command failure, stops the wait. Ordinary onboarding then follows the failed-creation cleanup path. Portable OpenClaw onboarding preserves the sandbox as described below. +If onboarding reports that the managed runtime commit completed but the same sandbox did not return to executable `Ready`, stop before retrying. +NemoClaw keeps the sandbox, saves its create-attempt label and a one-way durable identity fingerprint in the retained recovery record, and does not start dashboard forwarding. +Preserve that exact record, and do not delete the sandbox by its mutable name. +Give the create-attempt label and fingerprint to an OpenShell administrator so they can identify and remove the exact sandbox and reconcile the retained recovery state before you retry onboarding. + The 180-second default fits typical workstations but can be exceeded when: From 04ffb8a9a0708da08ad6003b34be285b3ed0ea32 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Mon, 31 Aug 2026 20:58:33 -0700 Subject: [PATCH 24/51] fix(onboard): close readiness review gaps Signed-off-by: Prekshi Vyas --- ci/source-architecture-budget.json | 2 +- docs/reference/troubleshooting.mdx | 8 +- src/lib/onboard/sandbox-create-step.test.ts | 456 ------------------ src/lib/onboard/sandbox-create-step.ts | 143 ------ .../pull-requests/pr-risk-plan.test.ts | 1 - test/security/shellquote-sandbox.test.ts | 25 - tools/advisors/risk-plan.mts | 1 - 7 files changed, 6 insertions(+), 630 deletions(-) delete mode 100644 src/lib/onboard/sandbox-create-step.test.ts delete mode 100644 src/lib/onboard/sandbox-create-step.ts diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index a57ecfa5c59..cbcd0e3541c 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -59,7 +59,7 @@ }, "allowedCycles": [], "maxRootFiles": { - "src/lib/onboard": 306, + "src/lib/onboard": 305, "src/lib/actions": 18, "src/lib/actions/sandbox": 182, "src/lib/state": 39, diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 0b1b5421a53..2fc7037f07d 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2005,9 +2005,11 @@ This is a separate budget from `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT`. It covers the For a newly created OpenClaw or Hermes sandbox, `Ready` is not the final acceptance signal. Within this same budget, NemoClaw also requires OpenShell to return a durable sandbox ID and accept `openshell sandbox exec --name -- true`. NemoClaw keeps waiting only when OpenShell returns its exact `sandbox is not ready` response. A missing or malformed ID, or another command failure, stops the wait. Ordinary onboarding then follows the failed-creation cleanup path. Portable OpenClaw onboarding preserves the sandbox as described below. If onboarding reports that the managed runtime commit completed but the same sandbox did not return to executable `Ready`, stop before retrying. -NemoClaw keeps the sandbox, saves its create-attempt label and a one-way durable identity fingerprint in the retained recovery record, and does not start dashboard forwarding. -Preserve that exact record, and do not delete the sandbox by its mutable name. -Give the create-attempt label and fingerprint to an OpenShell administrator so they can identify and remove the exact sandbox and reconcile the retained recovery state before you retry onboarding. +NemoClaw keeps the sandbox, prints its create-attempt label and a one-way durable identity fingerprint, and does not start dashboard forwarding. +When recovery persistence succeeds, NemoClaw also saves that evidence in the retained recovery record. +If NemoClaw reports that it could not save the recovery record, preserve the terminal output. +Do not delete the sandbox by its mutable name. +Give the printed create-attempt label and fingerprint to an OpenShell administrator so they can identify and remove the exact sandbox and reconcile the retained recovery state before you retry onboarding. diff --git a/src/lib/onboard/sandbox-create-step.test.ts b/src/lib/onboard/sandbox-create-step.test.ts deleted file mode 100644 index c4ed01887ca..00000000000 --- a/src/lib/onboard/sandbox-create-step.test.ts +++ /dev/null @@ -1,456 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { afterEach, describe, expect, it, vi } from "vitest"; - -import { streamSandboxCreate } from "../sandbox/create-stream"; -import { - dockerEnv, - FakeChild, - makePollingOptions, - vmEnv, -} from "../sandbox/create-stream-test-fixtures"; -import { - runSandboxCreateStep, - type SandboxCreateStepContext, - type SandboxCreateStepDeps, -} from "./sandbox-create-step"; - -afterEach(() => { - vi.unstubAllEnvs(); -}); - -function makeLaunch(overrides: Record = {}) { - return { - createCommand: "openshell sandbox create alpha", - effectiveDashboardPort: "18789", - createArgv: ["openshell", "sandbox", "create", "alpha"], - envArgs: [], - sandboxEnv: { FOO: "bar" }, - sandboxStartupCommand: ["run", "alpha"], - prebuild: { imageRef: "img:tag", createArgs: ["sandbox", "create", "alpha"] }, - ...overrides, - }; -} - -function makePatch() { - return { - maybeApplyDuringCreate: vi.fn(), - createFailureMessage: vi.fn(() => null), - ensureApplied: vi.fn(), - }; -} - -function makeContext(overrides: Partial = {}): SandboxCreateStepContext { - // Cast once at the boundary: hermesDashboardState / openshellShellCommand / - // prebuild are structural seams this orchestration test does not exercise. - const base = { - agent: null, - observabilityEnabled: false, - chatUiUrl: "", - createArgs: ["sandbox", "create", "alpha"], - sandboxName: "alpha", - env: {}, - extraPlaceholderKeys: [], - getDashboardForwardPort: () => "18789", - hermesDashboardState: null, - manageDashboard: false, - openshellShellCommand: null, - prebuild: { buildCtx: "/tmp/ctx", buildId: "b1", dockerDriverGateway: null, origin: "local" }, - useDockerGpuPatch: false, - gpuDevice: null, - gpuBackend: "generic" as const, - timeoutSecs: 300, - }; - return { ...base, ...overrides } as unknown as SandboxCreateStepContext; -} - -function makeDeps( - launch: ReturnType, - patch: ReturnType, - createResult: { status: number; output: string }, - overrides: Partial = {}, -): SandboxCreateStepDeps { - return { - prepareCreateLaunch: vi.fn(async () => launch), - createDockerGpuPatch: vi.fn(() => patch), - streamCreate: vi.fn(async () => createResult), - isSandboxReady: vi.fn(() => false), - isTerminalAgent: vi.fn(() => false), - addTraceEvent: vi.fn(), - runOpenshell: vi.fn(() => ({ status: 0, output: "" })), - runCaptureOpenshell: vi.fn(() => "sandbox-list"), - sleepSeconds: vi.fn(), - ...overrides, - } as unknown as SandboxCreateStepDeps; -} - -describe("runSandboxCreateStep", () => { - it("threads the prebuild handoff into launch, GPU patch, and stream, and returns the handles", async () => { - const launch = makeLaunch(); - const patch = makePatch(); - const createResult = { status: 0, output: "created" }; - const deps = makeDeps(launch, patch, createResult); - - const result = await runSandboxCreateStep( - makeContext({ - useDockerGpuPatch: true, - gpuDevice: "nvidia.com/gpu=all", - gpuBackend: "jetson", - }), - deps, - ); - - // prepareCreateLaunch receives the assembled launch input incl. the prebuild handoff. - expect(deps.prepareCreateLaunch).toHaveBeenCalledWith( - expect.objectContaining({ - sandboxName: "alpha", - prebuild: { - buildCtx: "/tmp/ctx", - buildId: "b1", - dockerDriverGateway: null, - origin: "local", - }, - }), - ); - // GPU patch is created with the startup command from the launch result + backend/device. - expect(deps.createDockerGpuPatch).toHaveBeenCalledWith( - expect.objectContaining({ - route: "compatibility", - openshellSandboxCommand: ["run", "alpha"], - gpuDevice: "nvidia.com/gpu=all", - backend: "jetson", - }), - ); - // stream is fed the launch command + env. - expect(deps.streamCreate).toHaveBeenCalledWith( - "openshell", - ["sandbox", "create", "alpha"], - { FOO: "bar" }, - expect.objectContaining({ traceEvent: deps.addTraceEvent }), - ); - // Handles returned for downstream consumers. - expect(result).toEqual({ - createResult, - prebuild: launch.prebuild, - effectiveDashboardPort: "18789", - dockerGpuCreatePatch: patch, - }); - }); - - it.each([ - { label: "OpenClaw", agent: null }, - { label: "Hermes", agent: { name: "hermes" } as SandboxCreateStepContext["agent"] }, - ])("persists the $label startup command for Docker-driver container restarts", async ({ - agent, - }) => { - const launch = makeLaunch({ - sandboxStartupCommand: ["env", "CHAT_UI_URL=http://127.0.0.1:8642", "nemoclaw-start"], - }); - const patch = makePatch(); - const deps = makeDeps(launch, patch, { status: 0, output: "created" }); - - await runSandboxCreateStep( - makeContext({ - agent, - prebuild: { - buildCtx: "/tmp/ctx", - buildId: "b1", - dockerDriverGateway: true, - origin: "generated", - }, - }), - deps, - ); - - expect(deps.createDockerGpuPatch).toHaveBeenCalledWith( - expect.objectContaining({ - route: "native", - persistStartupCommand: true, - openshellSandboxCommand: ["env", "CHAT_UI_URL=http://127.0.0.1:8642", "nemoclaw-start"], - }), - ); - }); - - it("gates restart-safe persistence on the step's own portable env, not process.env (#9462)", async () => { - vi.stubEnv("NEMOCLAW_EXPERIMENTAL_PROFILE", "default"); - const launch = makeLaunch({ - sandboxStartupCommand: ["env", "nemoclaw-start"], - }); - const patch = makePatch(); - const deps = makeDeps(launch, patch, { status: 0, output: "created" }); - - await runSandboxCreateStep( - makeContext({ - agent: { name: "hermes" } as SandboxCreateStepContext["agent"], - env: { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" }, - prebuild: { - buildCtx: "/tmp/ctx", - buildId: "b1", - dockerDriverGateway: true, - origin: "generated", - }, - }), - deps, - ); - - expect(deps.createDockerGpuPatch).toHaveBeenCalledWith( - expect.objectContaining({ - persistStartupCommand: false, - }), - ); - }); - - it("persists DCode startup with its exact Docker resource limits", async () => { - const launch = makeLaunch({ - sandboxStartupCommand: ["env", "nemoclaw-start"], - }); - const patch = makePatch(); - const deps = makeDeps(launch, patch, { status: 0, output: "created" }); - - await runSandboxCreateStep( - makeContext({ - agent: { - name: "langchain-deepagents-code", - } as SandboxCreateStepContext["agent"], - prebuild: { - buildCtx: "/tmp/ctx", - buildId: "b1", - dockerDriverGateway: true, - origin: "generated", - }, - }), - deps, - ); - - expect(deps.createDockerGpuPatch).toHaveBeenCalledWith( - expect.objectContaining({ - persistStartupCommand: true, - requiredUlimits: [ - { name: "nproc", soft: 512, hard: 512 }, - { name: "nofile", soft: 65_536, hard: 65_536 }, - ], - }), - ); - }); - - it("separates readiness detection from GPU patch polling", async () => { - const launch = makeLaunch(); - const patch = makePatch(); - const deps = makeDeps( - launch, - patch, - { status: 0, output: "" }, - { isSandboxReady: vi.fn(() => true) }, - ); - - await runSandboxCreateStep(makeContext(), deps); - const streamOpts = (deps.streamCreate as unknown as { mock: { calls: unknown[][] } }).mock - .calls[0][3] as { readyCheck: () => boolean; onPoll: () => void }; - - expect(streamOpts.readyCheck()).toBe(true); - expect(patch.maybeApplyDuringCreate).not.toHaveBeenCalled(); - - (deps.isSandboxReady as unknown as ReturnType).mockReturnValue(false); - expect(streamOpts.readyCheck()).toBe(false); - expect(patch.maybeApplyDuringCreate).not.toHaveBeenCalled(); - - streamOpts.onPoll(); - expect(patch.maybeApplyDuringCreate).toHaveBeenCalledTimes(1); - }); - - it("waits for the create ownership handoff before restart-safe recreation (#8720)", async () => { - vi.useFakeTimers(); - const child = new FakeChild(); - const patch = makePatch(); - let ready = false; - let resolved = false; - const deps = makeDeps( - makeLaunch({ sandboxEnv: dockerEnv }), - patch, - { status: 0, output: "" }, - { - streamCreate: ((command, args, sandboxEnv, options) => - streamSandboxCreate(command, args, sandboxEnv, { - ...options, - ...makePollingOptions(child), - })) as SandboxCreateStepDeps["streamCreate"], - isSandboxReady: vi.fn(() => ready), - }, - ); - - const create = runSandboxCreateStep( - makeContext({ - prebuild: { - buildCtx: "/tmp/ctx", - buildId: "b1", - dockerDriverGateway: true, - origin: "generated", - }, - }), - deps, - ).then((result) => { - resolved = true; - return result; - }); - - child.stdout.emit("data", Buffer.from("Created sandbox: alpha\n")); - ready = true; - await vi.advanceTimersByTimeAsync(6); - - expect(resolved).toBe(false); - expect(child.kill).not.toHaveBeenCalled(); - child.stderr.emit("data", Buffer.from("Setting up NemoClaw...\n")); - await vi.advanceTimersByTimeAsync(6); - - expect(child.kill).toHaveBeenCalledWith("SIGTERM"); - expect(patch.maybeApplyDuringCreate).not.toHaveBeenCalled(); - - child.emit("close", 143); - await expect(create).resolves.toMatchObject({ - createResult: { status: 0, forcedReady: true }, - }); - }); - - it("threads the terminal-agent early-ready gate into stream options", async () => { - const terminalDeps = makeDeps( - makeLaunch(), - makePatch(), - { status: 0, output: "" }, - { - isTerminalAgent: vi.fn(() => true), - }, - ); - await runSandboxCreateStep(makeContext(), terminalDeps); - expect( - (terminalDeps.streamCreate as unknown as { mock: { calls: unknown[][] } }).mock.calls[0][3], - ).toMatchObject({ readyCheckOutputPatterns: [] }); - - const nonTerminalDeps = makeDeps(makeLaunch({ sandboxEnv: vmEnv }), makePatch(), { - status: 0, - output: "", - }); - await runSandboxCreateStep(makeContext(), nonTerminalDeps); - expect( - (nonTerminalDeps.streamCreate as unknown as { mock: { calls: unknown[][] } }).mock - .calls[0][3], - ).toMatchObject({ readyCheckOutputPatterns: [expect.any(RegExp)] }); - }); - - it.each([ - ["terminal VM", true, vmEnv], - ["terminal Docker", true, dockerEnv], - ])("detaches immediately for %s", async (_label, isTerminalAgent, env) => { - vi.useFakeTimers(); - - const child = new FakeChild(); - const logLine = vi.fn(); - const streamOptions = makePollingOptions(child, { logLine }); - const deps = makeDeps( - makeLaunch({ sandboxEnv: env }), - makePatch(), - { status: 0, output: "" }, - { - streamCreate: ((command, args, sandboxEnv, options) => - streamSandboxCreate(command, args, sandboxEnv, { - ...options, - ...streamOptions, - })) as SandboxCreateStepDeps["streamCreate"], - isTerminalAgent: vi.fn(() => isTerminalAgent), - }, - ); - let ready = false; - deps.isSandboxReady = vi.fn(() => ready); - deps.addTraceEvent = vi.fn(); - - const promise = runSandboxCreateStep(makeContext(), deps); - child.stdout.emit("data", Buffer.from("Created sandbox: alpha\n")); - ready = true; - await vi.advanceTimersByTimeAsync(6); - - expect(logLine).not.toHaveBeenCalledWith( - " Sandbox reported Ready; waiting for startup command output before detaching.", - ); - await expect(promise).resolves.toMatchObject({ - createResult: expect.objectContaining({ status: 0, forcedReady: true }), - }); - expect(child.kill).toHaveBeenCalledWith("SIGTERM"); - vi.useRealTimers(); - }); - - it.each([ - ["VM", vmEnv], - ["Docker", dockerEnv], - ])("waits for startup output for non-terminal %s creates", async (_label, env) => { - vi.useFakeTimers(); - - const child = new FakeChild(); - const logLine = vi.fn(); - const streamOptions = makePollingOptions(child, { logLine }); - const deps = makeDeps( - makeLaunch({ sandboxEnv: env }), - makePatch(), - { status: 0, output: "" }, - { - streamCreate: ((command, args, sandboxEnv, options) => - streamSandboxCreate(command, args, sandboxEnv, { - ...options, - ...streamOptions, - })) as SandboxCreateStepDeps["streamCreate"], - }, - ); - let ready = false; - deps.isSandboxReady = vi.fn(() => ready); - deps.addTraceEvent = vi.fn(); - - const promise = runSandboxCreateStep(makeContext(), deps); - child.stdout.emit("data", Buffer.from("Created sandbox: alpha\n")); - ready = true; - await vi.advanceTimersByTimeAsync(6); - - expect(child.kill).not.toHaveBeenCalled(); - expect(logLine).toHaveBeenCalledWith( - " Sandbox reported Ready; waiting for startup command output before detaching.", - ); - - child.stderr.emit("data", Buffer.from("Setting up NemoClaw (Hermes)...\n")); - await vi.advanceTimersByTimeAsync(6); - - await expect(promise).resolves.toMatchObject({ - createResult: expect.objectContaining({ status: 0, forcedReady: true }), - }); - expect(child.kill).toHaveBeenCalledWith("SIGTERM"); - vi.useRealTimers(); - }); - - it("recovers SSH 255 exits when the sandbox is ready", async () => { - vi.useFakeTimers(); - - const child = new FakeChild(); - const streamOptions = makePollingOptions(child, { pollIntervalMs: 60_000 }); - const deps = makeDeps( - makeLaunch({ sandboxEnv: dockerEnv }), - makePatch(), - { status: 0, output: "" }, - { - streamCreate: ((command, args, sandboxEnv, options) => - streamSandboxCreate(command, args, sandboxEnv, { - ...options, - ...streamOptions, - })) as SandboxCreateStepDeps["streamCreate"], - isSandboxReady: vi.fn(() => true), - }, - ); - - const promise = runSandboxCreateStep(makeContext(), deps); - await vi.advanceTimersByTimeAsync(0); - child.stdout.emit("data", Buffer.from("Created sandbox: alpha\n")); - child.stderr.emit("data", Buffer.from("Setting up NemoClaw...\n")); - child.emit("close", 255); - - await expect(promise).resolves.toMatchObject({ - createResult: expect.objectContaining({ status: 0, forcedReady: true }), - }); - vi.useRealTimers(); - }); -}); diff --git a/src/lib/onboard/sandbox-create-step.ts b/src/lib/onboard/sandbox-create-step.ts deleted file mode 100644 index 0cef3edfc7a..00000000000 --- a/src/lib/onboard/sandbox-create-step.ts +++ /dev/null @@ -1,143 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { AgentDefinition } from "../agent/defs"; -import type { - StreamSandboxCreateOptions, - StreamSandboxCreateResult, - streamSandboxCreate, -} from "../sandbox/create-stream"; -import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-ready-gate"; -import type { - createDockerGpuSandboxCreatePatch, - DockerGpuSandboxCreatePatch, -} from "./docker-gpu-sandbox-create"; -import { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; -import type { - prepareSandboxCreateLaunchWithPrebuild, - SandboxCreateLaunchWithPrebuild, - SandboxCreateLaunchWithPrebuildInput, -} from "./sandbox-create-launch"; - -type LaunchInput = SandboxCreateLaunchWithPrebuildInput; -type GpuPatchDeps = Parameters[0]["deps"]; - -export type SandboxCreateStepContext = { - agent: LaunchInput["agent"]; - observabilityEnabled: boolean; - chatUiUrl: string; - createArgs: LaunchInput["createArgs"]; - sandboxName: string; - env: NodeJS.ProcessEnv; - extraPlaceholderKeys: LaunchInput["extraPlaceholderKeys"]; - getDashboardForwardPort: LaunchInput["getDashboardForwardPort"]; - hermesDashboardState: LaunchInput["hermesDashboardState"]; - manageDashboard: boolean; - openshellShellCommand: LaunchInput["openshellShellCommand"]; - openshellArgv?: LaunchInput["openshellArgv"]; - prebuild: LaunchInput["prebuild"]; - useDockerGpuPatch: boolean; - gpuDevice: string | null | undefined; - gpuBackend: "jetson" | "generic"; - timeoutSecs: number; -}; - -export type SandboxCreateStepDeps = { - prepareCreateLaunch: typeof prepareSandboxCreateLaunchWithPrebuild; - createDockerGpuPatch: typeof createDockerGpuSandboxCreatePatch; - streamCreate: typeof streamSandboxCreate; - isSandboxReady(output: string, sandboxName: string): boolean; - isTerminalAgent(agent: AgentDefinition | null | undefined): boolean; - addTraceEvent: NonNullable; - runOpenshell: GpuPatchDeps["runOpenshell"]; - runCaptureOpenshell: NonNullable; - sleepSeconds: GpuPatchDeps["sleep"]; -}; - -export type SandboxCreateStepResult = { - createResult: StreamSandboxCreateResult; - prebuild: SandboxCreateLaunchWithPrebuild["prebuild"]; - effectiveDashboardPort: string; - dockerGpuCreatePatch: DockerGpuSandboxCreatePatch; -}; - -/** - * Resolve the BuildKit prebuild handoff into the launch command, provision the - * Docker-GPU create patch, and stream the sandbox create. Returns the create - * result plus the handles the caller needs downstream (prebuild identity, the - * dashboard port, and the GPU patch for its ready/verify hooks). Build-context - * and exit-listener cleanup stay with the caller that armed them. - */ -export async function runSandboxCreateStep( - context: SandboxCreateStepContext, - deps: SandboxCreateStepDeps, -): Promise { - const { - createCommand, - createArgv, - effectiveDashboardPort, - prebuild, - sandboxEnv, - sandboxStartupCommand, - } = await deps.prepareCreateLaunch({ - agent: context.agent, - observabilityEnabled: context.observabilityEnabled, - chatUiUrl: context.chatUiUrl, - createArgs: context.createArgs, - sandboxName: context.sandboxName, - env: context.env, - extraPlaceholderKeys: context.extraPlaceholderKeys, - getDashboardForwardPort: context.getDashboardForwardPort, - hermesDashboardState: context.hermesDashboardState, - manageDashboard: context.manageDashboard, - openshellShellCommand: context.openshellShellCommand, - openshellArgv: context.openshellArgv, - prebuild: context.prebuild, - }); - const startupCommandPatch = resolveDockerStartupCommandPatch( - context.agent, - context.prebuild.dockerDriverGateway, - context.env, - ); - const deferRestartSafeCutover = - startupCommandPatch.persistStartupCommand && !context.useDockerGpuPatch; - const dockerGpuCreatePatch = deps.createDockerGpuPatch({ - route: context.useDockerGpuPatch ? "compatibility" : "native", - persistStartupCommand: startupCommandPatch.persistStartupCommand, - requiredUlimits: startupCommandPatch.requiredUlimits, - sandboxName: context.sandboxName, - gpuDevice: context.gpuDevice, - openshellSandboxCommand: sandboxStartupCommand, - timeoutSecs: context.timeoutSecs, - backend: context.gpuBackend, - deps: { - runOpenshell: deps.runOpenshell, - runCaptureOpenshell: deps.runCaptureOpenshell, - sleep: deps.sleepSeconds, - }, - }); - const [createExecutable, ...createExecutableArgs] = createArgv; - const createResult = await deps.streamCreate( - createExecutable ?? createCommand, - createExecutableArgs, - sandboxEnv, - { - readyCheck: () => { - const list = deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); - return deps.isSandboxReady(list, context.sandboxName); - }, - onPoll: () => { - if (!deferRestartSafeCutover) dockerGpuCreatePatch.maybeApplyDuringCreate(); - }, - readyCheckOutputPatterns: getReadyCheckOutputPatternsForAgent({ - isTerminalAgent: deps.isTerminalAgent(context.agent), - startupRunsDuringCreate: true, - env: sandboxEnv, - }), - failureCheck: dockerGpuCreatePatch.createFailureMessage, - traceEvent: deps.addTraceEvent, - waitForReadyTermination: deferRestartSafeCutover, - }, - ); - return { createResult, prebuild, effectiveDashboardPort, dockerGpuCreatePatch }; -} diff --git a/test/automation/pull-requests/pr-risk-plan.test.ts b/test/automation/pull-requests/pr-risk-plan.test.ts index 5ce6a3fe972..6a5af0964c1 100644 --- a/test/automation/pull-requests/pr-risk-plan.test.ts +++ b/test/automation/pull-requests/pr-risk-plan.test.ts @@ -752,7 +752,6 @@ describe("deterministic PR risk plan", () => { "src/lib/actions/sandbox/status-snapshot.ts", "src/lib/onboard/docker-driver-sandbox-recovery.ts", "src/lib/onboard/docker-startup-command-agent.ts", - "src/lib/onboard/sandbox-create-step.ts", ])("selects post-reboot recovery for Docker delivery changes in %s (#7824)", (changedFile) => { const result = plan(changedFile); const adjacentStatusFile = plan("src/lib/actions/sandbox/status-text.ts"); diff --git a/test/security/shellquote-sandbox.test.ts b/test/security/shellquote-sandbox.test.ts index f548e4c6ed5..81a55cfc7ba 100644 --- a/test/security/shellquote-sandbox.test.ts +++ b/test/security/shellquote-sandbox.test.ts @@ -11,31 +11,6 @@ import { describe, expect, it } from "vitest"; import { writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; describe("sandboxName command hardening in onboard.js", () => { - it("rejects a marker-only security inventory fixture probe", async () => { - const helper = (await import("../helpers/onboard-script-mocks.cjs")) as { - isOpenClawSecurityInventoryProbe: (command: unknown) => boolean; - }; - - expect( - helper.isOpenClawSecurityInventoryProbe([ - "run", - "--rm", - "--network", - "none", - "--cap-drop", - "ALL", - "--security-opt", - "no-new-privileges", - "--read-only", - "--entrypoint", - "/bin/sh", - "nemoclaw:test", - "-c", - "echo nemoclaw-security-inventory-ok", - ]), - ).toBe(false); - }); - it("re-validates sandboxName at the createSandbox boundary", async () => { const onboardModule = await import("../../src/lib/onboard.js"); const { createSandbox } = onboardModule as unknown as { diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index 4f7dbc32401..72f7e7c9fe8 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -46,7 +46,6 @@ const POST_REBOOT_DELIVERY_RUNTIME_FILES = new Set([ "src/lib/actions/sandbox/status-snapshot.ts", "src/lib/onboard/docker-driver-sandbox-recovery.ts", "src/lib/onboard/docker-startup-command-agent.ts", - "src/lib/onboard/sandbox-create-step.ts", "tools/e2e/onboard-timeout-contract.mts", ]); export const GATEWAY_TOPOLOGY_FILES = [ From 92bc83f5b94ca248c524a094dd9036119e3bef7a Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 1 Sep 2026 00:44:38 -0400 Subject: [PATCH 25/51] docs: clarify retained recovery evidence Signed-off-by: Julie Yaunches --- docs/reference/troubleshooting.mdx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 0b1b5421a53..78e150cbec8 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2005,9 +2005,11 @@ This is a separate budget from `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT`. It covers the For a newly created OpenClaw or Hermes sandbox, `Ready` is not the final acceptance signal. Within this same budget, NemoClaw also requires OpenShell to return a durable sandbox ID and accept `openshell sandbox exec --name -- true`. NemoClaw keeps waiting only when OpenShell returns its exact `sandbox is not ready` response. A missing or malformed ID, or another command failure, stops the wait. Ordinary onboarding then follows the failed-creation cleanup path. Portable OpenClaw onboarding preserves the sandbox as described below. If onboarding reports that the managed runtime commit completed but the same sandbox did not return to executable `Ready`, stop before retrying. -NemoClaw keeps the sandbox, saves its create-attempt label and a one-way durable identity fingerprint in the retained recovery record, and does not start dashboard forwarding. -Preserve that exact record, and do not delete the sandbox by its mutable name. -Give the create-attempt label and fingerprint to an OpenShell administrator so they can identify and remove the exact sandbox and reconcile the retained recovery state before you retry onboarding. +NemoClaw keeps the sandbox, prints its create-attempt label and a one-way durable identity fingerprint, and does not start dashboard forwarding. +It saves that evidence in the retained recovery record when persistence succeeds. +If persistence fails, preserve the terminal output instead. +Do not delete the sandbox by its mutable name. +Give the printed create-attempt label and fingerprint to an OpenShell administrator so they can identify and remove the exact sandbox and reconcile the retained recovery state before you retry onboarding. From f7186c37e001dd262bf72be6a42743ddc7a6ddf1 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 1 Sep 2026 01:10:42 -0400 Subject: [PATCH 26/51] fix(onboard): stop on publication probe errors Signed-off-by: Julie Yaunches --- docs/reference/troubleshooting.mdx | 6 +- src/lib/onboard/created-sandbox-failure.ts | 6 +- .../sandbox-gpu-create-identity-gate.test.ts | 108 +++++++++++------- .../onboard/sandbox-gpu-create-run-attempt.ts | 60 +++++++++- 4 files changed, 130 insertions(+), 50 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 78e150cbec8..61a6a7c49bb 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2002,14 +2002,14 @@ This is a separate budget from `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT`. It covers the -For a newly created OpenClaw or Hermes sandbox, `Ready` is not the final acceptance signal. Within this same budget, NemoClaw also requires OpenShell to return a durable sandbox ID and accept `openshell sandbox exec --name -- true`. NemoClaw keeps waiting only when OpenShell returns its exact `sandbox is not ready` response. A missing or malformed ID, or another command failure, stops the wait. Ordinary onboarding then follows the failed-creation cleanup path. Portable OpenClaw onboarding preserves the sandbox as described below. +For a newly created OpenClaw or Hermes sandbox, `Ready` is not the final acceptance signal. Within this same budget, NemoClaw also requires OpenShell to return a durable sandbox ID and accept `openshell sandbox exec --name -- true`. NemoClaw keeps waiting only when OpenShell reports exact sandbox absence or its exact `sandbox is not ready` response. A missing or malformed ID, or another command failure, stops the wait. Ordinary onboarding then follows the failed-creation cleanup path. Portable OpenClaw onboarding preserves the sandbox as described below. If onboarding reports that the managed runtime commit completed but the same sandbox did not return to executable `Ready`, stop before retrying. NemoClaw keeps the sandbox, prints its create-attempt label and a one-way durable identity fingerprint, and does not start dashboard forwarding. It saves that evidence in the retained recovery record when persistence succeeds. -If persistence fails, preserve the terminal output instead. +The terminal output also preserves the create-attempt evidence, but a recovery-only session remains blocked until NemoClaw can save the durable recovery record. Do not delete the sandbox by its mutable name. -Give the printed create-attempt label and fingerprint to an OpenShell administrator so they can identify and remove the exact sandbox and reconcile the retained recovery state before you retry onboarding. +When NemoClaw confirms that it saved the record, give the printed create-attempt label and fingerprint to an OpenShell administrator so they can identify and remove the exact sandbox and reconcile the retained recovery state before you retry onboarding. diff --git a/src/lib/onboard/created-sandbox-failure.ts b/src/lib/onboard/created-sandbox-failure.ts index 820c4fddbdd..84ac07cd5bf 100644 --- a/src/lib/onboard/created-sandbox-failure.ts +++ b/src/lib/onboard/created-sandbox-failure.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { redact } from "../security/redact"; +import { redact, redactFullWithUrls } from "../security/redact"; import type { CreatedSandboxReadinessResult } from "./sandbox-readiness-tracing"; export type SandboxCreateFailureReportOptions = { @@ -25,6 +25,10 @@ export type SandboxCreateFailureReportDeps = { exitProcess(code: number): never; }; +export function redactCreatedSandboxFailureDiagnostic(value: string, limit: number): string { + return redactFullWithUrls(value).replace(/\s+/gu, " ").trim().slice(0, limit); +} + /** * Report a non-zero sandbox create-stream exit. A mere "create incomplete" * (the sandbox exists in the gateway but the stream exited non-zero, e.g. SSH diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 5bcf84cc087..50ab363b535 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -288,6 +288,7 @@ describe("created sandbox identity gate", () => { "did not return to Ready after its managed runtime commit", ); + expect(patch.commitAfterReady).toHaveBeenCalledOnce(); expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( expect.stringContaining("did not return to executable Ready state"), sandboxIdentityFingerprint, @@ -310,49 +311,37 @@ describe("created sandbox identity gate", () => { expect(recoveryOutput).not.toContain("After OpenShell confirms the sandbox is absent"); }); - it("blocks committed-readiness recovery when durable persistence returns false (#9211)", async () => { - const { deps, error, input, patch } = createCommittedReadinessPersistenceFixture(); - const persist = vi.fn(() => false); - input.persistRetainedSandboxRecovery = persist; - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "the recovery-only session remains blocked", - ); - - expect(persist).toHaveBeenCalledOnce(); - expect(error.mock.calls.flat().join("\n")).toContain( - "The recovery-only session remains blocked until its durable recovery record can be saved.", - ); - expect(error.mock.calls.flat().join("\n")).not.toContain("Preserve the terminal output"); - expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); - expect(deps.runOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), - ); - }); - - it("blocks committed-readiness recovery when durable persistence throws (#9211)", async () => { - const { deps, error, input, patch } = createCommittedReadinessPersistenceFixture(); - const persist = vi.fn(() => { - throw new Error("durable writer failed"); - }); - input.persistRetainedSandboxRecovery = persist; + it.each([ + ["returns false", () => false], + [ + "throws", + () => { + throw new Error("durable writer failed"); + }, + ], + ])( + "blocks committed-readiness recovery when durable persistence %s (#9211)", + async (_name, writer) => { + const { deps, error, input, patch } = createCommittedReadinessPersistenceFixture(); + const persist = vi.fn(writer); + input.persistRetainedSandboxRecovery = persist; - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "the recovery-only session remains blocked", - ); + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "the recovery-only session remains blocked", + ); - expect(persist).toHaveBeenCalledOnce(); - expect(error.mock.calls.flat().join("\n")).toContain( - "The recovery-only session remains blocked until its durable recovery record can be saved.", - ); - expect(error.mock.calls.flat().join("\n")).not.toContain("Preserve the terminal output"); - expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); - expect(deps.runOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), - ); - }); + expect(persist).toHaveBeenCalledOnce(); + expect(error.mock.calls.flat().join("\n")).toContain( + "The recovery-only session remains blocked until its durable recovery record can be saved.", + ); + expect(error.mock.calls.flat().join("\n")).not.toContain("Preserve the terminal output"); + expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + }, + ); it("resumes the exact verified sandbox without issuing another create (#9833)", async () => { const events: string[] = []; @@ -834,6 +823,43 @@ describe("created sandbox identity gate", () => { expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); }); + it("stops publication probing on a non-transient OpenShell failure (#9833)", async () => { + let nonce = ""; + const input = noGpuInput(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + vi.mocked(deps.runOpenshell).mockReturnValue({ + status: 1, + stdout: "", + stderr: "permission denied: NVIDIA_API_KEY=nvapi-publication-secret", + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "OpenShell could not verify publication", + ); + + expect(deps.sleep).not.toHaveBeenCalled(); + expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringMatching(/OpenShell detail: .*permission denied: NVIDIA_API_KEY=/u), + fingerprintSandboxRecreateValue("alpha-sandbox-id"), + nonce, + ); + const recoveryMessage = + vi.mocked(input.persistRetainedSandboxRecovery!).mock.calls[0]?.[0] ?? ""; + expect(recoveryMessage).not.toContain("nvapi-publication-secret"); + }); + it("rejects a different owner-scoped sandbox identity before post-create effects (#9833)", async () => { let nonce = ""; const input = noGpuInput(); diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 11bdd463de8..256deba7220 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -23,7 +23,10 @@ import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-re import { isSandboxReady } from "../state/gateway"; import type { SandboxGpuProofResult } from "../state/registry"; import { classifySandboxCreateFailure } from "../validation"; -import { reportSandboxCreateFailure } from "./created-sandbox-failure"; +import { + redactCreatedSandboxFailureDiagnostic, + reportSandboxCreateFailure, +} from "./created-sandbox-failure"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; @@ -47,7 +50,10 @@ import type { } from "./sandbox-gpu-create-flow"; import { fingerprintSandboxRecreateValue } from "./sandbox-recreate-transaction"; import * as sandboxGpuPreflight from "./sandbox-gpu-preflight"; -import { SANDBOX_RECREATE_PROBE_TIMEOUT_MS } from "./sandbox-recreate-probe"; +import { + isExplicitMissingSandboxGatewayOutput, + SANDBOX_RECREATE_PROBE_TIMEOUT_MS, +} from "./sandbox-recreate-probe"; import type { CreatedSandboxReadyIdentityCheck } from "./sandbox-readiness-tracing"; import * as sandboxReadinessTracing from "./sandbox-readiness-tracing"; import { addTraceEvent } from "./tracing"; @@ -70,6 +76,7 @@ export type SandboxGpuCreateAttemptState = { const REPLACEMENT_STABLE_READY_POLLS = 2; const SANDBOX_READY_PROBE_TIMEOUT_MS = 5_000; const CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_MS = 1_000; +const CREATED_SANDBOX_PUBLICATION_DIAGNOSTIC_LIMIT = 1_000; async function streamSandboxCreateWithPublicImageCredentialIsolation( isolate: boolean, @@ -192,6 +199,23 @@ function normalizedOpenShellCommandOutput(result: OpenShellCommandResult): strin .trim(); } +function boundedPublicationDiagnostic(value: string): string { + return redactCreatedSandboxFailureDiagnostic( + value, + CREATED_SANDBOX_PUBLICATION_DIAGNOSTIC_LIMIT, + ); +} + +function publicationFailureDiagnostic(result: OpenShellCommandResult): string { + const commandOutput = normalizedOpenShellCommandOutput(result); + const processError = + result.error instanceof Error ? result.error.message : String(result.error ?? ""); + const status = result.status === null ? "no exit status" : `exit ${result.status}`; + return boundedPublicationDiagnostic( + [status, processError, commandOutput].filter((value) => value.length > 0).join(": "), + ); +} + type OpenShellSandboxIdentityProbe = | { state: "identified"; sandboxId: string } | { state: "not_ready" } @@ -310,6 +334,7 @@ function persistIdentitySettlementRecoveryEvidence(options: { readonly input: SandboxGpuCreateFlowInput; readonly createAttemptNonce: string | null; readonly sandboxIdentityFingerprint: string | null; + readonly failureDiagnostic?: string; }): void { const { input, createAttemptNonce, sandboxIdentityFingerprint } = options; const identityEvidence = sandboxIdentityFingerprint @@ -320,6 +345,7 @@ function persistIdentitySettlementRecoveryEvidence(options: { createAttemptNonce, detail: identityEvidence + + (options.failureDiagnostic ? `OpenShell detail: ${options.failureDiagnostic}. ` : "") + "Do not delete a sandbox by mutable name; preserve it until an OpenShell administrator resolves the create-attempt label to one sandbox.", sandboxIdentityFingerprint: sandboxIdentityFingerprint ?? undefined, }); @@ -463,7 +489,23 @@ function waitForCreatedOpenShellSandboxPublication( } return true; } - return false; + const output = normalizedOpenShellCommandOutput(result); + const failedCleanly = + !result.error && + result.status !== null && + !("signal" in result && result.signal) && + result.status !== 0; + if ( + failedCleanly && + (OPENSHELL_SANDBOX_NOT_READY.test(output) || + isExplicitMissingSandboxGatewayOutput(output, input.sandboxName)) + ) { + return false; + } + const diagnostic = publicationFailureDiagnostic(result); + throw new Error( + `OpenShell could not verify publication of created sandbox '${input.sandboxName}'${diagnostic ? `: ${diagnostic}` : "."}`, + ); }, }); if (!published) { @@ -477,12 +519,18 @@ function waitForCreatedSandboxPublicationOrPersist( sandboxId: string, input: SandboxGpuCreateFlowInput, deps: SandboxGpuCreateFlowDeps, - persistIdentitySettlementRecovery: (sandboxIdentityFingerprint: string) => void, + persistIdentitySettlementRecovery: ( + sandboxIdentityFingerprint: string, + failureDiagnostic?: string, + ) => void, ): void { try { waitForCreatedOpenShellSandboxPublication(sandboxId, input, deps); } catch (error) { - persistIdentitySettlementRecovery(fingerprintSandboxRecreateValue(sandboxId)); + persistIdentitySettlementRecovery( + fingerprintSandboxRecreateValue(sandboxId), + boundedPublicationDiagnostic(error instanceof Error ? error.message : String(error)), + ); throw error; } } @@ -612,11 +660,13 @@ export function createSandboxGpuCreateAttemptRunner( const createAttemptNonce = resolveCreateAttemptNonce(input, deferPostCreateEffects); const persistIdentitySettlementRecovery = ( sandboxIdentityFingerprint: string | null = null, + failureDiagnostic?: string, ): void => { persistIdentitySettlementRecoveryEvidence({ input, createAttemptNonce, sandboxIdentityFingerprint, + ...(failureDiagnostic ? { failureDiagnostic } : {}), }); }; const waitForCreatedSandboxPublication = (sandboxId: string): void => From 37ce7874a1014b020a4868f95cb4b2c33e2b18da Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 00:43:01 -0700 Subject: [PATCH 27/51] fix(onboard): close retained recovery gaps Signed-off-by: Prekshi Vyas --- docs/reference/troubleshooting.mdx | 4 +- .../onboard/created-sandbox-failure.test.ts | 209 ------------------ src/lib/onboard/created-sandbox-failure.ts | 141 ------------ 3 files changed, 2 insertions(+), 352 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 4474b3d6216..b3588064301 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2007,9 +2007,9 @@ For a newly created OpenClaw or Hermes sandbox, `Ready` is not the final accepta If onboarding reports that the managed runtime commit completed but the same sandbox did not return to executable `Ready`, stop before retrying. NemoClaw keeps the sandbox, prints its create-attempt label and a one-way durable identity fingerprint, and does not start dashboard forwarding. It saves that evidence in the retained recovery record when persistence succeeds. -If NemoClaw reports that it could not save the record, preserve the terminal output; the recovery-only session remains blocked until NemoClaw can save the durable recovery record. Do not delete the sandbox by its mutable name. -When NemoClaw confirms that it saved the record, give the printed create-attempt label and fingerprint to an OpenShell administrator so they can identify and remove the exact sandbox and reconcile the retained recovery state before you retry onboarding. +When NemoClaw confirms that it saved the record, run `$$nemoclaw destroy` and follow [Recover a retained sandbox](commands#recover-a-retained-sandbox) for the result-specific recovery steps. +If NemoClaw reports that it could not save the record, preserve the terminal output and ask an OpenShell administrator to identify the exact sandbox from gateway or controller evidence; the recovery-only session remains blocked until NemoClaw can save the durable recovery record. diff --git a/src/lib/onboard/created-sandbox-failure.test.ts b/src/lib/onboard/created-sandbox-failure.test.ts index 69fe3edc5d2..23ccc515c91 100644 --- a/src/lib/onboard/created-sandbox-failure.test.ts +++ b/src/lib/onboard/created-sandbox-failure.test.ts @@ -5,13 +5,9 @@ import { describe, expect, it, vi } from "vitest"; import { reportSandboxCreateFailure, - reportSandboxReadinessFailure, type SandboxCreateFailureReportDeps, type SandboxCreateFailureReportOptions, - type SandboxReadinessFailureReportDeps, - type SandboxReadinessFailureReportOptions, } from "./created-sandbox-failure"; -import type { CreatedSandboxReadinessResult } from "./sandbox-readiness-tracing"; class ExitSignal extends Error { constructor(readonly code: number) { @@ -149,208 +145,3 @@ describe("reportSandboxCreateFailure", () => { expect(deps.exitProcess).toHaveBeenCalledWith(1); }); }); - -const NOT_READY: CreatedSandboxReadinessResult = { - ready: false, - reason: "timeout", - failurePhase: null, -}; - -function readinessDeps( - overrides: Partial = {}, -): SandboxReadinessFailureReportDeps { - return { - printReadinessFailure: vi.fn(), - printCreateFailureDiagnostics: vi.fn(), - printDockerGpuReadinessFailure: vi.fn(), - deleteSandbox: vi.fn(() => ({ status: 0 })), - cliName: vi.fn(() => "nemoclaw"), - error: vi.fn(), - exitProcess: vi.fn((code: number): never => { - throw new ExitSignal(code); - }), - ...overrides, - }; -} - -function readinessOptions( - overrides: Partial = {}, -): SandboxReadinessFailureReportOptions { - return { - sandboxName: "alpha", - readiness: NOT_READY, - createStatus: 0, - timeoutSecs: 300, - restoreBackupPath: null, - useDockerGpuPatch: false, - ...overrides, - }; -} - -function errorLines(deps: SandboxReadinessFailureReportDeps): string[] { - return (deps.error as ReturnType).mock.calls.map((call) => String(call[0])); -} - -function expectReceiptBlock( - deps: SandboxReadinessFailureReportDeps, - expected: readonly string[], -): void { - const lines = errorLines(deps); - const start = lines.indexOf(" Sandbox lifecycle receipt:"); - expect(start).toBeGreaterThanOrEqual(0); - expect(lines.slice(start, start + expected.length)).toEqual(expected); -} - -describe("reportSandboxReadinessFailure", () => { - it("deletes the failed sandbox on the non-GPU path and exits 1", () => { - const deps = readinessDeps(); - expect(() => reportSandboxReadinessFailure(readinessOptions(), deps)).toThrow(ExitSignal); - expect(deps.printReadinessFailure).toHaveBeenCalledWith(NOT_READY, "alpha", 300); - expect(deps.printCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { backupPath: null }); - expect(deps.deleteSandbox).toHaveBeenCalledWith("alpha"); - expect(deps.printDockerGpuReadinessFailure).not.toHaveBeenCalled(); - expectReceiptBlock(deps, [ - " Sandbox lifecycle receipt:", - " state: created_but_not_ready", - " sandbox: alpha", - " readiness_gate: sandbox_list:not_ready_timeout", - " readiness_reason: timeout", - " create_stream_status: 0", - " timeout_seconds: 300", - " terminal_resolution: timed_out_deleted", - ]); - expect(deps.error).toHaveBeenCalledWith( - " Deleted sandbox 'alpha' after the readiness gate failed; retry will recreate it.", - ); - expect(deps.error).toHaveBeenCalledWith(" Retry: nemoclaw onboard"); - expect(deps.exitProcess).toHaveBeenCalledWith(1); - }); - - it("surfaces manual cleanup when deletion fails", () => { - const deps = readinessDeps({ deleteSandbox: vi.fn(() => ({ status: 1 })) }); - expect(() => reportSandboxReadinessFailure(readinessOptions(), deps)).toThrow(ExitSignal); - expectReceiptBlock(deps, [ - " Sandbox lifecycle receipt:", - " state: created_but_not_ready", - " sandbox: alpha", - " readiness_gate: sandbox_list:not_ready_timeout", - " readiness_reason: timeout", - " create_stream_status: 0", - " timeout_seconds: 300", - " terminal_resolution: timed_out_retained", - ]); - expect(deps.error).toHaveBeenCalledWith( - " Could not remove the failed sandbox. Manual cleanup:", - ); - expect(deps.error).toHaveBeenCalledWith(' openshell sandbox delete "alpha"'); - }); - - it("defers cleanup to the Docker-GPU patch and never deletes the sandbox", () => { - const deps = readinessDeps(); - expect(() => - reportSandboxReadinessFailure(readinessOptions({ useDockerGpuPatch: true }), deps), - ).toThrow(ExitSignal); - expect(deps.printDockerGpuReadinessFailure).toHaveBeenCalledTimes(1); - expect(deps.deleteSandbox).not.toHaveBeenCalled(); - expectReceiptBlock(deps, [ - " Sandbox lifecycle receipt:", - " state: created_but_not_ready", - " sandbox: alpha", - " readiness_gate: sandbox_list:not_ready_timeout", - " readiness_reason: timeout", - " create_stream_status: 0", - " timeout_seconds: 300", - " terminal_resolution: deferred_to_docker_gpu_patch", - ]); - expect(deps.exitProcess).toHaveBeenCalledWith(1); - }); - - it("names the terminal readiness phase in the lifecycle receipt", () => { - const deps = readinessDeps(); - expect(() => - reportSandboxReadinessFailure( - readinessOptions({ - readiness: { - ready: false, - reason: "terminal_failure_phase", - failurePhase: "CrashLoopBackOff", - }, - }), - deps, - ), - ).toThrow(ExitSignal); - expectReceiptBlock(deps, [ - " Sandbox lifecycle receipt:", - " state: created_but_not_ready", - " sandbox: alpha", - " readiness_gate: sandbox_list:CrashLoopBackOff", - " readiness_reason: terminal_failure_phase", - " create_stream_status: 0", - " timeout_seconds: 300", - " terminal_resolution: terminal_failure_deleted", - ]); - }); - - it("reports retained cleanup for terminal readiness failures when delete fails", () => { - const deps = readinessDeps({ deleteSandbox: vi.fn(() => ({ status: 1 })) }); - expect(() => - reportSandboxReadinessFailure( - readinessOptions({ - readiness: { - ready: false, - reason: "terminal_failure_phase", - failurePhase: "Error", - }, - }), - deps, - ), - ).toThrow(ExitSignal); - expectReceiptBlock(deps, [ - " Sandbox lifecycle receipt:", - " state: created_but_not_ready", - " sandbox: alpha", - " readiness_gate: sandbox_list:Error", - " readiness_reason: terminal_failure_phase", - " create_stream_status: 0", - " timeout_seconds: 300", - " terminal_resolution: terminal_failure_retained", - ]); - }); - - it.each([ - null, - "", - ])("falls back to a stable terminal readiness gate for missing phase %s", (failurePhase) => { - const deps = readinessDeps(); - expect(() => - reportSandboxReadinessFailure( - readinessOptions({ - readiness: { - ready: false, - reason: "terminal_failure_phase", - failurePhase, - }, - }), - deps, - ), - ).toThrow(ExitSignal); - expectReceiptBlock(deps, [ - " Sandbox lifecycle receipt:", - " state: created_but_not_ready", - " sandbox: alpha", - " readiness_gate: sandbox_list:terminal_failure", - " readiness_reason: terminal_failure_phase", - " create_stream_status: 0", - " timeout_seconds: 300", - " terminal_resolution: terminal_failure_deleted", - ]); - }); - - it("preserves a non-zero create-stream status when readiness later fails", () => { - const deps = readinessDeps(); - expect(() => - reportSandboxReadinessFailure(readinessOptions({ createStatus: 255 }), deps), - ).toThrow(ExitSignal); - expect(deps.exitProcess).toHaveBeenCalledWith(255); - }); -}); diff --git a/src/lib/onboard/created-sandbox-failure.ts b/src/lib/onboard/created-sandbox-failure.ts index 84ac07cd5bf..99680d085ee 100644 --- a/src/lib/onboard/created-sandbox-failure.ts +++ b/src/lib/onboard/created-sandbox-failure.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { redact, redactFullWithUrls } from "../security/redact"; -import type { CreatedSandboxReadinessResult } from "./sandbox-readiness-tracing"; export type SandboxCreateFailureReportOptions = { sandboxName: string; @@ -65,143 +64,3 @@ export function reportSandboxCreateFailure( deps.printRecoveryHints(redactedCreateOutput, { createArgs: options.createArgs }); return deps.exitProcess(options.createStatus === 0 ? 1 : options.createStatus); } - -export type SandboxReadinessFailureReportOptions = { - sandboxName: string; - readiness: CreatedSandboxReadinessResult; - /** Exit status reported by the sandbox create stream before readiness polling. */ - createStatus: number; - timeoutSecs: number; - restoreBackupPath: string | null; - /** When the Docker-GPU create patch is active, cleanup is deferred to the patch. */ - useDockerGpuPatch: boolean; -}; - -export type SandboxReadinessFailureReportDeps = { - printReadinessFailure( - readiness: CreatedSandboxReadinessResult, - sandboxName: string, - timeoutSecs: number, - ): void; - printCreateFailureDiagnostics(sandboxName: string, options: { backupPath: string | null }): void; - printDockerGpuReadinessFailure(): void; - deleteSandbox(sandboxName: string): { status: number | null }; - cliName(): string; - error(message: string): void; - exitProcess(code: number): never; -}; - -export type SandboxReadinessTerminalResolution = - | "deferred_to_docker_gpu_patch" - | "terminal_failure_deleted" - | "terminal_failure_retained" - | "timed_out_deleted" - | "timed_out_retained"; - -/** Map the readiness reason and cleanup outcome into the receipt terminal state. */ -function readinessTerminalResolution( - readiness: CreatedSandboxReadinessResult, - deleted: boolean, -): SandboxReadinessTerminalResolution { - if (readiness.reason === "terminal_failure_phase") { - return deleted ? "terminal_failure_deleted" : "terminal_failure_retained"; - } - return deleted ? "timed_out_deleted" : "timed_out_retained"; -} - -/** Name the readiness gate that blocked the created sandbox from becoming Ready. */ -function readinessGate(readiness: CreatedSandboxReadinessResult): string { - if (readiness.reason === "terminal_failure_phase") { - const phase = - typeof readiness.failurePhase === "string" && readiness.failurePhase.length > 0 - ? readiness.failurePhase - : "terminal_failure"; - return `sandbox_list:${phase}`; - } - return "sandbox_list:not_ready_timeout"; -} - -/** - * Format the created-but-not-ready receipt so day-0 onboard failures retain a - * stable terminal state: the created sandbox identity, last readiness gate, - * cleanup result, and retry boundary are all visible in one block (#3344). - */ -function formatCreatedSandboxReadinessReceipt(options: { - sandboxName: string; - readiness: CreatedSandboxReadinessResult; - createStatus: number; - timeoutSecs: number; - terminalResolution: SandboxReadinessTerminalResolution; -}): readonly string[] { - return [ - " Sandbox lifecycle receipt:", - ` state: created_but_not_ready`, - ` sandbox: ${options.sandboxName}`, - ` readiness_gate: ${readinessGate(options.readiness)}`, - ` readiness_reason: ${options.readiness.reason}`, - ` create_stream_status: ${options.createStatus}`, - ` timeout_seconds: ${options.timeoutSecs}`, - ` terminal_resolution: ${options.terminalResolution}`, - ]; -} - -/** - * Report a sandbox that never reached Ready: print the readiness failure and - * create diagnostics, then either defer cleanup to the Docker-GPU patch or - * delete the failed sandbox so a same-name retry does not collide, and exit. - */ -export function reportSandboxReadinessFailure( - options: SandboxReadinessFailureReportOptions, - deps: SandboxReadinessFailureReportDeps, -): never { - deps.error(""); - deps.printReadinessFailure(options.readiness, options.sandboxName, options.timeoutSecs); - deps.printCreateFailureDiagnostics(options.sandboxName, { - backupPath: options.restoreBackupPath, - }); - if (options.useDockerGpuPatch) { - for (const line of formatCreatedSandboxReadinessReceipt({ - sandboxName: options.sandboxName, - readiness: options.readiness, - createStatus: options.createStatus, - timeoutSecs: options.timeoutSecs, - terminalResolution: "deferred_to_docker_gpu_patch", - })) { - deps.error(line); - } - deps.printDockerGpuReadinessFailure(); - } else { - // Clean up non-GPU failures after preserving local diagnostics so the - // next onboard retry with the same name does not fail on "sandbox already exists". - const delResult = deps.deleteSandbox(options.sandboxName); - if (delResult.status === 0) { - for (const line of formatCreatedSandboxReadinessReceipt({ - sandboxName: options.sandboxName, - readiness: options.readiness, - createStatus: options.createStatus, - timeoutSecs: options.timeoutSecs, - terminalResolution: readinessTerminalResolution(options.readiness, true), - })) { - deps.error(line); - } - deps.error( - ` Deleted sandbox '${options.sandboxName}' after the readiness gate failed; retry will recreate it.`, - ); - } else { - for (const line of formatCreatedSandboxReadinessReceipt({ - sandboxName: options.sandboxName, - readiness: options.readiness, - createStatus: options.createStatus, - timeoutSecs: options.timeoutSecs, - terminalResolution: readinessTerminalResolution(options.readiness, false), - })) { - deps.error(line); - } - deps.error(" Could not remove the failed sandbox. Manual cleanup:"); - deps.error(` openshell sandbox delete "${options.sandboxName}"`); - } - } - deps.error(` Retry: ${deps.cliName()} onboard`); - const exitCode = options.createStatus === 0 ? 1 : options.createStatus; - return deps.exitProcess(exitCode); -} From 7c398b6a1970c73610ceadf641118ca8c274fc2c Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 01:11:42 -0700 Subject: [PATCH 28/51] fix(onboard): preserve APF recovery persistence Signed-off-by: Prekshi Vyas --- src/lib/onboard/sandbox-gpu-create-flow.ts | 17 +- .../sandbox-gpu-create-identity-gate.test.ts | 158 +++++++++--------- 2 files changed, 94 insertions(+), 81 deletions(-) diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index a290809ed8f..7f8454fef7c 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -578,7 +578,8 @@ export async function runSandboxGpuCreateFlow( `APF sandbox '${input.sandboxName}' may have been retained after native GPU fallback stopped. ` + `Gateway '${input.gatewayName}'. ${identityGuidance} ` + "Do not delete a sandbox by mutable name; use an identity-bound administrator recovery procedure."; - let persisted = false; + console.error(` ${message}`); + let persisted: boolean; try { persisted = evidence.liveIdentityFingerprint ? persistRetainedSandboxRecovery( @@ -587,14 +588,22 @@ export async function runSandboxGpuCreateFlow( evidence.createAttemptNonce, ) : persistRetainedSandboxRecovery(message, undefined, evidence.createAttemptNonce); - } catch { - persisted = false; + } catch (error) { + console.error( + " APF recovery is blocked because NemoClaw could not save this create-attempt evidence. Preserve the terminal output for an OpenShell administrator.", + ); + throw new Error( + "The APF recovery-only session remains blocked until its durable recovery record can be saved.", + { cause: error }, + ); } - console.error(` ${message}`); if (!persisted) { console.error( " APF recovery is blocked because NemoClaw could not save this create-attempt evidence. Preserve the terminal output for an OpenShell administrator.", ); + throw new Error( + "The APF recovery-only session remains blocked until its durable recovery record can be saved.", + ); } } } diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 50ab363b535..705a887766d 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -94,6 +94,14 @@ function createAttemptNonce(args: readonly string[]): string { return (args[labelIndex + 1] ?? "").slice(NEMOCLAW_CREATE_ATTEMPT_LABEL.length + 1); } +function expectNoSandboxDelete(deps: ReturnType): void { + expect( + vi + .mocked(deps.runOpenshell) + .mock.calls.some(([args]) => args[0] === "sandbox" && args[1] === "delete"), + ).toBe(false); +} + function noGpuInput() { const input = createGpuFlowInput(); input.sandboxGpuConfig = { @@ -161,6 +169,42 @@ function createCommittedReadinessPersistenceFixture() { return { deps, error, input, patch }; } +function createApfFallbackRecoveryFixture(captureExactIdentity = true) { + let nonce = ""; + const input = createGpuFlowInput(); + input.requirePolicylessCreate = true; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + input.persistRetainedSandboxRecovery = vi.fn(() => true); + mocks.streamSandboxCreate.mockImplementationOnce(async (_command, args) => { + nonce = createAttemptNonce(args); + return { + status: 1, + output: "native runtime failed after sandbox creation", + sawProgress: true, + }; + }); + mocks.queryOpenShellDockerSandboxRuntimeSnapshot.mockReturnValue({ + ok: true, + imageId: "sha256:" + "a".repeat(64), + bookkeepingImageRef: "openshell/sandbox-from:test", + stateError: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + deviceRequests: null, + devices: null, + runtime: "runc", + nvidiaVisibleDevices: null, + nativeGpuAttachmentState: "absent", + containerId: "container-a", + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementation(() => + captureExactIdentity + ? sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }) + : "[]", + ); + return { deps, input, readNonce: () => nonce }; +} + beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); @@ -259,10 +303,7 @@ describe("created sandbox identity gate", () => { expect.objectContaining({ suppressOutput: true }), ); expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); - expect(deps.runOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), - ); + expectNoSandboxDelete(deps); }); it("retains exact recovery when committed managed readiness does not return (#9211)", async () => { @@ -295,10 +336,7 @@ describe("created sandbox identity gate", () => { "a".repeat(62), ); expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); - expect(deps.runOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), - ); + expectNoSandboxDelete(deps); expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { backupPath: null, }); @@ -336,10 +374,7 @@ describe("created sandbox identity gate", () => { ); expect(error.mock.calls.flat().join("\n")).not.toContain("Preserve the terminal output"); expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); - expect(deps.runOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), - ); + expectNoSandboxDelete(deps); }, ); @@ -1114,42 +1149,14 @@ describe("created sandbox identity gate", () => { }); it("persists exact APF recovery evidence before refusing native fallback (#9833)", async () => { - let nonce = ""; - const input = createGpuFlowInput(); - input.requirePolicylessCreate = true; - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - input.persistRetainedSandboxRecovery = vi.fn(() => true); - mocks.streamSandboxCreate.mockImplementationOnce(async (_command, args) => { - nonce = createAttemptNonce(args); - return { - status: 1, - output: "native runtime failed after sandbox creation", - sawProgress: true, - }; - }); - mocks.queryOpenShellDockerSandboxRuntimeSnapshot.mockReturnValue({ - ok: true, - imageId: "sha256:" + "a".repeat(64), - bookkeepingImageRef: "openshell/sandbox-from:test", - stateError: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", - deviceRequests: null, - devices: null, - runtime: "runc", - nvidiaVisibleDevices: null, - nativeGpuAttachmentState: "absent", - containerId: "container-a", - }); - const deps = createGpuFlowDeps(); - vi.mocked(deps.runCaptureOpenshell).mockImplementation(() => - sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), - ); + const { deps, input, readNonce } = createApfFallbackRecoveryFixture(); const exit = vi.spyOn(process, "exit").mockImplementation(() => { throw new Error("process.exit:1"); }); await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1"); + const nonce = readNonce(); const fingerprint = fingerprintSandboxRecreateValue("alpha-sandbox-id"); expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( expect.stringMatching( @@ -1166,48 +1173,19 @@ describe("created sandbox identity gate", () => { expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); expect(output).toContain(`Durable sandbox identity fingerprint: ${fingerprint}`); expect(output).not.toContain("alpha-sandbox-id"); - expect(deps.runOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), - ); + expectNoSandboxDelete(deps); expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); }); it("persists the APF create-attempt label when exact recovery identity is unavailable (#9833)", async () => { - let nonce = ""; - const input = createGpuFlowInput(); - input.requirePolicylessCreate = true; - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - input.persistRetainedSandboxRecovery = vi.fn(() => true); - mocks.streamSandboxCreate.mockImplementationOnce(async (_command, args) => { - nonce = createAttemptNonce(args); - return { - status: 1, - output: "native runtime failed after sandbox creation", - sawProgress: true, - }; - }); - mocks.queryOpenShellDockerSandboxRuntimeSnapshot.mockReturnValue({ - ok: true, - imageId: "sha256:" + "a".repeat(64), - bookkeepingImageRef: "openshell/sandbox-from:test", - stateError: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", - deviceRequests: null, - devices: null, - runtime: "runc", - nvidiaVisibleDevices: null, - nativeGpuAttachmentState: "absent", - containerId: "container-a", - }); - const deps = createGpuFlowDeps(); - vi.mocked(deps.runCaptureOpenshell).mockReturnValue("[]"); + const { deps, input, readNonce } = createApfFallbackRecoveryFixture(false); vi.spyOn(process, "exit").mockImplementation(() => { throw new Error("process.exit:1"); }); await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1"); + const nonce = readNonce(); expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( expect.stringMatching( new RegExp( @@ -1221,10 +1199,36 @@ describe("created sandbox identity gate", () => { const output = vi.mocked(console.error).mock.calls.flat().join("\n"); expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); expect(output).toContain("Recovery is blocked"); - expect(deps.runOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), + expectNoSandboxDelete(deps); + }); + + it.each([ + ["returns false", () => false], + [ + "throws", + () => { + throw new Error("durable writer failed"); + }, + ], + ])("blocks APF fallback when durable recovery persistence %s (#9833)", async (_name, writer) => { + const { deps, input, readNonce } = createApfFallbackRecoveryFixture(); + const persist = vi.fn(writer); + input.persistRetainedSandboxRecovery = persist; + const exit = vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("process.exit:1"); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "The APF recovery-only session remains blocked until its durable recovery record can be saved.", ); + + expect(persist).toHaveBeenCalledOnce(); + expect(exit).not.toHaveBeenCalled(); + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${readNonce()}`); + expect(output).toContain("APF recovery is blocked because NemoClaw could not save"); + expectNoSandboxDelete(deps); + expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); }); it("stops before a runtime patch when the durable checkpoint drifts (#9833)", async () => { From e9bd3eefa081021aeb283333c6142fe9aea4026d Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 01:45:29 -0700 Subject: [PATCH 29/51] fix(onboard): close remaining recovery review gaps Signed-off-by: Prekshi Vyas --- .../onboard/created-sandbox-failure.test.ts | 19 ---- src/lib/onboard/created-sandbox-failure.ts | 21 +--- .../onboard/sandbox-gpu-create-flow.test.ts | 16 --- src/lib/onboard/sandbox-gpu-create-flow.ts | 61 +++++----- .../sandbox-gpu-create-identity-gate.test.ts | 105 ++++++++++++------ .../onboard/sandbox-gpu-create-run-attempt.ts | 88 ++++++++++----- test/security/shellquote-sandbox.test.ts | 64 ++++++++--- 7 files changed, 215 insertions(+), 159 deletions(-) diff --git a/src/lib/onboard/created-sandbox-failure.test.ts b/src/lib/onboard/created-sandbox-failure.test.ts index 23ccc515c91..b0b56f591a0 100644 --- a/src/lib/onboard/created-sandbox-failure.test.ts +++ b/src/lib/onboard/created-sandbox-failure.test.ts @@ -19,10 +19,8 @@ function createFailureDeps( overrides: Partial = {}, ): SandboxCreateFailureReportDeps { return { - classifyCreateFailure: vi.fn(() => ({ kind: "unknown" })), printCreateFailureDiagnostics: vi.fn(), printRecoveryHints: vi.fn(), - warn: vi.fn(), error: vi.fn(), exitProcess: vi.fn((code: number): never => { throw new ExitSignal(code); @@ -45,19 +43,6 @@ function createFailureOptions( } describe("reportSandboxCreateFailure", () => { - it("warns and returns (does not exit) when the create is merely incomplete", () => { - const deps = createFailureDeps({ - classifyCreateFailure: vi.fn(() => ({ kind: "sandbox_create_incomplete" })), - }); - expect(() => reportSandboxCreateFailure(createFailureOptions(), deps)).not.toThrow(); - expect(deps.warn).toHaveBeenCalledWith( - " Create stream exited with code 3 after sandbox was created.", - ); - expect(deps.printCreateFailureDiagnostics).not.toHaveBeenCalled(); - expect(deps.printRecoveryHints).not.toHaveBeenCalled(); - expect(deps.exitProcess).not.toHaveBeenCalled(); - }); - it("prints diagnostics + recovery hints and exits with the create status on a hard failure", () => { const deps = createFailureDeps(); expect(() => @@ -73,7 +58,6 @@ describe("reportSandboxCreateFailure", () => { createArgs: ["sandbox", "create", "alpha"], }); expect(deps.exitProcess).toHaveBeenCalledWith(42); - expect(deps.warn).not.toHaveBeenCalled(); }); it("redacts create output before classification and echoing", () => { @@ -85,9 +69,6 @@ describe("reportSandboxCreateFailure", () => { withOutput, ), ).toThrow(ExitSignal); - expect(withOutput.classifyCreateFailure).toHaveBeenCalledWith( - "failed with Authorization: Bearer secr********", - ); expect(withOutput.error).toHaveBeenCalledWith("failed with Authorization: Bearer secr********"); expect(withOutput.error).not.toHaveBeenCalledWith( "failed with Authorization: Bearer secret-token", diff --git a/src/lib/onboard/created-sandbox-failure.ts b/src/lib/onboard/created-sandbox-failure.ts index 99680d085ee..90a8a986421 100644 --- a/src/lib/onboard/created-sandbox-failure.ts +++ b/src/lib/onboard/created-sandbox-failure.ts @@ -16,10 +16,8 @@ export type SandboxCreateFailureReportOptions = { }; export type SandboxCreateFailureReportDeps = { - classifyCreateFailure(output: string): { kind: string }; printCreateFailureDiagnostics(sandboxName: string, options: { backupPath: string | null }): void; printRecoveryHints(output: string, options: { createArgs: readonly string[] }): void; - warn(message: string): void; error(message: string): void; exitProcess(code: number): never; }; @@ -28,29 +26,12 @@ export function redactCreatedSandboxFailureDiagnostic(value: string, limit: numb return redactFullWithUrls(value).replace(/\s+/gu, " ").trim().slice(0, limit); } -/** - * Report a non-zero sandbox create-stream exit. A mere "create incomplete" - * (the sandbox exists in the gateway but the stream exited non-zero, e.g. SSH - * 255) warns and returns so the caller can fall through to the ready-wait loop; - * any other failure prints diagnostics + recovery hints and exits. - */ +/** Report a hard sandbox create-stream failure with diagnostics and recovery hints. */ export function reportSandboxCreateFailure( options: SandboxCreateFailureReportOptions, deps: SandboxCreateFailureReportDeps, ): void { const redactedCreateOutput = redact(options.createOutput); - const failure = deps.classifyCreateFailure(redactedCreateOutput); - if (failure.kind === "sandbox_create_incomplete") { - // The sandbox was created in the gateway but the create stream exited - // with a non-zero code (e.g. SSH 255). Fall through to the ready-wait - // loop — the sandbox may still reach Ready on its own. - deps.warn(""); - deps.warn( - ` Create stream exited with code ${options.createStatus} after sandbox was created.`, - ); - deps.warn(" Checking whether the sandbox reaches Ready state..."); - return; - } deps.error(""); deps.error(` Sandbox creation failed (exit ${options.createStatus}).`); if (options.createOutput) { diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index 6d6d90066eb..ff4e6c23a24 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -1059,22 +1059,6 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { ); }); - it("keeps a created sandbox when portable lifecycle setup fails (#8441)", async () => { - const deps = createDeps(); - deps.installPortableDemoLifecycle = vi.fn(() => { - throw new Error("Authorization: Bearer portable-secret"); - }); - - await expect(runSandboxGpuCreateFlow(createInput(), deps)).resolves.toMatchObject({ - route: "native", - }); - - const warning = vi.mocked(console.warn).mock.calls.flat().join("\n"); - expect(warning).toContain("Portable demo lifecycle setup did not complete"); - expect(warning).toContain("Authorization: Bearer "); - expect(warning).not.toContain("portable-secret"); - }); - it("uses the exact portable lifecycle without Docker container substitution (#9068)", async () => { const input = createInput(); input.gpuRoutePlan = "native-only"; diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 7f8454fef7c..89da6971046 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -55,7 +55,10 @@ import type { RuntimeProviderManagedImageBootstrapSurface, } from "./runtime-provider/contract"; import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; -import { createSandboxGpuCreateAttemptRunner } from "./sandbox-gpu-create-run-attempt"; +import { + createSandboxGpuCreateAttemptRunner, + persistRetainedSandboxRecoveryOrBlock, +} from "./sandbox-gpu-create-run-attempt"; import { managedBootstrapCreateArgs } from "./sandbox-create-launch"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; import { @@ -578,33 +581,18 @@ export async function runSandboxGpuCreateFlow( `APF sandbox '${input.sandboxName}' may have been retained after native GPU fallback stopped. ` + `Gateway '${input.gatewayName}'. ${identityGuidance} ` + "Do not delete a sandbox by mutable name; use an identity-bound administrator recovery procedure."; - console.error(` ${message}`); - let persisted: boolean; - try { - persisted = evidence.liveIdentityFingerprint - ? persistRetainedSandboxRecovery( - message, - evidence.liveIdentityFingerprint, - evidence.createAttemptNonce, - ) - : persistRetainedSandboxRecovery(message, undefined, evidence.createAttemptNonce); - } catch (error) { - console.error( + persistRetainedSandboxRecoveryOrBlock({ + persist: persistRetainedSandboxRecovery, + message, + createAttemptNonce: evidence.createAttemptNonce, + ...(evidence.liveIdentityFingerprint + ? { sandboxIdentityFingerprint: evidence.liveIdentityFingerprint } + : {}), + persistenceFailureDiagnostic: " APF recovery is blocked because NemoClaw could not save this create-attempt evidence. Preserve the terminal output for an OpenShell administrator.", - ); - throw new Error( + persistenceFailureMessage: "The APF recovery-only session remains blocked until its durable recovery record can be saved.", - { cause: error }, - ); - } - if (!persisted) { - console.error( - " APF recovery is blocked because NemoClaw could not save this create-attempt evidence. Preserve the terminal output for an OpenShell administrator.", - ); - throw new Error( - "The APF recovery-only session remains blocked until its durable recovery record can be saved.", - ); - } + }); } } process.exit(1); @@ -635,7 +623,26 @@ export async function runSandboxGpuCreateFlow( 0, 500, ); - console.warn(` Portable demo lifecycle setup did not complete: ${detail}`); + const identity = attemptRunner.state.verifiedCreatedSandboxIdentity; + const persist = input.persistRetainedSandboxRecovery; + if (!identity || !persist) { + throw new Error( + `Portable demo lifecycle setup failed after sandbox creation without exact durable recovery authority: ${detail}`, + ); + } + const message = + `Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${identity.createAttemptNonce}. ` + + `Durable sandbox identity fingerprint: ${identity.liveIdentityFingerprint}. ` + + `Portable lifecycle receipt setup did not complete for sandbox '${input.sandboxName}' on gateway '${input.gatewayName}'. ` + + "NemoClaw stopped before registry publication and success output. " + + `Run the retained identity-bound destroy action for sandbox '${input.sandboxName}'; stop if destroy cannot prove that identity.`; + persistRetainedSandboxRecoveryOrBlock({ + persist, + message, + createAttemptNonce: identity.createAttemptNonce, + sandboxIdentityFingerprint: identity.liveIdentityFingerprint, + }); + throw new Error(`Portable demo lifecycle setup did not complete: ${detail}`); } } diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 705a887766d..1ac7340c04b 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -94,6 +94,16 @@ function createAttemptNonce(args: readonly string[]): string { return (args[labelIndex + 1] ?? "").slice(NEMOCLAW_CREATE_ATTEMPT_LABEL.length + 1); } +const durableRecoveryWriterFailures = [ + ["returns false", () => false], + [ + "throws", + () => { + throw new Error("durable writer failed"); + }, + ], +] as const; + function expectNoSandboxDelete(deps: ReturnType): void { expect( vi @@ -349,15 +359,7 @@ describe("created sandbox identity gate", () => { expect(recoveryOutput).not.toContain("After OpenShell confirms the sandbox is absent"); }); - it.each([ - ["returns false", () => false], - [ - "throws", - () => { - throw new Error("durable writer failed"); - }, - ], - ])( + it.each(durableRecoveryWriterFailures)( "blocks committed-readiness recovery when durable persistence %s (#9211)", async (_name, writer) => { const { deps, error, input, patch } = createCommittedReadinessPersistenceFixture(); @@ -432,6 +434,7 @@ describe("created sandbox identity gate", () => { ["sandbox", "get", "-g", gatewayName, "alpha"], expect.objectContaining({ ignoreError: true, suppressOutput: true }), ); + expect(deps.installPortableDemoLifecycle).toHaveBeenCalledOnce(); expect(events).toEqual([ "verify-created", "revalidate:activate managed sandbox network for 'alpha'", @@ -449,6 +452,43 @@ describe("created sandbox identity gate", () => { ]); }); + it("retains exact recovery when portable lifecycle setup fails after resume (#8441)", async () => { + const sandboxId = "alpha-sandbox-id"; + const createAttemptNonce = "a".repeat(62); + const liveIdentityFingerprint = fingerprintSandboxRecreateValue(sandboxId); + const input = noGpuInput(); + input.resumeVerifiedCreate = { + route: "none", + liveIdentityFingerprint, + createAttemptNonce, + }; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const deps = createGpuFlowDeps(sandboxId); + deps.installPortableDemoLifecycle = vi.fn(() => { + throw new Error("Authorization: Bearer portable-secret"); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "Portable demo lifecycle setup did not complete: Authorization: Bearer ", + ); + + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining("Run the retained identity-bound destroy action for sandbox 'alpha'"), + liveIdentityFingerprint, + createAttemptNonce, + ); + expect(deps.installPortableDemoLifecycle).toHaveBeenCalledOnce(); + expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); + expectNoSandboxDelete(deps); + expect(vi.mocked(console.error).mock.calls.flat().join("\n")).not.toContain( + "portable-secret", + ); + expect(vi.mocked(console.log).mock.calls.flat().join("\n")).not.toContain( + "Sandbox 'alpha' created", + ); + }); + it("refuses a changed live identity before resumed effects (#9833)", async () => { const input = noGpuInput(); input.resumeVerifiedCreate = { @@ -1202,34 +1242,29 @@ describe("created sandbox identity gate", () => { expectNoSandboxDelete(deps); }); - it.each([ - ["returns false", () => false], - [ - "throws", - () => { - throw new Error("durable writer failed"); - }, - ], - ])("blocks APF fallback when durable recovery persistence %s (#9833)", async (_name, writer) => { - const { deps, input, readNonce } = createApfFallbackRecoveryFixture(); - const persist = vi.fn(writer); - input.persistRetainedSandboxRecovery = persist; - const exit = vi.spyOn(process, "exit").mockImplementation(() => { - throw new Error("process.exit:1"); - }); + it.each(durableRecoveryWriterFailures)( + "blocks APF fallback when durable recovery persistence %s (#9833)", + async (_name, writer) => { + const { deps, input, readNonce } = createApfFallbackRecoveryFixture(); + const persist = vi.fn(writer); + input.persistRetainedSandboxRecovery = persist; + const exit = vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("process.exit:1"); + }); - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "The APF recovery-only session remains blocked until its durable recovery record can be saved.", - ); + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "The APF recovery-only session remains blocked until its durable recovery record can be saved.", + ); - expect(persist).toHaveBeenCalledOnce(); - expect(exit).not.toHaveBeenCalled(); - const output = vi.mocked(console.error).mock.calls.flat().join("\n"); - expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${readNonce()}`); - expect(output).toContain("APF recovery is blocked because NemoClaw could not save"); - expectNoSandboxDelete(deps); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - }); + expect(persist).toHaveBeenCalledOnce(); + expect(exit).not.toHaveBeenCalled(); + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${readNonce()}`); + expect(output).toContain("APF recovery is blocked because NemoClaw could not save"); + expectNoSandboxDelete(deps); + expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); + }, + ); it("stops before a runtime patch when the durable checkpoint drifts (#9833)", async () => { let nonce = ""; diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 256deba7220..0a398a3e1ba 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -45,6 +45,7 @@ import { import { printSandboxCreateFailureDiagnostics } from "./sandbox-create-failure"; import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; import type { + CreatedSandboxIdentity, SandboxGpuCreateFlowDeps, SandboxGpuCreateFlowInput, } from "./sandbox-gpu-create-flow"; @@ -68,6 +69,7 @@ export type SandboxGpuCreateAttemptState = { allowUnbuiltCompatibilitySource: boolean; nativeRuntimeSnapshot: NativeRuntimeSnapshot | null; portableLifecycleGeneration: string | null; + verifiedCreatedSandboxIdentity: CreatedSandboxIdentity | null; }; // A runtime-managed container replacement can briefly observe the original @@ -278,14 +280,47 @@ async function verifyCreatedSandboxBeforeEffects( createAttemptNonce: string, route: SelectedDockerGpuRoute, input: SandboxGpuCreateFlowInput, -): Promise { - if (!input.verifyCreatedSandboxBeforeEffects) return; - await input.verifyCreatedSandboxBeforeEffects({ +): Promise { + const identity = { sandboxId, liveIdentityFingerprint: fingerprintSandboxRecreateValue(sandboxId), createAttemptNonce, route, - }); + }; + await input.verifyCreatedSandboxBeforeEffects?.(identity); + return identity; +} + +export function persistRetainedSandboxRecoveryOrBlock(options: { + readonly persist: NonNullable; + readonly message: string; + readonly createAttemptNonce: string; + readonly sandboxIdentityFingerprint?: string; + readonly persistenceFailureDiagnostic?: string; + readonly persistenceFailureMessage?: string; +}): void { + const { persist, message, createAttemptNonce, sandboxIdentityFingerprint } = options; + let persistenceFailure: unknown = null; + try { + if (!persist(message, sandboxIdentityFingerprint, createAttemptNonce)) { + persistenceFailure = new Error( + "The retained sandbox recovery writer did not confirm durable persistence.", + ); + } + } catch (error) { + persistenceFailure = error; + } + console.error(` ${message}`); + if (!persistenceFailure) return; + console.error( + options.persistenceFailureDiagnostic ?? + " NemoClaw could not save this create-attempt evidence. The recovery-only session remains blocked until its durable recovery record can be saved.", + ); + throw new Error( + options.persistenceFailureMessage ?? + "NemoClaw could not save the retained sandbox recovery record; the recovery-only session remains blocked.", + { cause: persistenceFailure }, + ); } function persistCreateAttemptRecovery(options: { @@ -308,26 +343,12 @@ function persistCreateAttemptRecovery(options: { ? `Durable sandbox identity fingerprint: ${sandboxIdentityFingerprint}. ` : "") + detail; - let persistenceFailure: unknown = null; - try { - if (!persist(message, sandboxIdentityFingerprint, createAttemptNonce)) { - persistenceFailure = new Error( - "The retained sandbox recovery writer did not confirm durable persistence.", - ); - } - } catch (error) { - persistenceFailure = error; - } - console.error(` ${message}`); - if (persistenceFailure) { - console.error( - " NemoClaw could not save this create-attempt evidence. The recovery-only session remains blocked until its durable recovery record can be saved.", - ); - throw new Error( - "NemoClaw could not save the retained sandbox recovery record; the recovery-only session remains blocked.", - { cause: persistenceFailure }, - ); - } + persistRetainedSandboxRecoveryOrBlock({ + persist, + message, + createAttemptNonce, + ...(sandboxIdentityFingerprint ? { sandboxIdentityFingerprint } : {}), + }); } function persistIdentitySettlementRecoveryEvidence(options: { @@ -624,6 +645,7 @@ export function createSandboxGpuCreateAttemptRunner( allowUnbuiltCompatibilitySource: false, nativeRuntimeSnapshot: null, portableLifecycleGeneration: null, + verifiedCreatedSandboxIdentity: null, }; const revalidatePostCreateEffect = (operation: string): void => { if (!input.verifyCreatedSandboxBeforeEffects) return; @@ -900,7 +922,7 @@ export function createSandboxGpuCreateAttemptRunner( } resumedSandboxId = identity.sandboxId; verifiedCreatedSandboxId = identity.sandboxId; - await verifyCreatedSandboxBeforeEffects( + state.verifiedCreatedSandboxIdentity = await verifyCreatedSandboxBeforeEffects( identity.sandboxId, createAttemptNonce!, route, @@ -981,7 +1003,12 @@ export function createSandboxGpuCreateAttemptRunner( } waitForCreatedSandboxPublication(sandboxId); verifiedCreatedSandboxId = sandboxId; - await verifyCreatedSandboxBeforeEffects(sandboxId, createAttemptNonce!, route, input); + state.verifiedCreatedSandboxIdentity = await verifyCreatedSandboxBeforeEffects( + sandboxId, + createAttemptNonce!, + route, + input, + ); createdSandboxVerified = true; if (deferPostCreateEffects) { revalidatePostCreateEffect( @@ -1089,10 +1116,8 @@ export function createSandboxGpuCreateAttemptRunner( createArgs: input.prebuild.createArgs, }, { - classifyCreateFailure: classifySandboxCreateFailure, printCreateFailureDiagnostics, printRecoveryHints: printSandboxCreateRecoveryHints, - warn: (message) => console.warn(message), error: (message) => console.error(message), exitProcess: (code) => process.exit(code), }, @@ -1115,7 +1140,12 @@ export function createSandboxGpuCreateAttemptRunner( } waitForCreatedSandboxPublication(sandboxId); verifiedCreatedSandboxId = sandboxId; - await verifyCreatedSandboxBeforeEffects(sandboxId, createAttemptNonce!, route, input); + state.verifiedCreatedSandboxIdentity = await verifyCreatedSandboxBeforeEffects( + sandboxId, + createAttemptNonce!, + route, + input, + ); createdSandboxVerified = true; } if (deferPostCreateEffects) { diff --git a/test/security/shellquote-sandbox.test.ts b/test/security/shellquote-sandbox.test.ts index 81a55cfc7ba..aa4b3faa8de 100644 --- a/test/security/shellquote-sandbox.test.ts +++ b/test/security/shellquote-sandbox.test.ts @@ -8,6 +8,7 @@ import os from "os"; import path from "path"; import { describe, expect, it } from "vitest"; +import { completeOrdinaryOnboardSandboxCreation } from "../../src/lib/onboard/created-sandbox-finalization"; import { writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; describe("sandboxName command hardening in onboard.js", () => { @@ -28,7 +29,56 @@ describe("sandboxName command hardening in onboard.js", () => { ).rejects.toThrow(/Invalid sandbox name/); }); - it("runs setup-dns-proxy.sh through the argv helper instead of bash -c interpolation", () => { + it("passes DNS proxy gateway values as one literal argument", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dns-literal-")); + const argsFile = path.join(tmpDir, "dns-args.txt"); + const sideEffectFile = path.join(tmpDir, "shell-expanded"); + const gatewayName = `nemoclaw; touch ${sideEffectFile}; #`; + fs.writeFileSync( + path.join(tmpDir, "setup-dns-proxy.sh"), + '#!/usr/bin/env bash\nset -eu\nprintf \'%s\\n\' "$1" "$2" > "$NEMOCLAW_DNS_ARGS_FILE"\n', + ); + + try { + completeOrdinaryOnboardSandboxCreation( + { + sandboxName: "my-assistant", + sandboxWasLiveDefault: false, + gatewayPort: 8080, + runtimeFields: { openshellDriver: "kubernetes" }, + messagingProviders: [], + liveExists: true, + } as never, + { + setDefault: () => undefined, + runFile: (command: string, args: string[]) => + spawnSync(command, args, { + encoding: "utf-8", + env: { ...process.env, NEMOCLAW_DNS_ARGS_FILE: argsFile }, + }), + scriptsDir: tmpDir, + gatewayName, + providerExistsInGateway: () => true, + armCancelRollback: () => undefined, + markCancellationRecovery: () => undefined, + dockerInfoFormat: () => "", + runCapture: () => "", + revalidateSandboxIdentity: () => undefined, + applyVmDnsMonkeypatch: () => undefined, + } as never, + ); + + expect(fs.readFileSync(argsFile, "utf-8").trim().split("\n")).toEqual([ + gatewayName, + "my-assistant", + ]); + expect(fs.existsSync(sideEffectFile)).toBe(false); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("scopes created sandbox probes to the owning gateway", () => { const repoRoot = path.join(import.meta.dirname, "../.."); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dns-argv-")); const fakeBin = path.join(tmpDir, "bin"); @@ -183,18 +233,6 @@ try { .find((line) => line.startsWith("{") && line.endsWith("}")); expect(payloadLine).toBeTruthy(); const payload = JSON.parse(payloadLine!); - const dnsCommand = payload.commands.find( - (entry: { type: string; args: string[] }) => - entry.type === "runFile" && entry.args[0]?.endsWith("setup-dns-proxy.sh"), - ); - expect(dnsCommand).toBeTruthy(); - expect(dnsCommand.file).toBe("bash"); - expect(dnsCommand.args).toEqual([ - expect.stringMatching(/setup-dns-proxy\.sh$/), - "nemoclaw", - "my-assistant", - ]); - expect(dnsCommand.command).not.toContain("bash -c"); expect( payload.commands.some((entry: { command: string }) => entry.command.includes("sandbox get -g nemoclaw my-assistant"), From 657df801c8c8fc305ce7637b7bc669ba3527089c Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 02:03:47 -0700 Subject: [PATCH 30/51] fix(onboard): align retained recovery evidence Signed-off-by: Prekshi Vyas --- docs/reference/troubleshooting.mdx | 2 +- .../sandbox-gpu-create-identity-gate.test.ts | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index b3588064301..44d4d6d0bb3 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2002,7 +2002,7 @@ This is a separate budget from `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT`. It covers the -For a newly created OpenClaw or Hermes sandbox, `Ready` is not the final acceptance signal. Within this same budget, NemoClaw also requires OpenShell to return a durable sandbox ID and accept `openshell sandbox exec --name -- true`. NemoClaw keeps waiting only when OpenShell reports exact sandbox absence or its exact `sandbox is not ready` response. A missing or malformed ID, or another command failure, stops the wait. Ordinary onboarding then follows the failed-creation cleanup path. Portable OpenClaw onboarding preserves the sandbox as described below. +For a newly created OpenClaw or Hermes sandbox, `Ready` is not the final acceptance signal. Within this same budget, NemoClaw also requires OpenShell to return a durable sandbox ID and accept `openshell sandbox exec --name -- true`. NemoClaw keeps waiting only when OpenShell reports exact sandbox absence or its exact `sandbox is not ready` response. A missing or malformed ID, or another command failure, stops the wait. NemoClaw preserves the sandbox, saves create-attempt evidence when possible, and requires identity-bound recovery instead of ordinary failed-creation cleanup. If onboarding reports that the managed runtime commit completed but the same sandbox did not return to executable `Ready`, stop before retrying. NemoClaw keeps the sandbox, prints its create-attempt label and a one-way durable identity fingerprint, and does not start dashboard forwarding. diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 1ac7340c04b..1650d25bd19 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -103,6 +103,8 @@ const durableRecoveryWriterFailures = [ }, ], ] as const; +const ALPHA_SANDBOX_IDENTITY_FINGERPRINT = + "8174fa2a5d65755138d8339e086c03d736633130b22dca10952e80e74750c01d"; function expectNoSandboxDelete(deps: ReturnType): void { expect( @@ -165,7 +167,7 @@ function createCommittedReadinessPersistenceFixture() { const input = noGpuInput(); input.resumeVerifiedCreate = { route: "none", - liveIdentityFingerprint: fingerprintSandboxRecreateValue("alpha-sandbox-id"), + liveIdentityFingerprint: ALPHA_SANDBOX_IDENTITY_FINGERPRINT, createAttemptNonce: "a".repeat(62), }; input.verifyCreatedSandboxBeforeEffects = vi.fn(); @@ -319,7 +321,7 @@ describe("created sandbox identity gate", () => { it("retains exact recovery when committed managed readiness does not return (#9211)", async () => { const error = vi.spyOn(console, "error").mockImplementation(() => undefined); const sandboxId = "alpha-sandbox-id"; - const sandboxIdentityFingerprint = fingerprintSandboxRecreateValue(sandboxId); + const sandboxIdentityFingerprint = ALPHA_SANDBOX_IDENTITY_FINGERPRINT; const input = noGpuInput(); input.resumeVerifiedCreate = { route: "none", @@ -927,7 +929,7 @@ describe("created sandbox identity gate", () => { expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( expect.stringMatching(/OpenShell detail: .*permission denied: NVIDIA_API_KEY=/u), - fingerprintSandboxRecreateValue("alpha-sandbox-id"), + ALPHA_SANDBOX_IDENTITY_FINGERPRINT, nonce, ); const recoveryMessage = @@ -963,9 +965,9 @@ describe("created sandbox identity gate", () => { expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( expect.stringContaining( - `Durable sandbox identity fingerprint: ${fingerprintSandboxRecreateValue("alpha-sandbox-id")}`, + `Durable sandbox identity fingerprint: ${ALPHA_SANDBOX_IDENTITY_FINGERPRINT}`, ), - fingerprintSandboxRecreateValue("alpha-sandbox-id"), + ALPHA_SANDBOX_IDENTITY_FINGERPRINT, nonce, ); expect(patch.exitOnPatchError).not.toHaveBeenCalled(); @@ -1003,9 +1005,9 @@ describe("created sandbox identity gate", () => { expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( expect.stringContaining( - `Durable sandbox identity fingerprint: ${fingerprintSandboxRecreateValue("alpha-sandbox-id")}`, + `Durable sandbox identity fingerprint: ${ALPHA_SANDBOX_IDENTITY_FINGERPRINT}`, ), - fingerprintSandboxRecreateValue("alpha-sandbox-id"), + ALPHA_SANDBOX_IDENTITY_FINGERPRINT, nonce, ); expect(patch.exitOnPatchError).not.toHaveBeenCalled(); @@ -1197,7 +1199,7 @@ describe("created sandbox identity gate", () => { await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1"); const nonce = readNonce(); - const fingerprint = fingerprintSandboxRecreateValue("alpha-sandbox-id"); + const fingerprint = ALPHA_SANDBOX_IDENTITY_FINGERPRINT; expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( expect.stringMatching( new RegExp( From dd1cdc272624179080d35e19b8cae3162c60fda9 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 02:38:34 -0700 Subject: [PATCH 31/51] fix(onboard): share post-create readiness deadline Signed-off-by: Prekshi Vyas --- docs/reference/troubleshooting.mdx | 5 +- .../adapters/openshell/sandbox-identity.ts | 14 +++- src/lib/onboard/sandbox-gpu-create-flow.ts | 2 +- .../sandbox-gpu-create-identity-gate.test.ts | 78 +++++++++++++++++-- .../onboard/sandbox-gpu-create-run-attempt.ts | 49 ++++++++++-- 5 files changed, 131 insertions(+), 17 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 44d4d6d0bb3..6572b1cda79 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2026,7 +2026,10 @@ export NEMOCLAW_SANDBOX_READY_TIMEOUT=600 $$nemoclaw onboard ``` -The variable accepts seconds and applies to the readiness wait only. When the ordinary create deadline expires, NemoClaw tries to delete the partially created sandbox. After successful cleanup, the output ends with `Retry: $$nemoclaw onboard`. If cleanup fails, NemoClaw instead reports that the failed sandbox could not be removed and prints `Manual cleanup: openshell sandbox delete ""`. +The variable accepts seconds and sets the shared post-create readiness deadline. +For a post-create readiness failure, NemoClaw preserves the sandbox and does not delete it by mutable name. +Follow the retained-sandbox recovery procedure above. +NemoClaw removes the recovery state only after identity-bound cleanup succeeds. diff --git a/src/lib/adapters/openshell/sandbox-identity.ts b/src/lib/adapters/openshell/sandbox-identity.ts index a416ebd8435..4e8f3d7478c 100644 --- a/src/lib/adapters/openshell/sandbox-identity.ts +++ b/src/lib/adapters/openshell/sandbox-identity.ts @@ -305,6 +305,7 @@ export function settleCreatedOpenShellSandboxId(input: { readonly runCaptureOpenshell: (args: string[], options?: Record) => string; readonly priorSandboxId?: string | null; readonly now?: () => number; + readonly timeoutMs?: number; readonly sleep: (milliseconds: number) => void; }): string { assertCreateAttemptNonce(input.createAttemptNonce); @@ -315,9 +316,18 @@ export function settleCreatedOpenShellSandboxId(input: { } const now = input.now ?? (() => performance.now()); const startedAt = now(); - const deadlineMs = startedAt + CREATED_IDENTITY_SETTLEMENT_TIMEOUT_MS; + const timeoutMs = Math.min( + CREATED_IDENTITY_SETTLEMENT_TIMEOUT_MS, + input.timeoutMs ?? CREATED_IDENTITY_SETTLEMENT_TIMEOUT_MS, + ); + const deadlineMs = startedAt + timeoutMs; - if (!Number.isFinite(startedAt) || !Number.isFinite(deadlineMs) || deadlineMs <= startedAt) { + if ( + !Number.isFinite(startedAt) || + !Number.isFinite(timeoutMs) || + !Number.isFinite(deadlineMs) || + deadlineMs <= startedAt + ) { throw createdIdentityError(input.sandboxName); } diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 89da6971046..2b1b057dca4 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -298,7 +298,7 @@ export interface SandboxGpuCreateFlowDeps { runCaptureOpenshell: RunCaptureOpenshell; sandboxObserver: OpenShellSandboxObserver; sleep: Sleep; - /** Production callers use the system clock; tests may inject publication-wait time. */ + /** Production callers use a monotonic clock; tests may inject post-create deadline time. */ publicationNow?: () => number; openshellArgv(args: string[]): string[]; verifyDirectSandboxGpu(sandboxName: string): SandboxGpuProofResult; diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 1650d25bd19..777cf332494 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -1,8 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { performance } from "node:perf_hooks"; - import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ @@ -900,6 +898,66 @@ describe("created sandbox identity gate", () => { expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); }); + it("shares publication time with the final post-create readiness deadline (#10652)", async () => { + let nonce = ""; + let nowMs = 1_000; + const input = noGpuInput(); + input.sandboxReadyTimeoutSecs = 10; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + deps.publicationNow = () => nowMs; + vi.mocked(deps.sleep).mockImplementation((seconds) => { + nowMs += seconds * 1_000; + }); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + const missingSandbox = { + status: 1, + stdout: "", + stderr: + "Error: × code: 'Some requested entity was not found', message: \"sandbox not found\"", + }; + vi.mocked(deps.runOpenshell) + .mockReturnValueOnce(missingSandbox) + .mockReturnValueOnce(missingSandbox) + .mockReturnValueOnce(missingSandbox) + .mockReturnValueOnce(missingSandbox) + .mockReturnValue({ + status: 0, + stdout: "Name: alpha\nId: alpha-sandbox-id\nState: Ready\n", + stderr: "", + }); + let finalReadinessTimeoutSecs = Number.NaN; + mocks.waitForCreatedSandboxReadyWithTrace.mockImplementationOnce(async (options) => { + finalReadinessTimeoutSecs = options.timeoutSecs; + nowMs += options.timeoutSecs * 1_000; + return { ready: false, reason: "timeout", failurePhase: null }; + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "did not become ready after verified creation", + ); + + expect(deps.sleep).toHaveBeenCalledTimes(4); + expect(finalReadinessTimeoutSecs).toBe(6); + expect(nowMs).toBe(11_000); + expect(mocks.waitForCreatedSandboxReadyWithTrace).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + timeoutSecs: 6, + now: deps.publicationNow, + }), + ); + expect(patch.commitAfterReady).not.toHaveBeenCalled(); + }); + it("stops publication probing on a non-transient OpenShell failure (#9833)", async () => { let nonce = ""; const input = noGpuInput(); @@ -988,6 +1046,11 @@ describe("created sandbox identity gate", () => { return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; }); const deps = createGpuFlowDeps(); + let nowMs = 0; + deps.publicationNow = () => nowMs; + vi.mocked(deps.sleep).mockImplementation((seconds) => { + nowMs += seconds * 1_000; + }); vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), ); @@ -1063,11 +1126,12 @@ describe("created sandbox identity gate", () => { return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; }); const deps = createGpuFlowDeps(); - vi.mocked(deps.runCaptureOpenshell).mockReturnValue("[]"); - vi.spyOn(performance, "now") - .mockReturnValueOnce(0) - .mockReturnValueOnce(0) - .mockReturnValueOnce(30_000); + let nowMs = 0; + deps.publicationNow = () => nowMs; + vi.mocked(deps.runCaptureOpenshell).mockImplementation(() => { + nowMs = 30_000; + return "[]"; + }); const error = await runSandboxGpuCreateFlow(input, deps).catch((caught: unknown) => { events.push("rejected"); diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 0a398a3e1ba..e4103386bec 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { randomBytes } from "node:crypto"; +import { performance } from "node:perf_hooks"; import { mergeIsolatedDockerClientEnv, @@ -80,6 +81,26 @@ const SANDBOX_READY_PROBE_TIMEOUT_MS = 5_000; const CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_MS = 1_000; const CREATED_SANDBOX_PUBLICATION_DIAGNOSTIC_LIMIT = 1_000; +type PostCreateReadinessDeadline = Readonly<{ + deadlineMs: number; + now: () => number; +}>; + +function createPostCreateReadinessDeadline( + input: SandboxGpuCreateFlowInput, + deps: SandboxGpuCreateFlowDeps, +): PostCreateReadinessDeadline { + const now = deps.publicationNow ?? (() => performance.now()); + return { + deadlineMs: now() + Math.max(1, Math.round(input.sandboxReadyTimeoutSecs * 1_000)), + now, + }; +} + +function remainingPostCreateReadinessMs(deadline: PostCreateReadinessDeadline): number { + return Math.max(0, deadline.deadlineMs - deadline.now()); +} + async function streamSandboxCreateWithPublicImageCredentialIsolation( isolate: boolean, sandboxName: string, @@ -479,12 +500,12 @@ function waitForCreatedOpenShellSandboxPublication( sandboxId: string, input: SandboxGpuCreateFlowInput, deps: SandboxGpuCreateFlowDeps, + deadline: PostCreateReadinessDeadline, ): void { - const timeoutMs = Math.max(1, Math.round(input.sandboxReadyTimeoutSecs * 1_000)); const published = sandboxReadinessTracing.waitForCreatedSandboxPublication({ - budgetMs: timeoutMs, + budgetMs: remainingPostCreateReadinessMs(deadline), pollIntervalMs: CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_MS, - now: deps.publicationNow, + now: deadline.now, sleep: deps.sleep, probe: (getRemainingMs) => { const result = deps.runOpenshell( @@ -540,13 +561,14 @@ function waitForCreatedSandboxPublicationOrPersist( sandboxId: string, input: SandboxGpuCreateFlowInput, deps: SandboxGpuCreateFlowDeps, + deadline: PostCreateReadinessDeadline, persistIdentitySettlementRecovery: ( sandboxIdentityFingerprint: string, failureDiagnostic?: string, ) => void, ): void { try { - waitForCreatedOpenShellSandboxPublication(sandboxId, input, deps); + waitForCreatedOpenShellSandboxPublication(sandboxId, input, deps, deadline); } catch (error) { persistIdentitySettlementRecovery( fingerprintSandboxRecreateValue(sandboxId), @@ -680,6 +702,13 @@ export function createSandboxGpuCreateAttemptRunner( const unboundAttemptArgv = state.compatibilityArgv ?? input.createArgv; if (input.requirePolicylessCreate) assertPolicylessSandboxCreateArgv(unboundAttemptArgv); const createAttemptNonce = resolveCreateAttemptNonce(input, deferPostCreateEffects); + let postCreateReadinessDeadline: PostCreateReadinessDeadline | null = null; + const requirePostCreateReadinessDeadline = (): PostCreateReadinessDeadline => { + postCreateReadinessDeadline ??= createPostCreateReadinessDeadline(input, deps); + return postCreateReadinessDeadline; + }; + const remainingPostCreateReadinessSecs = (): number => + remainingPostCreateReadinessMs(requirePostCreateReadinessDeadline()) / 1_000; const persistIdentitySettlementRecovery = ( sandboxIdentityFingerprint: string | null = null, failureDiagnostic?: string, @@ -696,6 +725,7 @@ export function createSandboxGpuCreateAttemptRunner( sandboxId, input, deps, + requirePostCreateReadinessDeadline(), persistIdentitySettlementRecovery, ); const captureRetainedSandboxRecovery = () => { @@ -820,12 +850,15 @@ export function createSandboxGpuCreateAttemptRunner( }; const settleCreatedIdentity = (): string => { if (readyCheckCreatedIdentityFailure !== null) throw readyCheckCreatedIdentityFailure; + const deadline = requirePostCreateReadinessDeadline(); const sandboxId = settleCreatedOpenShellSandboxId({ sandboxName: input.sandboxName, gatewayName: input.gatewayName, createAttemptNonce: createAttemptNonce!, runCaptureOpenshell: deps.runCaptureOpenshell, priorSandboxId: readyCheckCreatedSandboxId, + now: deadline.now, + timeoutMs: remainingPostCreateReadinessMs(deadline), sleep: (milliseconds) => deps.sleep(milliseconds / 1000), }); if (readyCheckCreatedSandboxId && sandboxId !== readyCheckCreatedSandboxId) { @@ -905,6 +938,7 @@ export function createSandboxGpuCreateAttemptRunner( return process.exit(status); }; if (input.resumeVerifiedCreate) { + requirePostCreateReadinessDeadline(); if (route !== input.resumeVerifiedCreate.route) { throw new Error("Verified sandbox recovery route changed before continuation."); } @@ -947,6 +981,7 @@ export function createSandboxGpuCreateAttemptRunner( ); } const result = await streamCreate(); + requirePostCreateReadinessDeadline(); const createFailure = result.status === 0 ? null : classifySandboxCreateFailure(result.output); if (result.status !== 0 && createFailure?.kind !== "sandbox_create_incomplete") { @@ -955,11 +990,12 @@ export function createSandboxGpuCreateAttemptRunner( if (createFailure?.kind === "sandbox_create_incomplete") { const readiness = await sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ sandboxName: input.sandboxName, - timeoutSecs: input.sandboxReadyTimeoutSecs, + timeoutSecs: remainingPostCreateReadinessSecs(), observer: deps.sandboxObserver, target: { kind: "named", gatewayName: input.gatewayName }, stableReadyPolls: REPLACEMENT_STABLE_READY_POLLS, sleep: deps.sleep, + now: requirePostCreateReadinessDeadline().now, }); if (!readiness.ready) { if (createAttemptNonce) persistIdentitySettlementRecovery(); @@ -1181,7 +1217,7 @@ export function createSandboxGpuCreateAttemptRunner( console.log(" Waiting for sandbox to become ready..."); const readiness = await sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ sandboxName: input.sandboxName, - timeoutSecs: input.sandboxReadyTimeoutSecs, + timeoutSecs: remainingPostCreateReadinessSecs(), observer: deps.sandboxObserver, target: { kind: "named", gatewayName: input.gatewayName }, stableReadyPolls: @@ -1207,6 +1243,7 @@ export function createSandboxGpuCreateAttemptRunner( getRemainingMs, ), sleep: deps.sleep, + now: requirePostCreateReadinessDeadline().now, }); if (!readiness.ready) { console.error(""); From 3286fb566e182263afe22956508264ab123b5587 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 03:37:41 -0700 Subject: [PATCH 32/51] fix(onboard): complete shared readiness review Signed-off-by: Prekshi Vyas --- ci/source-architecture-budget.json | 2 +- docs/reference/commands.mdx | 2 +- .../onboard/created-sandbox-failure.test.ts | 128 ------------------ src/lib/onboard/created-sandbox-failure.ts | 47 ------- .../hermes-portable-onboarding.test.ts | 106 ++++++++------- .../onboard/sandbox-gpu-create-flow.test.ts | 21 ++- .../sandbox-gpu-create-identity-gate.test.ts | 102 ++++++++++++-- .../onboard/sandbox-gpu-create-run-attempt.ts | 99 +++++++++----- 8 files changed, 232 insertions(+), 275 deletions(-) delete mode 100644 src/lib/onboard/created-sandbox-failure.test.ts delete mode 100644 src/lib/onboard/created-sandbox-failure.ts diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index c50a54ab1d8..b5110dd6b48 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -59,7 +59,7 @@ }, "allowedCycles": [], "maxRootFiles": { - "src/lib/onboard": 305, + "src/lib/onboard": 304, "src/lib/actions": 18, "src/lib/actions/sandbox": 182, "src/lib/state": 39, diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index e4a06bc872f..da386c6f687 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3942,7 +3942,7 @@ The following environment variables tune onboard-time and recovery wall-clock li | `NEMOCLAW_OLLAMA_PULL_TIMEOUT` | `1800` (30 minutes) | Wall-clock timeout for `ollama pull` during onboard, in seconds. Accepts integer or float values. Already-downloaded layers are kept; re-running the pull resumes them. | | `NEMOCLAW_HF_DOWNLOAD_STALL_TIMEOUT` | `600` (10 minutes) | Maximum silence between Hugging Face download output during onboard. A positive finite value in seconds overrides the default, up to the Node.js timer limit of about 24.8 days. Blank, invalid, non-positive, sub-millisecond, and oversized values use the default. This is not a total download limit. Increase it only when a working download can produce no output for ten minutes. | | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | `180` | Wall-clock timeout for the inference-server validation probe during onboard, in seconds. Raise on slow networks or for very large prompts. | -| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Wall-clock timeout for post-create readiness, in seconds. Raise the timeout when the managed-image pull, explicit custom image build, gateway upload, or in-sandbox boot exceeds the default (typical on 70B+ models, first-time gateway uploads over slow links, or DGX Station / remote-VM first runs). Ordinary onboarding deletes the partially created sandbox when the deadline expires and prints the retry hint. Portable OpenClaw onboarding instead preserves the sandbox when NemoClaw cannot verify its runtime identity. | +| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Shared wall-clock deadline for post-create publication, durable identity settlement, and executable readiness. Raise the timeout when the managed-image pull, explicit custom image build, gateway upload, or in-sandbox boot exceeds the default (typical on 70B+ models, first-time gateway uploads over slow links, or DGX Station / remote-VM first runs). A post-create readiness failure preserves the sandbox for the identity-bound retained-sandbox recovery procedure above. | | `NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE` | `30` | Consecutive `Error`-phase polls the post-create readiness wait tolerates before treating `Error` as terminal. Polling starts at 250ms and backs off to a 2-second cap, while `NEMOCLAW_SANDBOX_READY_TIMEOUT` remains the overall deadline. The gateway can briefly report a just-created sandbox in `Error` while it re-registers the sandbox (seen on DGX Spark); the debounce lets that transient recover to `Ready`. Every terminal observation outside the `Error` phase, including one with no reported phase, fails immediately. Set to `1` to restore fast-fail on the first `Error` poll. | | `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` | `30`, `90`, or `120`, depending on the recovery phase | Wall-clock timeout for OpenShell command re-registration after policy application, plus gateway health and re-registration during managed OpenClaw or Hermes recovery. A valid finite, nonnegative value overrides the internal budget for the current recovery phase. | diff --git a/src/lib/onboard/created-sandbox-failure.test.ts b/src/lib/onboard/created-sandbox-failure.test.ts deleted file mode 100644 index b0b56f591a0..00000000000 --- a/src/lib/onboard/created-sandbox-failure.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it, vi } from "vitest"; - -import { - reportSandboxCreateFailure, - type SandboxCreateFailureReportDeps, - type SandboxCreateFailureReportOptions, -} from "./created-sandbox-failure"; - -class ExitSignal extends Error { - constructor(readonly code: number) { - super(`exit:${code}`); - } -} - -function createFailureDeps( - overrides: Partial = {}, -): SandboxCreateFailureReportDeps { - return { - printCreateFailureDiagnostics: vi.fn(), - printRecoveryHints: vi.fn(), - error: vi.fn(), - exitProcess: vi.fn((code: number): never => { - throw new ExitSignal(code); - }), - ...overrides, - }; -} - -function createFailureOptions( - overrides: Partial = {}, -): SandboxCreateFailureReportOptions { - return { - sandboxName: "alpha", - createStatus: 3, - createOutput: "boom", - restoreBackupPath: null, - createArgs: ["sandbox", "create", "alpha"], - ...overrides, - }; -} - -describe("reportSandboxCreateFailure", () => { - it("prints diagnostics + recovery hints and exits with the create status on a hard failure", () => { - const deps = createFailureDeps(); - expect(() => - reportSandboxCreateFailure( - createFailureOptions({ createStatus: 42, restoreBackupPath: "/tmp/backup" }), - deps, - ), - ).toThrow(ExitSignal); - expect(deps.printCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { - backupPath: "/tmp/backup", - }); - expect(deps.printRecoveryHints).toHaveBeenCalledWith("boom", { - createArgs: ["sandbox", "create", "alpha"], - }); - expect(deps.exitProcess).toHaveBeenCalledWith(42); - }); - - it("redacts create output before classification and echoing", () => { - // With output: leading blank + headline + blank + output echo + "Try:" hint = 5 error() calls. - const withOutput = createFailureDeps(); - expect(() => - reportSandboxCreateFailure( - createFailureOptions({ createOutput: "failed with Authorization: Bearer secret-token" }), - withOutput, - ), - ).toThrow(ExitSignal); - expect(withOutput.error).toHaveBeenCalledWith("failed with Authorization: Bearer secr********"); - expect(withOutput.error).not.toHaveBeenCalledWith( - "failed with Authorization: Bearer secret-token", - ); - expect(withOutput.printRecoveryHints).toHaveBeenCalledWith( - "failed with Authorization: Bearer secr********", - expect.any(Object), - ); - expect(withOutput.error).toHaveBeenCalledTimes(5); - - // Without output: the echo block is skipped, so only 3 error() calls remain. - const noOutput = createFailureDeps(); - expect(() => - reportSandboxCreateFailure(createFailureOptions({ createOutput: "" }), noOutput), - ).toThrow(ExitSignal); - expect(noOutput.error).toHaveBeenCalledTimes(3); - // still exits (createStatus || 1) - expect(noOutput.exitProcess).toHaveBeenCalledWith(3); - }); - - it("redacts multiple known token formats in create output", () => { - const deps = createFailureDeps(); - const createOutput = [ - "Authorization: Bearer secret-token", - "github ghp_abcdefghijklmnopqrstuvwxyz1234567890", - "openai sk-abcdefghijklmnopqrstuvwxyz1234567890", - "aws AKIAABCDEFGHIJKLMNOP", // gitleaks:allow - ].join("\n"); - - expect(() => reportSandboxCreateFailure(createFailureOptions({ createOutput }), deps)).toThrow( - ExitSignal, - ); - - const echoed = (deps.error as ReturnType).mock.calls - .map((call) => String(call[0])) - .join("\n"); - expect(echoed).not.toContain("secret-token"); - expect(echoed).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz1234567890"); - expect(echoed).not.toContain("sk-abcdefghijklmnopqrstuvwxyz1234567890"); - expect(echoed).not.toContain("AKIAABCDEFGHIJKLMNOP"); // gitleaks:allow - const hinted = (deps.printRecoveryHints as ReturnType).mock.calls - .map((call) => String(call[0])) - .join("\n"); - expect(hinted).not.toContain("secret-token"); - expect(hinted).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz1234567890"); - expect(hinted).not.toContain("sk-abcdefghijklmnopqrstuvwxyz1234567890"); - expect(hinted).not.toContain("AKIAABCDEFGHIJKLMNOP"); // gitleaks:allow - }); - - it("falls back to exit code 1 when the create status is zero", () => { - const deps = createFailureDeps(); - expect(() => - reportSandboxCreateFailure(createFailureOptions({ createStatus: 0 }), deps), - ).toThrow(ExitSignal); - expect(deps.exitProcess).toHaveBeenCalledWith(1); - }); -}); diff --git a/src/lib/onboard/created-sandbox-failure.ts b/src/lib/onboard/created-sandbox-failure.ts deleted file mode 100644 index 90a8a986421..00000000000 --- a/src/lib/onboard/created-sandbox-failure.ts +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { redact, redactFullWithUrls } from "../security/redact"; - -export type SandboxCreateFailureReportOptions = { - sandboxName: string; - /** Non-zero exit status from the create stream. */ - createStatus: number; - /** Raw create-stream output, used for failure classification and recovery hints. */ - createOutput: string; - /** Pre-recreate/pre-upgrade state backup path to surface in diagnostics, if any. */ - restoreBackupPath: string | null; - /** Resolved `openshell sandbox create` args, so recovery hints stay aligned with --from. */ - createArgs: readonly string[]; -}; - -export type SandboxCreateFailureReportDeps = { - printCreateFailureDiagnostics(sandboxName: string, options: { backupPath: string | null }): void; - printRecoveryHints(output: string, options: { createArgs: readonly string[] }): void; - error(message: string): void; - exitProcess(code: number): never; -}; - -export function redactCreatedSandboxFailureDiagnostic(value: string, limit: number): string { - return redactFullWithUrls(value).replace(/\s+/gu, " ").trim().slice(0, limit); -} - -/** Report a hard sandbox create-stream failure with diagnostics and recovery hints. */ -export function reportSandboxCreateFailure( - options: SandboxCreateFailureReportOptions, - deps: SandboxCreateFailureReportDeps, -): void { - const redactedCreateOutput = redact(options.createOutput); - deps.error(""); - deps.error(` Sandbox creation failed (exit ${options.createStatus}).`); - if (options.createOutput) { - deps.error(""); - deps.error(redactedCreateOutput); - } - deps.printCreateFailureDiagnostics(options.sandboxName, { - backupPath: options.restoreBackupPath, - }); - deps.error(" Try: openshell sandbox list # check gateway state"); - deps.printRecoveryHints(redactedCreateOutput, { createArgs: options.createArgs }); - return deps.exitProcess(options.createStatus === 0 ? 1 : options.createStatus); -} diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts b/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts index 5c8d2f40a67..cf375487afe 100644 --- a/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts +++ b/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts @@ -339,23 +339,27 @@ describe("Hermes portable onboarding transaction", () => { sandboxId: "sandbox-id-1", liveIdentityFingerprint: HERMES_PORTABLE_TEST_LIVE_IDENTITY, }; - const observations = [ - { kind: "absent" as const }, - { kind: "absent" as const }, - { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }, - { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }, - present, - ]; let nowMs = 0; - let boundedObservations = 0; + let classificationObservations = 0; + const boundedBudgets: Array<{ budgetMs: number; remainingMs: number }> = []; const observeSandbox = vi.fn((timeoutBudgetMs?: number) => { - const observation = observations.shift() ?? present; - nowMs += timeoutBudgetMs === undefined ? 0 : ([61_000][boundedObservations++] ?? 0); - return observation; + const budgetMs = timeoutBudgetMs ?? 0; + const classification = timeoutBudgetMs === undefined; + classificationObservations += Number(classification); + boundedBudgets.push({ budgetMs, remainingMs: 180_000 - nowMs }); + const observed = classification + ? classificationObservations <= 2 + ? { kind: "absent" as const } + : present + : nowMs >= 61_000 + ? present + : { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }; + nowMs += classification ? 0 : Math.min(budgetMs, Math.max(0, 61_000 - nowMs)); + return observed; }); - const delaySandboxReadyPublicationPoll = vi.fn(async (milliseconds: number) => { + const delaySandboxReadyPublicationPoll = async (milliseconds: number) => { nowMs += milliseconds; - }); + }; const fixture = deps({ observeSandbox, delaySandboxReadyPublicationPoll, @@ -367,11 +371,10 @@ describe("Hermes portable onboarding transaction", () => { expect(completed.active.receipt.phase).toBe("active"); expect(completed.created).toBe(true); expect(fixture.events.filter((event) => event === "create")).toHaveLength(1); - expect(delaySandboxReadyPublicationPoll).toHaveBeenCalledTimes(2); - expect(delaySandboxReadyPublicationPoll).toHaveBeenCalledWith(1_000); - expect( - observeSandbox.mock.calls.filter(([timeoutBudgetMs]) => timeoutBudgetMs !== undefined), - ).toEqual([[180_000], [118_000], [117_000]]); + expect(classificationObservations).toBeGreaterThan(0); + expect(nowMs).toBeGreaterThan(60_000); + expect(boundedBudgets.length).toBeGreaterThan(0); + expect(boundedBudgets.every(({ budgetMs, remainingMs }) => budgetMs <= remainingMs)).toBe(true); expect(fixture.events[0]).toBe("lock-enter"); expect(fixture.events.at(-1)).toBe("lock-exit"); }); @@ -434,16 +437,19 @@ describe("Hermes portable onboarding transaction", () => { }); it("fails closed when exact post-create Ready publication exceeds its bound (#9211)", async () => { - let observations = 0; - const observeSandbox = vi.fn(() => - observations++ < 2 - ? { kind: "absent" as const } - : { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }, - ); let nowMs = 0; - const delaySandboxReadyPublicationPoll = vi.fn(async (milliseconds: number) => { - nowMs += milliseconds; + const boundedBudgets: Array<{ budgetMs: number; remainingMs: number }> = []; + const observeSandbox = vi.fn((timeoutBudgetMs?: number) => { + const budgetMs = timeoutBudgetMs ?? 0; + boundedBudgets.push({ budgetMs, remainingMs: 180_000 - nowMs }); + nowMs += budgetMs; + return timeoutBudgetMs === undefined + ? { kind: "absent" as const } + : { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }; }); + const delaySandboxReadyPublicationPoll = async (milliseconds: number) => { + nowMs += milliseconds; + }; const fixture = deps({ observeSandbox, delaySandboxReadyPublicationPoll, @@ -455,32 +461,32 @@ describe("Hermes portable onboarding transaction", () => { ); expect(fixture.events.filter((event) => event === "create")).toHaveLength(1); - expect(delaySandboxReadyPublicationPoll).toHaveBeenCalledTimes(180); - expect(observeSandbox.mock.calls.slice(2)).toEqual( - Array.from({ length: 180 }, (_value, index) => [180_000 - index * 1_000]), - ); + expect(nowMs).toBe(180_000); + expect(boundedBudgets.length).toBeGreaterThan(0); + expect(boundedBudgets.every(({ budgetMs, remainingMs }) => budgetMs <= remainingMs)).toBe(true); expect(fixture.events).not.toContain("registry"); expect(fixture.events.at(-1)).toBe("lock-exit"); }); it("counts OpenShell observation time against the total Ready publication deadline (#9211)", async () => { let nowMs = 0; - let observationIndex = 0; - const observationDurationsMs = [0, 0, 166_000, 13_000] as const; - const observations = [ - { kind: "absent" as const }, - { kind: "absent" as const }, - { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }, - { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }, - ]; - const observeSandbox = vi.fn((_timeoutBudgetMs?: number) => { - const currentIndex = observationIndex++; - nowMs += observationDurationsMs[currentIndex] ?? 0; - return observations[currentIndex] ?? observations.at(-1)!; + let observedMs = 0; + let delayedMs = 0; + const boundedBudgets: Array<{ budgetMs: number; remainingMs: number }> = []; + const observeSandbox = vi.fn((timeoutBudgetMs?: number) => { + const budgetMs = timeoutBudgetMs ?? 0; + boundedBudgets.push({ budgetMs, remainingMs: 180_000 - nowMs }); + const elapsedMs = Math.min(budgetMs, observedMs === 0 ? 179_000 : budgetMs); + observedMs += elapsedMs; + nowMs += elapsedMs; + return timeoutBudgetMs === undefined + ? { kind: "absent" as const } + : { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }; }); - const delaySandboxReadyPublicationPoll = vi.fn(async (milliseconds: number) => { + const delaySandboxReadyPublicationPoll = async (milliseconds: number) => { + delayedMs += milliseconds; nowMs += milliseconds; - }); + }; const fixture = deps({ observeSandbox, delaySandboxReadyPublicationPoll, @@ -491,10 +497,10 @@ describe("Hermes portable onboarding transaction", () => { "cannot classify create result: exact OpenShell sandbox is not Ready", ); - expect(observeSandbox.mock.calls.slice(2)).toEqual([[180_000], [13_000]]); - expect(delaySandboxReadyPublicationPoll).toHaveBeenCalledTimes(1); - expect(delaySandboxReadyPublicationPoll).toHaveBeenCalledWith(1_000); expect(nowMs).toBe(180_000); + expect(observedMs).toBeGreaterThan(delayedMs); + expect(boundedBudgets.length).toBeGreaterThan(0); + expect(boundedBudgets.every(({ budgetMs, remainingMs }) => budgetMs <= remainingMs)).toBe(true); expect(fixture.events.filter((event) => event === "create")).toHaveLength(1); expect(fixture.events).not.toContain("registry"); }); @@ -1367,9 +1373,9 @@ network_policies: const run = createHermesPortableReadyRunner("alpha", "nemoclaw", capture); expect(run(["sandbox", "list", "-g", "nemoclaw"]).status).toBe(0); - expect( - run(["sandbox", "exec", "-g", "nemoclaw", "--name", "alpha", "--", "true"]).status, - ).toBe(0); + expect(run(["sandbox", "exec", "-g", "nemoclaw", "--name", "alpha", "--", "true"]).status).toBe( + 0, + ); expect(capture.mock.calls).toEqual([ [["sandbox", "list", "-g", "nemoclaw"]], [["sandbox", "exec", "-g", "nemoclaw", "--name", "alpha", "--", "true"]], diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index ff4e6c23a24..23310aaa900 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -405,8 +405,8 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { vi.mocked(deps.runCaptureOpenshell).mockClear(); await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); expect(deps.runCaptureOpenshell).toHaveBeenCalledWith( - ["sandbox", "list", "-g", "nemoclaw"], - READY_CHECK_OPTIONS, + ["sandbox", "get", "-g", "nemoclaw", "alpha"], + { ignoreError: false }, ); expect(vi.mocked(console.warn).mock.calls.flat().join("\n")).toContain( "unrelated sandbox 'bravo'", @@ -942,10 +942,15 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { expect(deps.runOpenshell).not.toHaveBeenCalled(); }); - it("redacts create errors and preserves their exact nonzero status (#6110)", async () => { + it("redacts hard create errors and preserves diagnostics, hints, and status (#6110)", async () => { mocks.streamSandboxCreate.mockResolvedValueOnce({ status: 19, - output: "provider failed with NVIDIA_API_KEY=super-secret-create-value", + output: [ + "provider failed with NVIDIA_API_KEY=super-secret-create-value", + "github ghp_abcdefghijklmnopqrstuvwxyz1234567890", + "openai sk-abcdefghijklmnopqrstuvwxyz1234567890", + "aws AKIAABCDEFGHIJKLMNOP", // gitleaks:allow + ].join("\n"), sawProgress: true, }); const exit = mockExit(19); @@ -957,7 +962,13 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { const output = vi.mocked(console.error).mock.calls.flat().join("\n"); expect(exit).toHaveBeenCalledWith(19); expect(output).toMatch(/NVIDIA_API_KEY=[^\n]*\*+/); - expect(output).not.toContain("super-secret-create-value"); + expect(output).not.toMatch( + /super-secret-create-value|ghp_abcdefghijklmnopqrstuvwxyz1234567890|sk-abcdefghijklmnopqrstuvwxyz1234567890|AKIAABCDEFGHIJKLMNOP/u, // gitleaks:allow + ); + expect(output).toContain("Try: openshell sandbox list"); + expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { + backupPath: null, + }); }); it("does not retry compatibility for a non-GPU native readiness failure (#6110)", async () => { diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 777cf332494..5193b57e383 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -132,6 +132,7 @@ function noGpuInput() { function attachManagedBootstrap( input: ReturnType, patch: ReturnType, + mode: { freshCreate?: boolean } = {}, ): void { input.managedBootstrap = { bootstrapIdentity: "b".repeat(64), @@ -140,14 +141,20 @@ function attachManagedBootstrap( identity: { id: "mxc" }, bootstrap: { createOnboardRouting: () => ({ nativeFallbackHasCleanBaseline: false }), - createLifecycle: (options: { launchArgv: readonly string[] }) => ({ - launchArgv: options.launchArgv, + createLifecycle: (lifecycleOptions: { + launchArgv: readonly string[]; + heldWorkloadArgv: readonly string[]; + bootstrapIdentity: string; + }) => ({ + launchArgv: lifecycleOptions.launchArgv, patch, recoverUnfinished: async () => null, prepareNetwork: async () => undefined, - runCreate: async () => { - throw new Error("resumed create must not launch"); - }, + runCreate: mode.freshCreate + ? async ( + start: (held: typeof lifecycleOptions) => Promise<{ readonly value: T }>, + ): Promise => (await start(lifecycleOptions)).value + : async () => Promise.reject(new Error("resumed create must not launch")), }), }, }, @@ -316,6 +323,87 @@ describe("created sandbox identity gate", () => { expectNoSandboxDelete(deps); }); + it("waits through managed bootstrap publication beyond the former five-second probe (#10652)", async () => { + const actualTracing = await vi.importActual( + "./sandbox-readiness-tracing", + ); + let nonce = ""; + let nowMs = 1_000; + const input = noGpuInput(); + input.sandboxReadyTimeoutSecs = 20; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + attachManagedBootstrap(input, createGpuPatchFixture(), { freshCreate: true }); + mocks.streamSandboxCreate.mockImplementationOnce(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + deps.publicationNow = () => nowMs; + vi.mocked(deps.sleep).mockImplementation((seconds) => { + nowMs += seconds * 1_000; + }); + let readyObservations = 0; + vi.mocked(deps.runCaptureOpenshell).mockImplementation((args) => + args[1] !== "list" + ? "Name: alpha\nId: alpha-sandbox-id\nState: Ready\n" + : nowMs < 7_000 + ? "alpha Pending" + : readyObservations++ === 0 + ? "alpha Ready" + : sandboxListJson("alpha-sandbox-id", { + [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, + }), + ); + mocks.waitForCreatedSandboxReadyWithTrace + .mockImplementationOnce((options) => + actualTracing.waitForCreatedSandboxReadyWithTrace(options), + ) + .mockResolvedValue({ ready: true, reason: "ready", failurePhase: null }); + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); + expect(nowMs).toBeGreaterThanOrEqual(7_000); + const firstReadiness = mocks.waitForCreatedSandboxReadyWithTrace.mock.calls[0]?.[0]; + expect(firstReadiness?.timeoutSecs).toBe(20); + expect(firstReadiness?.now).toBe(deps.publicationNow); + expect(firstReadiness?.stableReadyPolls).toBe(1); + }); + + it("shares managed bootstrap time with post-commit readiness (#10652)", async () => { + let nonce = ""; + let nowMs = 0; + const input = noGpuInput(); + input.sandboxReadyTimeoutSecs = 10; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + attachManagedBootstrap(input, patch, { freshCreate: true }); + mocks.streamSandboxCreate.mockImplementationOnce(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + deps.publicationNow = () => nowMs; + vi.mocked(deps.runCaptureOpenshell).mockImplementation((args) => + args[1] === "list" + ? sandboxListJson("alpha-sandbox-id", { + [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, + }) + : "Name: alpha\nId: alpha-sandbox-id\nState: Ready\n", + ); + const readinessTimeouts: number[] = []; + mocks.waitForCreatedSandboxReadyWithTrace.mockImplementation(async (options) => { + readinessTimeouts.push(options.timeoutSecs); + nowMs += [3_000, 2_000, 0][readinessTimeouts.length - 1] ?? 0; + return { ready: true, reason: "ready", failurePhase: null }; + }); + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); + expect(readinessTimeouts).toEqual([10, 7, 5]); + expect( + mocks.waitForCreatedSandboxReadyWithTrace.mock.calls.map(([options]) => options.now), + ).toEqual([deps.publicationNow, deps.publicationNow, deps.publicationNow]); + expect(patch.commitAfterReady).toHaveBeenCalledOnce(); + }); + it("retains exact recovery when committed managed readiness does not return (#9211)", async () => { const error = vi.spyOn(console, "error").mockImplementation(() => undefined); const sandboxId = "alpha-sandbox-id"; @@ -481,9 +569,7 @@ describe("created sandbox identity gate", () => { expect(deps.installPortableDemoLifecycle).toHaveBeenCalledOnce(); expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); expectNoSandboxDelete(deps); - expect(vi.mocked(console.error).mock.calls.flat().join("\n")).not.toContain( - "portable-secret", - ); + expect(vi.mocked(console.error).mock.calls.flat().join("\n")).not.toContain("portable-secret"); expect(vi.mocked(console.log).mock.calls.flat().join("\n")).not.toContain( "Sandbox 'alpha' created", ); diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index e4103386bec..e33ef7261a1 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -21,13 +21,10 @@ import { import { printSandboxCreateRecoveryHints } from "../build-context"; import { streamSandboxCreate, type StreamSandboxCreateResult } from "../sandbox/create-stream"; import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-ready-gate"; +import { redact, redactFullWithUrls } from "../security/redact"; import { isSandboxReady } from "../state/gateway"; import type { SandboxGpuProofResult } from "../state/registry"; import { classifySandboxCreateFailure } from "../validation"; -import { - redactCreatedSandboxFailureDiagnostic, - reportSandboxCreateFailure, -} from "./created-sandbox-failure"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; @@ -101,6 +98,36 @@ function remainingPostCreateReadinessMs(deadline: PostCreateReadinessDeadline): return Math.max(0, deadline.deadlineMs - deadline.now()); } +function redactCreatedSandboxFailureDiagnostic(value: string, limit: number): string { + return redactFullWithUrls(value).replace(/\s+/gu, " ").trim().slice(0, limit); +} + +function reportSandboxCreateFailure(options: { + readonly sandboxName: string; + readonly createStatus: number; + readonly createOutput: string; + readonly restoreBackupPath: string | null; + readonly createArgs: readonly string[]; + readonly printCreateFailureDiagnostics: ( + sandboxName: string, + options: { backupPath: string | null }, + ) => void; +}): never { + const redactedCreateOutput = redact(options.createOutput); + console.error(""); + console.error(` Sandbox creation failed (exit ${options.createStatus}).`); + if (options.createOutput) { + console.error(""); + console.error(redactedCreateOutput); + } + options.printCreateFailureDiagnostics(options.sandboxName, { + backupPath: options.restoreBackupPath, + }); + console.error(" Try: openshell sandbox list # check gateway state"); + printSandboxCreateRecoveryHints(redactedCreateOutput, { createArgs: options.createArgs }); + return process.exit(options.createStatus === 0 ? 1 : options.createStatus); +} + async function streamSandboxCreateWithPublicImageCredentialIsolation( isolate: boolean, sandboxName: string, @@ -398,6 +425,7 @@ async function confirmManagedRuntimeCommitReadiness(options: { readonly deps: SandboxGpuCreateFlowDeps; readonly sandboxId: string | null; readonly createAttemptNonce: string | null; + readonly deadline: PostCreateReadinessDeadline; }): Promise { const { input, deps, sandboxId } = options; if (!sandboxId) return; @@ -406,7 +434,7 @@ async function confirmManagedRuntimeCommitReadiness(options: { ); const committedReadiness = await sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ sandboxName: input.sandboxName, - timeoutSecs: input.sandboxReadyTimeoutSecs, + timeoutSecs: remainingPostCreateReadinessMs(options.deadline) / 1_000, observer: deps.sandboxObserver, target: { kind: "named", gatewayName: input.gatewayName }, stableReadyPolls: REPLACEMENT_STABLE_READY_POLLS, @@ -419,6 +447,7 @@ async function confirmManagedRuntimeCommitReadiness(options: { getRemainingMs, ), sleep: deps.sleep, + now: options.deadline.now, }); if (committedReadiness.ready) return; console.error(""); @@ -455,23 +484,28 @@ async function requireManagedBootstrapCreatedSandboxReady(options: { readonly deps: SandboxGpuCreateFlowDeps; readonly createAttemptNonce: string | null; readonly persistIdentitySettlementRecovery: () => void; + readonly deadline: PostCreateReadinessDeadline; }): Promise { - const observation = await sandboxReadinessTracing.observeOpenShellSandbox( - options.deps.sandboxObserver, - { kind: "named", gatewayName: options.input.gatewayName }, - options.input.sandboxName, - SANDBOX_READY_PROBE_TIMEOUT_MS, + const readiness = await sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ + sandboxName: options.input.sandboxName, + timeoutSecs: remainingPostCreateReadinessMs(options.deadline) / 1_000, + observer: options.deps.sandboxObserver, + target: { kind: "named", gatewayName: options.input.gatewayName }, + stableReadyPolls: 1, + sleep: options.deps.sleep, + now: options.deadline.now, + }); + if (readiness.ready) return; + if (options.createAttemptNonce) options.persistIdentitySettlementRecovery(); + throw new Error( + sandboxReadinessTracing + .formatCreatedSandboxReadinessFailureMessage( + options.input.sandboxName, + readiness, + options.input.sandboxReadyTimeoutSecs, + ) + .trimStart(), ); - if (!observation.ok) { - if (options.createAttemptNonce) options.persistIdentitySettlementRecovery(); - throw new Error( - `Managed bootstrap create completed, but NemoClaw could not observe the sandbox. ${observation.error.message}`, - ); - } - if (observation.value.state !== "present" || observation.value.sandbox.readiness !== "ready") { - if (options.createAttemptNonce) options.persistIdentitySettlementRecovery(); - throw new Error("Managed bootstrap create completed without an authoritative Ready sandbox."); - } } function resolveCreateAttemptNonce( @@ -1015,6 +1049,7 @@ export function createSandboxGpuCreateAttemptRunner( deps, createAttemptNonce, persistIdentitySettlementRecovery, + deadline: requirePostCreateReadinessDeadline(), }); } let sandboxId: string; @@ -1143,21 +1178,14 @@ export function createSandboxGpuCreateAttemptRunner( } as const; } else { await runtimePatch.rollbackManagedStartupAfterCreateFailure(); - reportSandboxCreateFailure( - { - sandboxName: input.sandboxName, - createStatus: createResult.status, - createOutput: createResult.output, - restoreBackupPath: input.restoreBackupPath, - createArgs: input.prebuild.createArgs, - }, - { - printCreateFailureDiagnostics, - printRecoveryHints: printSandboxCreateRecoveryHints, - error: (message) => console.error(message), - exitProcess: (code) => process.exit(code), - }, - ); + reportSandboxCreateFailure({ + sandboxName: input.sandboxName, + createStatus: createResult.status, + createOutput: createResult.output, + restoreBackupPath: input.restoreBackupPath, + createArgs: input.prebuild.createArgs, + printCreateFailureDiagnostics, + }); } } if (!createdSandboxVerified && deferPostCreateEffects) { @@ -1395,6 +1423,7 @@ export function createSandboxGpuCreateAttemptRunner( deps, sandboxId: managedBootstrap ? verifiedCreatedSandboxId : null, createAttemptNonce, + deadline: requirePostCreateReadinessDeadline(), }); if (!input.sandboxGpuConfig.sandboxGpuEnabled) { await confirmCommittedRuntimeReadiness(); From 93f84e29cb9ecc134dd50f3ae7e7887f3efca6bb Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 04:03:46 -0700 Subject: [PATCH 33/51] fix(onboard): close final advisor findings Signed-off-by: Prekshi Vyas --- docs/reference/commands.mdx | 6 +-- docs/reference/troubleshooting.mdx | 11 +++-- .../onboard/sandbox-gpu-create-flow.test.ts | 2 +- test/security/shellquote-sandbox.test.ts | 46 ++++++++----------- 4 files changed, 28 insertions(+), 37 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index da386c6f687..c663cf46f35 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3942,7 +3942,7 @@ The following environment variables tune onboard-time and recovery wall-clock li | `NEMOCLAW_OLLAMA_PULL_TIMEOUT` | `1800` (30 minutes) | Wall-clock timeout for `ollama pull` during onboard, in seconds. Accepts integer or float values. Already-downloaded layers are kept; re-running the pull resumes them. | | `NEMOCLAW_HF_DOWNLOAD_STALL_TIMEOUT` | `600` (10 minutes) | Maximum silence between Hugging Face download output during onboard. A positive finite value in seconds overrides the default, up to the Node.js timer limit of about 24.8 days. Blank, invalid, non-positive, sub-millisecond, and oversized values use the default. This is not a total download limit. Increase it only when a working download can produce no output for ten minutes. | | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | `180` | Wall-clock timeout for the inference-server validation probe during onboard, in seconds. Raise on slow networks or for very large prompts. | -| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Shared wall-clock deadline for post-create publication, durable identity settlement, and executable readiness. Raise the timeout when the managed-image pull, explicit custom image build, gateway upload, or in-sandbox boot exceeds the default (typical on 70B+ models, first-time gateway uploads over slow links, or DGX Station / remote-VM first runs). A post-create readiness failure preserves the sandbox for the identity-bound retained-sandbox recovery procedure above. | +| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Shared wall-clock deadline for post-create publication, durable identity settlement, and executable readiness. Raise it only when OpenShell needs longer after the create command returns to publish the sandbox, settle its durable identity, or reach executable `Ready`. A post-create readiness failure preserves the sandbox for the identity-bound retained-sandbox recovery procedure above. | | `NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE` | `30` | Consecutive `Error`-phase polls the post-create readiness wait tolerates before treating `Error` as terminal. Polling starts at 250ms and backs off to a 2-second cap, while `NEMOCLAW_SANDBOX_READY_TIMEOUT` remains the overall deadline. The gateway can briefly report a just-created sandbox in `Error` while it re-registers the sandbox (seen on DGX Spark); the debounce lets that transient recover to `Ready`. Every terminal observation outside the `Error` phase, including one with no reported phase, fails immediately. Set to `1` to restore fast-fail on the first `Error` poll. | | `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` | `30`, `90`, or `120`, depending on the recovery phase | Wall-clock timeout for OpenShell command re-registration after policy application, plus gateway health and re-registration during managed OpenClaw or Hermes recovery. A valid finite, nonnegative value overrides the internal budget for the current recovery phase. | @@ -3977,11 +3977,11 @@ $$nemoclaw recover -If the Ollama pull or post-create readiness timeout fires, onboarding emits the elapsed budget plus a hint to raise the relevant variable. The Ollama pull preserves its partial download for the next attempt. The ordinary post-create readiness wait deletes the orphaned sandbox first so the next `$$nemoclaw onboard` starts without that partially created sandbox. +If the Ollama pull or post-create readiness timeout fires, onboarding emits the elapsed budget plus a hint to raise the relevant variable. The Ollama pull preserves its partial download for the next attempt. A post-create readiness failure preserves the sandbox and records recovery evidence when possible. The sandbox name remains blocked until the identity-bound retained-sandbox recovery procedure succeeds. -For portable OpenClaw onboarding, NemoClaw instead leaves the sandbox in place when it cannot verify the exact runtime identity. Inspect it with `openshell sandbox list` and `$$nemoclaw status`, then follow the recovery guidance from `status`. +Portable OpenClaw onboarding also preserves the sandbox when NemoClaw cannot verify the exact runtime identity. Inspect it with `openshell sandbox list` and `$$nemoclaw status`, then follow the recovery guidance from `status`. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 6572b1cda79..b60b6599b5a 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2008,16 +2008,17 @@ If onboarding reports that the managed runtime commit completed but the same san NemoClaw keeps the sandbox, prints its create-attempt label and a one-way durable identity fingerprint, and does not start dashboard forwarding. It saves that evidence in the retained recovery record when persistence succeeds. Do not delete the sandbox by its mutable name. -When NemoClaw confirms that it saved the record, run `$$nemoclaw destroy` and follow [Recover a retained sandbox](commands#recover-a-retained-sandbox) for the result-specific recovery steps. +When NemoClaw confirms that it saved the record and the record contains a durable identity fingerprint, run `$$nemoclaw destroy` and follow [Recover a retained sandbox](commands#recover-a-retained-sandbox) for the result-specific recovery steps. +If the saved record has no durable identity fingerprint, preserve the terminal output and give the create-attempt label to an OpenShell administrator so they can identify and remove the exact sandbox. If NemoClaw reports that it could not save the record, preserve the terminal output and ask an OpenShell administrator to identify the exact sandbox from gateway or controller evidence; the recovery-only session remains blocked until NemoClaw can save the durable recovery record. -The 180-second default fits typical workstations but can be exceeded when: +The 180-second default can be exceeded after the create command returns when: -- The host is building or uploading the sandbox image for the first time (cold caches, slow link). -- The selected model is large (70B+ parameters or 4-bit/8-bit quantisations that take time to memory-map). -- Onboarding runs on a remote VM where image upload to the gateway streams over the network (for example DGX Station first-run installer). +- The owning gateway needs longer to publish the new sandbox and its durable identity. +- The in-sandbox agent, policy, or model runtime needs longer to become executable. +- A managed runtime commit needs longer to re-register the same sandbox as executable `Ready`. Raise the budget before re-running onboard: diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index 23310aaa900..4cb35f3fe0a 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -1260,7 +1260,7 @@ describe("runSandboxGpuCreateFlow fallback ordering", () => { expect(mocks.streamSandboxCreate.mock.calls.flat()).not.toContain("-lc"); }); - it("discloses the compatibility container-swap confinement tradeoff and native-only opt-out", async () => { + it("discloses the compatibility container-swap tradeoff and fallback authorization", async () => { failNativeCreate(); const deps = createDeps(); diff --git a/test/security/shellquote-sandbox.test.ts b/test/security/shellquote-sandbox.test.ts index aa4b3faa8de..df2e2557602 100644 --- a/test/security/shellquote-sandbox.test.ts +++ b/test/security/shellquote-sandbox.test.ts @@ -233,40 +233,30 @@ try { .find((line) => line.startsWith("{") && line.endsWith("}")); expect(payloadLine).toBeTruthy(); const payload = JSON.parse(payloadLine!); + const sandboxGetCommands = payload.commands.filter( + (entry: { command: string }) => + entry.command.includes("sandbox get") && entry.command.includes("my-assistant"), + ); + const sandboxExecCommands = payload.commands.filter( + (entry: { command: string }) => + entry.command.includes("sandbox exec") && entry.command.includes("my-assistant"), + ); + expect(sandboxGetCommands).not.toHaveLength(0); + expect(sandboxExecCommands).not.toHaveLength(0); expect( - payload.commands.some((entry: { command: string }) => - entry.command.includes("sandbox get -g nemoclaw my-assistant"), + sandboxGetCommands.every( + (entry: { command: string }) => + entry.command.includes("sandbox get -g nemoclaw") || + entry.command.includes("sandbox get --gateway nemoclaw"), ), ).toBe(true); expect( - payload.commands.some((entry: { command: string }) => - entry.command.includes("sandbox exec -g nemoclaw --name my-assistant -- true"), + sandboxExecCommands.every( + (entry: { command: string }) => + entry.command.includes("sandbox exec -g nemoclaw") || + entry.command.includes("--gateway nemoclaw"), ), ).toBe(true); - expect( - payload.commands - .filter( - (entry: { command: string }) => - entry.command.includes("sandbox get") && entry.command.includes("my-assistant"), - ) - .every( - (entry: { command: string }) => - entry.command.includes("sandbox get -g nemoclaw") || - entry.command.includes("sandbox get --gateway nemoclaw"), - ), - ).toBe(true); - expect( - payload.commands - .filter( - (entry: { command: string }) => - entry.command.includes("sandbox exec") && entry.command.includes("my-assistant"), - ) - .every( - (entry: { command: string }) => - entry.command.includes("sandbox exec -g nemoclaw") || - entry.command.includes("--gateway nemoclaw"), - ), - ).toBe(true); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } From 8bc3dd1576188fe1effe69c5e1d2331e7f0d78e5 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 04:24:21 -0700 Subject: [PATCH 34/51] refactor(onboard): move create cleanup ownership Signed-off-by: Prekshi Vyas --- .../configure-inference-timeouts.mdx | 9 ++++-- .../onboard/sandbox-create/orchestration.ts | 27 ++++++++++++---- src/lib/onboard/sandbox-gpu-create-flow.ts | 30 ----------------- .../onboard-prepared-build-context.test.ts | 32 +++++++++++++++---- 4 files changed, 53 insertions(+), 45 deletions(-) diff --git a/docs/inference/configure-inference-timeouts.mdx b/docs/inference/configure-inference-timeouts.mdx index abeb4b2aaef..ee1e0538afa 100644 --- a/docs/inference/configure-inference-timeouts.mdx +++ b/docs/inference/configure-inference-timeouts.mdx @@ -20,7 +20,7 @@ Use the error location to select the correct setting. |---|---|---| | `NEMOCLAW_AGENT_TIMEOUT` | OpenClaw per-request inference | `600` seconds | | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | Ollama, vLLM, NIM, and compatible-endpoint onboarding validation paths that read this setting | `180` seconds | -| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | Image build, gateway upload, and in-sandbox boot after creation | `180` seconds | +| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | OpenShell publication, durable identity settlement, and executable readiness after creation | `180` seconds | | `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` | OpenShell command re-registration after policy application, plus gateway health and re-registration during managed OpenClaw or Hermes recovery | `30`, `90`, or `120` seconds, depending on the recovery phase | The readiness timeout does not govern inference requests or provider validation. @@ -94,14 +94,17 @@ This variable does not extend the later sandbox-readiness wait. ## Increase the Sandbox Readiness Timeout -Raise `NEMOCLAW_SANDBOX_READY_TIMEOUT` when onboarding creates the sandbox but image build, upload, or boot exceeds 180 seconds. -This can occur during a first run with cold caches or on a remote VM over a slow link. +Raise `NEMOCLAW_SANDBOX_READY_TIMEOUT` when OpenShell needs more than 180 seconds after the create command returns to publish the sandbox, settle its durable identity, or reach executable `Ready`. +This setting does not extend image build or gateway upload time. ```bash export NEMOCLAW_SANDBOX_READY_TIMEOUT=600 $$nemoclaw onboard ``` +If this deadline expires, NemoClaw preserves the sandbox and records recovery evidence when possible. +Follow [Recover a retained sandbox](../../reference/commands#recover-a-retained-sandbox) before reusing the sandbox name. + ## Increase the Recovery Wait Set `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` when OpenShell needs more than 120 seconds to re-register the sandbox after onboarding applies policy presets. diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index c33f37f034e..43cf9852100 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -2579,12 +2579,27 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche }); }; - const cleanupBuildContext = - sandboxGpuCreateFlow.createSandboxBuildContextCleanup(legacyBuildContext); - const cleanupInitialCreateSource = sandboxGpuCreateFlow.createSandboxCreateSourceCleanup( - initialSandboxPolicy, - agentCreateInput.hermesPortableLifecycle, - ); + let buildContextCleanupCompleted = false; + const cleanupBuildContext = (): boolean => { + if (buildContextCleanupCompleted || !legacyBuildContext?.cleanupBuildCtx) return true; + buildContextCleanupCompleted = legacyBuildContext.cleanupBuildCtx(); + if (buildContextCleanupCompleted) { + process.removeListener("exit", legacyBuildContext.cleanupBuildCtx); + } + return buildContextCleanupCompleted; + }; + let initialCreateSourceCleanupCompleted = false; + const cleanupInitialCreateSource = (): boolean => { + if (initialCreateSourceCleanupCompleted) return true; + initialCreateSourceCleanupCompleted = sandboxGpuCreateFlow.cleanupSandboxCreateSource( + initialSandboxPolicy.cleanup, + { + exactCleanup: initialSandboxPolicy.cleanupExact, + requireExact: agentCreateInput.hermesPortableLifecycle, + }, + ); + return initialCreateSourceCleanupCompleted; + }; const cleanupSandboxCreateSources = (): void => { const cleanupErrors: Error[] = []; try { diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 2b1b057dca4..b152b7d9ffb 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -103,36 +103,6 @@ export function cleanupSandboxCreateSource( return completed; } -/** Bind the exact create-source retirement decision without moving its execution point. */ -export function createSandboxCreateSourceCleanup( - source: { readonly cleanup?: () => boolean; readonly cleanupExact?: () => boolean }, - requireExact: boolean, -): () => boolean { - let completed = false; - return () => { - if (completed) return true; - completed = cleanupSandboxCreateSource(source.cleanup, { - exactCleanup: source.cleanupExact, - requireExact, - }); - return completed; - }; -} - -/** Bind cleanup for the one staged build context owned by this create attempt. */ -export function createSandboxBuildContextCleanup( - context: { readonly cleanupBuildCtx?: () => boolean } | null, -): () => boolean { - let completed = false; - return () => { - if (completed) return true; - if (!context?.cleanupBuildCtx) return true; - completed = context.cleanupBuildCtx(); - if (completed) process.removeListener("exit", context.cleanupBuildCtx); - return completed; - }; -} - export function resolvePortableLifecycleMode( agent: AgentDefinition | null, env: NodeJS.ProcessEnv = process.env, diff --git a/test/onboarding/onboard-prepared-build-context.test.ts b/test/onboarding/onboard-prepared-build-context.test.ts index 9224fc8e4be..9c747860418 100644 --- a/test/onboarding/onboard-prepared-build-context.test.ts +++ b/test/onboarding/onboard-prepared-build-context.test.ts @@ -9,11 +9,12 @@ import path from "node:path"; import { describe, it } from "vitest"; import { writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; -type PreparedContextScenario = "create" | "custom-dockerfile"; +type PreparedContextScenario = "create" | "custom-dockerfile" | "cleanup-incomplete"; type PreparedContextResult = { buildCtx: string; buildId: string; + buildCleanupListenerRegistered: boolean; cleanupCalls: number; commands: string[]; errorMessage: string | null; @@ -200,16 +201,19 @@ childProcess.spawn = (...args) => { return child; }; +const cleanupBuildCtx = () => { + cleanupCalls += 1; + if (scenario === "cleanup-incomplete") return false; + fs.rmSync(buildCtx, { recursive: true, force: true }); + return true; +}; +process.on("exit", cleanupBuildCtx); const preparedBuildContext = { buildCtx, stagedDockerfile: buildCtx + "/Dockerfile", buildId, origin: "generated", - cleanupBuildCtx: () => { - cleanupCalls += 1; - fs.rmSync(buildCtx, { recursive: true, force: true }); - return true; - }, + cleanupBuildCtx, }; const { createSandbox } = require(${onboardPath}); @@ -251,6 +255,7 @@ const { createSandbox } = require(${onboardPath}); console.log(JSON.stringify({ buildCtx, buildId, + buildCleanupListenerRegistered: process.listeners("exit").includes(cleanupBuildCtx), cleanupCalls, commands, errorMessage, @@ -305,6 +310,7 @@ describe("onboard prepared DCode build context", () => { assert.deepEqual(result.planFromRefs, [`${result.buildCtx}/Dockerfile`]); assert.deepEqual(result.resolvedBuildIds, [result.buildId]); assert.equal(result.cleanupCalls, 1); + assert.equal(result.buildCleanupListenerRegistered, false); assert.ok( result.commands.some((command) => command.includes(`sandbox create --from ${result.buildCtx}/Dockerfile`), @@ -320,6 +326,20 @@ describe("onboard prepared DCode build context", () => { }, ); + it( + "retains exit cleanup ownership when build-context retirement is incomplete (#10652)", + { + timeout: 90_000, + }, + () => { + const result = runPreparedContextScenario("cleanup-incomplete"); + + assert.equal(result.errorMessage, null); + assert.equal(result.cleanupCalls, 1); + assert.equal(result.buildCleanupListenerRegistered, true); + }, + ); + it( "passes the seconds-based sleep helper to the Docker GPU patch during prepared-context onboarding (#9218)", { From 249896e7fac0eb5841d31358dbf8eedffa15834d Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 04:54:24 -0700 Subject: [PATCH 35/51] fix(onboard): isolate portable operation recovery Signed-off-by: Prekshi Vyas --- ...-portable-onboarding-ready-timeout.test.ts | 80 +++++++++++++++++++ .../hermes-portable-onboarding.test.ts | 62 ++++---------- .../hermes-portable-onboarding.ts | 20 +++-- .../sandbox-create/orchestration.test.ts | 49 ++++++++++++ .../onboard/sandbox-create/orchestration.ts | 3 +- .../hermes-portable-onboarding-fixture.ts | 1 + .../install-hermes-portable-active.test.ts | 1 + 7 files changed, 161 insertions(+), 55 deletions(-) create mode 100644 src/lib/onboard/experimental/hermes-portable-onboarding-ready-timeout.test.ts diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding-ready-timeout.test.ts b/src/lib/onboard/experimental/hermes-portable-onboarding-ready-timeout.test.ts new file mode 100644 index 00000000000..afcbecdaa86 --- /dev/null +++ b/src/lib/onboard/experimental/hermes-portable-onboarding-ready-timeout.test.ts @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + createHermesPortableTestInput, + createHermesPortableTransactionFixture, + HERMES_PORTABLE_TEST_LIVE_IDENTITY, + HERMES_PORTABLE_TEST_POLICY, +} from "../../../../test/helpers/hermes-portable-onboarding-fixture"; +import { runHermesPortableOnboardingTransaction } from "./hermes-portable-onboarding"; + +let stateDir: string; +let policyPath: string; + +beforeEach(() => { + stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-ready-timeout-")); + policyPath = path.join(stateDir, "create.yaml"); + fs.writeFileSync(policyPath, HERMES_PORTABLE_TEST_POLICY, { mode: 0o600 }); +}); + +afterEach(() => fs.rmSync(stateDir, { recursive: true, force: true })); + +describe("Hermes portable onboarding readiness timeout", () => { + it("uses the configured timeout to settle identity after the old Ready deadline (#9211)", async () => { + const currentInput = { + ...createHermesPortableTestInput(stateDir, policyPath), + sandboxReadyTimeoutSecs: 90, + }; + const present = { + kind: "present" as const, + sandboxId: "sandbox-id-1", + liveIdentityFingerprint: HERMES_PORTABLE_TEST_LIVE_IDENTITY, + }; + let nowMs = 0; + let classificationObservations = 0; + const boundedBudgets: Array<{ budgetMs: number; remainingMs: number }> = []; + const observeSandbox = vi.fn((timeoutBudgetMs?: number) => { + const budgetMs = timeoutBudgetMs ?? 0; + const classification = timeoutBudgetMs === undefined; + classificationObservations += Number(classification); + boundedBudgets.push({ budgetMs, remainingMs: 90_000 - nowMs }); + const observed = classification + ? classificationObservations <= 2 + ? { kind: "absent" as const } + : present + : nowMs >= 61_000 + ? present + : { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }; + nowMs += classification ? 0 : Math.min(budgetMs, Math.max(0, 61_000 - nowMs)); + return observed; + }); + const delaySandboxReadyPublicationPoll = async (milliseconds: number) => { + nowMs += milliseconds; + }; + const fixture = createHermesPortableTransactionFixture(currentInput, { + observeSandbox, + delaySandboxReadyPublicationPoll, + readSandboxReadyPublicationClockMs: () => nowMs, + }); + + const completed = await runHermesPortableOnboardingTransaction(currentInput, fixture.value); + + expect(completed.active.receipt.phase).toBe("active"); + expect(completed.created).toBe(true); + expect(fixture.events.filter((event) => event === "create")).toHaveLength(1); + expect(classificationObservations).toBeGreaterThan(0); + expect(nowMs).toBeGreaterThan(60_000); + expect(boundedBudgets.length).toBeGreaterThan(0); + expect(boundedBudgets.find(({ budgetMs }) => budgetMs > 0)?.budgetMs).toBe(90_000); + expect(boundedBudgets.every(({ budgetMs, remainingMs }) => budgetMs <= remainingMs)).toBe(true); + expect(fixture.events[0]).toBe("lock-enter"); + expect(fixture.events.at(-1)).toBe("lock-exit"); + }); +}); diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts b/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts index cf375487afe..4547b44e5ae 100644 --- a/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts +++ b/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts @@ -333,52 +333,6 @@ describe("Hermes portable onboarding transaction", () => { ); }); - it("settles the exact post-create sandbox identity after the old Ready deadline (#9211)", async () => { - const present = { - kind: "present" as const, - sandboxId: "sandbox-id-1", - liveIdentityFingerprint: HERMES_PORTABLE_TEST_LIVE_IDENTITY, - }; - let nowMs = 0; - let classificationObservations = 0; - const boundedBudgets: Array<{ budgetMs: number; remainingMs: number }> = []; - const observeSandbox = vi.fn((timeoutBudgetMs?: number) => { - const budgetMs = timeoutBudgetMs ?? 0; - const classification = timeoutBudgetMs === undefined; - classificationObservations += Number(classification); - boundedBudgets.push({ budgetMs, remainingMs: 180_000 - nowMs }); - const observed = classification - ? classificationObservations <= 2 - ? { kind: "absent" as const } - : present - : nowMs >= 61_000 - ? present - : { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }; - nowMs += classification ? 0 : Math.min(budgetMs, Math.max(0, 61_000 - nowMs)); - return observed; - }); - const delaySandboxReadyPublicationPoll = async (milliseconds: number) => { - nowMs += milliseconds; - }; - const fixture = deps({ - observeSandbox, - delaySandboxReadyPublicationPoll, - readSandboxReadyPublicationClockMs: () => nowMs, - }); - - const completed = await runHermesPortableOnboardingTransaction(input(), fixture.value); - - expect(completed.active.receipt.phase).toBe("active"); - expect(completed.created).toBe(true); - expect(fixture.events.filter((event) => event === "create")).toHaveLength(1); - expect(classificationObservations).toBeGreaterThan(0); - expect(nowMs).toBeGreaterThan(60_000); - expect(boundedBudgets.length).toBeGreaterThan(0); - expect(boundedBudgets.every(({ budgetMs, remainingMs }) => budgetMs <= remainingMs)).toBe(true); - expect(fixture.events[0]).toBe("lock-enter"); - expect(fixture.events.at(-1)).toBe("lock-exit"); - }); - it("resumes a pending post-create receipt while Ready publication lags (#9203)", async () => { let firstNowMs = 0; let firstObservations = 0; @@ -642,7 +596,16 @@ network_policies: ...input(), createPolicySourceBytes: Buffer.from(regeneratedPolicy), }; - const second = createHermesPortableTransactionFixture(resumedInput); + let consumedPolicyPath: string | null = null; + let consumedPolicyBytes: Buffer | null = null; + const second = createHermesPortableTransactionFixture(resumedInput, { + createSandbox: async (argv, _buildContextPath, effectivePolicySourcePath) => { + consumedPolicyPath = effectivePolicySourcePath; + consumedPolicyBytes = fs.readFileSync(effectivePolicySourcePath); + expect(argv[argv.indexOf("--policy") + 1]).toBe(effectivePolicySourcePath); + return { ready: true }; + }, + }); const resumed = await runHermesPortableOnboardingTransaction(resumedInput, second.value); @@ -654,6 +617,11 @@ network_policies: ).toBe(false); expect(resumed.created).toBe(true); expect(second.events.filter((event) => event === "create")).toHaveLength(1); + expect(consumedPolicyPath).toBe( + hermesPortablePolicySourcePath("alpha", resumed.active.receipt.transactionId, stateDir), + ); + expect(consumedPolicyBytes).toEqual(Buffer.from(POLICY)); + expect(consumedPolicyBytes).not.toEqual(Buffer.from(regeneratedPolicy)); }); it("rejects changed non-policy create intent on pending reentry before effects (#9203)", async () => { diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding.ts b/src/lib/onboard/experimental/hermes-portable-onboarding.ts index 091d34bec82..9c6c8dce362 100644 --- a/src/lib/onboard/experimental/hermes-portable-onboarding.ts +++ b/src/lib/onboard/experimental/hermes-portable-onboarding.ts @@ -109,6 +109,7 @@ export interface HermesPortableOnboardingInput { readonly sandboxName: string; readonly gatewayName: string; readonly lifecycleGeneration: string; + readonly sandboxReadyTimeoutSecs: number; readonly runtimeAuthority: CheckpointPortableRuntimeAuthority; readonly openshellExecutableAuthority: HermesPortableOpenShellExecutableAuthority; readonly stateDir: string; @@ -705,10 +706,6 @@ function parseHermesPortableSandboxJson( } const HERMES_PORTABLE_READY_PUBLICATION_POLL_INTERVAL_MS = 1_000; -const HERMES_PORTABLE_READY_PUBLICATION_TIMEOUT_MS = 180_000; -const HERMES_PORTABLE_READY_PUBLICATION_MAX_POLLS = Math.ceil( - HERMES_PORTABLE_READY_PUBLICATION_TIMEOUT_MS / HERMES_PORTABLE_READY_PUBLICATION_POLL_INTERVAL_MS, -); const HERMES_PORTABLE_NOT_READY_DETAIL = "exact OpenShell sandbox is not Ready"; const HERMES_PORTABLE_READY_PUBLICATION_TIMEOUT_DETAIL = "exact OpenShell sandbox Ready publication exceeded its total deadline"; @@ -791,13 +788,16 @@ async function settleCreatedHermesPortableSandboxReadyPublication( observeSandbox: (timeoutBudgetMs?: number) => HermesPortableSandboxObservation, delayPoll: (milliseconds: number) => Promise, readClockMs: () => number, + timeoutMs: number, ): Promise { - const deadlineMs = readClockMs() + HERMES_PORTABLE_READY_PUBLICATION_TIMEOUT_MS; + const boundedTimeoutMs = Math.max(1, Math.round(timeoutMs)); + const maxPolls = Math.ceil(boundedTimeoutMs / HERMES_PORTABLE_READY_PUBLICATION_POLL_INTERVAL_MS); + const deadlineMs = readClockMs() + boundedTimeoutMs; let observation: HermesPortableSandboxObservation = { kind: "ambiguous", detail: HERMES_PORTABLE_READY_PUBLICATION_TIMEOUT_DETAIL, }; - for (let poll = 0; poll <= HERMES_PORTABLE_READY_PUBLICATION_MAX_POLLS; poll += 1) { + for (let poll = 0; poll <= maxPolls; poll += 1) { const observationBudgetMs = Math.floor(deadlineMs - readClockMs()); if (observationBudgetMs < 1) return observation; observation = observeSandbox(observationBudgetMs); @@ -807,7 +807,7 @@ async function settleCreatedHermesPortableSandboxReadyPublication( ) { return observation; } - if (poll === HERMES_PORTABLE_READY_PUBLICATION_MAX_POLLS) return observation; + if (poll === maxPolls) return observation; const delayBudgetMs = Math.floor(deadlineMs - readClockMs()); if (delayBudgetMs < 1) return observation; await delayPoll(Math.min(HERMES_PORTABLE_READY_PUBLICATION_POLL_INTERVAL_MS, delayBudgetMs)); @@ -1083,6 +1083,7 @@ export async function runHermesPortableOnboardingTransaction( deps: HermesPortableOnboardingDeps, ): Promise> { return await deps.withLifecycleLock(input.sandboxName, async () => { + const readyPublicationTimeoutMs = input.sandboxReadyTimeoutSecs * 1_000; assertHermesPortableUninstallCompleteForOnboarding(input.stateDir); const assertOpenShellExecutableAuthority = (): void => deps.assertOpenShellExecutableAuthority(input.openshellExecutableAuthority); @@ -1429,6 +1430,7 @@ export async function runHermesPortableOnboardingTransaction( observeSandbox, deps.delaySandboxReadyPublicationPoll ?? delayHermesPortableReadyPublicationPoll, deps.readSandboxReadyPublicationClockMs ?? performance.now.bind(performance), + readyPublicationTimeoutMs, ); } if (observation.kind === "ambiguous") @@ -1470,6 +1472,7 @@ export async function runHermesPortableOnboardingTransaction( observeSandbox, deps.delaySandboxReadyPublicationPoll ?? delayHermesPortableReadyPublicationPoll, deps.readSandboxReadyPublicationClockMs ?? performance.now.bind(performance), + readyPublicationTimeoutMs, ); buildContext.assertCurrent(); input.buildContext.assertCurrentSource(); @@ -1617,6 +1620,7 @@ export interface HermesPortableOnboardingFromOnboardInput { readonly sandboxName: string; readonly gatewayName: string; readonly lifecycleGeneration: string; + readonly sandboxReadyTimeoutSecs: number; readonly portableRuntime: PortableOnboardRuntimeContext; readonly createArgv: readonly string[]; readonly createPolicyPath: string; @@ -1679,6 +1683,7 @@ export async function runHermesPortableOnboardingFromOnboard( sandboxName, gatewayName, lifecycleGeneration, + sandboxReadyTimeoutSecs, portableRuntime, createArgv, createPolicyPath, @@ -1738,6 +1743,7 @@ export async function runHermesPortableOnboardingFromOnboard( sandboxName, gatewayName, lifecycleGeneration, + sandboxReadyTimeoutSecs, runtimeAuthority, openshellExecutableAuthority, stateDir: defaultPortableDemoStateDir(process.env), diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 153518c1117..999481f5ea5 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -13,6 +13,7 @@ import { assertApfCreateIntent, completeHermesPortableSandboxRegistration, createProviderEffectBoundary, + createSandboxWithBaseImageResolution, finalizeCreatedSandboxBeforeHermesCredentialReconciliation, hasManagedMcpRebuildHandoff, installPostCreateRecoveryRetryOwner, @@ -418,6 +419,54 @@ describe("retained create recovery persistence", () => { expect(recordRecovery).toHaveBeenCalledTimes(2); }, ); + + it("installs independent recovery retry owners for two operations from one factory (#10652)", async () => { + const exitHandlers: Array<() => void> = []; + const captureExitHandler = ((event: string | symbol, handler: (...args: unknown[]) => void) => { + event === "exit" && exitHandlers.push(handler as () => void); + return process; + }) as typeof process.on; + const processOn = vi.spyOn(process, "on").mockImplementation(captureExitHandler); + const createSandbox = createSandboxWithBaseImageResolution({} as never) as unknown as ( + ...args: unknown[] + ) => Promise; + + try { + await expect(createSandbox()).rejects.toThrow(); + await expect(createSandbox()).rejects.toThrow(); + } finally { + processOn.mockRestore(); + } + + expect(exitHandlers).toHaveLength(2); + expect(exitHandlers[0]).not.toBe(exitHandlers[1]); + }); + + it("retries both failed writers held by independent operation owners (#10652)", () => { + const exitHandlers: Array<() => void> = []; + const owners = [0, 1].map(() => + installPostCreateRecoveryRetryOwner({ + log: vi.fn(), + registerExitHandler: (handler) => exitHandlers.push(handler), + }), + ); + const writers = [0, 1].map((index) => + vi + .fn() + .mockImplementationOnce(() => { + throw new Error(`operation ${index + 1} recovery write failed`); + }) + .mockImplementationOnce(() => undefined), + ); + + expect(() => owners[0]?.record(writers[0]!)).toThrow(/operation 1/u); + expect(() => owners[1]?.record(writers[1]!)).toThrow(/operation 2/u); + expect(writers.map((writer) => writer.mock.calls.length)).toEqual([1, 1]); + + exitHandlers.forEach((handler) => handler()); + + expect(writers.map((writer) => writer.mock.calls.length)).toEqual([2, 2]); + }); }); describe("APF create policy selection", () => { diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 43cf9852100..b0fd3aca4cf 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -1163,7 +1163,6 @@ function readHermesPortableLifecycleGeneration(input: { } export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrchestrationRuntime) { - const postCreateRecoveryRetryOwner = installPostCreateRecoveryRetryOwner(); return async function createSandboxWithBaseImageResolution( baseImageResolutionContext: import("../base-image-resolution-flow").BaseImageResolutionContext, portableRuntimeContext: PortableOnboardRuntimeContext | null, @@ -1192,6 +1191,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche runVerifiedSandboxCreateEffects: import("../types").VerifiedSandboxCreateEffects | null = null, preparedBuildContext: PreparedSandboxBuildContext | null = null, ) { + const postCreateRecoveryRetryOwner = installPostCreateRecoveryRetryOwner(); const portableRuntimeAuthority = portableRuntimeContext?.authority ?? null; const { DASHBOARD_PORT, @@ -2757,6 +2757,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche sandboxName, gatewayName: GATEWAY_NAME, lifecycleGeneration: createdSandboxLifecycle.generation, + sandboxReadyTimeoutSecs, portableRuntime: portableRuntimeContext, createArgv, createPolicyPath: initialSandboxPolicy.policyPath, diff --git a/test/helpers/hermes-portable-onboarding-fixture.ts b/test/helpers/hermes-portable-onboarding-fixture.ts index 4f7a34f008e..585bb730fdb 100644 --- a/test/helpers/hermes-portable-onboarding-fixture.ts +++ b/test/helpers/hermes-portable-onboarding-fixture.ts @@ -204,6 +204,7 @@ export function createHermesPortableTestInput(stateDir: string, policyPath: stri sandboxName: "alpha", gatewayName: "nemoclaw", lifecycleGeneration: "generation-1", + sandboxReadyTimeoutSecs: 180, stateDir, createPolicyPath: policyPath, createArgv: [ diff --git a/test/installer-integration/install-hermes-portable-active.test.ts b/test/installer-integration/install-hermes-portable-active.test.ts index 7dedcdf040c..19ca8c889ef 100644 --- a/test/installer-integration/install-hermes-portable-active.test.ts +++ b/test/installer-integration/install-hermes-portable-active.test.ts @@ -258,6 +258,7 @@ describe("Hermes portable installer admission", testTimeoutOptions(60_000), () = sandboxName, gatewayName, lifecycleGeneration, + sandboxReadyTimeoutSecs: 180, runtimeAuthority, openshellExecutableAuthority: hermesPortableTestOpenShellAuthority(), stateDir, From 6b969becc936464a83cee7e4a433774543056f60 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 05:27:49 -0700 Subject: [PATCH 36/51] fix(onboard): honor shared readiness deadline Signed-off-by: Prekshi Vyas --- docs/reference/troubleshooting.mdx | 2 +- .../openshell/sandbox-identity.test.ts | 31 ++++ .../adapters/openshell/sandbox-identity.ts | 5 +- .../onboard/sandbox-create/orchestration.ts | 34 +++- src/lib/onboard/sandbox-gpu-create-flow.ts | 29 ---- ...ndbox-gpu-create-identity-deadline.test.ts | 154 +++++++++++++++++ .../sandbox-gpu-create-identity-gate.test.ts | 159 +----------------- 7 files changed, 219 insertions(+), 195 deletions(-) create mode 100644 src/lib/onboard/sandbox-gpu-create-identity-deadline.test.ts diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index b60b6599b5a..1508ee6d06c 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2047,7 +2047,7 @@ openshell sandbox list $$nemoclaw status ``` -If onboarding instead reports that the sandbox "did not re-register with OpenShell after policy application," the same timeout controls that post-policy command-readiness probe. Raise the budget before retrying, then inspect the same gateway and sandbox status if re-registration still fails. +If onboarding instead reports that the sandbox "did not re-register with OpenShell after policy application," `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` controls that post-policy command-readiness probe. Increase that recovery wait before retrying, then inspect the same gateway and sandbox status if re-registration still fails. ### Sandbox onboard fails with "entered Error phase before it became ready" diff --git a/src/lib/adapters/openshell/sandbox-identity.test.ts b/src/lib/adapters/openshell/sandbox-identity.test.ts index 507aef25bdd..8d58e0d7305 100644 --- a/src/lib/adapters/openshell/sandbox-identity.test.ts +++ b/src/lib/adapters/openshell/sandbox-identity.test.ts @@ -341,6 +341,37 @@ describe("OpenShell sandbox identity reading", () => { expect(sleep).toHaveBeenCalledExactlyOnceWith(250); }); + it("honors a caller-supplied identity settlement deadline beyond the default (#10652)", () => { + let nowMs = 0; + const sleep = vi.fn((milliseconds: number) => { + nowMs += milliseconds; + }); + const runCaptureOpenshell = vi + .fn<(args: string[], options?: Record) => string>() + .mockImplementationOnce(() => { + nowMs += 31_000; + return "[]"; + }) + .mockReturnValueOnce(sandboxListJson()); + + expect( + settleCreatedOpenShellSandboxId({ + sandboxName: "alpha", + gatewayName: "nemoclaw", + createAttemptNonce: CREATE_ATTEMPT_NONCE, + runCaptureOpenshell, + now: () => nowMs, + timeoutMs: 60_000, + sleep, + }), + ).toBe("sandbox-alpha"); + + expect(runCaptureOpenshell.mock.calls.map(([, options]) => options?.timeout)).toEqual([ + 60_000, 28_750, + ]); + expect(sleep).toHaveBeenCalledExactlyOnceWith(250); + }); + it("settles one exact nonce identity while its publication metadata becomes complete (#10423)", () => { let nowMs = 0; const sleep = vi.fn((milliseconds: number) => { diff --git a/src/lib/adapters/openshell/sandbox-identity.ts b/src/lib/adapters/openshell/sandbox-identity.ts index 4e8f3d7478c..6cd488f2b55 100644 --- a/src/lib/adapters/openshell/sandbox-identity.ts +++ b/src/lib/adapters/openshell/sandbox-identity.ts @@ -316,10 +316,7 @@ export function settleCreatedOpenShellSandboxId(input: { } const now = input.now ?? (() => performance.now()); const startedAt = now(); - const timeoutMs = Math.min( - CREATED_IDENTITY_SETTLEMENT_TIMEOUT_MS, - input.timeoutMs ?? CREATED_IDENTITY_SETTLEMENT_TIMEOUT_MS, - ); + const timeoutMs = input.timeoutMs ?? CREATED_IDENTITY_SETTLEMENT_TIMEOUT_MS; const deadlineMs = startedAt + timeoutMs; if ( diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index b0fd3aca4cf..20e84f5df86 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -29,6 +29,14 @@ import type { HermesAuthMethod } from "../hermes-auth"; import type { PreparedSandboxBuildContext } from "../build-context-stage"; import type { DcodeSelectionDriftReader } from "../dcode-selection-drift"; import { assertProviderlessInterceptorEnvironment } from "../entry-options"; +import { + createHermesPortableReadyCapture, + createHermesPortableReadyRunner, + defaultHermesPortableStateDir, + runHermesPortableOnboardingFromOnboard, + shouldManageHermesPortableDashboard, +} from "../experimental/hermes-portable-onboarding"; +import { inspectPortableAgentReceiptAuthorityForClassification } from "../experimental/hermes-portable-receipt"; import type { ManagedHermesStateVolumeCleanupResult, ManagedHermesStateVolumeContext, @@ -1149,6 +1157,22 @@ type PortableAgentReceiptGenerationObservation = readonly lifecycleGeneration: string; }; +function inspectPortableAgentReceiptGeneration( + sandboxName: string, +): PortableAgentReceiptGenerationObservation { + const authority = inspectPortableAgentReceiptAuthorityForClassification( + sandboxName, + defaultHermesPortableStateDir(process.env), + ); + if (authority.kind === "none") return { kind: "absent" }; + if (authority.kind === "openclaw") return { kind: "openclaw" }; + return { + kind: "hermes", + gatewayName: authority.snapshot.receipt.gatewayName, + lifecycleGeneration: authority.snapshot.receipt.lifecycleGeneration, + }; +} + function readHermesPortableLifecycleGeneration(input: { readonly enabled: boolean; readonly sandboxName: string; @@ -1351,7 +1375,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche ); const resolvedCreateIntent = preparedCreateIntent.intent; const messagingCapabilities = preparedCreateIntent.messagingCapabilities; - const manageDashboard = sandboxGpuCreateFlow.shouldManageHermesPortableDashboard( + const manageDashboard = shouldManageHermesPortableDashboard( dashboardRuntime.shouldManageDashboardForAgent(agent), agent, ); @@ -2220,7 +2244,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche enabled: agentCreateInput.hermesPortableLifecycle, sandboxName, gatewayName: GATEWAY_NAME, - inspect: sandboxGpuCreateFlow.inspectPortableAgentReceiptDisposition, + inspect: inspectPortableAgentReceiptGeneration, }); const createdSandboxLifecycle = sandboxRecreateTransaction.createCreatedSandboxLifecycle( recreateRuntime, @@ -2429,8 +2453,8 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche ); const runCreateFlow = async ( attemptCreateArgv: string[], - hermesPortableReadyCapture?: import("../sandbox-gpu-create-flow").HermesPortableReadyCapture, - hermesPortableReadyRunner?: import("../sandbox-gpu-create-flow").HermesPortableReadyRunner, + hermesPortableReadyCapture?: ReturnType, + hermesPortableReadyRunner?: ReturnType, createWorkingDirectory?: string, effectivePolicySourcePath?: string, runDeferredProviderEffects?: (context: VerifiedSandboxCreateEffectsContext) => Promise, @@ -2751,7 +2775,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche sessionId: inferenceRouteReservationAuthority.sessionId, selection: inferenceRouteReservationAuthority.selection, }; - await sandboxGpuCreateFlow.runHermesPortableOnboardingFromOnboard< + await runHermesPortableOnboardingFromOnboard< import("../sandbox-gpu-create-flow").SandboxGpuCreateFlowResult >({ sandboxName, diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index b152b7d9ffb..134305ff221 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -17,24 +17,12 @@ import { renderCompatibilityFallbackCreateArgs } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; import { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; import { - classifyHermesPortableRegistry, createHermesPortableChildEnvironment, - createHermesPortableContainerDeps, - createHermesPortableOpenShellCapture, - createHermesPortableReadyCapture, - createHermesPortableReadyRunner, - defaultHermesPortableStateDir, isHermesPortableLifecycleMode, - observeHermesPortableSandbox, - runHermesPortableOnboardingFromOnboard, - runHermesPortableOnboardingTransaction, - shouldManageHermesPortableDashboard, } from "./experimental/hermes-portable-onboarding"; import { installPortableDemoSandboxLifecycle } from "./experimental/portable-demo-lifecycle"; import { - buildHermesPortableCommandAuthority, buildHermesPortableOnboardingCommandAuthority, - inspectPortableAgentReceiptDisposition, } from "./experimental/portable-agent-lifecycle"; import { isPortableExperimentalProfile } from "./experimental/portable-profile"; import { @@ -70,23 +58,6 @@ import type { SandboxPrebuildResult } from "./sandbox-prebuild"; import { addTraceEvent } from "./tracing"; export { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; -export { - classifyHermesPortableRegistry, - createHermesPortableChildEnvironment, - createHermesPortableContainerDeps, - createHermesPortableOpenShellCapture, - createHermesPortableReadyCapture, - createHermesPortableReadyRunner, - defaultHermesPortableStateDir, - observeHermesPortableSandbox, - runHermesPortableOnboardingFromOnboard, - runHermesPortableOnboardingTransaction, - shouldManageHermesPortableDashboard, - buildHermesPortableCommandAuthority, - inspectPortableAgentReceiptDisposition, -}; -export type HermesPortableReadyCapture = ReturnType; -export type HermesPortableReadyRunner = ReturnType; /** Release the exit cleanup listener only after its exact create source was retired. */ export function cleanupSandboxCreateSource( diff --git a/src/lib/onboard/sandbox-gpu-create-identity-deadline.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-deadline.test.ts new file mode 100644 index 00000000000..9cebdea24e3 --- /dev/null +++ b/src/lib/onboard/sandbox-gpu-create-identity-deadline.test.ts @@ -0,0 +1,154 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + streamSandboxCreate: vi.fn(), + waitForCreatedSandboxReadyWithTrace: vi.fn(), + printReadinessFailure: vi.fn(), + enforceDockerGpuPatchPreserveNetwork: vi.fn(), + verifyGpuSandboxAccessAfterReady: vi.fn(), + createDockerGpuSandboxCreatePatch: vi.fn(), + printSandboxCreateFailureDiagnostics: vi.fn(), + collectDockerGpuPatchDiagnostics: vi.fn(), + queryOpenShellDockerSandboxContainers: vi.fn(), + queryOpenShellDockerSandboxRuntimeSnapshot: vi.fn(), +})); + +vi.mock("../sandbox/create-stream", () => ({ + streamSandboxCreate: mocks.streamSandboxCreate, +})); +vi.mock("./sandbox-readiness-tracing", async (importOriginal) => ({ + ...(await importOriginal()), + waitForCreatedSandboxReadyWithTrace: mocks.waitForCreatedSandboxReadyWithTrace, + printReadinessFailure: mocks.printReadinessFailure, +})); +vi.mock("./docker-gpu-local-inference", () => ({ + enforceDockerGpuPatchPreserveNetwork: mocks.enforceDockerGpuPatchPreserveNetwork, + verifyGpuSandboxAccessAfterReady: mocks.verifyGpuSandboxAccessAfterReady, +})); +vi.mock("./docker-gpu-sandbox-create", () => ({ + createDockerGpuSandboxCreatePatch: mocks.createDockerGpuSandboxCreatePatch, +})); +vi.mock("./sandbox-create-failure", () => ({ + printSandboxCreateFailureDiagnostics: mocks.printSandboxCreateFailureDiagnostics, +})); +vi.mock("./docker-gpu-patch", async (importOriginal) => ({ + ...(await importOriginal()), + collectDockerGpuPatchDiagnostics: mocks.collectDockerGpuPatchDiagnostics, +})); +vi.mock("./openshell-docker-sandbox-containers", async (importOriginal) => ({ + ...(await importOriginal()), + queryOpenShellDockerSandboxContainers: mocks.queryOpenShellDockerSandboxContainers, + queryOpenShellDockerSandboxRuntimeSnapshot: mocks.queryOpenShellDockerSandboxRuntimeSnapshot, +})); + +import { + NEMOCLAW_CREATE_ATTEMPT_LABEL, + NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH, +} from "../adapters/openshell/sandbox-identity"; +import { + createGpuFlowDeps, + createGpuFlowInput, + createGpuPatchFixture, + resetGpuFlowMocks, + setupGpuFlowMocks, +} from "./__test-helpers__/sandbox-gpu-create-flow"; +import { runSandboxGpuCreateFlow } from "./sandbox-gpu-create-flow"; + +function sandboxListJson( + sandboxId: string, + createAttemptNonce: string, + incomplete = false, +): string { + return JSON.stringify([ + { + id: sandboxId, + name: "alpha", + labels: { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: createAttemptNonce }, + resource_version: incomplete ? null : 1, + created_at: incomplete ? null : "2026-08-25T00:00:00Z", + phase: incomplete ? null : "Ready", + current_policy_version: incomplete ? null : 1, + }, + ]); +} + +function createAttemptNonce(args: readonly string[]): string { + const labelIndex = args.indexOf("--label"); + return (args[labelIndex + 1] ?? "").slice(NEMOCLAW_CREATE_ATTEMPT_LABEL.length + 1); +} + +beforeEach(() => setupGpuFlowMocks(mocks)); +afterEach(resetGpuFlowMocks); + +describe("created sandbox identity settlement deadline", () => { + it("allows identity-bound post-create effects after the former 30-second cap (#10652)", async () => { + let nonce = ""; + let nowMs = 0; + const input = createGpuFlowInput(); + input.sandboxGpuConfig = { + mode: "0", + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }; + input.gpuRoutePlan = "none"; + input.initialGpuRoute = "none"; + input.createArgv = ["openshell", "sandbox", "create", "--name", "alpha", "--", "agent"]; + input.sandboxReadyTimeoutSecs = 90; + input.persistRetainedSandboxRecovery = vi.fn(() => true); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { + nonce = createAttemptNonce(args); + expect(nonce).toMatch(/^[0-9a-f]{62}$/u); + expect(nonce).toHaveLength(NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH); + expect(options.readyCheck?.()).toBe(false); + expect(options.readyCheck?.()).toBe(true); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + + const deps = createGpuFlowDeps(); + deps.publicationNow = () => nowMs; + vi.mocked(deps.sleep).mockImplementation((seconds) => { + nowMs += seconds * 1_000; + }); + deps.installPortableDemoLifecycle = vi.fn(() => "generation-1"); + vi.mocked(deps.runCaptureOpenshell) + .mockReturnValueOnce("alpha Ready") + .mockImplementationOnce(() => sandboxListJson("alpha-sandbox-id", nonce, true)) + .mockReturnValueOnce("alpha Ready") + .mockImplementationOnce(() => sandboxListJson("alpha-sandbox-id", nonce)) + .mockImplementationOnce(() => { + nowMs += 31_000; + return sandboxListJson("alpha-sandbox-id", nonce, true); + }) + .mockImplementationOnce(() => sandboxListJson("alpha-sandbox-id", nonce)); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); + + expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledExactlyOnceWith({ + sandboxId: "alpha-sandbox-id", + liveIdentityFingerprint: expect.stringMatching(/^[0-9a-f]{64}$/u), + createAttemptNonce: nonce, + route: "none", + }); + expect(input.revalidateVerifiedSandboxBeforeEffect).toHaveBeenCalledWith( + "apply runtime patch for sandbox 'alpha'", + ); + expect(patch.ensureApplied).toHaveBeenCalledOnce(); + expect(patch.commitAfterReady).toHaveBeenCalledOnce(); + const postCreateIdentityTimeout = vi.mocked(deps.runCaptureOpenshell).mock.calls[4]?.[1] + ?.timeout; + expect(postCreateIdentityTimeout).toBeGreaterThan(30_000); + expect(postCreateIdentityTimeout).toBeLessThanOrEqual(90_000); + expect(deps.sleep).toHaveBeenCalledExactlyOnceWith(0.25); + }); +}); diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 5193b57e383..95915dc21da 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -630,160 +630,6 @@ describe("created sandbox identity gate", () => { expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); }); - it("settles the exact created sandbox before ending the Ready handoff and post-create effects (#9211)", async () => { - const events: string[] = []; - let nonce = ""; - const input = noGpuInput(); - const patch = createGpuPatchFixture(); - input.verifyCreatedSandboxBeforeEffects = vi.fn(async (identity) => { - events.push("verify-created"); - expect(identity).toEqual({ - sandboxId: "alpha-sandbox-id", - liveIdentityFingerprint: expect.stringMatching(/^[0-9a-f]{64}$/u), - createAttemptNonce: expect.stringMatching(/^[0-9a-f]{62}$/u), - route: "none", - }); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); - }); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn((operation) => - events.push(`revalidate:${operation}`), - ); - patch.exitOnPatchError.mockImplementation(() => events.push("runtime-check")); - patch.ensureApplied.mockImplementation(() => events.push("runtime-patch")); - patch.waitForSupervisorReconnectIfNeeded.mockImplementation(() => events.push("reconnect")); - patch.commitAfterReady.mockImplementation(() => events.push("commit")); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { - events.push("create"); - expect(options.onPoll).toBeUndefined(); - expect(options.waitForReadyTermination).toBe(true); - expect(args.indexOf("--label")).toBeGreaterThan(0); - expect(args.indexOf("--label")).toBeLessThan(args.indexOf("--")); - nonce = createAttemptNonce(args); - expect(nonce).toMatch(/^[0-9a-f]{62}$/u); - expect(nonce).toHaveLength(NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH); - expect(nonce.length).toBeLessThanOrEqual(63); - expect(options.readyCheck?.()).toBe(false); - expect(options.readyCheck?.()).toBe(true); - return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; - }); - mocks.waitForCreatedSandboxReadyWithTrace.mockImplementation(() => { - events.push("readiness"); - return { ready: true, reason: "ready", failurePhase: null }; - }); - const deps = createGpuFlowDeps(); - vi.mocked(deps.sleep).mockImplementation(() => { - events.push("identity-settle"); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - }); - deps.installPortableDemoLifecycle = vi.fn(() => { - events.push("portable-lifecycle"); - return "generation-1"; - }); - vi.mocked(deps.runCaptureOpenshell) - .mockImplementationOnce((args) => { - expect(args).not.toContain("--selector"); - events.push("ready-visible"); - return "alpha Ready"; - }) - .mockImplementationOnce((args) => { - expect(args).toContain("--selector"); - events.push("identity-metadata-pending"); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - return sandboxListJson( - "alpha-sandbox-id", - { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }, - { - resource_version: null, - created_at: null, - phase: null, - current_policy_version: null, - }, - ); - }) - .mockImplementationOnce((args) => { - expect(args).not.toContain("--selector"); - events.push("ready-visible-again"); - return "alpha Ready"; - }) - .mockImplementationOnce((args) => { - expect(args).toContain("--selector"); - events.push("identity-matched"); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - return sandboxListJson("alpha-sandbox-id", { - [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, - }); - }) - .mockImplementationOnce((args) => { - expect(args).toContain("--selector"); - events.push("identity-revalidated"); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - return sandboxListJson("alpha-sandbox-id", { - [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, - }); - }); - - await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); - - expect(events).toEqual([ - "create", - "ready-visible", - "identity-metadata-pending", - "ready-visible-again", - "identity-matched", - "identity-revalidated", - "verify-created", - "revalidate:validate runtime patch for sandbox 'alpha'", - "runtime-check", - "revalidate:apply runtime patch for sandbox 'alpha'", - "runtime-patch", - "reconnect", - "revalidate:reconnect sandbox supervisor for 'alpha'", - "readiness", - "revalidate:commit runtime readiness for sandbox 'alpha'", - "commit", - "revalidate:record portable lifecycle for sandbox 'alpha'", - "portable-lifecycle", - ]); - expect(deps.runCaptureOpenshell).toHaveBeenNthCalledWith( - 2, - [ - "sandbox", - "list", - "-g", - "nemoclaw", - "--selector", - `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`, - "--output", - "json", - "--limit", - "2", - ], - { - ignoreError: false, - timeout: expect.any(Number), - maxBuffer: 1024 * 1024, - killSignal: "SIGKILL", - killProcessTreeOnTimeout: true, - }, - ); - const firstIdentityTimeout = vi.mocked(deps.runCaptureOpenshell).mock.calls[1]?.[1]?.timeout; - expect(firstIdentityTimeout).toEqual(expect.any(Number)); - expect(firstIdentityTimeout as number).toBeGreaterThan(0); - expect(firstIdentityTimeout as number).toBeLessThanOrEqual(30_000); - expect(deps.runCaptureOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "get", "-g", "nemoclaw", "alpha"], - expect.anything(), - ); - expect(deps.sleep).not.toHaveBeenCalled(); - }); - it("carries Hermes receipt authority from selector settlement through publication lookup (#10423)", async () => { const events: string[] = []; let nonce = ""; @@ -1196,10 +1042,11 @@ describe("created sandbox identity gate", () => { expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); }); - it("persists create-attempt recovery when Ready identity settlement reaches its deadline (#9211)", async () => { + it("persists create-attempt recovery when Ready identity settlement reaches its configured deadline (#9211)", async () => { const events: string[] = []; let nonce = ""; const input = noGpuInput(); + input.sandboxReadyTimeoutSecs = 60; input.verifyCreatedSandboxBeforeEffects = vi.fn(); input.persistRetainedSandboxRecovery = vi.fn(() => { events.push("persist-recovery"); @@ -1215,7 +1062,7 @@ describe("created sandbox identity gate", () => { let nowMs = 0; deps.publicationNow = () => nowMs; vi.mocked(deps.runCaptureOpenshell).mockImplementation(() => { - nowMs = 30_000; + nowMs = 60_000; return "[]"; }); From 9ec635c57520d4685181ee7e37a3ff386969cb7f Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 05:45:16 -0700 Subject: [PATCH 37/51] fix(onboard): register recovery retry lazily Signed-off-by: Prekshi Vyas --- .../sandbox-create/orchestration.test.ts | 23 +++++++++++++++---- .../onboard/sandbox-create/orchestration.ts | 22 +++++++++++------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 999481f5ea5..5d43ee56f86 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -410,17 +410,18 @@ describe("retained create recovery persistence", () => { expect(caught).toBeInstanceOf(AggregateError); expect(operation).toHaveBeenCalledOnce(); expect(recordRecovery).toHaveBeenCalledOnce(); + expect(exitHandlers).toHaveLength(1); - exitHandlers[0](); + exitHandlers[0]!(); expect(recordRecovery).toHaveBeenCalledTimes(2); expect(operation).toHaveBeenCalledOnce(); - exitHandlers[0](); + exitHandlers[0]!(); expect(recordRecovery).toHaveBeenCalledTimes(2); }, ); - it("installs independent recovery retry owners for two operations from one factory (#10652)", async () => { + it("does not install retry handlers for operations without failed recovery writes (#10652)", async () => { const exitHandlers: Array<() => void> = []; const captureExitHandler = ((event: string | symbol, handler: (...args: unknown[]) => void) => { event === "exit" && exitHandlers.push(handler as () => void); @@ -438,8 +439,18 @@ describe("retained create recovery persistence", () => { processOn.mockRestore(); } - expect(exitHandlers).toHaveLength(2); - expect(exitHandlers[0]).not.toBe(exitHandlers[1]); + expect(exitHandlers).toHaveLength(0); + }); + + it("does not install a retry handler when recovery persistence succeeds (#10652)", () => { + const registerExitHandler = vi.fn(); + const owner = installPostCreateRecoveryRetryOwner({ registerExitHandler }); + const writer = vi.fn(); + + owner.record(writer); + + expect(writer).toHaveBeenCalledOnce(); + expect(registerExitHandler).not.toHaveBeenCalled(); }); it("retries both failed writers held by independent operation owners (#10652)", () => { @@ -462,6 +473,8 @@ describe("retained create recovery persistence", () => { expect(() => owners[0]?.record(writers[0]!)).toThrow(/operation 1/u); expect(() => owners[1]?.record(writers[1]!)).toThrow(/operation 2/u); expect(writers.map((writer) => writer.mock.calls.length)).toEqual([1, 1]); + expect(exitHandlers).toHaveLength(2); + expect(exitHandlers[0]).not.toBe(exitHandlers[1]); exitHandlers.forEach((handler) => handler()); diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 20e84f5df86..0cf574e109f 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -315,20 +315,32 @@ export function installPostCreateRecoveryRetryOwner( } = {}, ): PostCreateRecoveryRetryOwner { let pending: (() => void) | null = null; + let exitHandlerRegistered = false; const log = options.log ?? ((message: string) => console.error(message)); - const attemptPending = (propagateFailure: boolean): void => { + const register = + options.registerExitHandler ?? + ((handler: () => void) => { + process.on("exit", handler); + }); + const ensureExitHandler = (): void => { + if (exitHandlerRegistered) return; + register(() => attemptPending(false)); + exitHandlerRegistered = true; + }; + function attemptPending(propagateFailure: boolean): void { if (pending === null) return; const attempt = pending; try { attempt(); if (pending === attempt) pending = null; } catch (error) { + ensureExitHandler(); if (propagateFailure) throw error; log( " NemoClaw still could not save the retained sandbox recovery record; the recovery-only session remains blocked for administrator recovery.", ); } - }; + } const owner: PostCreateRecoveryRetryOwner = { record(recordRecovery): void { attemptPending(true); @@ -336,12 +348,6 @@ export function installPostCreateRecoveryRetryOwner( attemptPending(true); }, }; - const register = - options.registerExitHandler ?? - ((handler: () => void) => { - process.on("exit", handler); - }); - register(() => attemptPending(false)); return owner; } From d925401a10b1fa812b99ac13be622038a712bbbd Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 06:05:23 -0700 Subject: [PATCH 38/51] test(onboard): prove lazy recovery retry registration Signed-off-by: Prekshi Vyas --- .../sandbox-create/orchestration.test.ts | 35 +++++-------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 5d43ee56f86..839d14bb5ea 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -13,7 +13,6 @@ import { assertApfCreateIntent, completeHermesPortableSandboxRegistration, createProviderEffectBoundary, - createSandboxWithBaseImageResolution, finalizeCreatedSandboxBeforeHermesCredentialReconciliation, hasManagedMcpRebuildHandoff, installPostCreateRecoveryRetryOwner, @@ -421,35 +420,19 @@ describe("retained create recovery persistence", () => { }, ); - it("does not install retry handlers for operations without failed recovery writes (#10652)", async () => { - const exitHandlers: Array<() => void> = []; - const captureExitHandler = ((event: string | symbol, handler: (...args: unknown[]) => void) => { - event === "exit" && exitHandlers.push(handler as () => void); - return process; - }) as typeof process.on; - const processOn = vi.spyOn(process, "on").mockImplementation(captureExitHandler); - const createSandbox = createSandboxWithBaseImageResolution({} as never) as unknown as ( - ...args: unknown[] - ) => Promise; - - try { - await expect(createSandbox()).rejects.toThrow(); - await expect(createSandbox()).rejects.toThrow(); - } finally { - processOn.mockRestore(); - } - - expect(exitHandlers).toHaveLength(0); - }); - - it("does not install a retry handler when recovery persistence succeeds (#10652)", () => { + it("does not install a retry handler until a recovery write fails (#10652)", () => { const registerExitHandler = vi.fn(); const owner = installPostCreateRecoveryRetryOwner({ registerExitHandler }); - const writer = vi.fn(); + const firstWriter = vi.fn(); + const secondWriter = vi.fn(); + + owner.record(firstWriter); + expect(firstWriter).toHaveBeenCalledOnce(); + expect(registerExitHandler).not.toHaveBeenCalled(); - owner.record(writer); + owner.record(secondWriter); - expect(writer).toHaveBeenCalledOnce(); + expect(secondWriter).toHaveBeenCalledOnce(); expect(registerExitHandler).not.toHaveBeenCalled(); }); From 0df0e81c538b1e0868bdc8cd3b85db9987ec83eb Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 06:28:38 -0700 Subject: [PATCH 39/51] refactor(onboard): inline sandbox publication wait Signed-off-by: Prekshi Vyas --- .../checks/run-managed-image-openshell-e2e.ts | 3 + src/lib/agent/gateway-readiness.ts | 5 +- src/lib/onboard.ts | 25 +- .../sandbox-gpu-create-flow.ts | 2 + src/lib/onboard/readiness-wait.ts | 6 +- .../onboard/sandbox-create/orchestration.ts | 2 + src/lib/onboard/sandbox-gpu-create-flow.ts | 245 +++++++++--------- .../onboard/sandbox-gpu-create-run-attempt.ts | 102 ++++---- src/lib/onboard/sandbox-readiness-tracing.ts | 26 +- ...d-image-protected-runtime-contract.test.ts | 2 + 10 files changed, 204 insertions(+), 214 deletions(-) diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 349a8f40005..44046e9358c 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -161,6 +161,7 @@ type OnboardModule = { openshellArgv(args: string[]): string[]; runOpenshell(args: string[], opts?: Record): ReturnType; runCaptureOpenshell(args: string[], opts?: Record): string; + isSandboxReady(output: string, sandboxName: string): boolean; sleepSeconds(seconds: number): void; startGatewayForRecovery(options: { gatewayName: string; gatewayPort: number }): Promise; }; @@ -169,6 +170,7 @@ const REQUIRED_ONBOARD_OPERATIONS = [ "openshellArgv", "runOpenshell", "runCaptureOpenshell", + "isSandboxReady", "sleepSeconds", "startGatewayForRecovery", ] as const satisfies readonly (keyof OnboardModule)[]; @@ -1053,6 +1055,7 @@ async function run options.sleepSeconds(ms / 1000), }); - return waitOptions !== null && waitUntil(options.probe, waitOptions); + return waitOptions !== null && runReadinessWait(options.probe, waitOptions); } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index b1eca0570db..05ae8dfc2a6 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -429,11 +429,7 @@ const promptValidatedSandboxName = sandboxAgent.createPromptValidatedSandboxName exit: process.exit, }); const modelRouter: typeof import("./onboard/model-router") = require("./onboard/model-router"); -const { - isRoutedInferenceProvider, - loadBlueprintProfile, - reconcileModelRouter, -} = modelRouter; +const { isRoutedInferenceProvider, loadBlueprintProfile, reconcileModelRouter } = modelRouter; const routedInference: typeof import("./onboard/routed-inference") = require("./onboard/routed-inference"); const { OnboardRuntimeBoundary, @@ -1508,17 +1504,13 @@ const gatewayRecovery = createGatewayRecoveryOrchestration({ startGatewayWithOptions: gatewayStart.startGatewayWithOptions, }); -const { - recoverGatewayRuntime, - startDockerDriverGateway, - startGateway, - startGatewayForRecovery, -} = createGatewayLifecycleApplication({ - dockerDriverStart: dockerDriverGatewayStart, - recovery: gatewayRecovery, - registration: gatewayRegistration, - start: gatewayStart, -}); +const { recoverGatewayRuntime, startDockerDriverGateway, startGateway, startGatewayForRecovery } = + createGatewayLifecycleApplication({ + dockerDriverStart: dockerDriverGatewayStart, + recovery: gatewayRecovery, + registration: gatewayRegistration, + start: gatewayStart, + }); const { getSandboxRuntimeRegistryFields, hasSandboxGpuDrift, updateReusedSandboxMetadata } = sandboxRegistryMetadata.createSandboxRegistryMetadataHelpers({ @@ -1577,6 +1569,7 @@ const sandboxCreateOrchestrationRuntime = { hasSandboxGpuDrift, inferenceConfig, inspectSandboxForCreate, + isSandboxReady, isLinuxDockerDriverGatewayEnabled, isNonInteractive, isRecreateSandbox, diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index 9d13143e74c..f23bbc5a2bd 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -8,6 +8,7 @@ import path from "node:path"; import { expect, vi } from "vitest"; import { createCliOpenShellSandboxObserver } from "../../adapters/openshell/sandbox-observer-cli"; +import { isSandboxReady } from "../../state/gateway"; import type { CheckpointPortableRuntimeAuthority } from "../../state/onboard-checkpoint-types"; import type { SandboxGpuProofResult } from "../../state/registry"; import type { ManagedBootstrapRuntimeCreateLifecycleInput } from "../managed-bootstrap/runtime-create"; @@ -111,6 +112,7 @@ export function createGpuFlowDeps( }), sleep: vi.fn(), openshellArgv: vi.fn((args: string[]) => ["openshell", ...args]), + isSandboxReady, verifyDirectSandboxGpu: vi.fn(() => VERIFIED_GPU_PROOF), }; } diff --git a/src/lib/onboard/readiness-wait.ts b/src/lib/onboard/readiness-wait.ts index f6d572c1730..2300ea971d0 100644 --- a/src/lib/onboard/readiness-wait.ts +++ b/src/lib/onboard/readiness-wait.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { WaitUntilOptions } from "../core/wait"; +import { waitUntil, type WaitUntilOptions } from "../core/wait"; const DEFAULT_INITIAL_INTERVAL_MS = 250; const DEFAULT_MAX_INTERVAL_MS = 2_000; @@ -78,6 +78,10 @@ export function createReadinessWaitOptions(options: { }; } +export function runReadinessWait(condition: () => boolean, options: WaitUntilOptions): boolean { + return waitUntil(condition, options); +} + export function formatReadinessDeadline(budgetMs: number): string { const normalizedBudgetMs = nonNegativeFinite(budgetMs); if (normalizedBudgetMs < 1000) return `${Math.ceil(normalizedBudgetMs)}ms`; diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 0cf574e109f..4edea2457bd 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -1263,6 +1263,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche hasSandboxGpuDrift, inferenceConfig, inspectSandboxForCreate, + isSandboxReady, isLinuxDockerDriverGatewayEnabled, isNonInteractive, isRecreateSandbox, @@ -2601,6 +2602,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche ), sleep: sleepSeconds, openshellArgv, + isSandboxReady, verifyDirectSandboxGpu: createGpuVerifier, }, ); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 134305ff221..0124cdae5a9 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -21,9 +21,7 @@ import { isHermesPortableLifecycleMode, } from "./experimental/hermes-portable-onboarding"; import { installPortableDemoSandboxLifecycle } from "./experimental/portable-demo-lifecycle"; -import { - buildHermesPortableOnboardingCommandAuthority, -} from "./experimental/portable-agent-lifecycle"; +import { buildHermesPortableOnboardingCommandAuthority } from "./experimental/portable-agent-lifecycle"; import { isPortableExperimentalProfile } from "./experimental/portable-profile"; import { createManagedBootstrapIdentity, @@ -242,6 +240,7 @@ export interface SandboxGpuCreateFlowDeps { /** Production callers use a monotonic clock; tests may inject post-create deadline time. */ publicationNow?: () => number; openshellArgv(args: string[]): string[]; + isSandboxReady(output: string, sandboxName: string): boolean; verifyDirectSandboxGpu(sandboxName: string): SandboxGpuProofResult; printCreateFailureDiagnostics?: ( sandboxName: string, @@ -358,129 +357,131 @@ export async function runSandboxGpuCreateFlow( } : deps, ); - const gpuCreateOutcome = await (input.resumeVerifiedCreate - ? attemptRunner.runAttempt(input.resumeVerifiedCreate.route) - : sandboxGpuCreateAttempt.executeSandboxGpuCreatePlan(input.gpuRoutePlan, { - runAttempt: attemptRunner.runAttempt, - captureNativeFailure: (failure) => { - const routeAdapter = adaptDockerGpuRouteForPatch(failure.route); - const diagnostics = collectDockerGpuPatchDiagnostics( - input.sandboxName, - { - error: failure.error, - additionalSummaryLines: routeAdapter.additionalSummaryLines, + const gpuCreateOutcome = await ( + input.resumeVerifiedCreate + ? attemptRunner.runAttempt(input.resumeVerifiedCreate.route) + : sandboxGpuCreateAttempt.executeSandboxGpuCreatePlan(input.gpuRoutePlan, { + runAttempt: attemptRunner.runAttempt, + captureNativeFailure: (failure) => { + const routeAdapter = adaptDockerGpuRouteForPatch(failure.route); + const diagnostics = collectDockerGpuPatchDiagnostics( + input.sandboxName, + { + error: failure.error, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }, + { runCaptureOpenshell: deps.runCaptureOpenshell }, + ); + if (diagnostics) console.error(` Native GPU diagnostics saved: ${diagnostics.dir}`); }, - { runCaptureOpenshell: deps.runCaptureOpenshell }, - ); - if (diagnostics) console.error(` Native GPU diagnostics saved: ${diagnostics.dir}`); - }, - cleanupNativeFailure: (failure) => { - if (input.requirePolicylessCreate) { - return refuseApfMutableNameFallbackCleanup(input.sandboxName); - } - return sandboxGpuCreateAttempt.cleanupNativeGpuFailureForFallback( - input.sandboxName, - failure, - { - runOpenshell: deps.runOpenshell, - sleep: deps.sleep, + cleanupNativeFailure: (failure) => { + if (input.requirePolicylessCreate) { + return refuseApfMutableNameFallbackCleanup(input.sandboxName); + } + return sandboxGpuCreateAttempt.cleanupNativeGpuFailureForFallback( + input.sandboxName, + failure, + { + runOpenshell: deps.runOpenshell, + sleep: deps.sleep, + }, + ); }, - ); - }, - prepareCompatibilityAttempt: async () => { - if (!input.compatibilityPolicyPath) { - throw new Error("Compatibility retry policy was not materialized."); - } - const nativeRuntimeSnapshot = attemptRunner.state.nativeRuntimeSnapshot; - if (attemptRunner.managedRouting) { - const managedBootstrap = input.managedBootstrap; - if (!managedBootstrap) { - throw new Error("Managed compatibility routing is missing bootstrap authority."); - } - const bootstrapIdentity = createManagedBootstrapIdentity(); - const heldWorkloadArgv = [ - ...renderManagedBootstrapHeldCommand( - managedBootstrap.request, - bootstrapIdentity, - managedBootstrap.intendedWorkloadArgv, - ), - ]; - const prepared = attemptRunner.managedRouting.prepareCompatibilityLaunch({ - createArgs: managedBootstrapCreateArgs(input.prebuild.createArgs, bootstrapIdentity), - currentRegistryImageRef: registryImageRef, - managedImageReference: `${managedBootstrap.image.repository}@${managedBootstrap.image.manifestDigest}`, - prebuildImageId: input.prebuild.imageId, - allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, - compatibilityPolicyPath: input.compatibilityPolicyPath, - startupCommand: heldWorkloadArgv, - runtimeSnapshot: nativeRuntimeSnapshot, - }); - attemptRunner.state.compatibilityArgv = [...prepared.createArgv]; - attemptRunner.state.compatibilityBootstrapIdentity = bootstrapIdentity; - attemptRunner.state.compatibilityHeldWorkloadArgv = heldWorkloadArgv; - registryImageRef = prepared.registryImageRef; - } else { - const prebuildImageId = input.prebuild.imageId; - const imageId = - nativeRuntimeSnapshot?.imageId ?? - (prebuildImageId && isImmutableDockerImageId(prebuildImageId) - ? prebuildImageId.toLowerCase() - : null); - if ( - !registryImageRef && - nativeRuntimeSnapshot?.bookkeepingImageRef && - !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) - ) { - registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; - } - const compatibilityArgs = renderCompatibilityFallbackCreateArgs( - input.prebuild.createArgs, - { - imageRef: imageId, - allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, - compatibilityPolicyPath: input.compatibilityPolicyPath, - }, - ); - attemptRunner.state.compatibilityArgv = deps.openshellArgv([ - "sandbox", - "create", - ...compatibilityArgs, - "--", - ...input.sandboxStartupCommand, - ]); - } - if (attemptRunner.state.compatibilityArgv.length === 0) { - throw new Error("Compatibility sandbox create executable is missing."); - } - }, - activateCompatibilityAttempt: async () => { - if (!input.managedBootstrap) { - await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( - input.provider, - input.sandboxGpuConfig, - { - dockerDriverGateway: input.dockerDriverGateway, - selectedRoute: "compatibility", - gatewayPort: input.gatewayPort, - log: console.log, - }, - ); - } - input.sandboxGpuConfig.sandboxGpuProof = null; - }, - traceEvent: addTraceEvent, - })) - .catch((error: unknown) => { - if (error instanceof ManagedBootstrapRecoveryBlockedError) { - exitForManagedBootstrapRecovery(error); - } - throw error; - }); + prepareCompatibilityAttempt: async () => { + if (!input.compatibilityPolicyPath) { + throw new Error("Compatibility retry policy was not materialized."); + } + const nativeRuntimeSnapshot = attemptRunner.state.nativeRuntimeSnapshot; + if (attemptRunner.managedRouting) { + const managedBootstrap = input.managedBootstrap; + if (!managedBootstrap) { + throw new Error("Managed compatibility routing is missing bootstrap authority."); + } + const bootstrapIdentity = createManagedBootstrapIdentity(); + const heldWorkloadArgv = [ + ...renderManagedBootstrapHeldCommand( + managedBootstrap.request, + bootstrapIdentity, + managedBootstrap.intendedWorkloadArgv, + ), + ]; + const prepared = attemptRunner.managedRouting.prepareCompatibilityLaunch({ + createArgs: managedBootstrapCreateArgs( + input.prebuild.createArgs, + bootstrapIdentity, + ), + currentRegistryImageRef: registryImageRef, + managedImageReference: `${managedBootstrap.image.repository}@${managedBootstrap.image.manifestDigest}`, + prebuildImageId: input.prebuild.imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + startupCommand: heldWorkloadArgv, + runtimeSnapshot: nativeRuntimeSnapshot, + }); + attemptRunner.state.compatibilityArgv = [...prepared.createArgv]; + attemptRunner.state.compatibilityBootstrapIdentity = bootstrapIdentity; + attemptRunner.state.compatibilityHeldWorkloadArgv = heldWorkloadArgv; + registryImageRef = prepared.registryImageRef; + } else { + const prebuildImageId = input.prebuild.imageId; + const imageId = + nativeRuntimeSnapshot?.imageId ?? + (prebuildImageId && isImmutableDockerImageId(prebuildImageId) + ? prebuildImageId.toLowerCase() + : null); + if ( + !registryImageRef && + nativeRuntimeSnapshot?.bookkeepingImageRef && + !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) + ) { + registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; + } + const compatibilityArgs = renderCompatibilityFallbackCreateArgs( + input.prebuild.createArgs, + { + imageRef: imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + }, + ); + attemptRunner.state.compatibilityArgv = deps.openshellArgv([ + "sandbox", + "create", + ...compatibilityArgs, + "--", + ...input.sandboxStartupCommand, + ]); + } + if (attemptRunner.state.compatibilityArgv.length === 0) { + throw new Error("Compatibility sandbox create executable is missing."); + } + }, + activateCompatibilityAttempt: async () => { + if (!input.managedBootstrap) { + await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( + input.provider, + input.sandboxGpuConfig, + { + dockerDriverGateway: input.dockerDriverGateway, + selectedRoute: "compatibility", + gatewayPort: input.gatewayPort, + log: console.log, + }, + ); + } + input.sandboxGpuConfig.sandboxGpuProof = null; + }, + traceEvent: addTraceEvent, + }) + ).catch((error: unknown) => { + if (error instanceof ManagedBootstrapRecoveryBlockedError) { + exitForManagedBootstrapRecovery(error); + } + throw error; + }); if (!gpuCreateOutcome.ok) { const preparationRefused = - "preparationRefused" in gpuCreateOutcome - ? gpuCreateOutcome.preparationRefused - : undefined; + "preparationRefused" in gpuCreateOutcome ? gpuCreateOutcome.preparationRefused : undefined; const cleanupRefused = "cleanupRefused" in gpuCreateOutcome ? gpuCreateOutcome.cleanupRefused : undefined; const nativeCleanupHandoff = diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index e33ef7261a1..b8e667afa37 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -22,7 +22,6 @@ import { printSandboxCreateRecoveryHints } from "../build-context"; import { streamSandboxCreate, type StreamSandboxCreateResult } from "../sandbox/create-stream"; import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-ready-gate"; import { redact, redactFullWithUrls } from "../security/redact"; -import { isSandboxReady } from "../state/gateway"; import type { SandboxGpuProofResult } from "../state/registry"; import { classifySandboxCreateFailure } from "../validation"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; @@ -55,6 +54,7 @@ import { } from "./sandbox-recreate-probe"; import type { CreatedSandboxReadyIdentityCheck } from "./sandbox-readiness-tracing"; import * as sandboxReadinessTracing from "./sandbox-readiness-tracing"; +import { createReadinessWaitOptions, runReadinessWait } from "./readiness-wait"; import { addTraceEvent } from "./tracing"; type NativeRuntimeSnapshot = ManagedBootstrapRuntimeSnapshot; @@ -250,10 +250,7 @@ function normalizedOpenShellCommandOutput(result: OpenShellCommandResult): strin } function boundedPublicationDiagnostic(value: string): string { - return redactCreatedSandboxFailureDiagnostic( - value, - CREATED_SANDBOX_PUBLICATION_DIAGNOSTIC_LIMIT, - ); + return redactCreatedSandboxFailureDiagnostic(value, CREATED_SANDBOX_PUBLICATION_DIAGNOSTIC_LIMIT); } function publicationFailureDiagnostic(result: OpenShellCommandResult): string { @@ -536,54 +533,61 @@ function waitForCreatedOpenShellSandboxPublication( deps: SandboxGpuCreateFlowDeps, deadline: PostCreateReadinessDeadline, ): void { - const published = sandboxReadinessTracing.waitForCreatedSandboxPublication({ + const waitOptions = createReadinessWaitOptions({ budgetMs: remainingPostCreateReadinessMs(deadline), - pollIntervalMs: CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_MS, + initialIntervalMs: CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_MS, + maxIntervalMs: CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_MS, now: deadline.now, - sleep: deps.sleep, - probe: (getRemainingMs) => { - const result = deps.runOpenshell( - ["sandbox", "get", "-g", input.gatewayName, input.sandboxName], - { - ignoreError: true, - suppressOutput: true, - timeout: Math.min(SANDBOX_READY_PROBE_TIMEOUT_MS, getRemainingMs()), - killSignal: "SIGKILL", - }, - ); - if (result.status === 0 && !result.error) { - const publishedSandboxId = parseOpenShellSandboxId(String(result.stdout ?? "")); - if (!publishedSandboxId) { - throw new Error( - `OpenShell returned no exact durable ID for created sandbox '${input.sandboxName}'.`, + sleep: (milliseconds) => deps.sleep(milliseconds / 1_000), + }); + const deadlineMs = waitOptions?.deadlineMs; + const now = waitOptions?.now; + const published = + waitOptions && deadlineMs !== undefined && now + ? runReadinessWait(() => { + const getRemainingMs = () => Math.max(1, deadlineMs - now()); + const result = deps.runOpenshell( + ["sandbox", "get", "-g", input.gatewayName, input.sandboxName], + { + ignoreError: true, + suppressOutput: true, + timeout: Math.min(SANDBOX_READY_PROBE_TIMEOUT_MS, getRemainingMs()), + killSignal: "SIGKILL", + }, ); - } - if (publishedSandboxId !== sandboxId) { + if (result.status === 0 && !result.error) { + const publishedSandboxId = parseOpenShellSandboxId(String(result.stdout ?? "")); + if (!publishedSandboxId) { + throw new Error( + `OpenShell returned no exact durable ID for created sandbox '${input.sandboxName}'.`, + ); + } + if (publishedSandboxId !== sandboxId) { + throw new Error( + `Created sandbox '${input.sandboxName}' changed identity before identity verification completed.`, + ); + } + return true; + } + const output = normalizedOpenShellCommandOutput(result); + const failedCleanly = + !result.error && + result.status !== null && + !("signal" in result && result.signal) && + result.status !== 0; + if ( + failedCleanly && + (OPENSHELL_SANDBOX_NOT_READY.test(output) || + isExplicitMissingSandboxGatewayOutput(output, input.sandboxName)) + ) { + return false; + } + const diagnostic = publicationFailureDiagnostic(result); throw new Error( - `Created sandbox '${input.sandboxName}' changed identity before identity verification completed.`, + `OpenShell could not verify publication of created sandbox '${input.sandboxName}'${diagnostic ? `: ${diagnostic}` : "."}`, ); - } - return true; - } - const output = normalizedOpenShellCommandOutput(result); - const failedCleanly = - !result.error && - result.status !== null && - !("signal" in result && result.signal) && - result.status !== 0; - if ( - failedCleanly && - (OPENSHELL_SANDBOX_NOT_READY.test(output) || - isExplicitMissingSandboxGatewayOutput(output, input.sandboxName)) - ) { - return false; - } - const diagnostic = publicationFailureDiagnostic(result); - throw new Error( - `OpenShell could not verify publication of created sandbox '${input.sandboxName}'${diagnostic ? `: ${diagnostic}` : "."}`, - ); - }, - }); + }, waitOptions) + : false; if (!published) { throw new Error( `Created sandbox '${input.sandboxName}' did not become visible through its owning gateway before identity verification completed.`, @@ -913,7 +917,7 @@ export function createSandboxGpuCreateAttemptRunner( ignoreError: true, timeout: SANDBOX_READY_PROBE_TIMEOUT_MS, }); - const ready = isSandboxReady(list, input.sandboxName); + const ready = deps.isSandboxReady(list, input.sandboxName); if (!ready || !createAttemptNonce) return ready; const observation = observeCreatedOpenShellSandboxId( { diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index 84463d889f8..f597e60aea6 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -15,12 +15,13 @@ import { createCliOpenShellSandboxObserver, type CliOpenShellSandboxObserverDeps, } from "../adapters/openshell/sandbox-observer-cli"; -import { waitUntil, waitUntilAsync } from "../core/wait"; +import { waitUntilAsync } from "../core/wait"; import { envInt } from "./env"; import { createReadinessWaitOptions, formatReadinessDeadline, getLegacyPollDeadlineBudgetMs, + runReadinessWait, } from "./readiness-wait"; import { addTraceEvent, withDashboardReadinessTrace, withSandboxReadinessTrace } from "./tracing"; @@ -130,27 +131,6 @@ export interface SandboxReadyWaitOptions extends SandboxReadyWaitDeps { delaySeconds: number; } -/** Wait for one created-sandbox publication condition inside a shared bounded deadline. */ -export function waitForCreatedSandboxPublication(options: { - budgetMs: number; - pollIntervalMs: number; - probe: (getRemainingMs: () => number) => boolean; - sleep: (seconds: number) => void; - now?: () => number; -}): boolean { - const waitOptions = createReadinessWaitOptions({ - budgetMs: options.budgetMs, - initialIntervalMs: options.pollIntervalMs, - maxIntervalMs: options.pollIntervalMs, - now: options.now, - sleep: (milliseconds) => options.sleep(milliseconds / 1_000), - }); - const deadlineMs = waitOptions?.deadlineMs; - const now = waitOptions?.now; - if (!waitOptions || deadlineMs === undefined || !now) return false; - return waitUntil(() => options.probe(() => Math.max(1, deadlineMs - now())), waitOptions); -} - export async function observeOpenShellSandbox( observer: OpenShellSandboxObserver, target: OpenShellGatewayTarget, @@ -671,7 +651,7 @@ export function waitForDashboardReadyWithTrace(options: { } const ready = waitOptions !== null && - waitUntil(() => { + runReadinessWait(() => { attempt += 1; const readyOutput = runCaptureOpenshell( [ diff --git a/test/inference/managed/managed-image-protected-runtime-contract.test.ts b/test/inference/managed/managed-image-protected-runtime-contract.test.ts index d338b17447a..d9c82904396 100644 --- a/test/inference/managed/managed-image-protected-runtime-contract.test.ts +++ b/test/inference/managed/managed-image-protected-runtime-contract.test.ts @@ -160,6 +160,7 @@ describe("protected managed-image runtime contract", () => { "openshellArgv", "runOpenshell", "runCaptureOpenshell", + "isSandboxReady", "sleepSeconds", "startGatewayForRecovery", ] as const)( @@ -175,6 +176,7 @@ describe("protected managed-image runtime contract", () => { default: { openshellArgv: () => [], runCaptureOpenshell: () => "", + isSandboxReady: () => false, sleepSeconds: () => undefined, startGatewayForRecovery: async () => undefined, }, From f25e9b3ce6c6a0a137bfb449e11c81449b14ce17 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 06:38:01 -0700 Subject: [PATCH 40/51] test(onboard): keep identity deadline proof behavioral Signed-off-by: Prekshi Vyas --- src/lib/onboard/sandbox-gpu-create-identity-deadline.test.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/lib/onboard/sandbox-gpu-create-identity-deadline.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-deadline.test.ts index 9cebdea24e3..85ce71f2238 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-deadline.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-deadline.test.ts @@ -145,10 +145,6 @@ describe("created sandbox identity settlement deadline", () => { ); expect(patch.ensureApplied).toHaveBeenCalledOnce(); expect(patch.commitAfterReady).toHaveBeenCalledOnce(); - const postCreateIdentityTimeout = vi.mocked(deps.runCaptureOpenshell).mock.calls[4]?.[1] - ?.timeout; - expect(postCreateIdentityTimeout).toBeGreaterThan(30_000); - expect(postCreateIdentityTimeout).toBeLessThanOrEqual(90_000); expect(deps.sleep).toHaveBeenCalledExactlyOnceWith(0.25); }); }); From b257062ffc6228b521ebbde59617d0c509b6dae5 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 07:16:46 -0700 Subject: [PATCH 41/51] refactor(onboard): centralize readiness ownership Signed-off-by: Prekshi Vyas --- ci/source-architecture-budget.json | 4 +- .../checks/run-managed-image-openshell-e2e.ts | 6 +++ src/lib/agent/gateway-readiness.ts | 5 ++- .../{onboard => core}/readiness-wait.test.ts | 2 +- src/lib/{onboard => core}/readiness-wait.ts | 6 +-- src/lib/onboard.ts | 1 + .../sandbox-gpu-create-flow.ts | 1 + src/lib/onboard/gateway-health-wait.ts | 2 +- src/lib/onboard/gateway-recovery.ts | 2 +- .../onboard/sandbox-create/orchestration.ts | 19 +++++--- ...ox-gpu-create-flow-hermes-portable.test.ts | 45 +------------------ src/lib/onboard/sandbox-gpu-create-flow.ts | 19 ++------ .../onboard/sandbox-gpu-create-run-attempt.ts | 13 ++++-- src/lib/onboard/sandbox-readiness-tracing.ts | 9 ++-- ...d-image-protected-runtime-contract.test.ts | 2 + 15 files changed, 49 insertions(+), 87 deletions(-) rename src/lib/{onboard => core}/readiness-wait.test.ts (97%) rename src/lib/{onboard => core}/readiness-wait.ts (93%) diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index b5110dd6b48..2152bc22acc 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -18,7 +18,7 @@ "src/lib/core/ports.ts": 89, "src/lib/core/shell-quote.ts": 28, "src/lib/core/url-utils.ts": 30, - "src/lib/core/wait.ts": 36, + "src/lib/core/wait.ts": 37, "src/lib/credentials/store.ts": 45, "src/lib/inference/config.ts": 30, "src/lib/messaging/channels/index.ts": 25, @@ -59,7 +59,7 @@ }, "allowedCycles": [], "maxRootFiles": { - "src/lib/onboard": 304, + "src/lib/onboard": 303, "src/lib/actions": 18, "src/lib/actions/sandbox": 182, "src/lib/state": 39, diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 44046e9358c..8f10e925ee0 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -162,6 +162,10 @@ type OnboardModule = { runOpenshell(args: string[], opts?: Record): ReturnType; runCaptureOpenshell(args: string[], opts?: Record): string; isSandboxReady(output: string, sandboxName: string): boolean; + printSandboxCreateRecoveryHints( + output: string, + options: { readonly createArgs: readonly string[] }, + ): void; sleepSeconds(seconds: number): void; startGatewayForRecovery(options: { gatewayName: string; gatewayPort: number }): Promise; }; @@ -171,6 +175,7 @@ const REQUIRED_ONBOARD_OPERATIONS = [ "runOpenshell", "runCaptureOpenshell", "isSandboxReady", + "printSandboxCreateRecoveryHints", "sleepSeconds", "startGatewayForRecovery", ] as const satisfies readonly (keyof OnboardModule)[]; @@ -1056,6 +1061,7 @@ async function run options.sleepSeconds(ms / 1000), }); - return waitOptions !== null && runReadinessWait(options.probe, waitOptions); + return waitOptions !== null && waitUntil(options.probe, waitOptions); } diff --git a/src/lib/onboard/readiness-wait.test.ts b/src/lib/core/readiness-wait.test.ts similarity index 97% rename from src/lib/onboard/readiness-wait.test.ts rename to src/lib/core/readiness-wait.test.ts index 9ccac571a05..f32763d8798 100644 --- a/src/lib/onboard/readiness-wait.test.ts +++ b/src/lib/core/readiness-wait.test.ts @@ -3,12 +3,12 @@ import { describe, expect, it, vi } from "vitest"; -import { waitUntil } from "../core/wait"; import { createReadinessWaitOptions, formatReadinessDeadline, getLegacyPollDeadlineBudgetMs, } from "./readiness-wait"; +import { waitUntil } from "./wait"; describe("readiness deadline options", () => { it("starts fast, backs off to the cap, and consumes the full deadline", () => { diff --git a/src/lib/onboard/readiness-wait.ts b/src/lib/core/readiness-wait.ts similarity index 93% rename from src/lib/onboard/readiness-wait.ts rename to src/lib/core/readiness-wait.ts index 2300ea971d0..33750ba321d 100644 --- a/src/lib/onboard/readiness-wait.ts +++ b/src/lib/core/readiness-wait.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { waitUntil, type WaitUntilOptions } from "../core/wait"; +import type { WaitUntilOptions } from "./wait"; const DEFAULT_INITIAL_INTERVAL_MS = 250; const DEFAULT_MAX_INTERVAL_MS = 2_000; @@ -78,10 +78,6 @@ export function createReadinessWaitOptions(options: { }; } -export function runReadinessWait(condition: () => boolean, options: WaitUntilOptions): boolean { - return waitUntil(condition, options); -} - export function formatReadinessDeadline(budgetMs: number): string { const normalizedBudgetMs = nonNegativeFinite(budgetMs); if (normalizedBudgetMs < 1000) return `${Math.ceil(normalizedBudgetMs)}ms`; diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 05ae8dfc2a6..2fa4025364d 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -1588,6 +1588,7 @@ const sandboxCreateOrchestrationRuntime = { path, planRegisteredExtraProviders, preparedDcodeRebuild, + printSandboxCreateRecoveryHints, promptValidatedSandboxName, promptYesNoOrDefault, providerExistsInGateway, diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index f23bbc5a2bd..4a895d146e8 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -113,6 +113,7 @@ export function createGpuFlowDeps( sleep: vi.fn(), openshellArgv: vi.fn((args: string[]) => ["openshell", ...args]), isSandboxReady, + printCreateRecoveryHints: vi.fn(), verifyDirectSandboxGpu: vi.fn(() => VERIFIED_GPU_PROOF), }; } diff --git a/src/lib/onboard/gateway-health-wait.ts b/src/lib/onboard/gateway-health-wait.ts index db1d0f88e20..600bf4055f5 100644 --- a/src/lib/onboard/gateway-health-wait.ts +++ b/src/lib/onboard/gateway-health-wait.ts @@ -3,7 +3,7 @@ import { type WaitUntilOptions, waitUntilAsync } from "../core/wait"; import { envInt } from "./env"; -import { createReadinessWaitOptions, getLegacyPollDeadlineBudgetMs } from "./readiness-wait"; +import { createReadinessWaitOptions, getLegacyPollDeadlineBudgetMs } from "../core/readiness-wait"; type RunCaptureOpenshell = (args: string[], opts?: { ignoreError?: boolean }) => string; diff --git a/src/lib/onboard/gateway-recovery.ts b/src/lib/onboard/gateway-recovery.ts index c48a2a535b8..ddc4c2df096 100644 --- a/src/lib/onboard/gateway-recovery.ts +++ b/src/lib/onboard/gateway-recovery.ts @@ -34,7 +34,7 @@ import { createReadinessWaitOptions, formatReadinessDeadline, getLegacyPollDeadlineBudgetMs, -} from "./readiness-wait"; +} from "../core/readiness-wait"; export type StartGatewayForRecoveryOptions = { gatewayName?: string; diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 4edea2457bd..c1f2191d824 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -1282,6 +1282,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche path, planRegisteredExtraProviders, preparedDcodeRebuild, + printSandboxCreateRecoveryHints, promptValidatedSandboxName, promptYesNoOrDefault, providerExistsInGateway, @@ -2603,6 +2604,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche sleep: sleepSeconds, openshellArgv, isSandboxReady, + printCreateRecoveryHints: printSandboxCreateRecoveryHints, verifyDirectSandboxGpu: createGpuVerifier, }, ); @@ -2623,13 +2625,16 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche let initialCreateSourceCleanupCompleted = false; const cleanupInitialCreateSource = (): boolean => { if (initialCreateSourceCleanupCompleted) return true; - initialCreateSourceCleanupCompleted = sandboxGpuCreateFlow.cleanupSandboxCreateSource( - initialSandboxPolicy.cleanup, - { - exactCleanup: initialSandboxPolicy.cleanupExact, - requireExact: agentCreateInput.hermesPortableLifecycle, - }, - ); + const cleanup = initialSandboxPolicy.cleanup; + const exactCleanup = initialSandboxPolicy.cleanupExact; + if (agentCreateInput.hermesPortableLifecycle && cleanup && !exactCleanup) { + throw new Error("Hermes portable temporary policy source has no exact cleanup authority."); + } + const selectedCleanup = exactCleanup ?? cleanup; + initialCreateSourceCleanupCompleted = selectedCleanup?.() ?? true; + if (initialCreateSourceCleanupCompleted && cleanup) { + process.removeListener("exit", cleanup); + } return initialCreateSourceCleanupCompleted; }; const cleanupSandboxCreateSources = (): void => { diff --git a/src/lib/onboard/sandbox-gpu-create-flow-hermes-portable.test.ts b/src/lib/onboard/sandbox-gpu-create-flow-hermes-portable.test.ts index 861333626c5..8a10541dba7 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow-hermes-portable.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow-hermes-portable.test.ts @@ -48,11 +48,7 @@ import { resetGpuFlowMocks, setupGpuFlowMocks, } from "./__test-helpers__/sandbox-gpu-create-flow"; -import { - cleanupSandboxCreateSource, - runSandboxGpuCreateFlow, - type SandboxGpuCreateFlowInput, -} from "./sandbox-gpu-create-flow"; +import { runSandboxGpuCreateFlow, type SandboxGpuCreateFlowInput } from "./sandbox-gpu-create-flow"; import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; const PORTABLE_RUNTIME_AUTHORITY: CheckpointPortableRuntimeAuthority = { @@ -70,45 +66,6 @@ beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); describe("Hermes portable sandbox create flow", () => { - it("releases exit cleanup ownership only after successful retirement (#9203)", () => { - const cleanup = vi.fn(() => true); - process.on("exit", cleanup); - try { - expect(cleanupSandboxCreateSource(cleanup)).toBe(true); - expect(process.listeners("exit")).not.toContain(cleanup); - } finally { - process.removeListener("exit", cleanup); - } - }); - - it("preserves exit cleanup ownership when retirement is incomplete (#9203)", () => { - const cleanup = vi.fn(() => false); - process.on("exit", cleanup); - try { - expect(cleanupSandboxCreateSource(cleanup)).toBe(false); - expect(process.listeners("exit")).toContain(cleanup); - } finally { - process.removeListener("exit", cleanup); - } - }); - - it("requires and uses exact source cleanup for Hermes portable custody (#9203)", () => { - const cleanup = vi.fn(() => true); - const exactCleanup = vi.fn(() => true); - process.on("exit", cleanup); - try { - expect(cleanupSandboxCreateSource(cleanup, { exactCleanup, requireExact: true })).toBe(true); - expect(exactCleanup).toHaveBeenCalledOnce(); - expect(cleanup).not.toHaveBeenCalled(); - expect(process.listeners("exit")).not.toContain(cleanup); - expect(() => cleanupSandboxCreateSource(cleanup, { requireExact: true })).toThrow( - "has no exact cleanup authority", - ); - } finally { - process.removeListener("exit", cleanup); - } - }); - it("keeps non-OpenClaw portable creation on the existing runtime patch (#9068)", async () => { const input = createInput(); input.hostEnv = { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" }; diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 0124cdae5a9..069e73aecdc 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -57,21 +57,6 @@ import { addTraceEvent } from "./tracing"; export { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; -/** Release the exit cleanup listener only after its exact create source was retired. */ -export function cleanupSandboxCreateSource( - cleanup: (() => boolean) | undefined, - options: { readonly exactCleanup?: () => boolean; readonly requireExact?: boolean } = {}, -): boolean { - if (options.requireExact && cleanup && !options.exactCleanup) { - throw new Error("Hermes portable temporary policy source has no exact cleanup authority."); - } - const selected = options.exactCleanup ?? cleanup; - if (!selected) return true; - const completed = selected(); - if (completed && cleanup) process.removeListener("exit", cleanup); - return completed; -} - export function resolvePortableLifecycleMode( agent: AgentDefinition | null, env: NodeJS.ProcessEnv = process.env, @@ -242,6 +227,10 @@ export interface SandboxGpuCreateFlowDeps { openshellArgv(args: string[]): string[]; isSandboxReady(output: string, sandboxName: string): boolean; verifyDirectSandboxGpu(sandboxName: string): SandboxGpuProofResult; + printCreateRecoveryHints( + output: string, + options: { readonly createArgs: readonly string[] }, + ): void; printCreateFailureDiagnostics?: ( sandboxName: string, options: { readonly backupPath?: string | null }, diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index b8e667afa37..9e0bf995a16 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -18,7 +18,8 @@ import { resolveOpenShellSandboxId, settleCreatedOpenShellSandboxId, } from "../adapters/openshell/sandbox-identity"; -import { printSandboxCreateRecoveryHints } from "../build-context"; +import { createReadinessWaitOptions } from "../core/readiness-wait"; +import { waitUntil } from "../core/wait"; import { streamSandboxCreate, type StreamSandboxCreateResult } from "../sandbox/create-stream"; import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-ready-gate"; import { redact, redactFullWithUrls } from "../security/redact"; @@ -54,7 +55,6 @@ import { } from "./sandbox-recreate-probe"; import type { CreatedSandboxReadyIdentityCheck } from "./sandbox-readiness-tracing"; import * as sandboxReadinessTracing from "./sandbox-readiness-tracing"; -import { createReadinessWaitOptions, runReadinessWait } from "./readiness-wait"; import { addTraceEvent } from "./tracing"; type NativeRuntimeSnapshot = ManagedBootstrapRuntimeSnapshot; @@ -112,6 +112,10 @@ function reportSandboxCreateFailure(options: { sandboxName: string, options: { backupPath: string | null }, ) => void; + readonly printCreateRecoveryHints: ( + output: string, + options: { readonly createArgs: readonly string[] }, + ) => void; }): never { const redactedCreateOutput = redact(options.createOutput); console.error(""); @@ -124,7 +128,7 @@ function reportSandboxCreateFailure(options: { backupPath: options.restoreBackupPath, }); console.error(" Try: openshell sandbox list # check gateway state"); - printSandboxCreateRecoveryHints(redactedCreateOutput, { createArgs: options.createArgs }); + options.printCreateRecoveryHints(redactedCreateOutput, { createArgs: options.createArgs }); return process.exit(options.createStatus === 0 ? 1 : options.createStatus); } @@ -544,7 +548,7 @@ function waitForCreatedOpenShellSandboxPublication( const now = waitOptions?.now; const published = waitOptions && deadlineMs !== undefined && now - ? runReadinessWait(() => { + ? waitUntil(() => { const getRemainingMs = () => Math.max(1, deadlineMs - now()); const result = deps.runOpenshell( ["sandbox", "get", "-g", input.gatewayName, input.sandboxName], @@ -1189,6 +1193,7 @@ export function createSandboxGpuCreateAttemptRunner( restoreBackupPath: input.restoreBackupPath, createArgs: input.prebuild.createArgs, printCreateFailureDiagnostics, + printCreateRecoveryHints: deps.printCreateRecoveryHints, }); } } diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index f597e60aea6..52fbfe7bfb7 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -15,14 +15,13 @@ import { createCliOpenShellSandboxObserver, type CliOpenShellSandboxObserverDeps, } from "../adapters/openshell/sandbox-observer-cli"; -import { waitUntilAsync } from "../core/wait"; -import { envInt } from "./env"; import { createReadinessWaitOptions, formatReadinessDeadline, getLegacyPollDeadlineBudgetMs, - runReadinessWait, -} from "./readiness-wait"; +} from "../core/readiness-wait"; +import { waitUntil, waitUntilAsync } from "../core/wait"; +import { envInt } from "./env"; import { addTraceEvent, withDashboardReadinessTrace, withSandboxReadinessTrace } from "./tracing"; type RunCaptureOpenshell = (args: string[], options?: { ignoreError?: boolean }) => string; @@ -651,7 +650,7 @@ export function waitForDashboardReadyWithTrace(options: { } const ready = waitOptions !== null && - runReadinessWait(() => { + waitUntil(() => { attempt += 1; const readyOutput = runCaptureOpenshell( [ diff --git a/test/inference/managed/managed-image-protected-runtime-contract.test.ts b/test/inference/managed/managed-image-protected-runtime-contract.test.ts index d9c82904396..d80ca839e51 100644 --- a/test/inference/managed/managed-image-protected-runtime-contract.test.ts +++ b/test/inference/managed/managed-image-protected-runtime-contract.test.ts @@ -161,6 +161,7 @@ describe("protected managed-image runtime contract", () => { "runOpenshell", "runCaptureOpenshell", "isSandboxReady", + "printSandboxCreateRecoveryHints", "sleepSeconds", "startGatewayForRecovery", ] as const)( @@ -177,6 +178,7 @@ describe("protected managed-image runtime contract", () => { openshellArgv: () => [], runCaptureOpenshell: () => "", isSandboxReady: () => false, + printSandboxCreateRecoveryHints: () => undefined, sleepSeconds: () => undefined, startGatewayForRecovery: async () => undefined, }, From aeba4fe7fd72d9afe989ce119f90229a5b4af679 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 07:46:29 -0700 Subject: [PATCH 42/51] fix(onboard): preserve resumed readiness deadline Signed-off-by: Prekshi Vyas --- .../hermes-portable-onboarding.test.ts | 24 +++-- .../hermes-portable-onboarding.ts | 39 +++++--- test/security/shellquote-sandbox.test.ts | 88 ++++++++++++++----- 3 files changed, 110 insertions(+), 41 deletions(-) diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts b/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts index 4547b44e5ae..4d66b883e2f 100644 --- a/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts +++ b/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts @@ -360,13 +360,17 @@ describe("Hermes portable onboarding transaction", () => { sandboxId: "sandbox-id-1", liveIdentityFingerprint: HERMES_PORTABLE_TEST_LIVE_IDENTITY, }; + let resumeNowMs = 0; const resumeObservations = [ { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }, { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }, present, ]; - const resumeObserveSandbox = vi.fn(() => resumeObservations.shift() ?? present); - let resumeNowMs = 0; + const resumeObservationElapsedMs = [30_000, 0, 0]; + const resumeObserveSandbox = vi.fn(() => { + resumeNowMs += resumeObservationElapsedMs.shift() ?? 0; + return resumeObservations.shift() ?? present; + }); const delayResumePoll = vi.fn(async (milliseconds: number) => { resumeNowMs += milliseconds; }); @@ -384,20 +388,23 @@ describe("Hermes portable onboarding transaction", () => { expect(delayResumePoll).toHaveBeenCalledTimes(1); expect(delayResumePoll).toHaveBeenCalledWith(1_000); expect(resumeObserveSandbox.mock.calls.slice(0, 3)).toEqual([ - [undefined], [180_000], - [179_000], + [150_000], + [149_000], ]); + expect(resumeNowMs).toBe(31_000); }); it("fails closed when exact post-create Ready publication exceeds its bound (#9211)", async () => { let nowMs = 0; + const observationElapsedMs = [0, 0, 180_000]; const boundedBudgets: Array<{ budgetMs: number; remainingMs: number }> = []; const observeSandbox = vi.fn((timeoutBudgetMs?: number) => { const budgetMs = timeoutBudgetMs ?? 0; boundedBudgets.push({ budgetMs, remainingMs: 180_000 - nowMs }); - nowMs += budgetMs; - return timeoutBudgetMs === undefined + const elapsedMs = Math.min(budgetMs, observationElapsedMs.shift() ?? 0); + nowMs += elapsedMs; + return elapsedMs === 0 ? { kind: "absent" as const } : { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }; }); @@ -426,14 +433,15 @@ describe("Hermes portable onboarding transaction", () => { let nowMs = 0; let observedMs = 0; let delayedMs = 0; + const observationElapsedMs = [0, 0, 179_000]; const boundedBudgets: Array<{ budgetMs: number; remainingMs: number }> = []; const observeSandbox = vi.fn((timeoutBudgetMs?: number) => { const budgetMs = timeoutBudgetMs ?? 0; boundedBudgets.push({ budgetMs, remainingMs: 180_000 - nowMs }); - const elapsedMs = Math.min(budgetMs, observedMs === 0 ? 179_000 : budgetMs); + const elapsedMs = Math.min(budgetMs, observationElapsedMs.shift() ?? 0); observedMs += elapsedMs; nowMs += elapsedMs; - return timeoutBudgetMs === undefined + return elapsedMs === 0 ? { kind: "absent" as const } : { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }; }); diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding.ts b/src/lib/onboard/experimental/hermes-portable-onboarding.ts index 9c6c8dce362..413f3d976ef 100644 --- a/src/lib/onboard/experimental/hermes-portable-onboarding.ts +++ b/src/lib/onboard/experimental/hermes-portable-onboarding.ts @@ -788,11 +788,12 @@ async function settleCreatedHermesPortableSandboxReadyPublication( observeSandbox: (timeoutBudgetMs?: number) => HermesPortableSandboxObservation, delayPoll: (milliseconds: number) => Promise, readClockMs: () => number, - timeoutMs: number, + deadlineMs: number, ): Promise { - const boundedTimeoutMs = Math.max(1, Math.round(timeoutMs)); - const maxPolls = Math.ceil(boundedTimeoutMs / HERMES_PORTABLE_READY_PUBLICATION_POLL_INTERVAL_MS); - const deadlineMs = readClockMs() + boundedTimeoutMs; + const maxPolls = Math.ceil( + Math.max(1, Math.round(deadlineMs - readClockMs())) / + HERMES_PORTABLE_READY_PUBLICATION_POLL_INTERVAL_MS, + ); let observation: HermesPortableSandboxObservation = { kind: "ambiguous", detail: HERMES_PORTABLE_READY_PUBLICATION_TIMEOUT_DETAIL, @@ -1410,6 +1411,12 @@ export async function runHermesPortableOnboardingTransaction( } if (snapshot.receipt.phase === "pending") { + const readReadyPublicationClockMs = + deps.readSandboxReadyPublicationClockMs ?? performance.now.bind(performance); + const delayReadyPublicationPoll = + deps.delaySandboxReadyPublicationPoll ?? delayHermesPortableReadyPublicationPoll; + const createReadyPublicationDeadline = () => + readReadyPublicationClockMs() + Math.max(1, Math.round(readyPublicationTimeoutMs)); const createPolicySourcePath = snapshot.receipt.policy.sourcePath; assertRegistryMissingBeforeConfiguration( snapshot.receipt, @@ -1421,16 +1428,26 @@ export async function runHermesPortableOnboardingTransaction( createIntentSha256: snapshot.receipt.createIntentSha256, stateDir: input.stateDir, }); - let observation = observeSandbox(); + const pendingObservationDeadlineMs = createReadyPublicationDeadline(); + const pendingObservationBudgetMs = Math.floor( + pendingObservationDeadlineMs - readReadyPublicationClockMs(), + ); + let observation = + pendingObservationBudgetMs < 1 + ? { + kind: "ambiguous" as const, + detail: HERMES_PORTABLE_READY_PUBLICATION_TIMEOUT_DETAIL, + } + : observeSandbox(pendingObservationBudgetMs); if ( observation.kind === "ambiguous" && observation.detail === HERMES_PORTABLE_NOT_READY_DETAIL ) { observation = await settleCreatedHermesPortableSandboxReadyPublication( observeSandbox, - deps.delaySandboxReadyPublicationPoll ?? delayHermesPortableReadyPublicationPoll, - deps.readSandboxReadyPublicationClockMs ?? performance.now.bind(performance), - readyPublicationTimeoutMs, + delayReadyPublicationPoll, + readReadyPublicationClockMs, + pendingObservationDeadlineMs, ); } if (observation.kind === "ambiguous") @@ -1470,9 +1487,9 @@ export async function runHermesPortableOnboardingTransaction( created = true; observation = await settleCreatedHermesPortableSandboxReadyPublication( observeSandbox, - deps.delaySandboxReadyPublicationPoll ?? delayHermesPortableReadyPublicationPoll, - deps.readSandboxReadyPublicationClockMs ?? performance.now.bind(performance), - readyPublicationTimeoutMs, + delayReadyPublicationPoll, + readReadyPublicationClockMs, + createReadyPublicationDeadline(), ); buildContext.assertCurrent(); input.buildContext.assertCurrentSource(); diff --git a/test/security/shellquote-sandbox.test.ts b/test/security/shellquote-sandbox.test.ts index df2e2557602..b0a12be936d 100644 --- a/test/security/shellquote-sandbox.test.ts +++ b/test/security/shellquote-sandbox.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "child_process"; +import { createHash } from "node:crypto"; // Verify sandbox names stay validated and out of raw shell command strings. import fs from "fs"; import os from "os"; @@ -114,7 +115,43 @@ for (const key of Object.keys(process.env)) { process.env.NEMOCLAW_OPENSHELL_BIN = ${JSON.stringify(path.join(fakeBin, "openshell"))}; const commands = []; const asText = (command) => Array.isArray(command) ? command.join(" ") : String(command); -const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); +const createdSandbox = fixtureMocks.createCreatedSandboxFixture({ + gatewayName: "nemoclaw", + sandboxId: "sandbox-owning-gateway", +}); +const foreignSandbox = fixtureMocks.createCreatedSandboxFixture({ + gatewayName: "foreign-gateway", + sandboxId: "sandbox-foreign-gateway", + lifecycleState: "created", +}); +const probeEffects = []; +const runCreatedSandboxProbe = (command) => { + const args = Array.isArray(command) ? command.map(String) : []; + const sandboxIndex = args.indexOf("sandbox"); + const action = sandboxIndex < 0 ? null : args[sandboxIndex + 1]; + if (action !== "get" && action !== "exec") return null; + const name = action === "get" ? args.at(-1) : args[args.indexOf("--name") + 1]; + if (name !== "my-assistant") return null; + const gatewayIndex = args.findIndex((arg) => arg === "-g" || arg === "--gateway"); + const gateway = gatewayIndex < 0 ? null : args[gatewayIndex + 1] ?? null; + const target = gateway === "nemoclaw" ? "owning" : "foreign"; + probeEffects.push({ action, gateway, target }); + if (action === "get") { + const result = (target === "owning" ? createdSandbox : foreignSandbox).run(command); + return result ?? { + status: 1, + stdout: Buffer.alloc(0), + stderr: Buffer.from("gateway-scoped fixture rejected sandbox get\n"), + }; + } + return target === "owning" + ? { status: 0, stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) } + : { + status: 1, + stdout: Buffer.alloc(0), + stderr: Buffer.from("foreign gateway cannot satisfy owning sandbox exec\n"), + }; +}; createdSandbox.installRuntimeObservation(); runner.run = (command, opts = {}) => { const text = asText(command); @@ -132,6 +169,8 @@ runner.run = (command, opts = {}) => { stderr: Buffer.alloc(0), }; } + const sandboxProbe = runCreatedSandboxProbe(command); + if (sandboxProbe !== null) return sandboxProbe; return createdSandbox.run(command) ?? { status: 0 }; }; runner.runFile = (file, args = [], opts = {}) => { @@ -151,13 +190,14 @@ runner.runCapture = (command) => { }; registry.getSandbox = () => null; registry.getDisabledChannels = () => []; -registry.registerSandbox = () => true; registry.removeSandbox = () => true; registry.updateSandbox = () => true; +let registeredSandbox = null; const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", provider: "nvidia-prod", model: "gpt-5.4", + registerSandbox: (entry) => { registeredSandbox = entry; }, }); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; @@ -197,7 +237,14 @@ try { createFixture, ), ); - console.log(JSON.stringify({ sandboxName, commands })); + console.log(JSON.stringify({ + sandboxName, + commands, + probeEffects, + registeredSandbox, + owningSandboxId: createdSandbox.state.sandboxId, + foreignSandbox: foreignSandbox.state, + })); } catch (error) { console.error(error && error.stack ? error.stack : String(error)); process.exit(1); @@ -233,30 +280,27 @@ try { .find((line) => line.startsWith("{") && line.endsWith("}")); expect(payloadLine).toBeTruthy(); const payload = JSON.parse(payloadLine!); - const sandboxGetCommands = payload.commands.filter( - (entry: { command: string }) => - entry.command.includes("sandbox get") && entry.command.includes("my-assistant"), + expect(payload.sandboxName).toBe("my-assistant"); + expect(payload.registeredSandbox.lifecycleLiveIdentityFingerprint).toBe( + createHash("sha256").update(payload.owningSandboxId).digest("hex"), ); - const sandboxExecCommands = payload.commands.filter( - (entry: { command: string }) => - entry.command.includes("sandbox exec") && entry.command.includes("my-assistant"), + expect(payload.probeEffects).toEqual( + expect.arrayContaining([ + expect.objectContaining({ action: "get", target: "owning" }), + expect.objectContaining({ action: "exec", target: "owning" }), + ]), ); - expect(sandboxGetCommands).not.toHaveLength(0); - expect(sandboxExecCommands).not.toHaveLength(0); - expect( - sandboxGetCommands.every( - (entry: { command: string }) => - entry.command.includes("sandbox get -g nemoclaw") || - entry.command.includes("sandbox get --gateway nemoclaw"), - ), - ).toBe(true); expect( - sandboxExecCommands.every( - (entry: { command: string }) => - entry.command.includes("sandbox exec -g nemoclaw") || - entry.command.includes("--gateway nemoclaw"), + payload.probeEffects.every( + (effect: { target: string }) => effect.target === "owning", ), ).toBe(true); + expect(payload.foreignSandbox).toMatchObject({ + sandboxName: "my-assistant", + sandboxId: "sandbox-foreign-gateway", + gatewayName: "foreign-gateway", + lifecycleState: "created", + }); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } From 1752bfd479ab7a16e29516f2ea16f7502376ae7e Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 08:09:05 -0700 Subject: [PATCH 43/51] test(e2e): execute managed startup probe Signed-off-by: Prekshi Vyas --- .../checks/run-managed-image-openshell-e2e.ts | 58 +++++--- ...d-image-protected-runtime-contract.test.ts | 134 +++++++++++++++--- 2 files changed, 155 insertions(+), 37 deletions(-) diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 8f10e925ee0..454e07eb612 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -409,19 +409,27 @@ function managedConfigPath(agent: ShippedManagedImageAgent): string { export function managedImageOpenShellProbe( agent: ShippedManagedImageAgent, model: string = MODEL, + options: { readonly rootPath?: string } = {}, ): string { + const rootPath = options.rootPath ? path.resolve(options.rootPath) : null; + const probePath = (absolutePath: string) => + rootPath ? path.join(rootPath, absolutePath.replace(/^\/+/, "")) : absolutePath; + const quotePath = (absolutePath: string) => JSON.stringify(probePath(absolutePath)); + const stat = rootPath ? quotePath("/usr/bin/stat") : "stat"; + const openssl = rootPath ? quotePath("/usr/bin/openssl") : "openssl"; + const curl = quotePath("/usr/bin/curl"); const healthProbe = agent === "openclaw" ? [ - "openclaw_health_code=\"$(/usr/bin/curl -sS -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:18789/health || true)\"", + `openclaw_health_code="$(${curl} -sS -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:18789/health || true)"`, 'case "$openclaw_health_code" in', " 200 | 401) ;;", " *) printf 'OpenClaw /health returned HTTP %s\\n' \"${openclaw_health_code:-000}\" >&2; exit 1 ;;", "esac", ].join("\n") : agent === "hermes" - ? "/usr/bin/curl -fsS --max-time 5 http://127.0.0.1:8642/health >/dev/null" - : "/usr/local/bin/dcode --version >/dev/null"; + ? `${curl} -fsS --max-time 5 http://127.0.0.1:8642/health >/dev/null` + : `${quotePath("/usr/local/bin/dcode")} --version >/dev/null`; const readinessLabel = agent === "openclaw" ? "OpenClaw health endpoint" @@ -436,61 +444,75 @@ export function managedImageOpenShellProbe( "set -u", probeStep( `${agent} executable`, - `test -x ${ + `test -x ${quotePath( agent === "openclaw" ? "/usr/local/bin/openclaw" : agent === "hermes" ? "/usr/local/bin/hermes" - : "/usr/local/bin/dcode" - }`, + : "/usr/local/bin/dcode", + )}`, ), probeStep( `${agent} managed model configuration`, - `grep -F ${JSON.stringify(model)} ${JSON.stringify(managedConfigPath(agent))} >/dev/null`, + `grep -F ${JSON.stringify(model)} ${quotePath(managedConfigPath(agent))} >/dev/null`, ), probeStep( "managed runtime environment must not be a symbolic link", - "test ! -L /run/nemoclaw/managed-startup-runtime.env", + `test ! -L ${quotePath("/run/nemoclaw/managed-startup-runtime.env")}`, ), probeStep( "managed runtime environment owner, group, and mode must equal 0:0:444", - 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-runtime.env)" = "0:0:444"', + `test "$(${stat} -c "%u:%g:%a" ${quotePath( + "/run/nemoclaw/managed-startup-runtime.env", + )})" = "0:0:444"`, ), probeStep( "managed startup completion must not be a symbolic link", - "test ! -L /run/nemoclaw/managed-startup-complete.json", + `test ! -L ${quotePath("/run/nemoclaw/managed-startup-complete.json")}`, ), probeStep( "managed startup completion owner, group, and mode must equal 0:0:444", - 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-complete.json)" = "0:0:444"', + `test "$(${stat} -c "%u:%g:%a" ${quotePath( + "/run/nemoclaw/managed-startup-complete.json", + )})" = "0:0:444"`, ), probeStep( "corporate CA file must exist and be nonempty", - "test -s /usr/local/share/nemoclaw/corporate-ca.pem", + `test -s ${quotePath("/usr/local/share/nemoclaw/corporate-ca.pem")}`, ), probeStep( "corporate CA owner, group, and mode must equal 0:0:444", - 'test "$(stat -c "%u:%g:%a" /usr/local/share/nemoclaw/corporate-ca.pem)" = "0:0:444"', + `test "$(${stat} -c "%u:%g:%a" ${quotePath( + "/usr/local/share/nemoclaw/corporate-ca.pem", + )})" = "0:0:444"`, ), probeStep( "corporate CA system anchor must match the managed material", - "cmp -s /usr/local/share/nemoclaw/corporate-ca.pem /usr/local/share/ca-certificates/nemoclaw-corporate-ca-01.crt", + `cmp -s ${quotePath("/usr/local/share/nemoclaw/corporate-ca.pem")} ${quotePath( + "/usr/local/share/ca-certificates/nemoclaw-corporate-ca-01.crt", + )}`, ), probeStep( "corporate CA system anchor owner, group, and mode must equal 0:0:444", - 'test "$(stat -c "%u:%g:%a" /usr/local/share/ca-certificates/nemoclaw-corporate-ca-01.crt)" = "0:0:444"', + `test "$(${stat} -c "%u:%g:%a" ${quotePath( + "/usr/local/share/ca-certificates/nemoclaw-corporate-ca-01.crt", + )})" = "0:0:444"`, ), probeStep( "system trust must verify the managed corporate CA", - "openssl verify -CAfile /etc/ssl/certs/ca-certificates.crt /usr/local/share/nemoclaw/corporate-ca.pem >/dev/null", + `${openssl} verify -CAfile ${quotePath("/etc/ssl/certs/ca-certificates.crt")} ${quotePath( + "/usr/local/share/nemoclaw/corporate-ca.pem", + )} >/dev/null`, ), probeStep( "managed startup CA bundle must exist and be nonempty", - "test -s /run/nemoclaw/managed-startup-ca-bundle.pem", + `test -s ${quotePath("/run/nemoclaw/managed-startup-ca-bundle.pem")}`, ), probeStep( "managed startup CA bundle owner, group, and mode must equal 0:0:444", - 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-ca-bundle.pem)" = "0:0:444"', + `test "$(${stat} -c "%u:%g:%a" ${quotePath( + "/run/nemoclaw/managed-startup-ca-bundle.pem", + )})" = "0:0:444"`, ), probeStep(readinessLabel, healthProbe), ].join("\n"); diff --git a/test/inference/managed/managed-image-protected-runtime-contract.test.ts b/test/inference/managed/managed-image-protected-runtime-contract.test.ts index d80ca839e51..8c709d8f7a5 100644 --- a/test/inference/managed/managed-image-protected-runtime-contract.test.ts +++ b/test/inference/managed/managed-image-protected-runtime-contract.test.ts @@ -38,6 +38,7 @@ import { resolveOnboardManagedBootstrapLaunch } from "../../../src/lib/onboard/m const IMAGE = `localhost:5000/nemoclaw-managed-protected/openclaw@sha256:${"a".repeat(64)}`; const VALID_SANDBOX = "managed-openclaw"; +const LOCAL_MODEL = "nvidia/nemotron-3-nano"; const MANAGED_IMAGE_ONBOARD = resolveManagedImageOnboardModule( await import("../../../src/lib/onboard.ts"), ); @@ -48,6 +49,93 @@ const SUCCESS_WITHOUT_OUTPUT: ManagedImageCommandResult = { stderr: "", }; +function managedProbeFixturePath(rootPath: string, absolutePath: string): string { + return path.join(rootPath, absolutePath.replace(/^\/+/, "")); +} + +function writeManagedProbeFixture( + rootPath: string, + absolutePath: string, + contents: string, + mode: number, +): void { + const fixturePath = managedProbeFixturePath(rootPath, absolutePath); + fs.mkdirSync(path.dirname(fixturePath), { recursive: true }); + fs.writeFileSync(fixturePath, contents, { mode }); +} + +function materializeManagedProbeFixture(rootPath: string): void { + const rootOwnedStatStub = `#!/usr/bin/env node +const fs = require("node:fs"); +const stat = fs.statSync(process.argv.at(-1)); +process.stdout.write(\`0:0:\${(stat.mode & 0o777).toString(8)}\\n\`); +`; + const curlStub = `#!/bin/sh +case "$*" in + *http_code*) printf '200' ;; +esac +exit 0 +`; + const successStub = "#!/bin/sh\nexit 0\n"; + const caMaterial = "managed corporate CA\n"; + + writeManagedProbeFixture( + rootPath, + "/sandbox/.openclaw/openclaw.json", + `${LOCAL_MODEL}\n`, + 0o444, + ); + writeManagedProbeFixture(rootPath, "/sandbox/.hermes/config.yaml", `${LOCAL_MODEL}\n`, 0o444); + writeManagedProbeFixture( + rootPath, + "/sandbox/.deepagents/config.toml", + `${LOCAL_MODEL}\n`, + 0o444, + ); + writeManagedProbeFixture(rootPath, "/usr/local/bin/openclaw", successStub, 0o555); + writeManagedProbeFixture(rootPath, "/usr/local/bin/hermes", successStub, 0o555); + writeManagedProbeFixture(rootPath, "/usr/local/bin/dcode", successStub, 0o555); + writeManagedProbeFixture( + rootPath, + "/run/nemoclaw/managed-startup-runtime.env", + "MODEL=managed\n", + 0o444, + ); + writeManagedProbeFixture( + rootPath, + "/run/nemoclaw/managed-startup-complete.json", + '{"status":"ready"}\n', + 0o444, + ); + writeManagedProbeFixture( + rootPath, + "/usr/local/share/nemoclaw/corporate-ca.pem", + caMaterial, + 0o444, + ); + writeManagedProbeFixture( + rootPath, + "/usr/local/share/ca-certificates/nemoclaw-corporate-ca-01.crt", + caMaterial, + 0o444, + ); + writeManagedProbeFixture( + rootPath, + "/etc/ssl/certs/ca-certificates.crt", + caMaterial, + 0o444, + ); + writeManagedProbeFixture( + rootPath, + "/run/nemoclaw/managed-startup-ca-bundle.pem", + caMaterial, + 0o444, + ); + writeManagedProbeFixture(rootPath, "/usr/bin/stat", rootOwnedStatStub, 0o555); + writeManagedProbeFixture(rootPath, "/usr/bin/curl", curlStub, 0o555); + writeManagedProbeFixture(rootPath, "/usr/bin/openssl", successStub, 0o555); +} + function managedContainerInspectResult( contentId: string, running: boolean, @@ -461,7 +549,7 @@ describe("protected managed-image runtime contract", () => { }); it.each(["openclaw", "hermes", "langchain-deepagents-code"] as const)( - "binds %s to an exact GPU/local-inference launch", + "binds %s to an exact GPU/local-inference launch and verified startup receipt", (agent) => { const parsed = parseManagedImageOpenShellE2eInputs([ "--agent", @@ -474,7 +562,7 @@ describe("protected managed-image runtime contract", () => { "--local-provider", "nim", "--model", - "nvidia/nemotron-3-nano", + LOCAL_MODEL, ]); expect(parsed).toEqual({ @@ -482,26 +570,34 @@ describe("protected managed-image runtime contract", () => { gpu: true, image: IMAGE, localProvider: "nim", - model: "nvidia/nemotron-3-nano", + model: LOCAL_MODEL, sandbox: managedImageProtectedSandboxName(agent, "nim"), }); expect(path.isAbsolute(managedImageOpenShellBasePolicyPath(agent))).toBe(true); - const probe = managedImageOpenShellProbe(agent); - const syntax = spawnSync("/bin/sh", ["-n", "-c", probe], { encoding: "utf8" }); - expect(syntax.status, syntax.stderr).toBe(0); - expect(probe).toContain("managed-startup-complete.json"); - expect(probe).toContain( - `managed-image startup probe failed: ${ - agent === "openclaw" - ? "OpenClaw health endpoint" - : agent === "hermes" - ? "Hermes health endpoint" - : "LangChain Deep Agents Code version command" - }`, - ); - expect(probe).toContain( - "managed-image startup probe failed: managed startup completion owner, group, and mode must equal 0:0:444", - ); + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-probe-")); + try { + materializeManagedProbeFixture(fixtureRoot); + const probe = managedImageOpenShellProbe(agent, LOCAL_MODEL, { rootPath: fixtureRoot }); + const syntax = spawnSync("/bin/sh", ["-n", "-c", probe], { encoding: "utf8" }); + expect(syntax.status, syntax.stderr).toBe(0); + + const verified = spawnSync("/bin/sh", ["-eu", "-c", probe], { encoding: "utf8" }); + expect(verified.status, verified.stderr).toBe(0); + + fs.chmodSync( + managedProbeFixturePath(fixtureRoot, "/run/nemoclaw/managed-startup-complete.json"), + 0o644, + ); + const mutableReceipt = spawnSync("/bin/sh", ["-eu", "-c", probe], { + encoding: "utf8", + }); + expect(mutableReceipt.status).not.toBe(0); + expect(mutableReceipt.stderr).toContain( + "managed startup completion owner, group, and mode must equal 0:0:444", + ); + } finally { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + } }, ); From 692e3cb9cc9bcf77b2c3ce82b70f1005d78a219d Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 08:32:27 -0700 Subject: [PATCH 44/51] fix(e2e): bind cleanup to durable sandbox identity Signed-off-by: Prekshi Vyas --- .../checks/run-managed-image-openshell-e2e.ts | 113 +++++++++++++++++- src/lib/onboard/gateway-recovery.test.ts | 15 +++ src/lib/onboard/gateway-recovery.ts | 22 +--- ...d-image-protected-runtime-contract.test.ts | 97 ++++++++++++++- 4 files changed, 217 insertions(+), 30 deletions(-) diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 454e07eb612..b0018a15867 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -518,10 +518,18 @@ export function managedImageOpenShellProbe( ].join("\n"); } -export function managedImageOpenShellCommittedProbe(): string { +export function managedImageOpenShellCommittedProbe( + options: { readonly rootPath?: string } = {}, +): string { + const transactionPath = options.rootPath + ? path.join( + path.resolve(options.rootPath), + "var/lib/nemoclaw/managed-startup-shared-state-transaction-v1", + ) + : "/var/lib/nemoclaw/managed-startup-shared-state-transaction-v1"; return [ "set -eu", - "test ! -e /var/lib/nemoclaw/managed-startup-shared-state-transaction-v1", + `test ! -e ${JSON.stringify(transactionPath)}`, ].join("\n"); } @@ -900,6 +908,74 @@ export function assertFailedSandboxOwnerCleanupRetention( } } +function queryManagedImageSandboxIdentity( + onboard: Pick, + input: Pick, + env: NodeJS.ProcessEnv, +): { readonly result: ManagedImageCommandResult; readonly sandboxId: string | null } { + const result = onboard.runOpenshell( + ["sandbox", "get", "-g", GATEWAY_NAME, input.sandbox], + { + ignoreError: true, + env, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + return { + result, + sandboxId: result.status === 0 ? parseOpenShellSandboxId(String(result.stdout ?? "")) : null, + }; +} + +function assertManagedImageSandboxIdentity( + onboard: Pick, + input: Pick, + expectedSandboxId: string, + env: NodeJS.ProcessEnv, + operation: string, +): void { + const observed = queryManagedImageSandboxIdentity(onboard, input, env); + if (observed.sandboxId !== expectedSandboxId) { + throw new Error( + `managed-image sandbox durable identity changed before ${operation}; refusing the post-create effect`, + ); + } +} + +export function removeManagedImageSandboxIfOwned( + onboard: Pick, + input: Pick, + expectedSandboxId: string | null, + env: NodeJS.ProcessEnv, + runCommand: ManagedImageCommandRunner = commandResult, +): string | null { + const observed = queryManagedImageSandboxIdentity(onboard, input, env); + if (!expectedSandboxId) { + return observed.result.status === 0 + ? "refusing managed-image sandbox cleanup because no durable sandbox ID was captured" + : null; + } + if (observed.sandboxId === null) { + return "refusing managed-image sandbox cleanup because its durable identity is unavailable"; + } + if (observed.sandboxId !== expectedSandboxId) { + return "refusing managed-image sandbox cleanup because its durable identity changed"; + } + + const remove = runCommand( + onboard.openshellArgv(["sandbox", "delete", "-g", GATEWAY_NAME, input.sandbox]), + env, + 15_000, + ); + if (remove.status !== 0) { + return `OpenShell sandbox cleanup failed with status ${String(remove.status)}`; + } + const verification = queryManagedImageSandboxIdentity(onboard, input, env); + return verification.result.status === 0 || verification.sandboxId !== null + ? "OpenShell sandbox cleanup did not prove exact absence" + : null; +} + async function run( input: Inputs, afterLocalInference?: (context: ManagedImageOpenShellE2eProbeContext) => Promise | T, @@ -922,6 +998,7 @@ async function run { expect(deps.sleepSeconds).toHaveBeenNthCalledWith(2, 0); }); + it("uses the shared extended health configuration for an existing gateway container (#10652)", async () => { + vi.stubEnv("NEMOCLAW_HEALTH_POLL_COUNT", "1"); + vi.stubEnv("NEMOCLAW_HEALTH_POLL_INTERVAL", "0"); + vi.stubEnv("NEMOCLAW_GATEWAY_START_POLL_COUNT", "3"); + vi.stubEnv("NEMOCLAW_GATEWAY_START_POLL_INTERVAL", "0"); + const deps = createDeps({ getGatewayClusterContainerState: () => "running starting" }); + + await expect(startGatewayForRecovery({ gatewayPort: 8091 }, deps)).rejects.toThrow( + "did not become ready within the configured 3 immediate health probes", + ); + + expect(deps.runCaptureOpenshell).toHaveBeenCalledTimes(9); + expect(deps.sleepSeconds).toHaveBeenCalledTimes(2); + }); + it("rejects non-canonical gateway recovery names before invoking OpenShell", async () => { const deps = createDeps(); diff --git a/src/lib/onboard/gateway-recovery.ts b/src/lib/onboard/gateway-recovery.ts index ddc4c2df096..ee0898c3ebe 100644 --- a/src/lib/onboard/gateway-recovery.ts +++ b/src/lib/onboard/gateway-recovery.ts @@ -27,7 +27,7 @@ import { isGatewayHealthy } from "../state/gateway"; import { isLinuxDockerDriverGatewayEnabled } from "./docker-driver-platform"; import { envInt } from "./env"; import { resolveGatewayName, resolveGatewayPortFromName } from "./gateway-binding"; -import { formatGatewayHealthWaitLimit } from "./gateway-health-wait"; +import { formatGatewayHealthWaitLimit, getGatewayHealthWaitConfig } from "./gateway-health-wait"; import { isGatewayHttpReady } from "./gateway-http-readiness"; import { getContainerRuntime } from "./local-inference-topology"; import { @@ -135,26 +135,6 @@ function getDefaultGatewayClusterContainerState(gatewayName: string): string { return state || "missing"; } -function getGatewayHealthWaitConfig(_startStatus = 0, containerState = "") { - const isArm64 = process.arch === "arm64"; - const standardCount = envInt("NEMOCLAW_HEALTH_POLL_COUNT", isArm64 ? 30 : 12); - const standardInterval = envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", isArm64 ? 10 : 5); - const extendedCount = envInt("NEMOCLAW_GATEWAY_START_POLL_COUNT", standardCount); - const extendedInterval = envInt("NEMOCLAW_GATEWAY_START_POLL_INTERVAL", standardInterval); - const normalizedState = String(containerState || "") - .trim() - .toLowerCase(); - const normalizedContainerState = normalizedState || "missing"; - const useExtendedWait = normalizedContainerState !== "missing"; - - return { - count: useExtendedWait ? extendedCount : standardCount, - interval: useExtendedWait ? extendedInterval : standardInterval, - extended: useExtendedWait, - containerState: normalizedContainerState, - }; -} - function getGatewayRecoveryWaitBudgetMs(pollCount: number, pollIntervalSeconds: number): number { return getLegacyPollDeadlineBudgetMs(pollCount, pollIntervalSeconds); } diff --git a/test/inference/managed/managed-image-protected-runtime-contract.test.ts b/test/inference/managed/managed-image-protected-runtime-contract.test.ts index 8c709d8f7a5..e39d456d4c7 100644 --- a/test/inference/managed/managed-image-protected-runtime-contract.test.ts +++ b/test/inference/managed/managed-image-protected-runtime-contract.test.ts @@ -31,6 +31,7 @@ import { managedImageOpenShellCommittedProbe, managedImageOpenShellProbe, parseManagedImageOpenShellE2eInputs, + removeManagedImageSandboxIfOwned, removeManagedImageGatewayStateIfSafe, resolveManagedImageOnboardModule, } from "../../../scripts/checks/run-managed-image-openshell-e2e.ts"; @@ -303,6 +304,80 @@ describe("protected managed-image runtime contract", () => { expect(fs.existsSync(stateDir)).toBe(false); }); + it("refuses to delete a same-name replacement during managed-image cleanup (#10652)", () => { + const runOpenshell = vi.fn(() => ({ + status: 0, + stdout: "Name: managed-openclaw\nId: sandbox-replacement\nPhase: Ready\n", + stderr: "", + })); + const runCommand = vi.fn(() => SUCCESS_WITHOUT_OUTPUT); + const input = parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + VALID_SANDBOX, + ]); + + expect( + removeManagedImageSandboxIfOwned( + { + openshellArgv: (argv: readonly string[]) => ["openshell", ...argv], + runOpenshell, + } as never, + input, + "sandbox-created-by-harness", + {}, + runCommand, + ), + ).toBe("refusing managed-image sandbox cleanup because its durable identity changed"); + expect(runOpenshell).toHaveBeenCalledWith( + ["sandbox", "get", "-g", "nemoclaw", VALID_SANDBOX], + expect.objectContaining({ ignoreError: true }), + ); + expect(runCommand).not.toHaveBeenCalled(); + }); + + it("deletes only after exact durable-ID comparison and verifies absence (#10652)", () => { + const runOpenshell = vi + .fn() + .mockReturnValueOnce({ + status: 0, + stdout: "Name: managed-openclaw\nId: sandbox-created-by-harness\nPhase: Ready\n", + stderr: "", + }) + .mockReturnValueOnce({ status: 1, stdout: "", stderr: "sandbox not found" }); + const runCommand = vi.fn(() => SUCCESS_WITHOUT_OUTPUT); + const input = parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + VALID_SANDBOX, + ]); + + expect( + removeManagedImageSandboxIfOwned( + { + openshellArgv: (argv: readonly string[]) => ["openshell", ...argv], + runOpenshell, + } as never, + input, + "sandbox-created-by-harness", + {}, + runCommand, + ), + ).toBeNull(); + expect(runCommand).toHaveBeenCalledWith( + ["openshell", "sandbox", "delete", "-g", "nemoclaw", VALID_SANDBOX], + {}, + 15_000, + ); + expect(runOpenshell).toHaveBeenCalledTimes(2); + }); + it("distinguishes the running image from exact quiescent rollback retention (#7744)", () => { const calls: string[][] = []; const contentId = `sha256:${"b".repeat(64)}`; @@ -725,10 +800,7 @@ describe("protected managed-image runtime contract", () => { ).toThrow(/cannot be combined/u); }); - it("keeps rollback cleanup distinct from initial readiness", () => { - expect(managedImageOpenShellCommittedProbe()).toContain( - "managed-startup-shared-state-transaction-v1", - ); + it("keeps rollback cleanup distinct from initial readiness (#10652)", () => { expect( parseManagedImageOpenShellE2eInputs([ "--agent", @@ -740,5 +812,22 @@ describe("protected managed-image runtime contract", () => { "--inject-bootstrap-completion-failure", ]), ).toMatchObject({ failureInjection: "bootstrap-completion" }); + + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-committed-probe-")); + try { + const probe = managedImageOpenShellCommittedProbe({ rootPath: fixtureRoot }); + const completed = spawnSync("/bin/sh", ["-eu", "-c", probe], { encoding: "utf8" }); + expect(completed.status, completed.stderr).toBe(0); + + const retainedTransaction = managedProbeFixturePath( + fixtureRoot, + "/var/lib/nemoclaw/managed-startup-shared-state-transaction-v1", + ); + fs.mkdirSync(retainedTransaction, { recursive: true }); + const incomplete = spawnSync("/bin/sh", ["-eu", "-c", probe], { encoding: "utf8" }); + expect(incomplete.status).not.toBe(0); + } finally { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + } }); }); From 87fb85d1f83e334c2159d1c16d1848394f5a5215 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 09:10:23 -0700 Subject: [PATCH 45/51] fix(e2e): refuse mutable-name sandbox deletion Signed-off-by: Prekshi Vyas --- .../checks/run-managed-image-openshell-e2e.ts | 28 ++++----- src/lib/onboard/gateway-recovery.test.ts | 6 -- ...d-image-protected-runtime-contract.test.ts | 62 +++++++------------ 3 files changed, 33 insertions(+), 63 deletions(-) diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index b0018a15867..308a0a6c2cc 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -942,12 +942,11 @@ function assertManagedImageSandboxIdentity( } } -export function removeManagedImageSandboxIfOwned( - onboard: Pick, +export function managedImageSandboxCleanupOwnershipError( + onboard: Pick, input: Pick, expectedSandboxId: string | null, env: NodeJS.ProcessEnv, - runCommand: ManagedImageCommandRunner = commandResult, ): string | null { const observed = queryManagedImageSandboxIdentity(onboard, input, env); if (!expectedSandboxId) { @@ -961,19 +960,11 @@ export function removeManagedImageSandboxIfOwned( if (observed.sandboxId !== expectedSandboxId) { return "refusing managed-image sandbox cleanup because its durable identity changed"; } - - const remove = runCommand( - onboard.openshellArgv(["sandbox", "delete", "-g", GATEWAY_NAME, input.sandbox]), - env, - 15_000, - ); - if (remove.status !== 0) { - return `OpenShell sandbox cleanup failed with status ${String(remove.status)}`; + const deleteBoundary = queryManagedImageSandboxIdentity(onboard, input, env); + if (deleteBoundary.sandboxId !== expectedSandboxId) { + return "refusing managed-image sandbox cleanup because its durable identity changed at the delete boundary"; } - const verification = queryManagedImageSandboxIdentity(onboard, input, env); - return verification.result.status === 0 || verification.sandboxId !== null - ? "OpenShell sandbox cleanup did not prove exact absence" - : null; + return null; } async function run( @@ -1288,7 +1279,12 @@ async function run { expect(process.env.OPENSHELL_GATEWAY).toBe("nemoclaw-8091"); expect(deps.sleepSeconds).not.toHaveBeenCalled(); - // First iteration only: 3 subprocess calls (status + gateway info -g + - // gateway info); loop returns before the next iteration would start. - expect(deps.runCaptureOpenshell).toHaveBeenCalledTimes(3); }); it("succeeds after retrying past unhealthy probes and still sets OPENSHELL_GATEWAY (#3768)", async () => { @@ -195,11 +192,8 @@ describe("gateway recovery", () => { await startGatewayForRecovery({ gatewayPort: 8091 }, deps); expect(process.env.OPENSHELL_GATEWAY).toBe("nemoclaw-8091"); - // Exactly one inter-attempt sleep between the unhealthy first probe - // and the healthy second probe. expect(deps.sleepSeconds).toHaveBeenCalledTimes(1); expect(deps.sleepSeconds).toHaveBeenNthCalledWith(1, 0.25); - expect(deps.runCaptureOpenshell).toHaveBeenCalledTimes(6); }); it("with NEMOCLAW_HEALTH_POLL_COUNT=0 fails fast without silently claiming healthy (#3768)", async () => { diff --git a/test/inference/managed/managed-image-protected-runtime-contract.test.ts b/test/inference/managed/managed-image-protected-runtime-contract.test.ts index e39d456d4c7..c2b48fec5a5 100644 --- a/test/inference/managed/managed-image-protected-runtime-contract.test.ts +++ b/test/inference/managed/managed-image-protected-runtime-contract.test.ts @@ -30,8 +30,8 @@ import { managedImageOpenShellBasePolicyPath, managedImageOpenShellCommittedProbe, managedImageOpenShellProbe, + managedImageSandboxCleanupOwnershipError, parseManagedImageOpenShellE2eInputs, - removeManagedImageSandboxIfOwned, removeManagedImageGatewayStateIfSafe, resolveManagedImageOnboardModule, } from "../../../scripts/checks/run-managed-image-openshell-e2e.ts"; @@ -310,7 +310,6 @@ describe("protected managed-image runtime contract", () => { stdout: "Name: managed-openclaw\nId: sandbox-replacement\nPhase: Ready\n", stderr: "", })); - const runCommand = vi.fn(() => SUCCESS_WITHOUT_OUTPUT); const input = parseManagedImageOpenShellE2eInputs([ "--agent", "openclaw", @@ -321,25 +320,21 @@ describe("protected managed-image runtime contract", () => { ]); expect( - removeManagedImageSandboxIfOwned( - { - openshellArgv: (argv: readonly string[]) => ["openshell", ...argv], - runOpenshell, - } as never, + managedImageSandboxCleanupOwnershipError( + { runOpenshell } as never, input, "sandbox-created-by-harness", {}, - runCommand, ), ).toBe("refusing managed-image sandbox cleanup because its durable identity changed"); expect(runOpenshell).toHaveBeenCalledWith( ["sandbox", "get", "-g", "nemoclaw", VALID_SANDBOX], expect.objectContaining({ ignoreError: true }), ); - expect(runCommand).not.toHaveBeenCalled(); + expect(runOpenshell).toHaveBeenCalledTimes(1); }); - it("deletes only after exact durable-ID comparison and verifies absence (#10652)", () => { + it("retains a same-name replacement that appears at the delete boundary (#10652)", () => { const runOpenshell = vi .fn() .mockReturnValueOnce({ @@ -347,8 +342,11 @@ describe("protected managed-image runtime contract", () => { stdout: "Name: managed-openclaw\nId: sandbox-created-by-harness\nPhase: Ready\n", stderr: "", }) - .mockReturnValueOnce({ status: 1, stdout: "", stderr: "sandbox not found" }); - const runCommand = vi.fn(() => SUCCESS_WITHOUT_OUTPUT); + .mockReturnValueOnce({ + status: 0, + stdout: "Name: managed-openclaw\nId: sandbox-replacement\nPhase: Ready\n", + stderr: "", + }); const input = parseManagedImageOpenShellE2eInputs([ "--agent", "openclaw", @@ -359,23 +357,20 @@ describe("protected managed-image runtime contract", () => { ]); expect( - removeManagedImageSandboxIfOwned( - { - openshellArgv: (argv: readonly string[]) => ["openshell", ...argv], - runOpenshell, - } as never, + managedImageSandboxCleanupOwnershipError( + { runOpenshell } as never, input, "sandbox-created-by-harness", {}, - runCommand, ), - ).toBeNull(); - expect(runCommand).toHaveBeenCalledWith( - ["openshell", "sandbox", "delete", "-g", "nemoclaw", VALID_SANDBOX], - {}, - 15_000, + ).toBe( + "refusing managed-image sandbox cleanup because its durable identity changed at the delete boundary", ); expect(runOpenshell).toHaveBeenCalledTimes(2); + expect(runOpenshell.mock.calls.map(([argv]) => argv)).toEqual([ + ["sandbox", "get", "-g", "nemoclaw", VALID_SANDBOX], + ["sandbox", "get", "-g", "nemoclaw", VALID_SANDBOX], + ]); }); it("distinguishes the running image from exact quiescent rollback retention (#7744)", () => { @@ -529,30 +524,15 @@ describe("protected managed-image runtime contract", () => { const qualifications = PROTECTED_MANAGED_IMAGE_AGENTS.flatMap((agent) => routeKinds.map((routeKind) => ({ agent, + routeKind, sandbox: managedImageProtectedSandboxName(agent, routeKind), })), ); const names = qualifications.map(({ sandbox }) => sandbox); - expect(names).toEqual([ - "nmc-mi-oc-lc", - "nmc-mi-oc-ol", - "nmc-mi-oc-ni", - "nmc-mi-oc-vl", - "nmc-mi-oc-rb", - "nmc-mi-he-lc", - "nmc-mi-he-ol", - "nmc-mi-he-ni", - "nmc-mi-he-vl", - "nmc-mi-he-rb", - "nmc-mi-dc-lc", - "nmc-mi-dc-ol", - "nmc-mi-dc-ni", - "nmc-mi-dc-vl", - "nmc-mi-dc-rb", - ]); expect(new Set(names).size).toBe(names.length); - qualifications.forEach(({ agent, sandbox: name }) => { + qualifications.forEach(({ agent, routeKind, sandbox: name }) => { + expect(managedImageProtectedSandboxName(agent, routeKind)).toBe(name); expect(name.startsWith(MANAGED_IMAGE_PROTECTED_SANDBOX_PREFIX)).toBe(true); expect(name.length).toBeLessThanOrEqual(19); expect(name).not.toContain("--"); From 950b63d20afb6b588001f0fdae1e8307e18fbc70 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 09:35:55 -0700 Subject: [PATCH 46/51] fix(onboard): scope recovery evidence to gateway Signed-off-by: Prekshi Vyas --- docs/reference/troubleshooting.mdx | 4 +- .../checks/run-managed-image-openshell-e2e.ts | 4 +- src/lib/onboard/gateway-recovery.test.ts | 57 ++++++------------- ...d-image-protected-runtime-contract.test.ts | 56 +++++++++++++++++- 4 files changed, 77 insertions(+), 44 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 1508ee6d06c..1bf4d0a9243 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2007,8 +2007,8 @@ For a newly created OpenClaw or Hermes sandbox, `Ready` is not the final accepta If onboarding reports that the managed runtime commit completed but the same sandbox did not return to executable `Ready`, stop before retrying. NemoClaw keeps the sandbox, prints its create-attempt label and a one-way durable identity fingerprint, and does not start dashboard forwarding. It saves that evidence in the retained recovery record when persistence succeeds. -Do not delete the sandbox by its mutable name. -When NemoClaw confirms that it saved the record and the record contains a durable identity fingerprint, run `$$nemoclaw destroy` and follow [Recover a retained sandbox](commands#recover-a-retained-sandbox) for the result-specific recovery steps. +Do not use OpenShell mutable-name deletion. +When NemoClaw confirms that it saved the record and the record contains a durable identity fingerprint, run `$$nemoclaw destroy`; NemoClaw verifies the retained sandbox identity before cleanup. Follow [Recover a retained sandbox](commands#recover-a-retained-sandbox) for the result-specific recovery steps. If the saved record has no durable identity fingerprint, preserve the terminal output and give the create-attempt label to an OpenShell administrator so they can identify and remove the exact sandbox. If NemoClaw reports that it could not save the record, preserve the terminal output and ask an OpenShell administrator to identify the exact sandbox from gateway or controller evidence; the recovery-only session remains blocked until NemoClaw can save the durable recovery record. diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 308a0a6c2cc..3ee8f92fd3e 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -886,12 +886,12 @@ export function assertFailedSandboxOwnerCleanupRetention( expectedSandboxId: string, env: NodeJS.ProcessEnv, ): void { - const get = onboard.runOpenshell(["sandbox", "get", input.sandbox], { + const get = onboard.runOpenshell(["sandbox", "get", "-g", GATEWAY_NAME, input.sandbox], { ignoreError: true, env, stdio: ["ignore", "pipe", "pipe"], }); - const list = onboard.runOpenshell(["sandbox", "list"], { + const list = onboard.runOpenshell(["sandbox", "list", "-g", GATEWAY_NAME], { ignoreError: true, env, stdio: ["ignore", "pipe", "pipe"], diff --git a/src/lib/onboard/gateway-recovery.test.ts b/src/lib/onboard/gateway-recovery.test.ts index c030d22b770..9983fd192b2 100644 --- a/src/lib/onboard/gateway-recovery.test.ts +++ b/src/lib/onboard/gateway-recovery.test.ts @@ -12,15 +12,15 @@ import { type GatewayRecoveryDeps, startGatewayForRecovery } from "./gateway-rec // actually sleeps. Tests get deterministic deadline expiration without any // real wall-clock waits or global timer state. // -// `advance` is exposed so a test can also advance the clock from inside a -// mocked probe. This is how the timeout test proves the loop is truly -// deadline-driven: if each probe advances the clock, then a maxAttempts=N -// cap would exit at a different observable count than a pure deadline -// would, so the assertions can only be satisfied by the deadline path. +// `advance` is exposed so a test can also account for time spent inside a +// mocked probe. `elapsedMs` lets the timeout test observe that the complete +// configured deadline was consumed without depending on an internal call +// count or the number of OpenShell observations in one recovery probe. function makeVirtualClock(startMs = 1_000_000_000_000) { let now = startMs; return { now: () => now, + elapsedMs: () => now - startMs, advance: (seconds: number) => { now += Math.max(0, seconds) * 1000; }, @@ -100,52 +100,31 @@ describe("gateway recovery", () => { }); it("polls until the configured recovery deadline and reports it in the timeout (#3768)", async () => { - // #3768: prove the loop is DEADLINE-driven, not just attempt-capped. - // Design: with count=10 and interval=1s the wait budget is 10s. Make - // each subprocess-probe advance the clock by ~1s so probes are the - // primary time-consumer, then sleeps at 1s add another second per - // iteration. Under a pure deadline: iterations run until ~2s per - // iteration cumulatively hits 10s -> ~5 probes. Under a hidden - // maxAttempts=count cap, the loop would exit at exactly 10 probes - // (attempt cap hits first because probes and sleeps take equal time), - // which is a different observable count from the deadline path. The - // strict upper bound `probeCount < 10` therefore only passes when the - // deadline (not an attempt cap) terminates the loop. + // #3768: prove the loop consumes its configured wall-clock deadline. + // Account for one second of work only when the externally visible + // status probe begins; the other OpenShell observations remain free to + // change without changing this test's oracle. vi.stubEnv("NEMOCLAW_HEALTH_POLL_COUNT", "10"); vi.stubEnv("NEMOCLAW_HEALTH_POLL_INTERVAL", "1"); const clock = makeVirtualClock(); - // Only advance the clock ONCE per probe iteration (three subprocess - // calls per probe): status is the first call, gateway-info-g the - // second, gateway-info the third. Use a modulo counter so the test - // body stays linear (per repo growth guardrail on if statements in - // changed test files). - let mockCallIndex = 0; - const advanceOnStatusCall = (index: number) => (index % 3 === 0 ? clock.advance(1) : undefined); + const statusProbe = vi.fn(() => { + clock.advance(1); + return "Disconnected"; + }); const deps = createDeps({ sleepSeconds: clock.sleeper, now: clock.now, - runCaptureOpenshell: vi.fn(() => { - advanceOnStatusCall(mockCallIndex); - mockCallIndex += 1; - return "Disconnected"; - }), + runCaptureOpenshell: vi.fn((argv) => + argv[0] === "status" ? statusProbe() : "Disconnected", + ), }); await expect(startGatewayForRecovery({ gatewayPort: 8091 }, deps)).rejects.toThrow( "configured 10s recovery deadline (1s poll interval)", ); - const runCaptureCalls = (deps.runCaptureOpenshell as ReturnType).mock.calls - .length; - const probeCount = runCaptureCalls / 3; - // The deadline (not an attempt cap) MUST have terminated the loop: - // probe advances 1s + sleep advances 1s = 2s per iteration, so under - // a 10s budget the loop runs ~5 iterations and cannot reach the 10 - // attempts a hidden attempt cap would permit. - expect(probeCount).toBeGreaterThan(0); - expect(probeCount).toBeLessThan(10); - // Sleeps happen after every probe except the last one (deadline check - // after the final probe short-circuits before an extra sleep). + expect(statusProbe).toHaveBeenCalled(); + expect(clock.elapsedMs()).toBe(10_000); expect(clock.sleeper).toHaveBeenCalled(); expect(clock.sleeper).toHaveBeenNthCalledWith(1, 0.25); expect(clock.sleeper.mock.calls.every(([s]) => s <= 1)).toBe(true); diff --git a/test/inference/managed/managed-image-protected-runtime-contract.test.ts b/test/inference/managed/managed-image-protected-runtime-contract.test.ts index c2b48fec5a5..6490c87feee 100644 --- a/test/inference/managed/managed-image-protected-runtime-contract.test.ts +++ b/test/inference/managed/managed-image-protected-runtime-contract.test.ts @@ -476,6 +476,16 @@ describe("protected managed-image runtime contract", () => { {}, ), ).not.toThrow(); + expect(runOpenshell).toHaveBeenNthCalledWith( + 1, + ["sandbox", "get", "-g", "nemoclaw", VALID_SANDBOX], + expect.objectContaining({ ignoreError: true }), + ); + expect(runOpenshell).toHaveBeenNthCalledWith( + 2, + ["sandbox", "list", "-g", "nemoclaw"], + expect.objectContaining({ ignoreError: true }), + ); }); it("rejects a containing sandbox name and an exact name mentioned only in stderr", () => { @@ -514,7 +524,51 @@ describe("protected managed-image runtime contract", () => { expect(assertion).toThrow("exact OpenShell owner-cleanup state"); expect(runOpenshell).toHaveBeenNthCalledWith( 2, - ["sandbox", "list"], + ["sandbox", "list", "-g", "nemoclaw"], + expect.objectContaining({ ignoreError: true }), + ); + }); + + it("rejects foreign-gateway rollback retention evidence (#10652)", () => { + const expectedSandboxId = "sandbox-id-123"; + const input = parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + VALID_SANDBOX, + ]); + const valid = { status: 0, stdout: `Id: ${expectedSandboxId}\n`, stderr: "" }; + const missing = { status: 1, stdout: "", stderr: "sandbox not found" }; + const responses = new Map([ + [JSON.stringify(["sandbox", "get", VALID_SANDBOX]), valid], + [JSON.stringify(["sandbox", "list"]), { + status: 0, + stdout: `NAME STATUS\n${VALID_SANDBOX} Ready\n`, + stderr: "", + }], + ]); + const runOpenshell = vi.fn( + (argv: readonly string[]) => responses.get(JSON.stringify(argv)) ?? missing, + ); + + expect(() => + assertFailedSandboxOwnerCleanupRetention( + { runOpenshell } as never, + input, + expectedSandboxId, + {}, + ), + ).toThrow("exact OpenShell owner-cleanup state"); + expect(runOpenshell).toHaveBeenNthCalledWith( + 1, + ["sandbox", "get", "-g", "nemoclaw", VALID_SANDBOX], + expect.objectContaining({ ignoreError: true }), + ); + expect(runOpenshell).toHaveBeenNthCalledWith( + 2, + ["sandbox", "list", "-g", "nemoclaw"], expect.objectContaining({ ignoreError: true }), ); }); From f6738ada4cb8687d003666aec49cf3c08ed2fde6 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 10:01:21 -0700 Subject: [PATCH 47/51] fix(onboard): preserve post-commit readiness budget Signed-off-by: Prekshi Vyas --- .../checks/run-managed-image-openshell-e2e.ts | 154 +++++++++++++-- src/lib/onboard/sandbox-gpu-create-flow.ts | 2 + .../sandbox-gpu-create-identity-gate.test.ts | 52 ++++- .../onboard/sandbox-gpu-create-run-attempt.ts | 16 +- ...d-image-protected-runtime-contract.test.ts | 183 ++++++++++++++++-- 5 files changed, 370 insertions(+), 37 deletions(-) diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 3ee8f92fd3e..d3de9874b33 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -8,7 +8,10 @@ import os from "node:os"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { resolveAgent } from "../../src/lib/agent/onboard.ts"; -import { parseOpenShellSandboxId } from "../../src/lib/adapters/openshell/sandbox-identity.ts"; +import { + NEMOCLAW_CREATE_ATTEMPT_LABEL, + parseOpenShellSandboxId, +} from "../../src/lib/adapters/openshell/sandbox-identity.ts"; import { createCliOpenShellSandboxObserverFromRunner } from "../../src/lib/adapters/openshell/sandbox-observer-cli.ts"; import { isValidName, NAME_ALLOWED_FORMAT } from "../../src/lib/name-validation.ts"; import { @@ -48,6 +51,7 @@ import { resolveDockerStartupCommandPatch, runSandboxGpuCreateFlow, type SandboxGpuCreateFlowInput, + waitForSandboxReadinessUntil, } from "../../src/lib/onboard/sandbox-gpu-create-flow.ts"; import { createDirectSandboxGpuVerifier } from "../../src/lib/onboard/sandbox-gpu-preflight.ts"; import { @@ -322,6 +326,90 @@ function isDockerNotFound(result: ManagedImageCommandResult): boolean { ); } +export type ManagedImageRetainedSandboxRecovery = Readonly<{ + schemaVersion: 1; + sandboxName: string; + createAttemptLabel: string; + sandboxIdentityFingerprint: string | null; + message: string; +}>; + +const MANAGED_IMAGE_RETAINED_RECOVERY_FILE = "retained-sandbox-recovery.json"; + +export function readManagedImageRetainedSandboxRecovery( + stateDir: string, +): ManagedImageRetainedSandboxRecovery | null { + try { + const value = JSON.parse( + fs.readFileSync(path.join(stateDir, MANAGED_IMAGE_RETAINED_RECOVERY_FILE), "utf8"), + ) as Partial; + if ( + value.schemaVersion !== 1 || + typeof value.sandboxName !== "string" || + typeof value.createAttemptLabel !== "string" || + (value.sandboxIdentityFingerprint !== null && + typeof value.sandboxIdentityFingerprint !== "string") || + typeof value.message !== "string" + ) { + return null; + } + return value as ManagedImageRetainedSandboxRecovery; + } catch { + return null; + } +} + +export function persistManagedImageRetainedSandboxRecovery(options: { + readonly stateDir: string; + readonly sandboxName: string; + readonly message: string; + readonly sandboxIdentityFingerprint?: string; + readonly createAttemptNonce?: string; +}): boolean { + if (!options.createAttemptNonce) return false; + const record: ManagedImageRetainedSandboxRecovery = { + schemaVersion: 1, + sandboxName: options.sandboxName, + createAttemptLabel: `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${options.createAttemptNonce}`, + sandboxIdentityFingerprint: options.sandboxIdentityFingerprint ?? null, + message: options.message, + }; + const recordPath = path.join(options.stateDir, MANAGED_IMAGE_RETAINED_RECOVERY_FILE); + const temporaryPath = `${recordPath}.tmp`; + let descriptor: number | null = null; + let directoryDescriptor: number | null = null; + try { + fs.writeFileSync(temporaryPath, `${JSON.stringify(record, null, 2)}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + descriptor = fs.openSync(temporaryPath, "r"); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = null; + fs.renameSync(temporaryPath, recordPath); + directoryDescriptor = fs.openSync(options.stateDir, "r"); + fs.fsyncSync(directoryDescriptor); + fs.closeSync(directoryDescriptor); + directoryDescriptor = null; + const persisted = readManagedImageRetainedSandboxRecovery(options.stateDir); + return ( + persisted?.schemaVersion === record.schemaVersion && + persisted.sandboxName === record.sandboxName && + persisted.createAttemptLabel === record.createAttemptLabel && + persisted.sandboxIdentityFingerprint === record.sandboxIdentityFingerprint && + persisted.message === record.message + ); + } catch { + return false; + } finally { + if (descriptor !== null) fs.closeSync(descriptor); + if (directoryDescriptor !== null) fs.closeSync(directoryDescriptor); + fs.rmSync(temporaryPath, { force: true }); + } +} + export function removeManagedImageGatewayStateIfSafe( stateDir: string, gatewayStop: Pick, @@ -533,17 +621,28 @@ export function managedImageOpenShellCommittedProbe( ].join("\n"); } -async function waitForCommittedSandboxProbe( +export type ManagedImageCommittedProbeWaitOptions = { + readonly budgetMs?: number; + readonly now?: () => number; + readonly runCommand?: ManagedImageCommandRunner; + readonly sleep?: (milliseconds: number) => void | Promise; +}; + +export async function waitForCommittedSandboxProbe( onboard: OnboardModule, input: Inputs, env: NodeJS.ProcessEnv, requireCommitted = true, + options: ManagedImageCommittedProbeWaitOptions = {}, ): Promise { const healthProbe = managedImageOpenShellProbe(input.agent, input.model ?? MODEL); const committedProbe = managedImageOpenShellCommittedProbe(); - const deadline = Date.now() + 240_000; + const now = options.now ?? Date.now; + const budgetMs = options.budgetMs ?? 240_000; + const deadlineMs = now() + budgetMs; + const runCommand = options.runCommand ?? commandResult; const runProbe = (probe: string, timeoutMs: number) => - commandResult( + runCommand( onboard.openshellArgv([ "sandbox", "exec", @@ -559,26 +658,36 @@ async function waitForCommittedSandboxProbe( timeoutMs, ); let lastHealthDetail = ""; - while (Date.now() < deadline) { - const remainingMs = deadline - Date.now(); - const health = runProbe(healthProbe, Math.max(1, Math.min(15_000, remainingMs))); - if (health.status === 0) { - if (!requireCommitted) return; + const ready = await waitForSandboxReadinessUntil( + () => { + const remainingMs = deadlineMs - now(); + const health = runProbe(healthProbe, Math.max(1, Math.min(15_000, remainingMs))); + if (health.status !== 0) { + lastHealthDetail = commandDetail(health); + return false; + } + if (!requireCommitted) return true; const committed = runProbe( committedProbe, - Math.max(1, Math.min(15_000, deadline - Date.now())), + Math.max(1, Math.min(15_000, deadlineMs - now())), ); if (committed.status !== 0) { throw new Error( `managed bootstrap committed, but transaction cleanup was not observable through the exact sandbox: ${commandDetail(committed)}`, ); } - return; - } - lastHealthDetail = commandDetail(health); - const sleepMs = Math.min(2_000, Math.max(0, deadline - Date.now())); - if (sleepMs > 0) await new Promise((resolve) => setTimeout(resolve, sleepMs)); - } + return true; + }, + { + deadlineMs, + initialIntervalMs: 2_000, + maxIntervalMs: 2_000, + backoffFactor: 1, + now, + ...(options.sleep ? { sleep: options.sleep } : {}), + }, + ); + if (ready) return; throw new Error( `OpenShell sandbox did not pass the exact-image managed-bootstrap probe within 240s: ${lastHealthDetail}`, ); @@ -1142,6 +1251,19 @@ async function run { expect(firstReadiness?.stableReadyPolls).toBe(1); }); - it("shares managed bootstrap time with post-commit readiness (#10652)", async () => { + it("starts a fresh readiness deadline after the managed runtime commit (#10652)", async () => { let nonce = ""; let nowMs = 0; const input = noGpuInput(); @@ -397,13 +397,61 @@ describe("created sandbox identity gate", () => { return { ready: true, reason: "ready", failurePhase: null }; }); await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); - expect(readinessTimeouts).toEqual([10, 7, 5]); + expect(readinessTimeouts).toEqual([10, 7, 10]); expect( mocks.waitForCreatedSandboxReadyWithTrace.mock.calls.map(([options]) => options.now), ).toEqual([deps.publicationNow, deps.publicationNow, deps.publicationNow]); expect(patch.commitAfterReady).toHaveBeenCalledOnce(); }); + it("does not charge slow GPU inference validation to post-commit readiness (#10652)", async () => { + let nonce = ""; + let nowMs = 0; + const input = noGpuInput(); + input.sandboxGpuConfig = { + mode: "1", + hostGpuDetected: true, + hostGpuPlatform: "linux", + sandboxGpuEnabled: true, + sandboxGpuDevice: null, + errors: [], + }; + input.gpuRoutePlan = "native-only"; + input.initialGpuRoute = "native"; + input.sandboxReadyTimeoutSecs = 10; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + attachManagedBootstrap(input, patch, { freshCreate: true }); + mocks.streamSandboxCreate.mockImplementationOnce(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + deps.publicationNow = () => nowMs; + vi.mocked(deps.runCaptureOpenshell).mockImplementation((args) => + args[1] === "list" + ? sandboxListJson("alpha-sandbox-id", { + [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, + }) + : "Name: alpha\nId: alpha-sandbox-id\nState: Ready\n", + ); + const readinessTimeouts: number[] = []; + mocks.waitForCreatedSandboxReadyWithTrace.mockImplementation(async (options) => { + readinessTimeouts.push(options.timeoutSecs); + return { ready: true, reason: "ready", failurePhase: null }; + }); + + const created = await runSandboxGpuCreateFlow(input, deps); + expect(patch.commitAfterReady).not.toHaveBeenCalled(); + nowMs += 9_500; + await created.runtimePatch.commitAfterReady(); + await created.confirmManagedRuntimeCommitReadiness(); + + expect(readinessTimeouts.at(-1)).toBe(10); + expect(patch.commitAfterReady).toHaveBeenCalledOnce(); + }); + it("retains exact recovery when committed managed readiness does not return (#9211)", async () => { const error = vi.spyOn(console, "error").mockImplementation(() => undefined); const sandboxId = "alpha-sandbox-id"; diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 9e0bf995a16..1f4137733d4 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -19,7 +19,7 @@ import { settleCreatedOpenShellSandboxId, } from "../adapters/openshell/sandbox-identity"; import { createReadinessWaitOptions } from "../core/readiness-wait"; -import { waitUntil } from "../core/wait"; +import { waitUntil, waitUntilAsync } from "../core/wait"; import { streamSandboxCreate, type StreamSandboxCreateResult } from "../sandbox/create-stream"; import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-ready-gate"; import { redact, redactFullWithUrls } from "../security/redact"; @@ -83,6 +83,8 @@ type PostCreateReadinessDeadline = Readonly<{ now: () => number; }>; +export { waitUntilAsync as waitForSandboxReadinessUntil }; + function createPostCreateReadinessDeadline( input: SandboxGpuCreateFlowInput, deps: SandboxGpuCreateFlowDeps, @@ -1426,14 +1428,20 @@ export function createSandboxGpuCreateAttemptRunner( revalidatePostCreateEffect(`commit runtime readiness for sandbox '${input.sandboxName}'`); await runtimePatch.commitAfterReady(); } - const confirmCommittedRuntimeReadiness = () => - confirmManagedRuntimeCommitReadiness({ + let committedRuntimeReadinessDeadline: PostCreateReadinessDeadline | null = null; + const confirmCommittedRuntimeReadiness = () => { + // GPU local-inference validation has its own timeout and runs before + // commit. Start this budget only when the caller has committed the + // runtime so slow inference validation cannot consume Ready recovery. + committedRuntimeReadinessDeadline ??= createPostCreateReadinessDeadline(input, deps); + return confirmManagedRuntimeCommitReadiness({ input, deps, sandboxId: managedBootstrap ? verifiedCreatedSandboxId : null, createAttemptNonce, - deadline: requirePostCreateReadinessDeadline(), + deadline: committedRuntimeReadinessDeadline, }); + }; if (!input.sandboxGpuConfig.sandboxGpuEnabled) { await confirmCommittedRuntimeReadiness(); } diff --git a/test/inference/managed/managed-image-protected-runtime-contract.test.ts b/test/inference/managed/managed-image-protected-runtime-contract.test.ts index 6490c87feee..44cd46bfc96 100644 --- a/test/inference/managed/managed-image-protected-runtime-contract.test.ts +++ b/test/inference/managed/managed-image-protected-runtime-contract.test.ts @@ -32,9 +32,13 @@ import { managedImageOpenShellProbe, managedImageSandboxCleanupOwnershipError, parseManagedImageOpenShellE2eInputs, + persistManagedImageRetainedSandboxRecovery, + readManagedImageRetainedSandboxRecovery, removeManagedImageGatewayStateIfSafe, resolveManagedImageOnboardModule, + waitForCommittedSandboxProbe, } from "../../../scripts/checks/run-managed-image-openshell-e2e.ts"; +import { persistRetainedSandboxRecoveryOrBlock } from "../../../src/lib/onboard/sandbox-gpu-create-run-attempt.ts"; import { resolveOnboardManagedBootstrapLaunch } from "../../../src/lib/onboard/managed-workload/onboard-orchestration.js"; const IMAGE = `localhost:5000/nemoclaw-managed-protected/openclaw@sha256:${"a".repeat(64)}`; @@ -245,21 +249,6 @@ describe("protected managed-image runtime contract", () => { expect(Object.isFrozen(protectedLaunch.expectedSupervisorArgv)).toBe(true); }); - it.each([ - "openshellArgv", - "runOpenshell", - "runCaptureOpenshell", - "isSandboxReady", - "printSandboxCreateRecoveryHints", - "sleepSeconds", - "startGatewayForRecovery", - ] as const)( - "loads every OpenShell operation required before protected image launch [%s] (#7744)", - (operation) => { - expect(MANAGED_IMAGE_ONBOARD[operation], operation).toBeTypeOf("function"); - }, - ); - it("rejects a missing protected OpenShell operation with a precise contract error (#8759)", () => { expect(() => resolveManagedImageOnboardModule({ @@ -275,6 +264,170 @@ describe("protected managed-image runtime contract", () => { ).toThrow("managed-image onboard module is missing required operation(s): runOpenshell"); }); + it("uses the shared wait contract for an immediate probe and fixed retries (#10652)", async () => { + const input = parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + VALID_SANDBOX, + ]); + const onboard = { + openshellArgv: (argv: string[]) => ["openshell", ...argv], + } as never; + let nowMs = 0; + let healthAttempt = 0; + const healthProbeTimes: number[] = []; + const sleeps: number[] = []; + const healthResult = () => { + healthProbeTimes.push(nowMs); + healthAttempt += 1; + return healthAttempt === 1 + ? { status: 1, stdout: "", stderr: "sandbox warming" } + : SUCCESS_WITHOUT_OUTPUT; + }; + const runCommand = vi.fn((argv: readonly string[]) => + String(argv.at(-1)).includes("managed-startup-shared-state-transaction-v1") + ? SUCCESS_WITHOUT_OUTPUT + : healthResult(), + ); + + await waitForCommittedSandboxProbe(onboard, input, {}, true, { + budgetMs: 6_000, + now: () => nowMs, + runCommand, + sleep: (milliseconds) => { + sleeps.push(milliseconds); + nowMs += milliseconds; + }, + }); + + expect(healthProbeTimes).toEqual([0, 2_000]); + expect(sleeps).toEqual([2_000]); + }); + + it("reports the final managed-image probe diagnostic at its deadline (#10652)", async () => { + const input = parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + VALID_SANDBOX, + ]); + const onboard = { + openshellArgv: (argv: string[]) => ["openshell", ...argv], + } as never; + let nowMs = 0; + const sleeps: number[] = []; + + await expect( + waitForCommittedSandboxProbe(onboard, input, {}, true, { + budgetMs: 4_000, + now: () => nowMs, + runCommand: () => ({ status: 1, stdout: "", stderr: "last sandbox diagnostic" }), + sleep: (milliseconds) => { + sleeps.push(milliseconds); + nowMs += milliseconds; + }, + }), + ).rejects.toThrow( + "OpenShell sandbox did not pass the exact-image managed-bootstrap probe within 240s: last sandbox diagnostic", + ); + expect(nowMs).toBe(4_000); + expect(sleeps).toEqual([2_000, 2_000]); + }); + + it("fails immediately when the healthy sandbox still has transaction state (#10652)", async () => { + const input = parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + VALID_SANDBOX, + ]); + const onboard = { + openshellArgv: (argv: string[]) => ["openshell", ...argv], + } as never; + const sleeps: number[] = []; + const runCommand = vi + .fn() + .mockReturnValueOnce(SUCCESS_WITHOUT_OUTPUT) + .mockReturnValueOnce({ status: 1, stdout: "", stderr: "transaction remains" }); + + await expect( + waitForCommittedSandboxProbe(onboard, input, {}, true, { + budgetMs: 4_000, + now: () => 0, + runCommand, + sleep: (milliseconds) => { + sleeps.push(milliseconds); + }, + }), + ).rejects.toThrow( + "managed bootstrap committed, but transaction cleanup was not observable through the exact sandbox: transaction remains", + ); + expect(runCommand).toHaveBeenCalledTimes(2); + expect(sleeps).toEqual([]); + }); + + it("durably preserves harness recovery evidence when acknowledgement blocks (#10652)", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-recovery-")); + const createAttemptNonce = "a".repeat(62); + const sandboxIdentityFingerprint = "b".repeat(64); + const message = "Retain this exact managed-image sandbox for recovery."; + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + const persist = vi.fn( + ( + persistedMessage: string, + persistedFingerprint?: string, + persistedNonce?: string, + ) => { + expect( + persistManagedImageRetainedSandboxRecovery({ + stateDir, + sandboxName: VALID_SANDBOX, + message: persistedMessage, + ...(persistedFingerprint + ? { sandboxIdentityFingerprint: persistedFingerprint } + : {}), + ...(persistedNonce ? { createAttemptNonce: persistedNonce } : {}), + }), + ).toBe(true); + return false; + }, + ); + + try { + expect(() => + persistRetainedSandboxRecoveryOrBlock({ + persist, + message, + createAttemptNonce, + sandboxIdentityFingerprint, + }), + ).toThrow("the recovery-only session remains blocked"); + expect(readManagedImageRetainedSandboxRecovery(stateDir)).toEqual({ + schemaVersion: 1, + sandboxName: VALID_SANDBOX, + createAttemptLabel: expect.stringMatching(new RegExp(`=${createAttemptNonce}$`, "u")), + sandboxIdentityFingerprint, + message, + }); + expect( + fs.statSync(path.join(stateDir, "retained-sandbox-recovery.json")).mode & 0o777, + ).toBe(0o600); + expect(persist).toHaveBeenCalledOnce(); + expect(error.mock.calls.flat().join("\n")).toContain( + "recovery-only session remains blocked", + ); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } + }); + it.each([ ["unknown ownership", { failed: [], ownershipFailures: ["status cannot be proven"] }, 0], ["denied signal", { failed: [9_999_601], ownershipFailures: [] }, 0], From 23e2b655499d1986b96f732ee6b59dcb1e0beaa7 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 10:29:21 -0700 Subject: [PATCH 48/51] fix(onboard): retain cleanup recovery evidence Signed-off-by: Prekshi Vyas --- .../configure-inference-timeouts.mdx | 5 +- docs/reference/commands.mdx | 2 +- docs/reference/troubleshooting.mdx | 4 +- .../checks/run-managed-image-openshell-e2e.ts | 41 ++++++++++------ src/lib/core/readiness-wait.ts | 4 +- src/lib/onboard/gateway-health-wait.ts | 8 ++- src/lib/onboard/sandbox-gpu-create-flow.ts | 2 - .../onboard/sandbox-gpu-create-run-attempt.ts | 4 +- src/lib/onboard/sandbox-readiness-tracing.ts | 3 +- ...d-image-protected-runtime-contract.test.ts | 49 ++++++++++++++++++- 10 files changed, 92 insertions(+), 30 deletions(-) diff --git a/docs/inference/configure-inference-timeouts.mdx b/docs/inference/configure-inference-timeouts.mdx index ee1e0538afa..25fdd8d8da3 100644 --- a/docs/inference/configure-inference-timeouts.mdx +++ b/docs/inference/configure-inference-timeouts.mdx @@ -20,7 +20,7 @@ Use the error location to select the correct setting. |---|---|---| | `NEMOCLAW_AGENT_TIMEOUT` | OpenClaw per-request inference | `600` seconds | | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | Ollama, vLLM, NIM, and compatible-endpoint onboarding validation paths that read this setting | `180` seconds | -| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | OpenShell publication, durable identity settlement, and executable readiness after creation | `180` seconds | +| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | OpenShell publication, durable identity settlement, and executable readiness after creation; a new full budget starts after a managed runtime commit | `180` seconds | | `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` | OpenShell command re-registration after policy application, plus gateway health and re-registration during managed OpenClaw or Hermes recovery | `30`, `90`, or `120` seconds, depending on the recovery phase | The readiness timeout does not govern inference requests or provider validation. @@ -95,6 +95,7 @@ This variable does not extend the later sandbox-readiness wait. ## Increase the Sandbox Readiness Timeout Raise `NEMOCLAW_SANDBOX_READY_TIMEOUT` when OpenShell needs more than 180 seconds after the create command returns to publish the sandbox, settle its durable identity, or reach executable `Ready`. +After a managed runtime commit, NemoClaw starts a new full budget with this value to verify that the same sandbox returns to executable `Ready`. Local-inference validation does not consume the post-commit budget. This setting does not extend image build or gateway upload time. ```bash @@ -102,7 +103,7 @@ export NEMOCLAW_SANDBOX_READY_TIMEOUT=600 $$nemoclaw onboard ``` -If this deadline expires, NemoClaw preserves the sandbox and records recovery evidence when possible. +If either readiness deadline expires, NemoClaw preserves the sandbox and records recovery evidence when possible. Follow [Recover a retained sandbox](../../reference/commands#recover-a-retained-sandbox) before reusing the sandbox name. ## Increase the Recovery Wait diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index c663cf46f35..29d8eb67907 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3942,7 +3942,7 @@ The following environment variables tune onboard-time and recovery wall-clock li | `NEMOCLAW_OLLAMA_PULL_TIMEOUT` | `1800` (30 minutes) | Wall-clock timeout for `ollama pull` during onboard, in seconds. Accepts integer or float values. Already-downloaded layers are kept; re-running the pull resumes them. | | `NEMOCLAW_HF_DOWNLOAD_STALL_TIMEOUT` | `600` (10 minutes) | Maximum silence between Hugging Face download output during onboard. A positive finite value in seconds overrides the default, up to the Node.js timer limit of about 24.8 days. Blank, invalid, non-positive, sub-millisecond, and oversized values use the default. This is not a total download limit. Increase it only when a working download can produce no output for ten minutes. | | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | `180` | Wall-clock timeout for the inference-server validation probe during onboard, in seconds. Raise on slow networks or for very large prompts. | -| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Shared wall-clock deadline for post-create publication, durable identity settlement, and executable readiness. Raise it only when OpenShell needs longer after the create command returns to publish the sandbox, settle its durable identity, or reach executable `Ready`. A post-create readiness failure preserves the sandbox for the identity-bound retained-sandbox recovery procedure above. | +| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Wall-clock deadline for initial post-create publication, durable identity settlement, and executable readiness. After a managed runtime commit, NemoClaw starts a new full deadline with this value to verify that the same sandbox returns to executable `Ready`; local-inference validation does not consume that new budget. A readiness failure preserves the sandbox. Follow the [identity-bound retained-sandbox recovery procedure](#recover-a-retained-sandbox) before reusing its name. | | `NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE` | `30` | Consecutive `Error`-phase polls the post-create readiness wait tolerates before treating `Error` as terminal. Polling starts at 250ms and backs off to a 2-second cap, while `NEMOCLAW_SANDBOX_READY_TIMEOUT` remains the overall deadline. The gateway can briefly report a just-created sandbox in `Error` while it re-registers the sandbox (seen on DGX Spark); the debounce lets that transient recover to `Ready`. Every terminal observation outside the `Error` phase, including one with no reported phase, fails immediately. Set to `1` to restore fast-fail on the first `Error` poll. | | `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` | `30`, `90`, or `120`, depending on the recovery phase | Wall-clock timeout for OpenShell command re-registration after policy application, plus gateway health and re-registration during managed OpenClaw or Hermes recovery. A valid finite, nonnegative value overrides the internal budget for the current recovery phase. | diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 1bf4d0a9243..97284b307f7 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2002,7 +2002,9 @@ This is a separate budget from `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT`. It covers the -For a newly created OpenClaw or Hermes sandbox, `Ready` is not the final acceptance signal. Within this same budget, NemoClaw also requires OpenShell to return a durable sandbox ID and accept `openshell sandbox exec --name -- true`. NemoClaw keeps waiting only when OpenShell reports exact sandbox absence or its exact `sandbox is not ready` response. A missing or malformed ID, or another command failure, stops the wait. NemoClaw preserves the sandbox, saves create-attempt evidence when possible, and requires identity-bound recovery instead of ordinary failed-creation cleanup. +For a newly created OpenClaw or Hermes sandbox, `Ready` is not the final acceptance signal. Within the initial post-create budget, NemoClaw also requires OpenShell to return a durable sandbox ID and accept `openshell sandbox exec --name -- true`. NemoClaw keeps waiting only when OpenShell reports exact sandbox absence or its exact `sandbox is not ready` response. A missing or malformed ID, or another command failure, stops the wait. NemoClaw preserves the sandbox, saves create-attempt evidence when possible, and requires identity-bound recovery instead of ordinary failed-creation cleanup. + +After a managed runtime commit, NemoClaw starts a new full `NEMOCLAW_SANDBOX_READY_TIMEOUT` budget to verify that the same durable sandbox returns to executable `Ready`. The local-inference validation timeout remains separate and does not consume this post-commit budget. If onboarding reports that the managed runtime commit completed but the same sandbox did not return to executable `Ready`, stop before retrying. NemoClaw keeps the sandbox, prints its create-attempt label and a one-way durable identity fingerprint, and does not start dashboard forwarding. diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index d3de9874b33..1a02133724e 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -13,6 +13,7 @@ import { parseOpenShellSandboxId, } from "../../src/lib/adapters/openshell/sandbox-identity.ts"; import { createCliOpenShellSandboxObserverFromRunner } from "../../src/lib/adapters/openshell/sandbox-observer-cli.ts"; +import { waitUntilAsync } from "../../src/lib/core/wait.ts"; import { isValidName, NAME_ALLOWED_FORMAT } from "../../src/lib/name-validation.ts"; import { type StopHostGatewayResult, @@ -37,12 +38,7 @@ import { } from "../../src/lib/onboard/managed-image/contract.ts"; import { encodeManagedStartupProfile } from "../../src/lib/onboard/managed-startup/profile.ts"; import { createManagedStartupRootApplyRequest } from "../../src/lib/onboard/managed-startup/root-apply.ts"; -import type { - RuntimeProviderBundle, - RuntimeProviderManagedImageBootstrapSurface, -} from "../../src/lib/onboard/runtime-provider/contract.ts"; import { createDockerRuntimeProviderBundle } from "../../src/lib/onboard/runtime-provider/docker.ts"; -import { parseLiveSandboxNames } from "../../src/lib/runtime-recovery.ts"; import { OPENSHELL_SANDBOX_SUPERVISOR_ARGV, prepareSandboxCreateLaunch, @@ -51,7 +47,6 @@ import { resolveDockerStartupCommandPatch, runSandboxGpuCreateFlow, type SandboxGpuCreateFlowInput, - waitForSandboxReadinessUntil, } from "../../src/lib/onboard/sandbox-gpu-create-flow.ts"; import { createDirectSandboxGpuVerifier } from "../../src/lib/onboard/sandbox-gpu-preflight.ts"; import { @@ -336,12 +331,16 @@ export type ManagedImageRetainedSandboxRecovery = Readonly<{ const MANAGED_IMAGE_RETAINED_RECOVERY_FILE = "retained-sandbox-recovery.json"; +export function managedImageRetainedSandboxRecoveryPath(stateDir: string): string { + return path.join(stateDir, MANAGED_IMAGE_RETAINED_RECOVERY_FILE); +} + export function readManagedImageRetainedSandboxRecovery( stateDir: string, ): ManagedImageRetainedSandboxRecovery | null { try { const value = JSON.parse( - fs.readFileSync(path.join(stateDir, MANAGED_IMAGE_RETAINED_RECOVERY_FILE), "utf8"), + fs.readFileSync(managedImageRetainedSandboxRecoveryPath(stateDir), "utf8"), ) as Partial; if ( value.schemaVersion !== 1 || @@ -374,7 +373,7 @@ export function persistManagedImageRetainedSandboxRecovery(options: { sandboxIdentityFingerprint: options.sandboxIdentityFingerprint ?? null, message: options.message, }; - const recordPath = path.join(options.stateDir, MANAGED_IMAGE_RETAINED_RECOVERY_FILE); + const recordPath = managedImageRetainedSandboxRecoveryPath(options.stateDir); const temporaryPath = `${recordPath}.tmp`; let descriptor: number | null = null; let directoryDescriptor: number | null = null; @@ -414,14 +413,17 @@ export function removeManagedImageGatewayStateIfSafe( stateDir: string, gatewayStop: Pick, gatewayRemovalStatus: number | null, + cleanupVerified = true, ): boolean { if ( gatewayStop.failed.length > 0 || (gatewayStop.ownershipFailures?.length ?? 0) > 0 || - gatewayRemovalStatus !== 0 + gatewayRemovalStatus !== 0 || + !cleanupVerified ) { return false; } + fs.rmSync(managedImageRetainedSandboxRecoveryPath(stateDir), { force: true }); fs.rmSync(stateDir, { recursive: true, force: true }); return true; } @@ -658,7 +660,7 @@ export async function waitForCommittedSandboxProbe( timeoutMs, ); let lastHealthDetail = ""; - const ready = await waitForSandboxReadinessUntil( + const ready = await waitUntilAsync( () => { const remainingMs = deadlineMs - now(); const health = runProbe(healthProbe, Math.max(1, Math.min(15_000, remainingMs))); @@ -1009,7 +1011,7 @@ export function assertFailedSandboxOwnerCleanupRetention( get.status !== 0 || parseOpenShellSandboxId(String(get.stdout ?? "")) !== expectedSandboxId || list.status !== 0 || - !parseLiveSandboxNames(String(list.stdout ?? "")).has(input.sandbox) + !onboard.isSandboxReady(String(list.stdout ?? ""), input.sandbox) ) { throw new Error( `managed-bootstrap rollback did not retain its exact OpenShell owner-cleanup state: get=${commandDetail(get)} list=${commandDetail(list)}`, @@ -1216,8 +1218,6 @@ async function run> | null = null; try { @@ -1526,9 +1526,20 @@ async function run string; diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index a173b6d3532..069e73aecdc 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -44,7 +44,6 @@ import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; import { createSandboxGpuCreateAttemptRunner, persistRetainedSandboxRecoveryOrBlock, - waitForSandboxReadinessUntil, } from "./sandbox-gpu-create-run-attempt"; import { managedBootstrapCreateArgs } from "./sandbox-create-launch"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; @@ -57,7 +56,6 @@ import type { SandboxPrebuildResult } from "./sandbox-prebuild"; import { addTraceEvent } from "./tracing"; export { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; -export { waitForSandboxReadinessUntil }; export function resolvePortableLifecycleMode( agent: AgentDefinition | null, diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 1f4137733d4..37820b8a516 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -19,7 +19,7 @@ import { settleCreatedOpenShellSandboxId, } from "../adapters/openshell/sandbox-identity"; import { createReadinessWaitOptions } from "../core/readiness-wait"; -import { waitUntil, waitUntilAsync } from "../core/wait"; +import { waitUntil } from "../core/wait"; import { streamSandboxCreate, type StreamSandboxCreateResult } from "../sandbox/create-stream"; import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-ready-gate"; import { redact, redactFullWithUrls } from "../security/redact"; @@ -83,8 +83,6 @@ type PostCreateReadinessDeadline = Readonly<{ now: () => number; }>; -export { waitUntilAsync as waitForSandboxReadinessUntil }; - function createPostCreateReadinessDeadline( input: SandboxGpuCreateFlowInput, deps: SandboxGpuCreateFlowDeps, diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index 52fbfe7bfb7..a99ccbe859e 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -19,8 +19,9 @@ import { createReadinessWaitOptions, formatReadinessDeadline, getLegacyPollDeadlineBudgetMs, + waitUntil, + waitUntilAsync, } from "../core/readiness-wait"; -import { waitUntil, waitUntilAsync } from "../core/wait"; import { envInt } from "./env"; import { addTraceEvent, withDashboardReadinessTrace, withSandboxReadinessTrace } from "./tracing"; diff --git a/test/inference/managed/managed-image-protected-runtime-contract.test.ts b/test/inference/managed/managed-image-protected-runtime-contract.test.ts index 44cd46bfc96..805de218a94 100644 --- a/test/inference/managed/managed-image-protected-runtime-contract.test.ts +++ b/test/inference/managed/managed-image-protected-runtime-contract.test.ts @@ -30,6 +30,7 @@ import { managedImageOpenShellBasePolicyPath, managedImageOpenShellCommittedProbe, managedImageOpenShellProbe, + managedImageRetainedSandboxRecoveryPath, managedImageSandboxCleanupOwnershipError, parseManagedImageOpenShellE2eInputs, persistManagedImageRetainedSandboxRecovery, @@ -447,9 +448,53 @@ describe("protected managed-image runtime contract", () => { } }); + it("retains exact recovery evidence when managed-image cleanup is unresolved (#10652)", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-state-recovery-")); + const createAttemptNonce = "c".repeat(62); + const sandboxIdentityFingerprint = "d".repeat(64); + expect( + persistManagedImageRetainedSandboxRecovery({ + stateDir, + sandboxName: VALID_SANDBOX, + message: "Retained after incomplete cleanup.", + createAttemptNonce, + sandboxIdentityFingerprint, + }), + ).toBe(true); + + try { + expect( + removeManagedImageGatewayStateIfSafe( + stateDir, + { failed: [], ownershipFailures: [] }, + 0, + false, + ), + ).toBe(false); + expect(managedImageRetainedSandboxRecoveryPath(stateDir)).toBe( + path.join(stateDir, "retained-sandbox-recovery.json"), + ); + expect(readManagedImageRetainedSandboxRecovery(stateDir)).toMatchObject({ + createAttemptLabel: expect.stringMatching(new RegExp(`=${createAttemptNonce}$`, "u")), + sandboxIdentityFingerprint, + }); + } finally { + fs.rmSync(stateDir, { force: true, recursive: true }); + } + }); + it("removes gateway state only after scoped stop and gateway removal succeed (#7744)", () => { const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-state-remove-")); fs.writeFileSync(path.join(stateDir, "openshell-gateway.pid"), "9999601\n"); + expect( + persistManagedImageRetainedSandboxRecovery({ + stateDir, + sandboxName: VALID_SANDBOX, + message: "Retire after exact cleanup.", + createAttemptNonce: "e".repeat(62), + sandboxIdentityFingerprint: "f".repeat(64), + }), + ).toBe(true); expect( removeManagedImageGatewayStateIfSafe(stateDir, { failed: [], ownershipFailures: [] }, 0), @@ -623,7 +668,7 @@ describe("protected managed-image runtime contract", () => { expect(() => assertFailedSandboxOwnerCleanupRetention( - { runOpenshell } as never, + { ...MANAGED_IMAGE_ONBOARD, runOpenshell }, input, expectedSandboxId, {}, @@ -668,7 +713,7 @@ describe("protected managed-image runtime contract", () => { ); const assertion = () => assertFailedSandboxOwnerCleanupRetention( - { runOpenshell } as never, + { ...MANAGED_IMAGE_ONBOARD, runOpenshell }, input, expectedSandboxId, {}, From 59ea6e23e691f311a247db217ba2520379e55b4f Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 12:14:54 -0700 Subject: [PATCH 49/51] test(onboard): consolidate identity deadline coverage Signed-off-by: Prekshi Vyas --- ...ndbox-gpu-create-identity-deadline.test.ts | 150 ------------------ .../sandbox-gpu-create-identity-gate.test.ts | 58 +++++++ 2 files changed, 58 insertions(+), 150 deletions(-) delete mode 100644 src/lib/onboard/sandbox-gpu-create-identity-deadline.test.ts diff --git a/src/lib/onboard/sandbox-gpu-create-identity-deadline.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-deadline.test.ts deleted file mode 100644 index 85ce71f2238..00000000000 --- a/src/lib/onboard/sandbox-gpu-create-identity-deadline.test.ts +++ /dev/null @@ -1,150 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -const mocks = vi.hoisted(() => ({ - streamSandboxCreate: vi.fn(), - waitForCreatedSandboxReadyWithTrace: vi.fn(), - printReadinessFailure: vi.fn(), - enforceDockerGpuPatchPreserveNetwork: vi.fn(), - verifyGpuSandboxAccessAfterReady: vi.fn(), - createDockerGpuSandboxCreatePatch: vi.fn(), - printSandboxCreateFailureDiagnostics: vi.fn(), - collectDockerGpuPatchDiagnostics: vi.fn(), - queryOpenShellDockerSandboxContainers: vi.fn(), - queryOpenShellDockerSandboxRuntimeSnapshot: vi.fn(), -})); - -vi.mock("../sandbox/create-stream", () => ({ - streamSandboxCreate: mocks.streamSandboxCreate, -})); -vi.mock("./sandbox-readiness-tracing", async (importOriginal) => ({ - ...(await importOriginal()), - waitForCreatedSandboxReadyWithTrace: mocks.waitForCreatedSandboxReadyWithTrace, - printReadinessFailure: mocks.printReadinessFailure, -})); -vi.mock("./docker-gpu-local-inference", () => ({ - enforceDockerGpuPatchPreserveNetwork: mocks.enforceDockerGpuPatchPreserveNetwork, - verifyGpuSandboxAccessAfterReady: mocks.verifyGpuSandboxAccessAfterReady, -})); -vi.mock("./docker-gpu-sandbox-create", () => ({ - createDockerGpuSandboxCreatePatch: mocks.createDockerGpuSandboxCreatePatch, -})); -vi.mock("./sandbox-create-failure", () => ({ - printSandboxCreateFailureDiagnostics: mocks.printSandboxCreateFailureDiagnostics, -})); -vi.mock("./docker-gpu-patch", async (importOriginal) => ({ - ...(await importOriginal()), - collectDockerGpuPatchDiagnostics: mocks.collectDockerGpuPatchDiagnostics, -})); -vi.mock("./openshell-docker-sandbox-containers", async (importOriginal) => ({ - ...(await importOriginal()), - queryOpenShellDockerSandboxContainers: mocks.queryOpenShellDockerSandboxContainers, - queryOpenShellDockerSandboxRuntimeSnapshot: mocks.queryOpenShellDockerSandboxRuntimeSnapshot, -})); - -import { - NEMOCLAW_CREATE_ATTEMPT_LABEL, - NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH, -} from "../adapters/openshell/sandbox-identity"; -import { - createGpuFlowDeps, - createGpuFlowInput, - createGpuPatchFixture, - resetGpuFlowMocks, - setupGpuFlowMocks, -} from "./__test-helpers__/sandbox-gpu-create-flow"; -import { runSandboxGpuCreateFlow } from "./sandbox-gpu-create-flow"; - -function sandboxListJson( - sandboxId: string, - createAttemptNonce: string, - incomplete = false, -): string { - return JSON.stringify([ - { - id: sandboxId, - name: "alpha", - labels: { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: createAttemptNonce }, - resource_version: incomplete ? null : 1, - created_at: incomplete ? null : "2026-08-25T00:00:00Z", - phase: incomplete ? null : "Ready", - current_policy_version: incomplete ? null : 1, - }, - ]); -} - -function createAttemptNonce(args: readonly string[]): string { - const labelIndex = args.indexOf("--label"); - return (args[labelIndex + 1] ?? "").slice(NEMOCLAW_CREATE_ATTEMPT_LABEL.length + 1); -} - -beforeEach(() => setupGpuFlowMocks(mocks)); -afterEach(resetGpuFlowMocks); - -describe("created sandbox identity settlement deadline", () => { - it("allows identity-bound post-create effects after the former 30-second cap (#10652)", async () => { - let nonce = ""; - let nowMs = 0; - const input = createGpuFlowInput(); - input.sandboxGpuConfig = { - mode: "0", - hostGpuDetected: false, - hostGpuPlatform: null, - sandboxGpuEnabled: false, - sandboxGpuDevice: null, - errors: [], - }; - input.gpuRoutePlan = "none"; - input.initialGpuRoute = "none"; - input.createArgv = ["openshell", "sandbox", "create", "--name", "alpha", "--", "agent"]; - input.sandboxReadyTimeoutSecs = 90; - input.persistRetainedSandboxRecovery = vi.fn(() => true); - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - - const patch = createGpuPatchFixture(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { - nonce = createAttemptNonce(args); - expect(nonce).toMatch(/^[0-9a-f]{62}$/u); - expect(nonce).toHaveLength(NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH); - expect(options.readyCheck?.()).toBe(false); - expect(options.readyCheck?.()).toBe(true); - return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; - }); - - const deps = createGpuFlowDeps(); - deps.publicationNow = () => nowMs; - vi.mocked(deps.sleep).mockImplementation((seconds) => { - nowMs += seconds * 1_000; - }); - deps.installPortableDemoLifecycle = vi.fn(() => "generation-1"); - vi.mocked(deps.runCaptureOpenshell) - .mockReturnValueOnce("alpha Ready") - .mockImplementationOnce(() => sandboxListJson("alpha-sandbox-id", nonce, true)) - .mockReturnValueOnce("alpha Ready") - .mockImplementationOnce(() => sandboxListJson("alpha-sandbox-id", nonce)) - .mockImplementationOnce(() => { - nowMs += 31_000; - return sandboxListJson("alpha-sandbox-id", nonce, true); - }) - .mockImplementationOnce(() => sandboxListJson("alpha-sandbox-id", nonce)); - - await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); - - expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledExactlyOnceWith({ - sandboxId: "alpha-sandbox-id", - liveIdentityFingerprint: expect.stringMatching(/^[0-9a-f]{64}$/u), - createAttemptNonce: nonce, - route: "none", - }); - expect(input.revalidateVerifiedSandboxBeforeEffect).toHaveBeenCalledWith( - "apply runtime patch for sandbox 'alpha'", - ); - expect(patch.ensureApplied).toHaveBeenCalledOnce(); - expect(patch.commitAfterReady).toHaveBeenCalledOnce(); - expect(deps.sleep).toHaveBeenCalledExactlyOnceWith(0.25); - }); -}); diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 1fab8e4ec0f..acd6f3bc927 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -678,6 +678,64 @@ describe("created sandbox identity gate", () => { expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); }); + it("allows identity-bound post-create effects after the former 30-second cap (#10652)", async () => { + let nonce = ""; + let nowMs = 0; + const input = noGpuInput(); + input.sandboxReadyTimeoutSecs = 90; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { + nonce = createAttemptNonce(args); + expect(nonce).toMatch(/^[0-9a-f]{62}$/u); + expect(nonce).toHaveLength(NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH); + expect(options.readyCheck?.()).toBe(true); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + + const deps = createGpuFlowDeps(); + deps.publicationNow = () => nowMs; + vi.mocked(deps.sleep).mockImplementation((seconds) => { + nowMs += seconds * 1_000; + }); + deps.installPortableDemoLifecycle = vi.fn(() => "generation-1"); + const identityLabels = () => ({ [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }); + const pendingMetadata = { + resource_version: null, + created_at: null, + phase: null, + current_policy_version: null, + }; + vi.mocked(deps.runCaptureOpenshell) + .mockReturnValueOnce("alpha Ready") + .mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", identityLabels(), pendingMetadata), + ) + .mockImplementationOnce(() => { + nowMs += 31_000; + return sandboxListJson("alpha-sandbox-id", identityLabels(), pendingMetadata); + }) + .mockImplementationOnce(() => sandboxListJson("alpha-sandbox-id", identityLabels())); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); + + expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledExactlyOnceWith({ + sandboxId: "alpha-sandbox-id", + liveIdentityFingerprint: expect.stringMatching(/^[0-9a-f]{64}$/u), + createAttemptNonce: nonce, + route: "none", + }); + expect(input.revalidateVerifiedSandboxBeforeEffect).toHaveBeenCalledWith( + "apply runtime patch for sandbox 'alpha'", + ); + expect(patch.ensureApplied).toHaveBeenCalledOnce(); + expect(patch.commitAfterReady).toHaveBeenCalledOnce(); + expect(deps.sleep).toHaveBeenCalledExactlyOnceWith(0.25); + }); + it("carries Hermes receipt authority from selector settlement through publication lookup (#10423)", async () => { const events: string[] = []; let nonce = ""; From 0cebdf3ee89d97f551a79bc39adcb3452b1b9f9e Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 13:07:01 -0700 Subject: [PATCH 50/51] test(onboard): consolidate create handoff coverage Signed-off-by: Prekshi Vyas --- src/lib/core/readiness-wait.test.ts | 11 +- .../sandbox-gpu-create-flow.ts | 73 +++ .../sandbox-gpu-create-apf-policyless.test.ts | 192 ++++++- ...ox-gpu-create-flow-hermes-portable.test.ts | 88 +++ .../sandbox-gpu-create-handoff.test.ts | 448 --------------- .../sandbox-gpu-create-identity-gate.test.ts | 516 +++++++++--------- 6 files changed, 632 insertions(+), 696 deletions(-) delete mode 100644 src/lib/onboard/sandbox-gpu-create-handoff.test.ts diff --git a/src/lib/core/readiness-wait.test.ts b/src/lib/core/readiness-wait.test.ts index f32763d8798..ba0ab3a9c23 100644 --- a/src/lib/core/readiness-wait.test.ts +++ b/src/lib/core/readiness-wait.test.ts @@ -31,13 +31,22 @@ describe("readiness deadline options", () => { }); it("honors a slower initial interval for readiness paths with a stability contract", () => { + let nowMs = 0; + const sleep = vi.fn((ms: number) => { + nowMs += ms; + }); const options = createReadinessWaitOptions({ budgetMs: 10_000, initialIntervalMs: 2_000, maxIntervalMs: 2_000, + now: () => nowMs, + sleep, }); - expect(options?.initialIntervalMs).toBe(2_000); + expect(waitUntil(() => false, options!)).toBe(false); + expect(sleep).toHaveBeenNthCalledWith(1, 2_000); + expect(sleep.mock.calls.every(([ms]) => ms <= 2_000)).toBe(true); + expect(sleep.mock.calls.reduce((total, [ms]) => total + ms, 0)).toBe(10_000); }); it("preserves bounded immediate probes for a zero-interval legacy configuration", () => { diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index 4a895d146e8..b92b57e3461 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -7,6 +7,7 @@ import path from "node:path"; import { expect, vi } from "vitest"; +import { NEMOCLAW_CREATE_ATTEMPT_LABEL } from "../../adapters/openshell/sandbox-identity"; import { createCliOpenShellSandboxObserver } from "../../adapters/openshell/sandbox-observer-cli"; import { isSandboxReady } from "../../state/gateway"; import type { CheckpointPortableRuntimeAuthority } from "../../state/onboard-checkpoint-types"; @@ -26,6 +27,41 @@ export const VERIFIED_GPU_PROOF: SandboxGpuProofResult = { at: "2026-07-06T00:00:00.000Z", }; export const GPU_IMAGE_ID = `sha256:${"a".repeat(64)}`; +export const ALPHA_SANDBOX_IDENTITY_FINGERPRINT = + "8174fa2a5d65755138d8339e086c03d736633130b22dca10952e80e74750c01d"; + +export function sandboxListJson( + sandboxId: string, + labels: Readonly>, + overrides: Readonly> = {}, +): string { + return JSON.stringify([ + { + id: sandboxId, + name: "alpha", + labels, + resource_version: 1, + created_at: "2026-08-25T00:00:00Z", + phase: "Ready", + current_policy_version: 1, + ...overrides, + }, + ]); +} + +export function createAttemptNonce(args: readonly string[]): string { + const labelIndex = args.indexOf("--label"); + return (args[labelIndex + 1] ?? "").slice(NEMOCLAW_CREATE_ATTEMPT_LABEL.length + 1); +} + +export function createTimedOutCreateResult(output: string) { + return { + status: 1, + output, + sawProgress: true, + readyTerminationTimedOut: true, + } as const; +} export function createGpuFlowInput(): SandboxGpuCreateFlowInput { return { @@ -59,6 +95,23 @@ export function createGpuFlowInput(): SandboxGpuCreateFlowInput { }; } +export function createNoGpuFlowInput(): SandboxGpuCreateFlowInput { + const input = createGpuFlowInput(); + input.sandboxGpuConfig = { + mode: "0", + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }; + input.gpuRoutePlan = "none"; + input.initialGpuRoute = "none"; + input.createArgv = ["openshell", "sandbox", "create", "--name", "alpha", "--", "agent"]; + input.persistRetainedSandboxRecovery = vi.fn(() => true); + return input; +} + export function createGpuFlowDeps(sandboxId?: string): SandboxGpuCreateFlowDeps; export function createGpuFlowDeps( expectedGatewayName: string, @@ -134,6 +187,26 @@ export function createGpuPatchFixture() { }; } +export function expectNoPostCreateEffects( + input: SandboxGpuCreateFlowInput, + patch: ReturnType, + deps: SandboxGpuCreateFlowDeps, + readinessWait: ReturnType, +): void { + for (const effect of [ + input.verifyCreatedSandboxBeforeEffects, + input.revalidateVerifiedSandboxBeforeEffect, + patch.exitOnPatchError, + patch.ensureApplied, + patch.waitForSupervisorReconnectIfNeeded, + patch.commitAfterReady, + readinessWait, + deps.installPortableDemoLifecycle, + ]) { + if (effect) expect(effect).not.toHaveBeenCalled(); + } +} + export function setupGpuFlowMocks(mocks: Record>): void { mocks.streamSandboxCreate.mockResolvedValue({ status: 0, diff --git a/src/lib/onboard/sandbox-gpu-create-apf-policyless.test.ts b/src/lib/onboard/sandbox-gpu-create-apf-policyless.test.ts index 8840f896230..e6667096fa8 100644 --- a/src/lib/onboard/sandbox-gpu-create-apf-policyless.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-apf-policyless.test.ts @@ -1,11 +1,113 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { refuseApfMutableNameFallbackCleanup } from "./sandbox-gpu-create-flow"; +const mocks = vi.hoisted(() => ({ + streamSandboxCreate: vi.fn(), + waitForCreatedSandboxReadyWithTrace: vi.fn(), + printReadinessFailure: vi.fn(), + enforceDockerGpuPatchPreserveNetwork: vi.fn(), + verifyGpuSandboxAccessAfterReady: vi.fn(), + createDockerGpuSandboxCreatePatch: vi.fn(), + printSandboxCreateFailureDiagnostics: vi.fn(), + collectDockerGpuPatchDiagnostics: vi.fn(), + queryOpenShellDockerSandboxContainers: vi.fn(), + queryOpenShellDockerSandboxRuntimeSnapshot: vi.fn(), +})); + +vi.mock("../sandbox/create-stream", () => ({ + streamSandboxCreate: mocks.streamSandboxCreate, +})); +vi.mock("./sandbox-readiness-tracing", async (importOriginal) => ({ + ...(await importOriginal()), + waitForCreatedSandboxReadyWithTrace: mocks.waitForCreatedSandboxReadyWithTrace, + printReadinessFailure: mocks.printReadinessFailure, +})); +vi.mock("./docker-gpu-local-inference", () => ({ + enforceDockerGpuPatchPreserveNetwork: mocks.enforceDockerGpuPatchPreserveNetwork, + verifyGpuSandboxAccessAfterReady: mocks.verifyGpuSandboxAccessAfterReady, +})); +vi.mock("./docker-gpu-sandbox-create", () => ({ + createDockerGpuSandboxCreatePatch: mocks.createDockerGpuSandboxCreatePatch, +})); +vi.mock("./sandbox-create-failure", () => ({ + printSandboxCreateFailureDiagnostics: mocks.printSandboxCreateFailureDiagnostics, +})); +vi.mock("./docker-gpu-patch", async (importOriginal) => ({ + ...(await importOriginal()), + collectDockerGpuPatchDiagnostics: mocks.collectDockerGpuPatchDiagnostics, +})); +vi.mock("./openshell-docker-sandbox-containers", async (importOriginal) => ({ + ...(await importOriginal()), + queryOpenShellDockerSandboxContainers: mocks.queryOpenShellDockerSandboxContainers, + queryOpenShellDockerSandboxRuntimeSnapshot: mocks.queryOpenShellDockerSandboxRuntimeSnapshot, +})); + +import { NEMOCLAW_CREATE_ATTEMPT_LABEL } from "../adapters/openshell/sandbox-identity"; +import { + ALPHA_SANDBOX_IDENTITY_FINGERPRINT, + createAttemptNonce, + createGpuFlowDeps, + createGpuFlowInput, + resetGpuFlowMocks, + sandboxListJson, + setupGpuFlowMocks, +} from "./__test-helpers__/sandbox-gpu-create-flow"; + +import { + refuseApfMutableNameFallbackCleanup, + runSandboxGpuCreateFlow, +} from "./sandbox-gpu-create-flow"; import { assertPolicylessSandboxCreateArgv } from "./sandbox-gpu-create-run-attempt"; +function expectNoSandboxDelete(deps: ReturnType): void { + expect( + vi + .mocked(deps.runOpenshell) + .mock.calls.some(([args]) => args[0] === "sandbox" && args[1] === "delete"), + ).toBe(false); +} + +function createApfFallbackRecoveryFixture(captureExactIdentity = true) { + let nonce = ""; + const input = createGpuFlowInput(); + input.requirePolicylessCreate = true; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + input.persistRetainedSandboxRecovery = vi.fn(() => true); + mocks.streamSandboxCreate.mockImplementationOnce(async (_command, args) => { + nonce = createAttemptNonce(args); + return { + status: 1, + output: "native runtime failed after sandbox creation", + sawProgress: true, + }; + }); + mocks.queryOpenShellDockerSandboxRuntimeSnapshot.mockReturnValue({ + ok: true, + imageId: "sha256:" + "a".repeat(64), + bookkeepingImageRef: "openshell/sandbox-from:test", + stateError: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + deviceRequests: null, + devices: null, + runtime: "runc", + nvidiaVisibleDevices: null, + nativeGpuAttachmentState: "absent", + containerId: "container-a", + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementation(() => + captureExactIdentity + ? sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }) + : "[]", + ); + return { deps, input, readNonce: () => nonce }; +} + +beforeEach(() => setupGpuFlowMocks(mocks)); +afterEach(resetGpuFlowMocks); + describe("APF policyless sandbox create attempts", () => { it("accepts create arguments without a caller policy (#9833)", () => { expect(() => @@ -56,4 +158,90 @@ describe("APF policyless sandbox create attempts", () => { containerIds: null, }); }); + + it("persists exact APF recovery evidence before refusing native fallback (#9833)", async () => { + const { deps, input, readNonce } = createApfFallbackRecoveryFixture(); + const exit = vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("process.exit:1"); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1"); + + const nonce = readNonce(); + const fingerprint = ALPHA_SANDBOX_IDENTITY_FINGERPRINT; + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringMatching( + new RegExp( + `^Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}\\. Durable sandbox identity fingerprint: ${fingerprint}\\.`, + "u", + ), + ), + fingerprint, + nonce, + ); + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledBefore(exit); + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); + expect(output).toContain(`Durable sandbox identity fingerprint: ${fingerprint}`); + expect(output).not.toContain("alpha-sandbox-id"); + expectNoSandboxDelete(deps); + expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); + }); + + it("persists the APF create-attempt label when exact recovery identity is unavailable (#9833)", async () => { + const { deps, input, readNonce } = createApfFallbackRecoveryFixture(false); + vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("process.exit:1"); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1"); + + const nonce = readNonce(); + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringMatching( + new RegExp( + `^Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}\\..*Recovery is blocked until an OpenShell administrator resolves the create-attempt label`, + "u", + ), + ), + undefined, + nonce, + ); + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); + expect(output).toContain("Recovery is blocked"); + expectNoSandboxDelete(deps); + }); + + it.each([ + ["returns false", (): boolean => false], + [ + "throws", + (): boolean => { + throw new Error("durable writer failed"); + }, + ], + ] as const)( + "blocks APF fallback when durable recovery persistence %s (#9833)", + async (_name, writer) => { + const { deps, input, readNonce } = createApfFallbackRecoveryFixture(); + const persist = vi.fn(writer); + input.persistRetainedSandboxRecovery = persist; + const exit = vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("process.exit:1"); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "The APF recovery-only session remains blocked until its durable recovery record can be saved.", + ); + + expect(persist).toHaveBeenCalledOnce(); + expect(exit).not.toHaveBeenCalled(); + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${readNonce()}`); + expect(output).toContain("APF recovery is blocked because NemoClaw could not save"); + expectNoSandboxDelete(deps); + expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); + }, + ); }); diff --git a/src/lib/onboard/sandbox-gpu-create-flow-hermes-portable.test.ts b/src/lib/onboard/sandbox-gpu-create-flow-hermes-portable.test.ts index 8a10541dba7..febb7d773ac 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow-hermes-portable.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow-hermes-portable.test.ts @@ -41,13 +41,22 @@ vi.mock("./openshell-docker-sandbox-containers", async (importOriginal) => ({ queryOpenShellDockerSandboxRuntimeSnapshot: mocks.queryOpenShellDockerSandboxRuntimeSnapshot, })); +import { NEMOCLAW_CREATE_ATTEMPT_LABEL } from "../adapters/openshell/sandbox-identity"; import type { CheckpointPortableRuntimeAuthority } from "../state/onboard-checkpoint-types"; import { + createAttemptNonce, createGpuFlowDeps as createDeps, createGpuFlowInput as createInput, + createGpuPatchFixture, + createNoGpuFlowInput as createNoGpuInput, resetGpuFlowMocks, + sandboxListJson, setupGpuFlowMocks, } from "./__test-helpers__/sandbox-gpu-create-flow"; +import { + createHermesPortableReadyCapture, + createHermesPortableReadyRunner, +} from "./experimental/hermes-portable-onboarding"; import { runSandboxGpuCreateFlow, type SandboxGpuCreateFlowInput } from "./sandbox-gpu-create-flow"; import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; @@ -182,4 +191,83 @@ describe("Hermes portable sandbox create flow", () => { expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); expect(mocks.createDockerGpuSandboxCreatePatch).not.toHaveBeenCalled(); }); + + it("carries Hermes receipt authority from selector settlement through publication lookup (#10423)", async () => { + let nonce = ""; + const input = createNoGpuInput(); + const patch = createGpuPatchFixture(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { + nonce = createAttemptNonce(args); + expect(options.readyCheck?.()).toBe(true); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValue({ + ready: true, + reason: "ready", + failurePhase: null, + }); + const capture = vi.fn((args: readonly string[]) => { + const results = { + [["sandbox", "list", "-g", "nemoclaw"].join("\0")]: () => ({ + status: 0, + stdout: Buffer.from("alpha Ready"), + stderr: Buffer.alloc(0), + }), + [[ + "sandbox", + "list", + "-g", + "nemoclaw", + "--selector", + `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`, + "--output", + "json", + "--limit", + "2", + ].join("\0")]: () => ({ + status: 0, + stdout: Buffer.from( + sandboxListJson("alpha-sandbox-id", { + [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, + }), + ), + stderr: Buffer.alloc(0), + }), + [["sandbox", "get", "-g", "nemoclaw", "alpha"].join("\0")]: () => ({ + status: 0, + stdout: Buffer.from("ID: alpha-sandbox-id\n"), + stderr: Buffer.alloc(0), + }), + } satisfies Readonly< + Record { status: number; stdout: Buffer; stderr: Buffer }> + >; + return ( + results[args.join("\0") as keyof typeof results] ?? + (() => ({ status: 1, stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) })) + )(); + }); + const deps = createDeps(); + deps.runOpenshell = createHermesPortableReadyRunner("alpha", "nemoclaw", capture); + deps.runCaptureOpenshell = createHermesPortableReadyCapture("alpha", "nemoclaw", capture); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); + + expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); + expect(capture).toHaveBeenCalledWith([ + "sandbox", + "list", + "-g", + "nemoclaw", + "--selector", + `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`, + "--output", + "json", + "--limit", + "2", + ]); + expect(capture).toHaveBeenCalledWith(["sandbox", "get", "-g", "nemoclaw", "alpha"]); + }); }); diff --git a/src/lib/onboard/sandbox-gpu-create-handoff.test.ts b/src/lib/onboard/sandbox-gpu-create-handoff.test.ts deleted file mode 100644 index 12990b29781..00000000000 --- a/src/lib/onboard/sandbox-gpu-create-handoff.test.ts +++ /dev/null @@ -1,448 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -const mocks = vi.hoisted(() => ({ - streamSandboxCreate: vi.fn(), - waitForCreatedSandboxReadyWithTrace: vi.fn(), - printReadinessFailure: vi.fn(), - enforceDockerGpuPatchPreserveNetwork: vi.fn(), - verifyGpuSandboxAccessAfterReady: vi.fn(), - createDockerGpuSandboxCreatePatch: vi.fn(), - printSandboxCreateFailureDiagnostics: vi.fn(), - collectDockerGpuPatchDiagnostics: vi.fn(), - queryOpenShellDockerSandboxContainers: vi.fn(), - queryOpenShellDockerSandboxRuntimeSnapshot: vi.fn(), -})); - -vi.mock("../sandbox/create-stream", () => ({ - streamSandboxCreate: mocks.streamSandboxCreate, -})); - -vi.mock("./sandbox-readiness-tracing", async (importOriginal) => ({ - ...(await importOriginal()), - waitForCreatedSandboxReadyWithTrace: mocks.waitForCreatedSandboxReadyWithTrace, - printReadinessFailure: mocks.printReadinessFailure, -})); - -vi.mock("./docker-gpu-local-inference", () => ({ - enforceDockerGpuPatchPreserveNetwork: mocks.enforceDockerGpuPatchPreserveNetwork, - verifyGpuSandboxAccessAfterReady: mocks.verifyGpuSandboxAccessAfterReady, -})); - -vi.mock("./docker-gpu-sandbox-create", () => ({ - createDockerGpuSandboxCreatePatch: mocks.createDockerGpuSandboxCreatePatch, -})); - -vi.mock("./sandbox-create-failure", () => ({ - printSandboxCreateFailureDiagnostics: mocks.printSandboxCreateFailureDiagnostics, -})); - -vi.mock("./docker-gpu-patch", async (importOriginal) => ({ - ...(await importOriginal()), - collectDockerGpuPatchDiagnostics: mocks.collectDockerGpuPatchDiagnostics, -})); - -vi.mock("./openshell-docker-sandbox-containers", async (importOriginal) => ({ - ...(await importOriginal()), - queryOpenShellDockerSandboxContainers: mocks.queryOpenShellDockerSandboxContainers, - queryOpenShellDockerSandboxRuntimeSnapshot: mocks.queryOpenShellDockerSandboxRuntimeSnapshot, -})); - -import { - NEMOCLAW_CREATE_ATTEMPT_LABEL, - NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH, -} from "../adapters/openshell/sandbox-identity"; -import { - createGpuFlowDeps, - createGpuFlowInput, - createGpuPatchFixture, - resetGpuFlowMocks, - setupGpuFlowMocks, -} from "./__test-helpers__/sandbox-gpu-create-flow"; -import { runSandboxGpuCreateFlow } from "./sandbox-gpu-create-flow"; - -function sandboxListJson( - sandboxId: string, - labels: Readonly>, - overrides: Readonly> = {}, -): string { - return JSON.stringify([ - { - id: sandboxId, - name: "alpha", - labels, - resource_version: 1, - created_at: "2026-08-25T00:00:00Z", - phase: "Ready", - current_policy_version: 1, - ...overrides, - }, - ]); -} - -function createAttemptNonce(args: readonly string[]): string { - const labelIndex = args.indexOf("--label"); - return (args[labelIndex + 1] ?? "").slice(NEMOCLAW_CREATE_ATTEMPT_LABEL.length + 1); -} - -const ALPHA_SANDBOX_IDENTITY_FINGERPRINT = - "8174fa2a5d65755138d8339e086c03d736633130b22dca10952e80e74750c01d"; - -function noGpuInput() { - const input = createGpuFlowInput(); - input.sandboxGpuConfig = { - mode: "0", - hostGpuDetected: false, - hostGpuPlatform: null, - sandboxGpuEnabled: false, - sandboxGpuDevice: null, - errors: [], - }; - input.gpuRoutePlan = "none"; - input.initialGpuRoute = "none"; - input.createArgv = ["openshell", "sandbox", "create", "--name", "alpha", "--", "agent"]; - input.persistRetainedSandboxRecovery = vi.fn(() => true); - return input; -} - -beforeEach(() => setupGpuFlowMocks(mocks)); -afterEach(resetGpuFlowMocks); - -describe("created sandbox create-client handoff", () => { - it("ends the create-client handoff after a nonce-owned ID appears and settles metadata before effects (#10769)", async () => { - const events: string[] = []; - let nonce = ""; - const input = noGpuInput(); - const patch = createGpuPatchFixture(); - input.verifyCreatedSandboxBeforeEffects = vi.fn(async (identity) => { - events.push("verify-created"); - expect(identity).toEqual({ - sandboxId: "alpha-sandbox-id", - liveIdentityFingerprint: expect.stringMatching(/^[0-9a-f]{64}$/u), - createAttemptNonce: expect.stringMatching(/^[0-9a-f]{62}$/u), - route: "none", - }); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); - }); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn((operation) => - events.push(`revalidate:${operation}`), - ); - patch.exitOnPatchError.mockImplementation(() => events.push("runtime-check")); - patch.ensureApplied.mockImplementation(() => events.push("runtime-patch")); - patch.waitForSupervisorReconnectIfNeeded.mockImplementation(() => events.push("reconnect")); - patch.commitAfterReady.mockImplementation(() => events.push("commit")); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { - events.push("create"); - expect(options.onPoll).toBeUndefined(); - expect(options.waitForReadyTermination).toBe(true); - expect(args.indexOf("--label")).toBeGreaterThan(0); - expect(args.indexOf("--label")).toBeLessThan(args.indexOf("--")); - nonce = createAttemptNonce(args); - expect(nonce).toMatch(/^[0-9a-f]{62}$/u); - expect(nonce).toHaveLength(NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH); - expect(nonce.length).toBeLessThanOrEqual(63); - expect(options.readyCheck?.()).toBe(true); - return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; - }); - mocks.waitForCreatedSandboxReadyWithTrace.mockImplementation(() => { - events.push("readiness"); - return { ready: true, reason: "ready", failurePhase: null }; - }); - const deps = createGpuFlowDeps(); - vi.mocked(deps.sleep).mockImplementation(() => { - events.push("identity-settle"); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - }); - deps.installPortableDemoLifecycle = vi.fn(() => { - events.push("portable-lifecycle"); - return "generation-1"; - }); - vi.mocked(deps.runCaptureOpenshell) - .mockImplementationOnce((args) => { - expect(args).not.toContain("--selector"); - events.push("ready-visible"); - return "alpha Ready"; - }) - .mockImplementationOnce((args) => { - expect(args).toContain("--selector"); - events.push("identity-metadata-pending"); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - return sandboxListJson( - "alpha-sandbox-id", - { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }, - { - resource_version: null, - created_at: null, - phase: null, - current_policy_version: null, - }, - ); - }) - .mockImplementationOnce((args) => { - expect(args).toContain("--selector"); - events.push("identity-matched"); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - return sandboxListJson("alpha-sandbox-id", { - [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, - }); - }); - - await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); - - expect(events).toEqual([ - "create", - "ready-visible", - "identity-metadata-pending", - "identity-matched", - "verify-created", - "revalidate:validate runtime patch for sandbox 'alpha'", - "runtime-check", - "revalidate:apply runtime patch for sandbox 'alpha'", - "runtime-patch", - "reconnect", - "revalidate:reconnect sandbox supervisor for 'alpha'", - "readiness", - "revalidate:commit runtime readiness for sandbox 'alpha'", - "commit", - "revalidate:record portable lifecycle for sandbox 'alpha'", - "portable-lifecycle", - ]); - expect(deps.runCaptureOpenshell).toHaveBeenNthCalledWith( - 2, - [ - "sandbox", - "list", - "-g", - "nemoclaw", - "--selector", - `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`, - "--output", - "json", - "--limit", - "2", - ], - { - ignoreError: false, - timeout: expect.any(Number), - maxBuffer: 1024 * 1024, - killSignal: "SIGKILL", - killProcessTreeOnTimeout: true, - }, - ); - const firstIdentityTimeout = vi.mocked(deps.runCaptureOpenshell).mock.calls[1]?.[1]?.timeout; - expect(firstIdentityTimeout).toEqual(expect.any(Number)); - expect(firstIdentityTimeout as number).toBeGreaterThan(0); - expect(firstIdentityTimeout as number).toBeLessThanOrEqual(30_000); - expect(deps.runCaptureOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "get", "-g", "nemoclaw", "alpha"], - expect.anything(), - ); - expect(deps.sleep).not.toHaveBeenCalled(); - }); - - it("returns false and blocks effects when the create-attempt selector returns no sandbox ID (#10769)", async () => { - const input = noGpuInput(); - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - const patch = createGpuPatchFixture(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, _args, _env, options) => { - expect(options.readyCheck?.()).toBe(false); - return { status: 0, output: "", sawProgress: true }; - }); - const deps = createGpuFlowDeps(); - deps.installPortableDemoLifecycle = vi.fn(); - vi.mocked(deps.runCaptureOpenshell) - .mockReturnValueOnce("alpha Ready") - .mockReturnValueOnce("[]"); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "did not return one exact durable sandbox identity before post-create effects", - ); - - expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledOnce(); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(input.revalidateVerifiedSandboxBeforeEffect).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - expect(patch.waitForSupervisorReconnectIfNeeded).not.toHaveBeenCalled(); - expect(patch.commitAfterReady).not.toHaveBeenCalled(); - expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); - expect(deps.installPortableDemoLifecycle).not.toHaveBeenCalled(); - }); - - it("persists recovery before reporting a handoff timeout that looks like an incomplete create (#10769)", async () => { - const events: string[] = []; - let nonce = ""; - const input = noGpuInput(); - input.persistRetainedSandboxRecovery = vi.fn(() => { - events.push("persist-recovery"); - return true; - }); - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - const patch = createGpuPatchFixture(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { - nonce = createAttemptNonce(args); - expect(options.readyCheck?.()).toBe(true); - return { - status: 1, - output: - "Created sandbox: alpha\nOpenShell create client did not exit after Ready; aborting cutover.", - sawProgress: true, - readyTerminationTimedOut: true, - }; - }); - const deps = createGpuFlowDeps(); - deps.installPortableDemoLifecycle = vi.fn(); - vi.mocked(deps.runCaptureOpenshell) - .mockReturnValueOnce("alpha Ready") - .mockImplementationOnce(() => - sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), - ); - vi.mocked(console.error).mockImplementation(() => { - events.push("report-recovery"); - }); - const exit = vi.spyOn(process, "exit").mockImplementation(() => { - throw new Error("process.exit:1"); - }); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "OpenShell create client did not exit after Ready for sandbox 'alpha'", - ); - - const fingerprint = ALPHA_SANDBOX_IDENTITY_FINGERPRINT; - expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( - expect.stringMatching( - new RegExp( - `^Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}\\. Durable sandbox identity fingerprint: ${fingerprint}\\.`, - "u", - ), - ), - fingerprint, - nonce, - ); - expect(events.slice(0, 2)).toEqual(["persist-recovery", "report-recovery"]); - expect(exit).not.toHaveBeenCalled(); - const output = vi.mocked(console.error).mock.calls.flat().join("\n"); - expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); - expect(output).toContain(`Durable sandbox identity fingerprint: ${fingerprint}`); - expect(output).toContain("Run 'nemoclaw alpha destroy'"); - expect(output).toContain("the command removes nothing and preserves the recovery record"); - expect(output).toContain("Give the create-attempt label to an OpenShell administrator"); - expect(output).toContain("After OpenShell confirms removal"); - expect(output).toContain("run 'nemoclaw alpha destroy --yes'"); - expect(output).not.toContain("alpha-sandbox-id"); - expect(output).not.toContain("Recovery:"); - expect(output).not.toContain("Or: nemoclaw onboard"); - expect(output).not.toContain("onboard --resume"); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(input.revalidateVerifiedSandboxBeforeEffect).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - expect(patch.waitForSupervisorReconnectIfNeeded).not.toHaveBeenCalled(); - expect(patch.commitAfterReady).not.toHaveBeenCalled(); - expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); - expect(deps.installPortableDemoLifecycle).not.toHaveBeenCalled(); - }); - - it("blocks a restart-safe handoff timeout without create-attempt identity (#10769)", async () => { - const input = noGpuInput(); - input.persistStartupCommand = true; - const patch = createGpuPatchFixture(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, _args, _env, options) => { - expect(options.waitForReadyTermination).toBe(true); - expect(options.readyCheck?.()).toBe(true); - return { - status: 1, - output: - "Created sandbox: alpha\nOpenShell create client did not exit after Ready; aborting cutover.", - sawProgress: true, - readyTerminationTimedOut: true, - }; - }); - const deps = createGpuFlowDeps(); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "No create-attempt identity was available for retained recovery", - ); - - expect(input.persistRetainedSandboxRecovery).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - expect(patch.waitForSupervisorReconnectIfNeeded).not.toHaveBeenCalled(); - expect(patch.commitAfterReady).not.toHaveBeenCalled(); - expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); - }); - - it.each([ - ["returns false", (): boolean => false], - [ - "throws", - (): boolean => { - throw new Error("recovery writer failed"); - }, - ], - ] as const)( - "blocks the create after retained recovery persistence %s (#10769)", - async (_failureMode, persistRecovery) => { - let nonce = ""; - const input = noGpuInput(); - input.persistRetainedSandboxRecovery = vi.fn(persistRecovery); - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - const patch = createGpuPatchFixture(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { - nonce = createAttemptNonce(args); - expect(options.readyCheck?.()).toBe(true); - return { - status: 1, - output: "OpenShell create client did not exit after Ready; aborting cutover.", - sawProgress: true, - readyTerminationTimedOut: true, - }; - }); - const deps = createGpuFlowDeps(); - vi.mocked(deps.runCaptureOpenshell) - .mockReturnValueOnce("alpha Ready") - .mockImplementationOnce(() => - sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), - ); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "NemoClaw could not save the retained sandbox recovery record for this create attempt", - ); - - expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( - expect.stringContaining( - `Durable sandbox identity fingerprint: ${ALPHA_SANDBOX_IDENTITY_FINGERPRINT}`, - ), - ALPHA_SANDBOX_IDENTITY_FINGERPRINT, - nonce, - ); - const output = vi.mocked(console.error).mock.calls.flat().join("\n"); - expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); - expect(output).toContain( - "NemoClaw could not save the retained sandbox recovery record for this create attempt", - ); - expect(output).not.toContain("alpha-sandbox-id"); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(input.revalidateVerifiedSandboxBeforeEffect).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - expect(patch.waitForSupervisorReconnectIfNeeded).not.toHaveBeenCalled(); - expect(patch.commitAfterReady).not.toHaveBeenCalled(); - }, - ); -}); diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index acd6f3bc927..6786b701bd5 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -55,43 +55,21 @@ import { NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH, } from "../adapters/openshell/sandbox-identity"; import { + ALPHA_SANDBOX_IDENTITY_FINGERPRINT, + createAttemptNonce, createGpuFlowDeps, createGpuFlowInput, createGpuPatchFixture, + createNoGpuFlowInput as noGpuInput, + createTimedOutCreateResult, + expectNoPostCreateEffects as expectNoEffects, resetGpuFlowMocks, + sandboxListJson, setupGpuFlowMocks, } from "./__test-helpers__/sandbox-gpu-create-flow"; -import { - createHermesPortableReadyCapture, - createHermesPortableReadyRunner, -} from "./experimental/hermes-portable-onboarding"; import { runSandboxGpuCreateFlow } from "./sandbox-gpu-create-flow"; import { fingerprintSandboxRecreateValue } from "./sandbox-recreate-transaction"; -function sandboxListJson( - sandboxId: string, - labels: Readonly>, - overrides: Readonly> = {}, -): string { - return JSON.stringify([ - { - id: sandboxId, - name: "alpha", - labels, - resource_version: 1, - created_at: "2026-08-25T00:00:00Z", - phase: "Ready", - current_policy_version: 1, - ...overrides, - }, - ]); -} - -function createAttemptNonce(args: readonly string[]): string { - const labelIndex = args.indexOf("--label"); - return (args[labelIndex + 1] ?? "").slice(NEMOCLAW_CREATE_ATTEMPT_LABEL.length + 1); -} - const durableRecoveryWriterFailures = [ ["returns false", () => false], [ @@ -101,8 +79,6 @@ const durableRecoveryWriterFailures = [ }, ], ] as const; -const ALPHA_SANDBOX_IDENTITY_FINGERPRINT = - "8174fa2a5d65755138d8339e086c03d736633130b22dca10952e80e74750c01d"; function expectNoSandboxDelete(deps: ReturnType): void { expect( @@ -112,23 +88,6 @@ function expectNoSandboxDelete(deps: ReturnType): void ).toBe(false); } -function noGpuInput() { - const input = createGpuFlowInput(); - input.sandboxGpuConfig = { - mode: "0", - hostGpuDetected: false, - hostGpuPlatform: null, - sandboxGpuEnabled: false, - sandboxGpuDevice: null, - errors: [], - }; - input.gpuRoutePlan = "none"; - input.initialGpuRoute = "none"; - input.createArgv = ["openshell", "sandbox", "create", "--name", "alpha", "--", "agent"]; - input.persistRetainedSandboxRecovery = vi.fn(() => true); - return input; -} - function attachManagedBootstrap( input: ReturnType, patch: ReturnType, @@ -186,42 +145,6 @@ function createCommittedReadinessPersistenceFixture() { return { deps, error, input, patch }; } -function createApfFallbackRecoveryFixture(captureExactIdentity = true) { - let nonce = ""; - const input = createGpuFlowInput(); - input.requirePolicylessCreate = true; - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - input.persistRetainedSandboxRecovery = vi.fn(() => true); - mocks.streamSandboxCreate.mockImplementationOnce(async (_command, args) => { - nonce = createAttemptNonce(args); - return { - status: 1, - output: "native runtime failed after sandbox creation", - sawProgress: true, - }; - }); - mocks.queryOpenShellDockerSandboxRuntimeSnapshot.mockReturnValue({ - ok: true, - imageId: "sha256:" + "a".repeat(64), - bookkeepingImageRef: "openshell/sandbox-from:test", - stateError: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", - deviceRequests: null, - devices: null, - runtime: "runc", - nvidiaVisibleDevices: null, - nativeGpuAttachmentState: "absent", - containerId: "container-a", - }); - const deps = createGpuFlowDeps(); - vi.mocked(deps.runCaptureOpenshell).mockImplementation(() => - captureExactIdentity - ? sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }) - : "[]", - ); - return { deps, input, readNonce: () => nonce }; -} - beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); @@ -736,90 +659,6 @@ describe("created sandbox identity gate", () => { expect(deps.sleep).toHaveBeenCalledExactlyOnceWith(0.25); }); - it("carries Hermes receipt authority from selector settlement through publication lookup (#10423)", async () => { - const events: string[] = []; - let nonce = ""; - const input = noGpuInput(); - const patch = createGpuPatchFixture(); - input.verifyCreatedSandboxBeforeEffects = vi.fn(async () => { - events.push("verify-policy"); - }); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { - events.push("create"); - nonce = createAttemptNonce(args); - expect(options.readyCheck?.()).toBe(true); - return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; - }); - mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValue({ - ready: true, - reason: "ready", - failurePhase: null, - }); - const capture = vi.fn((args: readonly string[]) => { - const results = { - [["sandbox", "list", "-g", "nemoclaw"].join("\0")]: () => { - events.push("ready-visible"); - return { status: 0, stdout: Buffer.from("alpha Ready"), stderr: Buffer.alloc(0) }; - }, - [[ - "sandbox", - "list", - "-g", - "nemoclaw", - "--selector", - `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`, - "--output", - "json", - "--limit", - "2", - ].join("\0")]: () => { - events.push("selector-settled"); - return { - status: 0, - stdout: Buffer.from( - sandboxListJson("alpha-sandbox-id", { - [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, - }), - ), - stderr: Buffer.alloc(0), - }; - }, - [["sandbox", "get", "-g", "nemoclaw", "alpha"].join("\0")]: () => { - events.push("publication-get"); - return { - status: 0, - stdout: Buffer.from("ID: alpha-sandbox-id\n"), - stderr: Buffer.alloc(0), - }; - }, - } satisfies Readonly< - Record { status: number; stdout: Buffer; stderr: Buffer }> - >; - return ( - results[args.join("\0") as keyof typeof results] ?? - (() => ({ status: 1, stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) })) - )(); - }); - const deps = createGpuFlowDeps(); - deps.runOpenshell = createHermesPortableReadyRunner("alpha", "nemoclaw", capture); - deps.runCaptureOpenshell = createHermesPortableReadyCapture("alpha", "nemoclaw", capture); - - await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); - - expect(events.slice(0, 6)).toEqual([ - "create", - "ready-visible", - "selector-settled", - "selector-settled", - "publication-get", - "verify-policy", - ]); - expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); - expect(capture).toHaveBeenCalledWith(["sandbox", "get", "-g", "nemoclaw", "alpha"]); - }); - it.each([ [ "changes durable ID", @@ -1293,84 +1132,6 @@ describe("created sandbox identity gate", () => { expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); }); - it("persists exact APF recovery evidence before refusing native fallback (#9833)", async () => { - const { deps, input, readNonce } = createApfFallbackRecoveryFixture(); - const exit = vi.spyOn(process, "exit").mockImplementation(() => { - throw new Error("process.exit:1"); - }); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1"); - - const nonce = readNonce(); - const fingerprint = ALPHA_SANDBOX_IDENTITY_FINGERPRINT; - expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( - expect.stringMatching( - new RegExp( - `^Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}\\. Durable sandbox identity fingerprint: ${fingerprint}\\.`, - "u", - ), - ), - fingerprint, - nonce, - ); - expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledBefore(exit); - const output = vi.mocked(console.error).mock.calls.flat().join("\n"); - expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); - expect(output).toContain(`Durable sandbox identity fingerprint: ${fingerprint}`); - expect(output).not.toContain("alpha-sandbox-id"); - expectNoSandboxDelete(deps); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - }); - - it("persists the APF create-attempt label when exact recovery identity is unavailable (#9833)", async () => { - const { deps, input, readNonce } = createApfFallbackRecoveryFixture(false); - vi.spyOn(process, "exit").mockImplementation(() => { - throw new Error("process.exit:1"); - }); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1"); - - const nonce = readNonce(); - expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( - expect.stringMatching( - new RegExp( - `^Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}\\..*Recovery is blocked until an OpenShell administrator resolves the create-attempt label`, - "u", - ), - ), - undefined, - nonce, - ); - const output = vi.mocked(console.error).mock.calls.flat().join("\n"); - expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); - expect(output).toContain("Recovery is blocked"); - expectNoSandboxDelete(deps); - }); - - it.each(durableRecoveryWriterFailures)( - "blocks APF fallback when durable recovery persistence %s (#9833)", - async (_name, writer) => { - const { deps, input, readNonce } = createApfFallbackRecoveryFixture(); - const persist = vi.fn(writer); - input.persistRetainedSandboxRecovery = persist; - const exit = vi.spyOn(process, "exit").mockImplementation(() => { - throw new Error("process.exit:1"); - }); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "The APF recovery-only session remains blocked until its durable recovery record can be saved.", - ); - - expect(persist).toHaveBeenCalledOnce(); - expect(exit).not.toHaveBeenCalled(); - const output = vi.mocked(console.error).mock.calls.flat().join("\n"); - expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${readNonce()}`); - expect(output).toContain("APF recovery is blocked because NemoClaw could not save"); - expectNoSandboxDelete(deps); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - }, - ); - it("stops before a runtime patch when the durable checkpoint drifts (#9833)", async () => { let nonce = ""; const input = noGpuInput(); @@ -1449,4 +1210,269 @@ describe("created sandbox identity gate", () => { expect(mocks.verifyGpuSandboxAccessAfterReady).not.toHaveBeenCalled(); expect(patch.commitAfterReady).not.toHaveBeenCalled(); }); + + it("ends the create-client handoff after a nonce-owned ID appears and settles metadata before effects (#10769)", async () => { + const metadataStates: string[] = []; + let nonce = ""; + const input = noGpuInput(); + const patch = createGpuPatchFixture(); + const deps = createGpuFlowDeps(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(async (identity) => { + expect(metadataStates).toEqual(["pending", "complete"]); + expect(identity).toEqual({ + sandboxId: "alpha-sandbox-id", + liveIdentityFingerprint: expect.stringMatching(/^[0-9a-f]{64}$/u), + createAttemptNonce: expect.stringMatching(/^[0-9a-f]{62}$/u), + route: "none", + }); + expect(patch.exitOnPatchError).not.toHaveBeenCalled(); + expect(patch.ensureApplied).not.toHaveBeenCalled(); + expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); + expect(deps.installPortableDemoLifecycle).not.toHaveBeenCalled(); + }); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { + expect(options.waitForReadyTermination).toBe(true); + expect(args.indexOf("--label")).toBeLessThan(args.indexOf("--")); + nonce = createAttemptNonce(args); + expect(nonce).toMatch(/^[0-9a-f]{62}$/u); + expect(options.readyCheck?.()).toBe(true); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValue({ + ready: true, + reason: "ready", + failurePhase: null, + }); + deps.installPortableDemoLifecycle = vi.fn(() => "generation-1"); + vi.mocked(deps.runCaptureOpenshell) + .mockImplementationOnce((args) => { + expect(args).not.toContain("--selector"); + return "alpha Ready"; + }) + .mockImplementationOnce((args) => { + expect(args).toContain("--selector"); + metadataStates.push("pending"); + expectNoEffects(input, patch, deps, mocks.waitForCreatedSandboxReadyWithTrace); + return sandboxListJson( + "alpha-sandbox-id", + { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }, + { + resource_version: null, + created_at: null, + phase: null, + current_policy_version: null, + }, + ); + }) + .mockImplementationOnce((args) => { + expect(args).toContain("--selector"); + metadataStates.push("complete"); + expectNoEffects(input, patch, deps, mocks.waitForCreatedSandboxReadyWithTrace); + return sandboxListJson("alpha-sandbox-id", { + [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, + }); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); + + expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); + expect(patch.exitOnPatchError).toHaveBeenCalledOnce(); + expect(patch.ensureApplied).toHaveBeenCalledOnce(); + expect(patch.waitForSupervisorReconnectIfNeeded).toHaveBeenCalledOnce(); + expect(mocks.waitForCreatedSandboxReadyWithTrace).toHaveBeenCalledOnce(); + expect(patch.commitAfterReady).toHaveBeenCalledOnce(); + expect(deps.installPortableDemoLifecycle).toHaveBeenCalledOnce(); + expect(deps.runCaptureOpenshell).toHaveBeenNthCalledWith( + 2, + [ + "sandbox", + "list", + "-g", + "nemoclaw", + "--selector", + `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`, + "--output", + "json", + "--limit", + "2", + ], + { + ignoreError: false, + timeout: expect.any(Number), + maxBuffer: 1024 * 1024, + killSignal: "SIGKILL", + killProcessTreeOnTimeout: true, + }, + ); + const firstIdentityTimeout = vi.mocked(deps.runCaptureOpenshell).mock.calls[1]?.[1]?.timeout; + expect(firstIdentityTimeout as number).toBeGreaterThan(0); + expect(firstIdentityTimeout as number).toBeLessThanOrEqual(30_000); + expect(deps.runCaptureOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "get", "-g", "nemoclaw", "alpha"], + expect.anything(), + ); + expect(deps.sleep).not.toHaveBeenCalled(); + }); + + it("returns false and blocks effects when the create-attempt selector returns no sandbox ID (#10769)", async () => { + const input = noGpuInput(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, _args, _env, options) => { + expect(options.readyCheck?.()).toBe(false); + return { status: 0, output: "", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell) + .mockReturnValueOnce("alpha Ready") + .mockReturnValueOnce("[]"); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "did not return one exact durable sandbox identity before post-create effects", + ); + + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledOnce(); + expectNoEffects(input, patch, deps, mocks.waitForCreatedSandboxReadyWithTrace); + }); + + it("persists recovery before reporting a handoff timeout that looks like an incomplete create (#10769)", async () => { + const events: string[] = []; + let nonce = ""; + const input = noGpuInput(); + input.persistRetainedSandboxRecovery = vi.fn(() => { + events.push("persist-recovery"); + return true; + }); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { + nonce = createAttemptNonce(args); + expect(options.readyCheck?.()).toBe(true); + return createTimedOutCreateResult( + "Created sandbox: alpha\nOpenShell create client did not exit after Ready; aborting cutover.", + ); + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell) + .mockReturnValueOnce("alpha Ready") + .mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + vi.mocked(console.error).mockImplementation(() => events.push("report-recovery")); + const exit = vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("process.exit:1"); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "OpenShell create client did not exit after Ready for sandbox 'alpha'", + ); + + const fingerprint = ALPHA_SANDBOX_IDENTITY_FINGERPRINT; + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringMatching( + new RegExp( + `^Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}\\. Durable sandbox identity fingerprint: ${fingerprint}\\.`, + "u", + ), + ), + fingerprint, + nonce, + ); + expect(events.slice(0, 2)).toEqual(["persist-recovery", "report-recovery"]); + expect(exit).not.toHaveBeenCalled(); + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); + expect(output).toContain(`Durable sandbox identity fingerprint: ${fingerprint}`); + expect(output).toContain("Run 'nemoclaw alpha destroy'"); + expect(output).toContain("the command removes nothing and preserves the recovery record"); + expect(output).toContain("Give the create-attempt label to an OpenShell administrator"); + expect(output).toContain("After OpenShell confirms removal"); + expect(output).toContain("run 'nemoclaw alpha destroy --yes'"); + expect(output).not.toContain("alpha-sandbox-id"); + expect(output).not.toContain("Recovery:"); + expect(output).not.toContain("Or: nemoclaw onboard"); + expect(output).not.toContain("onboard --resume"); + expectNoEffects(input, patch, deps, mocks.waitForCreatedSandboxReadyWithTrace); + }); + + it("blocks a restart-safe handoff timeout without create-attempt identity (#10769)", async () => { + const input = noGpuInput(); + input.persistStartupCommand = true; + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, _args, _env, options) => { + expect(options.waitForReadyTermination).toBe(true); + expect(options.readyCheck?.()).toBe(true); + return createTimedOutCreateResult( + "Created sandbox: alpha\nOpenShell create client did not exit after Ready; aborting cutover.", + ); + }); + const deps = createGpuFlowDeps(); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "No create-attempt identity was available for retained recovery", + ); + + expect(input.persistRetainedSandboxRecovery).not.toHaveBeenCalled(); + expectNoEffects(input, patch, deps, mocks.waitForCreatedSandboxReadyWithTrace); + }); + + it.each([ + ["returns false", (): boolean => false], + [ + "throws", + (): boolean => { + throw new Error("recovery writer failed"); + }, + ], + ] as const)( + "blocks the create after retained recovery persistence %s (#10769)", + async (_failureMode, persistRecovery) => { + let nonce = ""; + const input = noGpuInput(); + input.persistRetainedSandboxRecovery = vi.fn(persistRecovery); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { + nonce = createAttemptNonce(args); + expect(options.readyCheck?.()).toBe(true); + return createTimedOutCreateResult( + "OpenShell create client did not exit after Ready; aborting cutover.", + ); + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell) + .mockReturnValueOnce("alpha Ready") + .mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "NemoClaw could not save the retained sandbox recovery record for this create attempt", + ); + + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining( + `Durable sandbox identity fingerprint: ${ALPHA_SANDBOX_IDENTITY_FINGERPRINT}`, + ), + ALPHA_SANDBOX_IDENTITY_FINGERPRINT, + nonce, + ); + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); + expect(output).toContain( + "NemoClaw could not save the retained sandbox recovery record for this create attempt", + ); + expect(output).not.toContain("alpha-sandbox-id"); + expectNoEffects(input, patch, deps, mocks.waitForCreatedSandboxReadyWithTrace); + }, + ); }); From 8417d00fdb0e51ef3032977c438a3a1773a2938e Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 14:59:09 -0700 Subject: [PATCH 51/51] fix(onboard): persist verified readiness recovery Signed-off-by: Prekshi Vyas --- docs/reference/troubleshooting.mdx | 9 ++- .../sandbox-gpu-create-flow.ts | 25 +----- .../sandbox-gpu-create-identity-gate.test.ts | 78 ++++++++++++------- .../onboard/sandbox-gpu-create-run-attempt.ts | 39 ++++++++-- 4 files changed, 88 insertions(+), 63 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 97284b307f7..57bcd3d103b 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2022,16 +2022,17 @@ The 180-second default can be exceeded after the create command returns when: - The in-sandbox agent, policy, or model runtime needs longer to become executable. - A managed runtime commit needs longer to re-register the same sandbox as executable `Ready`. -Raise the budget before re-running onboard: +For a post-create readiness failure, complete the retained-sandbox identity-bound cleanup above before you retry the same sandbox name. +If that cleanup remains unresolved, use a different explicit sandbox name for a separate onboarding run. + +After identity-bound cleanup succeeds, raise the budget and rerun onboarding: ```bash export NEMOCLAW_SANDBOX_READY_TIMEOUT=600 -$$nemoclaw onboard +$$nemoclaw onboard --name ``` The variable accepts seconds and sets the shared post-create readiness deadline. -For a post-create readiness failure, NemoClaw preserves the sandbox and does not delete it by mutable name. -Follow the retained-sandbox recovery procedure above. NemoClaw removes the recovery state only after identity-bound cleanup succeeds. diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index b92b57e3461..969f9eb7dc8 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -112,30 +112,8 @@ export function createNoGpuFlowInput(): SandboxGpuCreateFlowInput { return input; } -export function createGpuFlowDeps(sandboxId?: string): SandboxGpuCreateFlowDeps; -export function createGpuFlowDeps( - expectedGatewayName: string, - requireTargetedSandboxProbes: boolean, -): SandboxGpuCreateFlowDeps; -export function createGpuFlowDeps( - sandboxIdOrGatewayName = "alpha-sandbox-id", - expectedGatewayNameOrRequireTargetedProbes: string | boolean = "nemoclaw", -): SandboxGpuCreateFlowDeps { - const requiresTargetedSandboxProbes = - typeof expectedGatewayNameOrRequireTargetedProbes === "boolean"; - const sandboxId = requiresTargetedSandboxProbes ? "alpha-sandbox-id" : sandboxIdOrGatewayName; - const expectedGatewayName = requiresTargetedSandboxProbes - ? sandboxIdOrGatewayName - : expectedGatewayNameOrRequireTargetedProbes; - const assertSandboxProbeTarget = (args: readonly string[]) => { - if (!requiresTargetedSandboxProbes) return; - if (args[0] !== "sandbox" || !["exec", "get", "list"].includes(args[1] ?? "")) return; - const gatewayFlag = args.indexOf("-g"); - expect(gatewayFlag).toBeGreaterThan(1); - expect(args[gatewayFlag + 1]).toBe(expectedGatewayName); - }; +export function createGpuFlowDeps(sandboxId = "alpha-sandbox-id"): SandboxGpuCreateFlowDeps { const runCaptureOpenshell = vi.fn((args: string[], _options?: Record) => { - assertSandboxProbeTarget(args); if (args[0] === "sandbox" && args[1] === "get") { return `Name: alpha\nId: ${sandboxId}\nState: Ready\n`; } @@ -144,7 +122,6 @@ export function createGpuFlowDeps( }); return { runOpenshell: vi.fn((args: string[]) => { - assertSandboxProbeTarget(args); return args[0] === "sandbox" && args[1] === "get" ? { status: 0, diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 6786b701bd5..386578ba1c4 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -145,6 +145,35 @@ function createCommittedReadinessPersistenceFixture() { return { deps, error, input, patch }; } +function createPostVerificationReadinessFailureFixture(writer: () => boolean = () => true) { + const events: string[] = []; + let nonce = ""; + const input = noGpuInput(); + input.persistRetainedSandboxRecovery = vi.fn(() => { + events.push("persist"); + return writer(); + }); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + patch.rollbackManagedStartupAfterCreateFailure.mockImplementation(() => events.push("rollback")); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValue({ + ready: false, + reason: "timeout", + failurePhase: null, + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + return { deps, events, input, nonce: () => nonce }; +} + beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); @@ -153,7 +182,7 @@ describe("created sandbox identity gate", () => { const gatewayName = "nemoclaw-18080"; const input = createGpuFlowInput(); input.gatewayName = gatewayName; - const deps = createGpuFlowDeps(gatewayName, true); + const deps = createGpuFlowDeps(); mocks.streamSandboxCreate.mockImplementationOnce(async (...args) => { expect(args[3].readyCheck()).toBe(true); return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; @@ -1063,37 +1092,30 @@ describe("created sandbox identity gate", () => { expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); }); - it("returns a post-verification readiness failure to the recovery owner (#9833)", async () => { - let nonce = ""; - const input = noGpuInput(); - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - const patch = createGpuPatchFixture(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { - nonce = createAttemptNonce(args); - return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; - }); - mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValue({ - ready: false, - reason: "timeout", - failurePhase: null, - }); - const deps = createGpuFlowDeps(); - vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => - sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), - ); - const exit = vi.spyOn(process, "exit").mockImplementation(() => { - throw new Error("direct process exit bypassed the recovery owner"); - }); - + it("persists recovery before rollback when post-verification readiness fails (#9833)", async () => { + const { deps, events, input, nonce } = createPostVerificationReadinessFailureFixture(); await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( "Sandbox 'alpha' did not become ready after verified creation", ); - - expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); - expect(exit).not.toHaveBeenCalled(); + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringMatching(/Post-verification readiness detail: .*within 60s\./u), + ALPHA_SANDBOX_IDENTITY_FINGERPRINT, + nonce(), + ); + expect(events).toEqual(["persist", "rollback"]); }); + it.each(durableRecoveryWriterFailures)( + "blocks rollback when post-verification recovery persistence %s (#9833)", + async (_failureMode, writer) => { + const { deps, events, input } = createPostVerificationReadinessFailureFixture(writer); + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "could not save the retained sandbox recovery record", + ); + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledOnce(); + expect(events).toEqual(["persist"]); + expect(mocks.printReadinessFailure).not.toHaveBeenCalled(); + }, + ); it("uses a distinct identity label for each create attempt (#9833)", async () => { const input = createGpuFlowInput(); diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index c35803b29f1..2db434675a5 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -56,7 +56,10 @@ import { isExplicitMissingSandboxGatewayOutput, SANDBOX_RECREATE_PROBE_TIMEOUT_MS, } from "./sandbox-recreate-probe"; -import type { CreatedSandboxReadyIdentityCheck } from "./sandbox-readiness-tracing"; +import type { + CreatedSandboxReadinessResult, + CreatedSandboxReadyIdentityCheck, +} from "./sandbox-readiness-tracing"; import * as sandboxReadinessTracing from "./sandbox-readiness-tracing"; import { addTraceEvent } from "./tracing"; @@ -437,6 +440,22 @@ function persistIdentitySettlementRecoveryEvidence(options: { }); } +function persistPostVerificationReadinessRecovery(options: { + readonly input: SandboxGpuCreateFlowInput; + readonly createAttemptNonce: string | null; + readonly verifiedIdentity: CreatedSandboxIdentity | null; + readonly readiness: CreatedSandboxReadinessResult; +}): void { + if (!options.verifiedIdentity) return; + const { input } = options; + persistCreateAttemptRecovery({ + input, + createAttemptNonce: options.createAttemptNonce, + detail: `Post-verification readiness detail: ${boundedPublicationDiagnostic(sandboxReadinessTracing.formatCreatedSandboxReadinessFailureMessage(input.sandboxName, options.readiness, input.sandboxReadyTimeoutSecs).trim())}`, + sandboxIdentityFingerprint: options.verifiedIdentity.liveIdentityFingerprint, + }); +} + async function confirmManagedRuntimeCommitReadiness(options: { readonly input: SandboxGpuCreateFlowInput; readonly deps: SandboxGpuCreateFlowDeps; @@ -1317,12 +1336,6 @@ export function createSandboxGpuCreateAttemptRunner( now: requirePostCreateReadinessDeadline().now, }); if (!readiness.ready) { - console.error(""); - sandboxReadinessTracing.printReadinessFailure( - readiness, - input.sandboxName, - input.sandboxReadyTimeoutSecs, - ); const canClassifyNativeReadiness = route === "native" && input.gpuRoutePlan === "native-with-fallback" && @@ -1358,6 +1371,18 @@ export function createSandboxGpuCreateAttemptRunner( ...nativeCleanup, } as const; } + persistPostVerificationReadinessRecovery({ + input, + createAttemptNonce, + verifiedIdentity: state.verifiedCreatedSandboxIdentity, + readiness, + }); + console.error(""); + sandboxReadinessTracing.printReadinessFailure( + readiness, + input.sandboxName, + input.sandboxReadyTimeoutSecs, + ); await runtimePatch.rollbackManagedStartupAfterCreateFailure(); printCreateFailureDiagnostics(input.sandboxName, { backupPath: input.restoreBackupPath,