diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index af133cd9fa5..fc3ef820ac3 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -18,7 +18,7 @@ "src/lib/core/ports.ts": 90, "src/lib/core/shell-quote.ts": 28, "src/lib/core/url-utils.ts": 30, - "src/lib/core/wait.ts": 36, + "src/lib/core/wait.ts": 37, "src/lib/credentials/store.ts": 45, "src/lib/inference/config.ts": 30, "src/lib/messaging/channels/index.ts": 25, @@ -59,7 +59,7 @@ }, "allowedCycles": [], "maxRootFiles": { - "src/lib/onboard": 306, + "src/lib/onboard": 303, "src/lib/actions": 18, "src/lib/actions/sandbox": 182, "src/lib/state": 39, diff --git a/docs/inference/configure-inference-timeouts.mdx b/docs/inference/configure-inference-timeouts.mdx index abeb4b2aaef..25fdd8d8da3 100644 --- a/docs/inference/configure-inference-timeouts.mdx +++ b/docs/inference/configure-inference-timeouts.mdx @@ -20,7 +20,7 @@ Use the error location to select the correct setting. |---|---|---| | `NEMOCLAW_AGENT_TIMEOUT` | OpenClaw per-request inference | `600` seconds | | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | Ollama, vLLM, NIM, and compatible-endpoint onboarding validation paths that read this setting | `180` seconds | -| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | Image build, gateway upload, and in-sandbox boot after creation | `180` seconds | +| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | OpenShell publication, durable identity settlement, and executable readiness after creation; a new full budget starts after a managed runtime commit | `180` seconds | | `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` | OpenShell command re-registration after policy application, plus gateway health and re-registration during managed OpenClaw or Hermes recovery | `30`, `90`, or `120` seconds, depending on the recovery phase | The readiness timeout does not govern inference requests or provider validation. @@ -94,14 +94,18 @@ This variable does not extend the later sandbox-readiness wait. ## Increase the Sandbox Readiness Timeout -Raise `NEMOCLAW_SANDBOX_READY_TIMEOUT` when onboarding creates the sandbox but image build, upload, or boot exceeds 180 seconds. -This can occur during a first run with cold caches or on a remote VM over a slow link. +Raise `NEMOCLAW_SANDBOX_READY_TIMEOUT` when OpenShell needs more than 180 seconds after the create command returns to publish the sandbox, settle its durable identity, or reach executable `Ready`. +After a managed runtime commit, NemoClaw starts a new full budget with this value to verify that the same sandbox returns to executable `Ready`. Local-inference validation does not consume the post-commit budget. +This setting does not extend image build or gateway upload time. ```bash export NEMOCLAW_SANDBOX_READY_TIMEOUT=600 $$nemoclaw onboard ``` +If either readiness deadline expires, NemoClaw preserves the sandbox and records recovery evidence when possible. +Follow [Recover a retained sandbox](../../reference/commands#recover-a-retained-sandbox) before reusing the sandbox name. + ## Increase the Recovery Wait Set `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` when OpenShell needs more than 120 seconds to re-register the sandbox after onboarding applies policy presets. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index de3a33321eb..e5dc8d88b26 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3954,7 +3954,7 @@ The following environment variables tune onboard-time and recovery wall-clock li | `NEMOCLAW_OLLAMA_PULL_TIMEOUT` | `1800` (30 minutes) | Wall-clock timeout for `ollama pull` during onboard, in seconds. Accepts integer or float values. Already-downloaded layers are kept; re-running the pull resumes them. | | `NEMOCLAW_HF_DOWNLOAD_STALL_TIMEOUT` | `600` (10 minutes) | Maximum silence between Hugging Face download output during onboard. A positive finite value in seconds overrides the default, up to the Node.js timer limit of about 24.8 days. Blank, invalid, non-positive, sub-millisecond, and oversized values use the default. This is not a total download limit. Increase it only when a working download can produce no output for ten minutes. | | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | `180` | Wall-clock timeout for the inference-server validation probe during onboard, in seconds. Raise on slow networks or for very large prompts. | -| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Wall-clock timeout for post-create readiness, in seconds. Raise the timeout when the managed-image pull, explicit custom image build, gateway upload, or in-sandbox boot exceeds the default (typical on 70B+ models, first-time gateway uploads over slow links, or DGX Station / remote-VM first runs). Ordinary onboarding deletes the partially created sandbox when the deadline expires and prints the retry hint. Portable OpenClaw onboarding instead preserves the sandbox when NemoClaw cannot verify its runtime identity. | +| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Wall-clock deadline for initial post-create publication, durable identity settlement, and executable readiness. After a managed runtime commit, NemoClaw starts a new full deadline with this value to verify that the same sandbox returns to executable `Ready`; local-inference validation does not consume that new budget. A readiness failure preserves the sandbox. Follow the [identity-bound retained-sandbox recovery procedure](#recover-a-retained-sandbox) before reusing its name. | | `NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE` | `30` | Consecutive `Error`-phase polls the post-create readiness wait tolerates before treating `Error` as terminal. Polling starts at 250ms and backs off to a 2-second cap, while `NEMOCLAW_SANDBOX_READY_TIMEOUT` remains the overall deadline. The gateway can briefly report a just-created sandbox in `Error` while it re-registers the sandbox (seen on DGX Spark); the debounce lets that transient recover to `Ready`. Every terminal observation outside the `Error` phase, including one with no reported phase, fails immediately. Set to `1` to restore fast-fail on the first `Error` poll. | | `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` | `30`, `90`, or `120`, depending on the recovery phase | Wall-clock timeout for OpenShell command re-registration after policy application, plus gateway health and re-registration during managed OpenClaw or Hermes recovery. A valid finite, nonnegative value overrides the internal budget for the current recovery phase. | @@ -3989,11 +3989,11 @@ $$nemoclaw recover -If the Ollama pull or post-create readiness timeout fires, onboarding emits the elapsed budget plus a hint to raise the relevant variable. The Ollama pull preserves its partial download for the next attempt. The ordinary post-create readiness wait deletes the orphaned sandbox first so the next `$$nemoclaw onboard` starts without that partially created sandbox. +If the Ollama pull or post-create readiness timeout fires, onboarding emits the elapsed budget plus a hint to raise the relevant variable. The Ollama pull preserves its partial download for the next attempt. A post-create readiness failure preserves the sandbox and records recovery evidence when possible. The sandbox name remains blocked until the identity-bound retained-sandbox recovery procedure succeeds. -For portable OpenClaw onboarding, NemoClaw instead leaves the sandbox in place when it cannot verify the exact runtime identity. Inspect it with `openshell sandbox list` and `$$nemoclaw status`, then follow the recovery guidance from `status`. +Portable OpenClaw onboarding also preserves the sandbox when NemoClaw cannot verify the exact runtime identity. Inspect it with `openshell sandbox list` and `$$nemoclaw status`, then follow the recovery guidance from `status`. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 745a5df75ef..57bcd3d103b 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2002,24 +2002,38 @@ This is a separate budget from `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT`. It covers the -For a newly created OpenClaw or Hermes sandbox, `Ready` is not the final acceptance signal. Within this same budget, NemoClaw also requires OpenShell to return a durable sandbox ID and accept `openshell sandbox exec --name -- true`. NemoClaw keeps waiting only when OpenShell returns its exact `sandbox is not ready` response. A missing or malformed ID, or another command failure, stops the wait. Ordinary onboarding then follows the failed-creation cleanup path. Portable OpenClaw onboarding preserves the sandbox as described below. +For a newly created OpenClaw or Hermes sandbox, `Ready` is not the final acceptance signal. Within the initial post-create budget, NemoClaw also requires OpenShell to return a durable sandbox ID and accept `openshell sandbox exec --name -- true`. NemoClaw keeps waiting only when OpenShell reports exact sandbox absence or its exact `sandbox is not ready` response. A missing or malformed ID, or another command failure, stops the wait. NemoClaw preserves the sandbox, saves create-attempt evidence when possible, and requires identity-bound recovery instead of ordinary failed-creation cleanup. + +After a managed runtime commit, NemoClaw starts a new full `NEMOCLAW_SANDBOX_READY_TIMEOUT` budget to verify that the same durable sandbox returns to executable `Ready`. The local-inference validation timeout remains separate and does not consume this post-commit budget. + +If onboarding reports that the managed runtime commit completed but the same sandbox did not return to executable `Ready`, stop before retrying. +NemoClaw keeps the sandbox, prints its create-attempt label and a one-way durable identity fingerprint, and does not start dashboard forwarding. +It saves that evidence in the retained recovery record when persistence succeeds. +Do not use OpenShell mutable-name deletion. +When NemoClaw confirms that it saved the record and the record contains a durable identity fingerprint, run `$$nemoclaw destroy`; NemoClaw verifies the retained sandbox identity before cleanup. Follow [Recover a retained sandbox](commands#recover-a-retained-sandbox) for the result-specific recovery steps. +If the saved record has no durable identity fingerprint, preserve the terminal output and give the create-attempt label to an OpenShell administrator so they can identify and remove the exact sandbox. +If NemoClaw reports that it could not save the record, preserve the terminal output and ask an OpenShell administrator to identify the exact sandbox from gateway or controller evidence; the recovery-only session remains blocked until NemoClaw can save the durable recovery record. -The 180-second default fits typical workstations but can be exceeded when: +The 180-second default can be exceeded after the create command returns when: + +- The owning gateway needs longer to publish the new sandbox and its durable identity. +- The in-sandbox agent, policy, or model runtime needs longer to become executable. +- A managed runtime commit needs longer to re-register the same sandbox as executable `Ready`. -- The host is building or uploading the sandbox image for the first time (cold caches, slow link). -- The selected model is large (70B+ parameters or 4-bit/8-bit quantisations that take time to memory-map). -- Onboarding runs on a remote VM where image upload to the gateway streams over the network (for example DGX Station first-run installer). +For a post-create readiness failure, complete the retained-sandbox identity-bound cleanup above before you retry the same sandbox name. +If that cleanup remains unresolved, use a different explicit sandbox name for a separate onboarding run. -Raise the budget before re-running onboard: +After identity-bound cleanup succeeds, raise the budget and rerun onboarding: ```bash export NEMOCLAW_SANDBOX_READY_TIMEOUT=600 -$$nemoclaw onboard +$$nemoclaw onboard --name ``` -The variable accepts seconds and applies to the readiness wait only. When the ordinary create deadline expires, NemoClaw tries to delete the partially created sandbox. After successful cleanup, the output ends with `Retry: $$nemoclaw onboard`. If cleanup fails, NemoClaw instead reports that the failed sandbox could not be removed and prints `Manual cleanup: openshell sandbox delete ""`. +The variable accepts seconds and sets the shared post-create readiness deadline. +NemoClaw removes the recovery state only after identity-bound cleanup succeeds. @@ -2036,7 +2050,7 @@ openshell sandbox list $$nemoclaw status ``` -If onboarding instead reports that the sandbox "did not re-register with OpenShell after policy application," the same timeout controls that post-policy command-readiness probe. Raise the budget before retrying, then inspect the same gateway and sandbox status if re-registration still fails. +If onboarding instead reports that the sandbox "did not re-register with OpenShell after policy application," `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` controls that post-policy command-readiness probe. Increase that recovery wait before retrying, then inspect the same gateway and sandbox status if re-registration still fails. ### Sandbox onboard fails with "entered Error phase before it became ready" @@ -2048,18 +2062,11 @@ Onboarding ends with: On a fresh onboard the OpenShell gateway can (re)start its supervisor session and re-register the just-created sandbox. During that window `openshell sandbox list` briefly reports the sandbox in the transient `Error` phase before it flips to `Ready`, as seen on DGX Spark when supervisor restart races the sandbox bootstrap. -NemoClaw polls immediately, starts retrying after 250ms, and backs off to a 2-second cap. +NemoClaw polls immediately. A single-observation wait retries after 250 ms and backs off to a two-second cap. A stable-readiness wait that requires two consecutive `Ready` observations checks every two seconds. It tolerates 30 consecutive `Error` observations by default so this transient recovers on its own. Only `Error` that persists through the debounce count is terminal, unless the overall `NEMOCLAW_SANDBOX_READY_TIMEOUT` deadline expires first. Every terminal observation outside the `Error` phase, including one with no reported phase, fails immediately. -If your host needs more observations for slower re-registration, raise the debounce. Raise `NEMOCLAW_SANDBOX_READY_TIMEOUT` too if the overall deadline is too short. To fail fast on the first `Error` poll, set the debounce to `1`: - -```bash -export NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE=1 -$$nemoclaw onboard -``` - If the failure persists after the debounce, the sandbox is stuck. Inspect the retained diagnostics and gateway state: ```bash diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 349a8f40005..1a02133724e 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -8,8 +8,12 @@ import os from "node:os"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { resolveAgent } from "../../src/lib/agent/onboard.ts"; -import { parseOpenShellSandboxId } from "../../src/lib/adapters/openshell/sandbox-identity.ts"; +import { + NEMOCLAW_CREATE_ATTEMPT_LABEL, + parseOpenShellSandboxId, +} from "../../src/lib/adapters/openshell/sandbox-identity.ts"; import { createCliOpenShellSandboxObserverFromRunner } from "../../src/lib/adapters/openshell/sandbox-observer-cli.ts"; +import { waitUntilAsync } from "../../src/lib/core/wait.ts"; import { isValidName, NAME_ALLOWED_FORMAT } from "../../src/lib/name-validation.ts"; import { type StopHostGatewayResult, @@ -34,12 +38,7 @@ import { } from "../../src/lib/onboard/managed-image/contract.ts"; import { encodeManagedStartupProfile } from "../../src/lib/onboard/managed-startup/profile.ts"; import { createManagedStartupRootApplyRequest } from "../../src/lib/onboard/managed-startup/root-apply.ts"; -import type { - RuntimeProviderBundle, - RuntimeProviderManagedImageBootstrapSurface, -} from "../../src/lib/onboard/runtime-provider/contract.ts"; import { createDockerRuntimeProviderBundle } from "../../src/lib/onboard/runtime-provider/docker.ts"; -import { parseLiveSandboxNames } from "../../src/lib/runtime-recovery.ts"; import { OPENSHELL_SANDBOX_SUPERVISOR_ARGV, prepareSandboxCreateLaunch, @@ -161,6 +160,11 @@ type OnboardModule = { openshellArgv(args: string[]): string[]; runOpenshell(args: string[], opts?: Record): ReturnType; runCaptureOpenshell(args: string[], opts?: Record): string; + isSandboxReady(output: string, sandboxName: string): boolean; + printSandboxCreateRecoveryHints( + output: string, + options: { readonly createArgs: readonly string[] }, + ): void; sleepSeconds(seconds: number): void; startGatewayForRecovery(options: { gatewayName: string; gatewayPort: number }): Promise; }; @@ -169,6 +173,8 @@ const REQUIRED_ONBOARD_OPERATIONS = [ "openshellArgv", "runOpenshell", "runCaptureOpenshell", + "isSandboxReady", + "printSandboxCreateRecoveryHints", "sleepSeconds", "startGatewayForRecovery", ] as const satisfies readonly (keyof OnboardModule)[]; @@ -315,18 +321,109 @@ function isDockerNotFound(result: ManagedImageCommandResult): boolean { ); } +export type ManagedImageRetainedSandboxRecovery = Readonly<{ + schemaVersion: 1; + sandboxName: string; + createAttemptLabel: string; + sandboxIdentityFingerprint: string | null; + message: string; +}>; + +const MANAGED_IMAGE_RETAINED_RECOVERY_FILE = "retained-sandbox-recovery.json"; + +export function managedImageRetainedSandboxRecoveryPath(stateDir: string): string { + return path.join(stateDir, MANAGED_IMAGE_RETAINED_RECOVERY_FILE); +} + +export function readManagedImageRetainedSandboxRecovery( + stateDir: string, +): ManagedImageRetainedSandboxRecovery | null { + try { + const value = JSON.parse( + fs.readFileSync(managedImageRetainedSandboxRecoveryPath(stateDir), "utf8"), + ) as Partial; + if ( + value.schemaVersion !== 1 || + typeof value.sandboxName !== "string" || + typeof value.createAttemptLabel !== "string" || + (value.sandboxIdentityFingerprint !== null && + typeof value.sandboxIdentityFingerprint !== "string") || + typeof value.message !== "string" + ) { + return null; + } + return value as ManagedImageRetainedSandboxRecovery; + } catch { + return null; + } +} + +export function persistManagedImageRetainedSandboxRecovery(options: { + readonly stateDir: string; + readonly sandboxName: string; + readonly message: string; + readonly sandboxIdentityFingerprint?: string; + readonly createAttemptNonce?: string; +}): boolean { + if (!options.createAttemptNonce) return false; + const record: ManagedImageRetainedSandboxRecovery = { + schemaVersion: 1, + sandboxName: options.sandboxName, + createAttemptLabel: `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${options.createAttemptNonce}`, + sandboxIdentityFingerprint: options.sandboxIdentityFingerprint ?? null, + message: options.message, + }; + const recordPath = managedImageRetainedSandboxRecoveryPath(options.stateDir); + const temporaryPath = `${recordPath}.tmp`; + let descriptor: number | null = null; + let directoryDescriptor: number | null = null; + try { + fs.writeFileSync(temporaryPath, `${JSON.stringify(record, null, 2)}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + descriptor = fs.openSync(temporaryPath, "r"); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = null; + fs.renameSync(temporaryPath, recordPath); + directoryDescriptor = fs.openSync(options.stateDir, "r"); + fs.fsyncSync(directoryDescriptor); + fs.closeSync(directoryDescriptor); + directoryDescriptor = null; + const persisted = readManagedImageRetainedSandboxRecovery(options.stateDir); + return ( + persisted?.schemaVersion === record.schemaVersion && + persisted.sandboxName === record.sandboxName && + persisted.createAttemptLabel === record.createAttemptLabel && + persisted.sandboxIdentityFingerprint === record.sandboxIdentityFingerprint && + persisted.message === record.message + ); + } catch { + return false; + } finally { + if (descriptor !== null) fs.closeSync(descriptor); + if (directoryDescriptor !== null) fs.closeSync(directoryDescriptor); + fs.rmSync(temporaryPath, { force: true }); + } +} + export function removeManagedImageGatewayStateIfSafe( stateDir: string, gatewayStop: Pick, gatewayRemovalStatus: number | null, + cleanupVerified = true, ): boolean { if ( gatewayStop.failed.length > 0 || (gatewayStop.ownershipFailures?.length ?? 0) > 0 || - gatewayRemovalStatus !== 0 + gatewayRemovalStatus !== 0 || + !cleanupVerified ) { return false; } + fs.rmSync(managedImageRetainedSandboxRecoveryPath(stateDir), { force: true }); fs.rmSync(stateDir, { recursive: true, force: true }); return true; } @@ -402,19 +499,27 @@ function managedConfigPath(agent: ShippedManagedImageAgent): string { export function managedImageOpenShellProbe( agent: ShippedManagedImageAgent, model: string = MODEL, + options: { readonly rootPath?: string } = {}, ): string { + const rootPath = options.rootPath ? path.resolve(options.rootPath) : null; + const probePath = (absolutePath: string) => + rootPath ? path.join(rootPath, absolutePath.replace(/^\/+/, "")) : absolutePath; + const quotePath = (absolutePath: string) => JSON.stringify(probePath(absolutePath)); + const stat = rootPath ? quotePath("/usr/bin/stat") : "stat"; + const openssl = rootPath ? quotePath("/usr/bin/openssl") : "openssl"; + const curl = quotePath("/usr/bin/curl"); const healthProbe = agent === "openclaw" ? [ - "openclaw_health_code=\"$(/usr/bin/curl -sS -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:18789/health || true)\"", + `openclaw_health_code="$(${curl} -sS -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:18789/health || true)"`, 'case "$openclaw_health_code" in', " 200 | 401) ;;", " *) printf 'OpenClaw /health returned HTTP %s\\n' \"${openclaw_health_code:-000}\" >&2; exit 1 ;;", "esac", ].join("\n") : agent === "hermes" - ? "/usr/bin/curl -fsS --max-time 5 http://127.0.0.1:8642/health >/dev/null" - : "/usr/local/bin/dcode --version >/dev/null"; + ? `${curl} -fsS --max-time 5 http://127.0.0.1:8642/health >/dev/null` + : `${quotePath("/usr/local/bin/dcode")} --version >/dev/null`; const readinessLabel = agent === "openclaw" ? "OpenClaw health endpoint" @@ -429,84 +534,117 @@ export function managedImageOpenShellProbe( "set -u", probeStep( `${agent} executable`, - `test -x ${ + `test -x ${quotePath( agent === "openclaw" ? "/usr/local/bin/openclaw" : agent === "hermes" ? "/usr/local/bin/hermes" - : "/usr/local/bin/dcode" - }`, + : "/usr/local/bin/dcode", + )}`, ), probeStep( `${agent} managed model configuration`, - `grep -F ${JSON.stringify(model)} ${JSON.stringify(managedConfigPath(agent))} >/dev/null`, + `grep -F ${JSON.stringify(model)} ${quotePath(managedConfigPath(agent))} >/dev/null`, ), probeStep( "managed runtime environment must not be a symbolic link", - "test ! -L /run/nemoclaw/managed-startup-runtime.env", + `test ! -L ${quotePath("/run/nemoclaw/managed-startup-runtime.env")}`, ), probeStep( "managed runtime environment owner, group, and mode must equal 0:0:444", - 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-runtime.env)" = "0:0:444"', + `test "$(${stat} -c "%u:%g:%a" ${quotePath( + "/run/nemoclaw/managed-startup-runtime.env", + )})" = "0:0:444"`, ), probeStep( "managed startup completion must not be a symbolic link", - "test ! -L /run/nemoclaw/managed-startup-complete.json", + `test ! -L ${quotePath("/run/nemoclaw/managed-startup-complete.json")}`, ), probeStep( "managed startup completion owner, group, and mode must equal 0:0:444", - 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-complete.json)" = "0:0:444"', + `test "$(${stat} -c "%u:%g:%a" ${quotePath( + "/run/nemoclaw/managed-startup-complete.json", + )})" = "0:0:444"`, ), probeStep( "corporate CA file must exist and be nonempty", - "test -s /usr/local/share/nemoclaw/corporate-ca.pem", + `test -s ${quotePath("/usr/local/share/nemoclaw/corporate-ca.pem")}`, ), probeStep( "corporate CA owner, group, and mode must equal 0:0:444", - 'test "$(stat -c "%u:%g:%a" /usr/local/share/nemoclaw/corporate-ca.pem)" = "0:0:444"', + `test "$(${stat} -c "%u:%g:%a" ${quotePath( + "/usr/local/share/nemoclaw/corporate-ca.pem", + )})" = "0:0:444"`, ), probeStep( "corporate CA system anchor must match the managed material", - "cmp -s /usr/local/share/nemoclaw/corporate-ca.pem /usr/local/share/ca-certificates/nemoclaw-corporate-ca-01.crt", + `cmp -s ${quotePath("/usr/local/share/nemoclaw/corporate-ca.pem")} ${quotePath( + "/usr/local/share/ca-certificates/nemoclaw-corporate-ca-01.crt", + )}`, ), probeStep( "corporate CA system anchor owner, group, and mode must equal 0:0:444", - 'test "$(stat -c "%u:%g:%a" /usr/local/share/ca-certificates/nemoclaw-corporate-ca-01.crt)" = "0:0:444"', + `test "$(${stat} -c "%u:%g:%a" ${quotePath( + "/usr/local/share/ca-certificates/nemoclaw-corporate-ca-01.crt", + )})" = "0:0:444"`, ), probeStep( "system trust must verify the managed corporate CA", - "openssl verify -CAfile /etc/ssl/certs/ca-certificates.crt /usr/local/share/nemoclaw/corporate-ca.pem >/dev/null", + `${openssl} verify -CAfile ${quotePath("/etc/ssl/certs/ca-certificates.crt")} ${quotePath( + "/usr/local/share/nemoclaw/corporate-ca.pem", + )} >/dev/null`, ), probeStep( "managed startup CA bundle must exist and be nonempty", - "test -s /run/nemoclaw/managed-startup-ca-bundle.pem", + `test -s ${quotePath("/run/nemoclaw/managed-startup-ca-bundle.pem")}`, ), probeStep( "managed startup CA bundle owner, group, and mode must equal 0:0:444", - 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-ca-bundle.pem)" = "0:0:444"', + `test "$(${stat} -c "%u:%g:%a" ${quotePath( + "/run/nemoclaw/managed-startup-ca-bundle.pem", + )})" = "0:0:444"`, ), probeStep(readinessLabel, healthProbe), ].join("\n"); } -export function managedImageOpenShellCommittedProbe(): string { +export function managedImageOpenShellCommittedProbe( + options: { readonly rootPath?: string } = {}, +): string { + const transactionPath = options.rootPath + ? path.join( + path.resolve(options.rootPath), + "var/lib/nemoclaw/managed-startup-shared-state-transaction-v1", + ) + : "/var/lib/nemoclaw/managed-startup-shared-state-transaction-v1"; return [ "set -eu", - "test ! -e /var/lib/nemoclaw/managed-startup-shared-state-transaction-v1", + `test ! -e ${JSON.stringify(transactionPath)}`, ].join("\n"); } -async function waitForCommittedSandboxProbe( +export type ManagedImageCommittedProbeWaitOptions = { + readonly budgetMs?: number; + readonly now?: () => number; + readonly runCommand?: ManagedImageCommandRunner; + readonly sleep?: (milliseconds: number) => void | Promise; +}; + +export async function waitForCommittedSandboxProbe( onboard: OnboardModule, input: Inputs, env: NodeJS.ProcessEnv, requireCommitted = true, + options: ManagedImageCommittedProbeWaitOptions = {}, ): Promise { const healthProbe = managedImageOpenShellProbe(input.agent, input.model ?? MODEL); const committedProbe = managedImageOpenShellCommittedProbe(); - const deadline = Date.now() + 240_000; + const now = options.now ?? Date.now; + const budgetMs = options.budgetMs ?? 240_000; + const deadlineMs = now() + budgetMs; + const runCommand = options.runCommand ?? commandResult; const runProbe = (probe: string, timeoutMs: number) => - commandResult( + runCommand( onboard.openshellArgv([ "sandbox", "exec", @@ -522,26 +660,36 @@ async function waitForCommittedSandboxProbe( timeoutMs, ); let lastHealthDetail = ""; - while (Date.now() < deadline) { - const remainingMs = deadline - Date.now(); - const health = runProbe(healthProbe, Math.max(1, Math.min(15_000, remainingMs))); - if (health.status === 0) { - if (!requireCommitted) return; + const ready = await waitUntilAsync( + () => { + const remainingMs = deadlineMs - now(); + const health = runProbe(healthProbe, Math.max(1, Math.min(15_000, remainingMs))); + if (health.status !== 0) { + lastHealthDetail = commandDetail(health); + return false; + } + if (!requireCommitted) return true; const committed = runProbe( committedProbe, - Math.max(1, Math.min(15_000, deadline - Date.now())), + Math.max(1, Math.min(15_000, deadlineMs - now())), ); if (committed.status !== 0) { throw new Error( `managed bootstrap committed, but transaction cleanup was not observable through the exact sandbox: ${commandDetail(committed)}`, ); } - return; - } - lastHealthDetail = commandDetail(health); - const sleepMs = Math.min(2_000, Math.max(0, deadline - Date.now())); - if (sleepMs > 0) await new Promise((resolve) => setTimeout(resolve, sleepMs)); - } + return true; + }, + { + deadlineMs, + initialIntervalMs: 2_000, + maxIntervalMs: 2_000, + backoffFactor: 1, + now, + ...(options.sleep ? { sleep: options.sleep } : {}), + }, + ); + if (ready) return; throw new Error( `OpenShell sandbox did not pass the exact-image managed-bootstrap probe within 240s: ${lastHealthDetail}`, ); @@ -849,12 +997,12 @@ export function assertFailedSandboxOwnerCleanupRetention( expectedSandboxId: string, env: NodeJS.ProcessEnv, ): void { - const get = onboard.runOpenshell(["sandbox", "get", input.sandbox], { + const get = onboard.runOpenshell(["sandbox", "get", "-g", GATEWAY_NAME, input.sandbox], { ignoreError: true, env, stdio: ["ignore", "pipe", "pipe"], }); - const list = onboard.runOpenshell(["sandbox", "list"], { + const list = onboard.runOpenshell(["sandbox", "list", "-g", GATEWAY_NAME], { ignoreError: true, env, stdio: ["ignore", "pipe", "pipe"], @@ -863,7 +1011,7 @@ export function assertFailedSandboxOwnerCleanupRetention( get.status !== 0 || parseOpenShellSandboxId(String(get.stdout ?? "")) !== expectedSandboxId || list.status !== 0 || - !parseLiveSandboxNames(String(list.stdout ?? "")).has(input.sandbox) + !onboard.isSandboxReady(String(list.stdout ?? ""), input.sandbox) ) { throw new Error( `managed-bootstrap rollback did not retain its exact OpenShell owner-cleanup state: get=${commandDetail(get)} list=${commandDetail(list)}`, @@ -871,6 +1019,65 @@ export function assertFailedSandboxOwnerCleanupRetention( } } +function queryManagedImageSandboxIdentity( + onboard: Pick, + input: Pick, + env: NodeJS.ProcessEnv, +): { readonly result: ManagedImageCommandResult; readonly sandboxId: string | null } { + const result = onboard.runOpenshell( + ["sandbox", "get", "-g", GATEWAY_NAME, input.sandbox], + { + ignoreError: true, + env, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + return { + result, + sandboxId: result.status === 0 ? parseOpenShellSandboxId(String(result.stdout ?? "")) : null, + }; +} + +function assertManagedImageSandboxIdentity( + onboard: Pick, + input: Pick, + expectedSandboxId: string, + env: NodeJS.ProcessEnv, + operation: string, +): void { + const observed = queryManagedImageSandboxIdentity(onboard, input, env); + if (observed.sandboxId !== expectedSandboxId) { + throw new Error( + `managed-image sandbox durable identity changed before ${operation}; refusing the post-create effect`, + ); + } +} + +export function managedImageSandboxCleanupOwnershipError( + onboard: Pick, + input: Pick, + expectedSandboxId: string | null, + env: NodeJS.ProcessEnv, +): string | null { + const observed = queryManagedImageSandboxIdentity(onboard, input, env); + if (!expectedSandboxId) { + return observed.result.status === 0 + ? "refusing managed-image sandbox cleanup because no durable sandbox ID was captured" + : null; + } + if (observed.sandboxId === null) { + return "refusing managed-image sandbox cleanup because its durable identity is unavailable"; + } + if (observed.sandboxId !== expectedSandboxId) { + return "refusing managed-image sandbox cleanup because its durable identity changed"; + } + const deleteBoundary = queryManagedImageSandboxIdentity(onboard, input, env); + if (deleteBoundary.sandboxId !== expectedSandboxId) { + return "refusing managed-image sandbox cleanup because its durable identity changed at the delete boundary"; + } + return null; +} + async function run( input: Inputs, afterLocalInference?: (context: ManagedImageOpenShellE2eProbeContext) => Promise | T, @@ -893,6 +1100,7 @@ async function run> | null = null; try { @@ -1045,6 +1251,39 @@ async function run { expect(sleep).toHaveBeenCalledExactlyOnceWith(250); }); + it("honors a caller-supplied identity settlement deadline beyond the default (#10652)", () => { + let nowMs = 0; + const sleep = vi.fn((milliseconds: number) => { + nowMs += milliseconds; + }); + const runCaptureOpenshell = vi + .fn<(args: string[], options?: Record) => string>() + .mockImplementationOnce(() => { + nowMs += 31_000; + return "[]"; + }) + .mockReturnValueOnce(sandboxListJson()); + + expect( + settleCreatedOpenShellSandboxId({ + sandboxName: "alpha", + gatewayName: "nemoclaw", + createAttemptNonce: CREATE_ATTEMPT_NONCE, + runCaptureOpenshell, + now: () => nowMs, + timeoutMs: 60_000, + sleep, + }), + ).toBe("sandbox-alpha"); + + expect(runCaptureOpenshell.mock.calls.map(([, options]) => options?.timeout)).toEqual([ + 60_000, 28_750, + ]); + expect(sleep).toHaveBeenCalledExactlyOnceWith(250); + }); + it("settles one exact nonce identity while its publication metadata becomes complete (#10423)", () => { let nowMs = 0; const sleep = vi.fn((milliseconds: number) => { diff --git a/src/lib/adapters/openshell/sandbox-identity.ts b/src/lib/adapters/openshell/sandbox-identity.ts index a416ebd8435..6cd488f2b55 100644 --- a/src/lib/adapters/openshell/sandbox-identity.ts +++ b/src/lib/adapters/openshell/sandbox-identity.ts @@ -305,6 +305,7 @@ export function settleCreatedOpenShellSandboxId(input: { readonly runCaptureOpenshell: (args: string[], options?: Record) => string; readonly priorSandboxId?: string | null; readonly now?: () => number; + readonly timeoutMs?: number; readonly sleep: (milliseconds: number) => void; }): string { assertCreateAttemptNonce(input.createAttemptNonce); @@ -315,9 +316,15 @@ export function settleCreatedOpenShellSandboxId(input: { } const now = input.now ?? (() => performance.now()); const startedAt = now(); - const deadlineMs = startedAt + CREATED_IDENTITY_SETTLEMENT_TIMEOUT_MS; + const timeoutMs = input.timeoutMs ?? CREATED_IDENTITY_SETTLEMENT_TIMEOUT_MS; + const deadlineMs = startedAt + timeoutMs; - if (!Number.isFinite(startedAt) || !Number.isFinite(deadlineMs) || deadlineMs <= startedAt) { + if ( + !Number.isFinite(startedAt) || + !Number.isFinite(timeoutMs) || + !Number.isFinite(deadlineMs) || + deadlineMs <= startedAt + ) { throw createdIdentityError(input.sandboxName); } diff --git a/src/lib/agent/gateway-readiness.ts b/src/lib/agent/gateway-readiness.ts index 3f7e4ea00fb..f4b503f33dd 100644 --- a/src/lib/agent/gateway-readiness.ts +++ b/src/lib/agent/gateway-readiness.ts @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createReadinessWaitOptions } from "../core/readiness-wait"; import { waitUntil } from "../core/wait"; -import { createReadinessWaitOptions } from "../onboard/readiness-wait"; export function waitForAgentGatewayReady(options: { timeoutSeconds: number; diff --git a/src/lib/onboard/readiness-wait.test.ts b/src/lib/core/readiness-wait.test.ts similarity index 82% rename from src/lib/onboard/readiness-wait.test.ts rename to src/lib/core/readiness-wait.test.ts index 9ccac571a05..ba0ab3a9c23 100644 --- a/src/lib/onboard/readiness-wait.test.ts +++ b/src/lib/core/readiness-wait.test.ts @@ -3,12 +3,12 @@ import { describe, expect, it, vi } from "vitest"; -import { waitUntil } from "../core/wait"; import { createReadinessWaitOptions, formatReadinessDeadline, getLegacyPollDeadlineBudgetMs, } from "./readiness-wait"; +import { waitUntil } from "./wait"; describe("readiness deadline options", () => { it("starts fast, backs off to the cap, and consumes the full deadline", () => { @@ -31,13 +31,22 @@ describe("readiness deadline options", () => { }); it("honors a slower initial interval for readiness paths with a stability contract", () => { + let nowMs = 0; + const sleep = vi.fn((ms: number) => { + nowMs += ms; + }); const options = createReadinessWaitOptions({ budgetMs: 10_000, initialIntervalMs: 2_000, maxIntervalMs: 2_000, + now: () => nowMs, + sleep, }); - expect(options?.initialIntervalMs).toBe(2_000); + expect(waitUntil(() => false, options!)).toBe(false); + expect(sleep).toHaveBeenNthCalledWith(1, 2_000); + expect(sleep.mock.calls.every(([ms]) => ms <= 2_000)).toBe(true); + expect(sleep.mock.calls.reduce((total, [ms]) => total + ms, 0)).toBe(10_000); }); it("preserves bounded immediate probes for a zero-interval legacy configuration", () => { diff --git a/src/lib/onboard/readiness-wait.ts b/src/lib/core/readiness-wait.ts similarity index 95% rename from src/lib/onboard/readiness-wait.ts rename to src/lib/core/readiness-wait.ts index f6d572c1730..2eef5db7fa7 100644 --- a/src/lib/onboard/readiness-wait.ts +++ b/src/lib/core/readiness-wait.ts @@ -1,7 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { WaitUntilOptions } from "../core/wait"; +import { type WaitUntilOptions, waitUntil, waitUntilAsync } from "./wait"; + +export { type WaitUntilOptions, waitUntil, waitUntilAsync }; const DEFAULT_INITIAL_INTERVAL_MS = 250; const DEFAULT_MAX_INTERVAL_MS = 2_000; diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index b1eca0570db..2fa4025364d 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -429,11 +429,7 @@ const promptValidatedSandboxName = sandboxAgent.createPromptValidatedSandboxName exit: process.exit, }); const modelRouter: typeof import("./onboard/model-router") = require("./onboard/model-router"); -const { - isRoutedInferenceProvider, - loadBlueprintProfile, - reconcileModelRouter, -} = modelRouter; +const { isRoutedInferenceProvider, loadBlueprintProfile, reconcileModelRouter } = modelRouter; const routedInference: typeof import("./onboard/routed-inference") = require("./onboard/routed-inference"); const { OnboardRuntimeBoundary, @@ -1508,17 +1504,13 @@ const gatewayRecovery = createGatewayRecoveryOrchestration({ startGatewayWithOptions: gatewayStart.startGatewayWithOptions, }); -const { - recoverGatewayRuntime, - startDockerDriverGateway, - startGateway, - startGatewayForRecovery, -} = createGatewayLifecycleApplication({ - dockerDriverStart: dockerDriverGatewayStart, - recovery: gatewayRecovery, - registration: gatewayRegistration, - start: gatewayStart, -}); +const { recoverGatewayRuntime, startDockerDriverGateway, startGateway, startGatewayForRecovery } = + createGatewayLifecycleApplication({ + dockerDriverStart: dockerDriverGatewayStart, + recovery: gatewayRecovery, + registration: gatewayRegistration, + start: gatewayStart, + }); const { getSandboxRuntimeRegistryFields, hasSandboxGpuDrift, updateReusedSandboxMetadata } = sandboxRegistryMetadata.createSandboxRegistryMetadataHelpers({ @@ -1577,6 +1569,7 @@ const sandboxCreateOrchestrationRuntime = { hasSandboxGpuDrift, inferenceConfig, inspectSandboxForCreate, + isSandboxReady, isLinuxDockerDriverGatewayEnabled, isNonInteractive, isRecreateSandbox, @@ -1595,6 +1588,7 @@ const sandboxCreateOrchestrationRuntime = { path, planRegisteredExtraProviders, preparedDcodeRebuild, + printSandboxCreateRecoveryHints, promptValidatedSandboxName, promptYesNoOrDefault, providerExistsInGateway, diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index 9d13143e74c..969f9eb7dc8 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -7,7 +7,9 @@ import path from "node:path"; import { expect, vi } from "vitest"; +import { NEMOCLAW_CREATE_ATTEMPT_LABEL } from "../../adapters/openshell/sandbox-identity"; import { createCliOpenShellSandboxObserver } from "../../adapters/openshell/sandbox-observer-cli"; +import { isSandboxReady } from "../../state/gateway"; import type { CheckpointPortableRuntimeAuthority } from "../../state/onboard-checkpoint-types"; import type { SandboxGpuProofResult } from "../../state/registry"; import type { ManagedBootstrapRuntimeCreateLifecycleInput } from "../managed-bootstrap/runtime-create"; @@ -25,6 +27,41 @@ export const VERIFIED_GPU_PROOF: SandboxGpuProofResult = { at: "2026-07-06T00:00:00.000Z", }; export const GPU_IMAGE_ID = `sha256:${"a".repeat(64)}`; +export const ALPHA_SANDBOX_IDENTITY_FINGERPRINT = + "8174fa2a5d65755138d8339e086c03d736633130b22dca10952e80e74750c01d"; + +export function sandboxListJson( + sandboxId: string, + labels: Readonly>, + overrides: Readonly> = {}, +): string { + return JSON.stringify([ + { + id: sandboxId, + name: "alpha", + labels, + resource_version: 1, + created_at: "2026-08-25T00:00:00Z", + phase: "Ready", + current_policy_version: 1, + ...overrides, + }, + ]); +} + +export function createAttemptNonce(args: readonly string[]): string { + const labelIndex = args.indexOf("--label"); + return (args[labelIndex + 1] ?? "").slice(NEMOCLAW_CREATE_ATTEMPT_LABEL.length + 1); +} + +export function createTimedOutCreateResult(output: string) { + return { + status: 1, + output, + sawProgress: true, + readyTerminationTimedOut: true, + } as const; +} export function createGpuFlowInput(): SandboxGpuCreateFlowInput { return { @@ -58,30 +95,25 @@ export function createGpuFlowInput(): SandboxGpuCreateFlowInput { }; } -export function createGpuFlowDeps(sandboxId?: string): SandboxGpuCreateFlowDeps; -export function createGpuFlowDeps( - expectedGatewayName: string, - requireTargetedSandboxProbes: boolean, -): SandboxGpuCreateFlowDeps; -export function createGpuFlowDeps( - sandboxIdOrGatewayName = "alpha-sandbox-id", - expectedGatewayNameOrRequireTargetedProbes: string | boolean = "nemoclaw", -): SandboxGpuCreateFlowDeps { - const requiresTargetedSandboxProbes = - typeof expectedGatewayNameOrRequireTargetedProbes === "boolean"; - const sandboxId = requiresTargetedSandboxProbes ? "alpha-sandbox-id" : sandboxIdOrGatewayName; - const expectedGatewayName = requiresTargetedSandboxProbes - ? sandboxIdOrGatewayName - : expectedGatewayNameOrRequireTargetedProbes; - const assertSandboxProbeTarget = (args: readonly string[]) => { - if (!requiresTargetedSandboxProbes) return; - if (args[0] !== "sandbox" || !["exec", "get", "list"].includes(args[1] ?? "")) return; - const gatewayFlag = args.indexOf("-g"); - expect(gatewayFlag).toBeGreaterThan(1); - expect(args[gatewayFlag + 1]).toBe(expectedGatewayName); +export function createNoGpuFlowInput(): SandboxGpuCreateFlowInput { + const input = createGpuFlowInput(); + input.sandboxGpuConfig = { + mode: "0", + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], }; + input.gpuRoutePlan = "none"; + input.initialGpuRoute = "none"; + input.createArgv = ["openshell", "sandbox", "create", "--name", "alpha", "--", "agent"]; + input.persistRetainedSandboxRecovery = vi.fn(() => true); + return input; +} + +export function createGpuFlowDeps(sandboxId = "alpha-sandbox-id"): SandboxGpuCreateFlowDeps { const runCaptureOpenshell = vi.fn((args: string[], _options?: Record) => { - assertSandboxProbeTarget(args); if (args[0] === "sandbox" && args[1] === "get") { return `Name: alpha\nId: ${sandboxId}\nState: Ready\n`; } @@ -90,7 +122,6 @@ export function createGpuFlowDeps( }); return { runOpenshell: vi.fn((args: string[]) => { - assertSandboxProbeTarget(args); return args[0] === "sandbox" && args[1] === "get" ? { status: 0, @@ -111,6 +142,8 @@ export function createGpuFlowDeps( }), sleep: vi.fn(), openshellArgv: vi.fn((args: string[]) => ["openshell", ...args]), + isSandboxReady, + printCreateRecoveryHints: vi.fn(), verifyDirectSandboxGpu: vi.fn(() => VERIFIED_GPU_PROOF), }; } @@ -131,6 +164,26 @@ export function createGpuPatchFixture() { }; } +export function expectNoPostCreateEffects( + input: SandboxGpuCreateFlowInput, + patch: ReturnType, + deps: SandboxGpuCreateFlowDeps, + readinessWait: ReturnType, +): void { + for (const effect of [ + input.verifyCreatedSandboxBeforeEffects, + input.revalidateVerifiedSandboxBeforeEffect, + patch.exitOnPatchError, + patch.ensureApplied, + patch.waitForSupervisorReconnectIfNeeded, + patch.commitAfterReady, + readinessWait, + deps.installPortableDemoLifecycle, + ]) { + if (effect) expect(effect).not.toHaveBeenCalled(); + } +} + export function setupGpuFlowMocks(mocks: Record>): void { mocks.streamSandboxCreate.mockResolvedValue({ status: 0, diff --git a/src/lib/onboard/created-sandbox-failure.test.ts b/src/lib/onboard/created-sandbox-failure.test.ts deleted file mode 100644 index 69fe3edc5d2..00000000000 --- a/src/lib/onboard/created-sandbox-failure.test.ts +++ /dev/null @@ -1,356 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it, vi } from "vitest"; - -import { - reportSandboxCreateFailure, - reportSandboxReadinessFailure, - type SandboxCreateFailureReportDeps, - type SandboxCreateFailureReportOptions, - type SandboxReadinessFailureReportDeps, - type SandboxReadinessFailureReportOptions, -} from "./created-sandbox-failure"; -import type { CreatedSandboxReadinessResult } from "./sandbox-readiness-tracing"; - -class ExitSignal extends Error { - constructor(readonly code: number) { - super(`exit:${code}`); - } -} - -function createFailureDeps( - overrides: Partial = {}, -): SandboxCreateFailureReportDeps { - return { - classifyCreateFailure: vi.fn(() => ({ kind: "unknown" })), - printCreateFailureDiagnostics: vi.fn(), - printRecoveryHints: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - exitProcess: vi.fn((code: number): never => { - throw new ExitSignal(code); - }), - ...overrides, - }; -} - -function createFailureOptions( - overrides: Partial = {}, -): SandboxCreateFailureReportOptions { - return { - sandboxName: "alpha", - createStatus: 3, - createOutput: "boom", - restoreBackupPath: null, - createArgs: ["sandbox", "create", "alpha"], - ...overrides, - }; -} - -describe("reportSandboxCreateFailure", () => { - it("warns and returns (does not exit) when the create is merely incomplete", () => { - const deps = createFailureDeps({ - classifyCreateFailure: vi.fn(() => ({ kind: "sandbox_create_incomplete" })), - }); - expect(() => reportSandboxCreateFailure(createFailureOptions(), deps)).not.toThrow(); - expect(deps.warn).toHaveBeenCalledWith( - " Create stream exited with code 3 after sandbox was created.", - ); - expect(deps.printCreateFailureDiagnostics).not.toHaveBeenCalled(); - expect(deps.printRecoveryHints).not.toHaveBeenCalled(); - expect(deps.exitProcess).not.toHaveBeenCalled(); - }); - - it("prints diagnostics + recovery hints and exits with the create status on a hard failure", () => { - const deps = createFailureDeps(); - expect(() => - reportSandboxCreateFailure( - createFailureOptions({ createStatus: 42, restoreBackupPath: "/tmp/backup" }), - deps, - ), - ).toThrow(ExitSignal); - expect(deps.printCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { - backupPath: "/tmp/backup", - }); - expect(deps.printRecoveryHints).toHaveBeenCalledWith("boom", { - createArgs: ["sandbox", "create", "alpha"], - }); - expect(deps.exitProcess).toHaveBeenCalledWith(42); - expect(deps.warn).not.toHaveBeenCalled(); - }); - - it("redacts create output before classification and echoing", () => { - // With output: leading blank + headline + blank + output echo + "Try:" hint = 5 error() calls. - const withOutput = createFailureDeps(); - expect(() => - reportSandboxCreateFailure( - createFailureOptions({ createOutput: "failed with Authorization: Bearer secret-token" }), - withOutput, - ), - ).toThrow(ExitSignal); - expect(withOutput.classifyCreateFailure).toHaveBeenCalledWith( - "failed with Authorization: Bearer secr********", - ); - expect(withOutput.error).toHaveBeenCalledWith("failed with Authorization: Bearer secr********"); - expect(withOutput.error).not.toHaveBeenCalledWith( - "failed with Authorization: Bearer secret-token", - ); - expect(withOutput.printRecoveryHints).toHaveBeenCalledWith( - "failed with Authorization: Bearer secr********", - expect.any(Object), - ); - expect(withOutput.error).toHaveBeenCalledTimes(5); - - // Without output: the echo block is skipped, so only 3 error() calls remain. - const noOutput = createFailureDeps(); - expect(() => - reportSandboxCreateFailure(createFailureOptions({ createOutput: "" }), noOutput), - ).toThrow(ExitSignal); - expect(noOutput.error).toHaveBeenCalledTimes(3); - // still exits (createStatus || 1) - expect(noOutput.exitProcess).toHaveBeenCalledWith(3); - }); - - it("redacts multiple known token formats in create output", () => { - const deps = createFailureDeps(); - const createOutput = [ - "Authorization: Bearer secret-token", - "github ghp_abcdefghijklmnopqrstuvwxyz1234567890", - "openai sk-abcdefghijklmnopqrstuvwxyz1234567890", - "aws AKIAABCDEFGHIJKLMNOP", // gitleaks:allow - ].join("\n"); - - expect(() => reportSandboxCreateFailure(createFailureOptions({ createOutput }), deps)).toThrow( - ExitSignal, - ); - - const echoed = (deps.error as ReturnType).mock.calls - .map((call) => String(call[0])) - .join("\n"); - expect(echoed).not.toContain("secret-token"); - expect(echoed).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz1234567890"); - expect(echoed).not.toContain("sk-abcdefghijklmnopqrstuvwxyz1234567890"); - expect(echoed).not.toContain("AKIAABCDEFGHIJKLMNOP"); // gitleaks:allow - const hinted = (deps.printRecoveryHints as ReturnType).mock.calls - .map((call) => String(call[0])) - .join("\n"); - expect(hinted).not.toContain("secret-token"); - expect(hinted).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz1234567890"); - expect(hinted).not.toContain("sk-abcdefghijklmnopqrstuvwxyz1234567890"); - expect(hinted).not.toContain("AKIAABCDEFGHIJKLMNOP"); // gitleaks:allow - }); - - it("falls back to exit code 1 when the create status is zero", () => { - const deps = createFailureDeps(); - expect(() => - reportSandboxCreateFailure(createFailureOptions({ createStatus: 0 }), deps), - ).toThrow(ExitSignal); - expect(deps.exitProcess).toHaveBeenCalledWith(1); - }); -}); - -const NOT_READY: CreatedSandboxReadinessResult = { - ready: false, - reason: "timeout", - failurePhase: null, -}; - -function readinessDeps( - overrides: Partial = {}, -): SandboxReadinessFailureReportDeps { - return { - printReadinessFailure: vi.fn(), - printCreateFailureDiagnostics: vi.fn(), - printDockerGpuReadinessFailure: vi.fn(), - deleteSandbox: vi.fn(() => ({ status: 0 })), - cliName: vi.fn(() => "nemoclaw"), - error: vi.fn(), - exitProcess: vi.fn((code: number): never => { - throw new ExitSignal(code); - }), - ...overrides, - }; -} - -function readinessOptions( - overrides: Partial = {}, -): SandboxReadinessFailureReportOptions { - return { - sandboxName: "alpha", - readiness: NOT_READY, - createStatus: 0, - timeoutSecs: 300, - restoreBackupPath: null, - useDockerGpuPatch: false, - ...overrides, - }; -} - -function errorLines(deps: SandboxReadinessFailureReportDeps): string[] { - return (deps.error as ReturnType).mock.calls.map((call) => String(call[0])); -} - -function expectReceiptBlock( - deps: SandboxReadinessFailureReportDeps, - expected: readonly string[], -): void { - const lines = errorLines(deps); - const start = lines.indexOf(" Sandbox lifecycle receipt:"); - expect(start).toBeGreaterThanOrEqual(0); - expect(lines.slice(start, start + expected.length)).toEqual(expected); -} - -describe("reportSandboxReadinessFailure", () => { - it("deletes the failed sandbox on the non-GPU path and exits 1", () => { - const deps = readinessDeps(); - expect(() => reportSandboxReadinessFailure(readinessOptions(), deps)).toThrow(ExitSignal); - expect(deps.printReadinessFailure).toHaveBeenCalledWith(NOT_READY, "alpha", 300); - expect(deps.printCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { backupPath: null }); - expect(deps.deleteSandbox).toHaveBeenCalledWith("alpha"); - expect(deps.printDockerGpuReadinessFailure).not.toHaveBeenCalled(); - expectReceiptBlock(deps, [ - " Sandbox lifecycle receipt:", - " state: created_but_not_ready", - " sandbox: alpha", - " readiness_gate: sandbox_list:not_ready_timeout", - " readiness_reason: timeout", - " create_stream_status: 0", - " timeout_seconds: 300", - " terminal_resolution: timed_out_deleted", - ]); - expect(deps.error).toHaveBeenCalledWith( - " Deleted sandbox 'alpha' after the readiness gate failed; retry will recreate it.", - ); - expect(deps.error).toHaveBeenCalledWith(" Retry: nemoclaw onboard"); - expect(deps.exitProcess).toHaveBeenCalledWith(1); - }); - - it("surfaces manual cleanup when deletion fails", () => { - const deps = readinessDeps({ deleteSandbox: vi.fn(() => ({ status: 1 })) }); - expect(() => reportSandboxReadinessFailure(readinessOptions(), deps)).toThrow(ExitSignal); - expectReceiptBlock(deps, [ - " Sandbox lifecycle receipt:", - " state: created_but_not_ready", - " sandbox: alpha", - " readiness_gate: sandbox_list:not_ready_timeout", - " readiness_reason: timeout", - " create_stream_status: 0", - " timeout_seconds: 300", - " terminal_resolution: timed_out_retained", - ]); - expect(deps.error).toHaveBeenCalledWith( - " Could not remove the failed sandbox. Manual cleanup:", - ); - expect(deps.error).toHaveBeenCalledWith(' openshell sandbox delete "alpha"'); - }); - - it("defers cleanup to the Docker-GPU patch and never deletes the sandbox", () => { - const deps = readinessDeps(); - expect(() => - reportSandboxReadinessFailure(readinessOptions({ useDockerGpuPatch: true }), deps), - ).toThrow(ExitSignal); - expect(deps.printDockerGpuReadinessFailure).toHaveBeenCalledTimes(1); - expect(deps.deleteSandbox).not.toHaveBeenCalled(); - expectReceiptBlock(deps, [ - " Sandbox lifecycle receipt:", - " state: created_but_not_ready", - " sandbox: alpha", - " readiness_gate: sandbox_list:not_ready_timeout", - " readiness_reason: timeout", - " create_stream_status: 0", - " timeout_seconds: 300", - " terminal_resolution: deferred_to_docker_gpu_patch", - ]); - expect(deps.exitProcess).toHaveBeenCalledWith(1); - }); - - it("names the terminal readiness phase in the lifecycle receipt", () => { - const deps = readinessDeps(); - expect(() => - reportSandboxReadinessFailure( - readinessOptions({ - readiness: { - ready: false, - reason: "terminal_failure_phase", - failurePhase: "CrashLoopBackOff", - }, - }), - deps, - ), - ).toThrow(ExitSignal); - expectReceiptBlock(deps, [ - " Sandbox lifecycle receipt:", - " state: created_but_not_ready", - " sandbox: alpha", - " readiness_gate: sandbox_list:CrashLoopBackOff", - " readiness_reason: terminal_failure_phase", - " create_stream_status: 0", - " timeout_seconds: 300", - " terminal_resolution: terminal_failure_deleted", - ]); - }); - - it("reports retained cleanup for terminal readiness failures when delete fails", () => { - const deps = readinessDeps({ deleteSandbox: vi.fn(() => ({ status: 1 })) }); - expect(() => - reportSandboxReadinessFailure( - readinessOptions({ - readiness: { - ready: false, - reason: "terminal_failure_phase", - failurePhase: "Error", - }, - }), - deps, - ), - ).toThrow(ExitSignal); - expectReceiptBlock(deps, [ - " Sandbox lifecycle receipt:", - " state: created_but_not_ready", - " sandbox: alpha", - " readiness_gate: sandbox_list:Error", - " readiness_reason: terminal_failure_phase", - " create_stream_status: 0", - " timeout_seconds: 300", - " terminal_resolution: terminal_failure_retained", - ]); - }); - - it.each([ - null, - "", - ])("falls back to a stable terminal readiness gate for missing phase %s", (failurePhase) => { - const deps = readinessDeps(); - expect(() => - reportSandboxReadinessFailure( - readinessOptions({ - readiness: { - ready: false, - reason: "terminal_failure_phase", - failurePhase, - }, - }), - deps, - ), - ).toThrow(ExitSignal); - expectReceiptBlock(deps, [ - " Sandbox lifecycle receipt:", - " state: created_but_not_ready", - " sandbox: alpha", - " readiness_gate: sandbox_list:terminal_failure", - " readiness_reason: terminal_failure_phase", - " create_stream_status: 0", - " timeout_seconds: 300", - " terminal_resolution: terminal_failure_deleted", - ]); - }); - - it("preserves a non-zero create-stream status when readiness later fails", () => { - const deps = readinessDeps(); - expect(() => - reportSandboxReadinessFailure(readinessOptions({ createStatus: 255 }), deps), - ).toThrow(ExitSignal); - expect(deps.exitProcess).toHaveBeenCalledWith(255); - }); -}); diff --git a/src/lib/onboard/created-sandbox-failure.ts b/src/lib/onboard/created-sandbox-failure.ts deleted file mode 100644 index f0c136cd1fd..00000000000 --- a/src/lib/onboard/created-sandbox-failure.ts +++ /dev/null @@ -1,231 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { redact } from "../security/redact"; -import { cliName } from "./branding"; -import type { CreatedSandboxReadinessResult } from "./sandbox-readiness-tracing"; - -/** Format recovery without authorizing mutable-name deletion or an unsafe onboarding retry. */ -export function formatRetainedSandboxRecoveryMessage(input: { - sandboxName: string; - gatewayName: string; - createAttemptLabel: string; - sandboxIdentityFingerprint: string | null; -}): string { - const createAttemptEvidence = `Create-attempt label: ${input.createAttemptLabel}. `; - if (!input.sandboxIdentityFingerprint) { - return ( - createAttemptEvidence + - `Sandbox '${input.sandboxName}' reached Ready before OpenShell returned one exact durable create identity. Gateway '${input.gatewayName}'. ` + - "OpenShell did not return one exact durable sandbox identity for this create attempt. " + - "Do not delete a sandbox by mutable name; preserve it until an OpenShell administrator resolves the create-attempt label to one sandbox." - ); - } - return ( - createAttemptEvidence + - `Durable sandbox identity fingerprint: ${input.sandboxIdentityFingerprint}. ` + - `NemoClaw stopped before owning-gateway publication and identity verification completed for sandbox '${input.sandboxName}' through gateway '${input.gatewayName}'. ` + - `Do not delete the sandbox by mutable name. Run '${cliName()} ${input.sandboxName} destroy'. ` + - "If OpenShell reports the sandbox present, the command removes nothing and preserves the recovery record. " + - "Give the create-attempt label to an OpenShell administrator for identity-bound removal. " + - `After OpenShell confirms removal, run '${cliName()} ${input.sandboxName} destroy --yes' to reconcile the recovery record.` - ); -} - -export type SandboxCreateFailureReportOptions = { - sandboxName: string; - /** Non-zero exit status from the create stream. */ - createStatus: number; - /** Raw create-stream output, used for failure classification and recovery hints. */ - createOutput: string; - /** Pre-recreate/pre-upgrade state backup path to surface in diagnostics, if any. */ - restoreBackupPath: string | null; - /** Resolved `openshell sandbox create` args, so recovery hints stay aligned with --from. */ - createArgs: readonly string[]; -}; - -export type SandboxCreateFailureReportDeps = { - classifyCreateFailure(output: string): { kind: string }; - printCreateFailureDiagnostics(sandboxName: string, options: { backupPath: string | null }): void; - printRecoveryHints(output: string, options: { createArgs: readonly string[] }): void; - warn(message: string): void; - error(message: string): void; - exitProcess(code: number): never; -}; - -/** - * Report a non-zero sandbox create-stream exit. A mere "create incomplete" - * (the sandbox exists in the gateway but the stream exited non-zero, e.g. SSH - * 255) warns and returns so the caller can fall through to the ready-wait loop; - * any other failure prints diagnostics + recovery hints and exits. - */ -export function reportSandboxCreateFailure( - options: SandboxCreateFailureReportOptions, - deps: SandboxCreateFailureReportDeps, -): void { - const redactedCreateOutput = redact(options.createOutput); - const failure = deps.classifyCreateFailure(redactedCreateOutput); - if (failure.kind === "sandbox_create_incomplete") { - // The sandbox was created in the gateway but the create stream exited - // with a non-zero code (e.g. SSH 255). Fall through to the ready-wait - // loop — the sandbox may still reach Ready on its own. - deps.warn(""); - deps.warn( - ` Create stream exited with code ${options.createStatus} after sandbox was created.`, - ); - deps.warn(" Checking whether the sandbox reaches Ready state..."); - return; - } - deps.error(""); - deps.error(` Sandbox creation failed (exit ${options.createStatus}).`); - if (options.createOutput) { - deps.error(""); - deps.error(redactedCreateOutput); - } - deps.printCreateFailureDiagnostics(options.sandboxName, { - backupPath: options.restoreBackupPath, - }); - deps.error(" Try: openshell sandbox list # check gateway state"); - deps.printRecoveryHints(redactedCreateOutput, { createArgs: options.createArgs }); - return deps.exitProcess(options.createStatus === 0 ? 1 : options.createStatus); -} - -export type SandboxReadinessFailureReportOptions = { - sandboxName: string; - readiness: CreatedSandboxReadinessResult; - /** Exit status reported by the sandbox create stream before readiness polling. */ - createStatus: number; - timeoutSecs: number; - restoreBackupPath: string | null; - /** When the Docker-GPU create patch is active, cleanup is deferred to the patch. */ - useDockerGpuPatch: boolean; -}; - -export type SandboxReadinessFailureReportDeps = { - printReadinessFailure( - readiness: CreatedSandboxReadinessResult, - sandboxName: string, - timeoutSecs: number, - ): void; - printCreateFailureDiagnostics(sandboxName: string, options: { backupPath: string | null }): void; - printDockerGpuReadinessFailure(): void; - deleteSandbox(sandboxName: string): { status: number | null }; - cliName(): string; - error(message: string): void; - exitProcess(code: number): never; -}; - -export type SandboxReadinessTerminalResolution = - | "deferred_to_docker_gpu_patch" - | "terminal_failure_deleted" - | "terminal_failure_retained" - | "timed_out_deleted" - | "timed_out_retained"; - -/** Map the readiness reason and cleanup outcome into the receipt terminal state. */ -function readinessTerminalResolution( - readiness: CreatedSandboxReadinessResult, - deleted: boolean, -): SandboxReadinessTerminalResolution { - if (readiness.reason === "terminal_failure_phase") { - return deleted ? "terminal_failure_deleted" : "terminal_failure_retained"; - } - return deleted ? "timed_out_deleted" : "timed_out_retained"; -} - -/** Name the readiness gate that blocked the created sandbox from becoming Ready. */ -function readinessGate(readiness: CreatedSandboxReadinessResult): string { - if (readiness.reason === "terminal_failure_phase") { - const phase = - typeof readiness.failurePhase === "string" && readiness.failurePhase.length > 0 - ? readiness.failurePhase - : "terminal_failure"; - return `sandbox_list:${phase}`; - } - return "sandbox_list:not_ready_timeout"; -} - -/** - * Format the created-but-not-ready receipt so day-0 onboard failures retain a - * stable terminal state: the created sandbox identity, last readiness gate, - * cleanup result, and retry boundary are all visible in one block (#3344). - */ -function formatCreatedSandboxReadinessReceipt(options: { - sandboxName: string; - readiness: CreatedSandboxReadinessResult; - createStatus: number; - timeoutSecs: number; - terminalResolution: SandboxReadinessTerminalResolution; -}): readonly string[] { - return [ - " Sandbox lifecycle receipt:", - ` state: created_but_not_ready`, - ` sandbox: ${options.sandboxName}`, - ` readiness_gate: ${readinessGate(options.readiness)}`, - ` readiness_reason: ${options.readiness.reason}`, - ` create_stream_status: ${options.createStatus}`, - ` timeout_seconds: ${options.timeoutSecs}`, - ` terminal_resolution: ${options.terminalResolution}`, - ]; -} - -/** - * Report a sandbox that never reached Ready: print the readiness failure and - * create diagnostics, then either defer cleanup to the Docker-GPU patch or - * delete the failed sandbox so a same-name retry does not collide, and exit. - */ -export function reportSandboxReadinessFailure( - options: SandboxReadinessFailureReportOptions, - deps: SandboxReadinessFailureReportDeps, -): never { - deps.error(""); - deps.printReadinessFailure(options.readiness, options.sandboxName, options.timeoutSecs); - deps.printCreateFailureDiagnostics(options.sandboxName, { - backupPath: options.restoreBackupPath, - }); - if (options.useDockerGpuPatch) { - for (const line of formatCreatedSandboxReadinessReceipt({ - sandboxName: options.sandboxName, - readiness: options.readiness, - createStatus: options.createStatus, - timeoutSecs: options.timeoutSecs, - terminalResolution: "deferred_to_docker_gpu_patch", - })) { - deps.error(line); - } - deps.printDockerGpuReadinessFailure(); - } else { - // Clean up non-GPU failures after preserving local diagnostics so the - // next onboard retry with the same name does not fail on "sandbox already exists". - const delResult = deps.deleteSandbox(options.sandboxName); - if (delResult.status === 0) { - for (const line of formatCreatedSandboxReadinessReceipt({ - sandboxName: options.sandboxName, - readiness: options.readiness, - createStatus: options.createStatus, - timeoutSecs: options.timeoutSecs, - terminalResolution: readinessTerminalResolution(options.readiness, true), - })) { - deps.error(line); - } - deps.error( - ` Deleted sandbox '${options.sandboxName}' after the readiness gate failed; retry will recreate it.`, - ); - } else { - for (const line of formatCreatedSandboxReadinessReceipt({ - sandboxName: options.sandboxName, - readiness: options.readiness, - createStatus: options.createStatus, - timeoutSecs: options.timeoutSecs, - terminalResolution: readinessTerminalResolution(options.readiness, false), - })) { - deps.error(line); - } - deps.error(" Could not remove the failed sandbox. Manual cleanup:"); - deps.error(` openshell sandbox delete "${options.sandboxName}"`); - } - } - deps.error(` Retry: ${deps.cliName()} onboard`); - const exitCode = options.createStatus === 0 ? 1 : options.createStatus; - return deps.exitProcess(exitCode); -} diff --git a/src/lib/onboard/created-sandbox-finalization.test.ts b/src/lib/onboard/created-sandbox-finalization.test.ts index 28ec63eb85c..def9ddc1634 100644 --- a/src/lib/onboard/created-sandbox-finalization.test.ts +++ b/src/lib/onboard/created-sandbox-finalization.test.ts @@ -19,6 +19,7 @@ import { finalizeCreatedSandbox, } from "./created-sandbox-finalization"; import { getDcodeSelectionDrift } from "./dcode-selection-drift"; +import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import type { HermesPortableConfiguredReceipt } from "./experimental/hermes-portable-receipt"; import { pendingSandboxCreateIdentityForBoundary } from "./sandbox-create/identity-boundary"; import type { SandboxGpuCreateFlowResult } from "./sandbox-gpu-create-flow"; @@ -984,6 +985,99 @@ describe("created OpenClaw sandbox finalization", () => { }); describe("created sandbox completion actions", () => { + it("stops GPU dashboard and registry effects when committed readiness does not return", async () => { + vi.spyOn( + dockerGpuLocalInference, + "verifyGpuSandboxLocalInferenceAndCommitAfterReady", + ).mockResolvedValue(); + const ensureForward = vi.fn(); + const registerCreatedSandbox = vi.fn(); + const lifecycleRegistration = { + lifecycleGeneration: "generation-1", + lifecycleLiveIdentityFingerprint: "a".repeat(64), + }; + const verifiedCreateBoundary = { + sandboxName: "alpha", + gatewayName: "nemoclaw", + gatewayPort: 8080, + ...lifecycleRegistration, + route: "native" as const, + }; + const completion = createCreatedSandboxCompletionActions( + { + finalization: { sandboxName: "alpha" }, + registration: { gatewayName: "nemoclaw", gatewayPort: 8080 }, + policy: { + initialPolicyPath: "/private/initial-policy.yaml", + compatibilityPolicyPath: null, + getVerifiedCreateBoundary: () => verifiedCreateBoundary, + getVerifiedCreateRegistrationAuthority: vi.fn(), + }, + gpu: { + config: {} as SandboxGpuConfig, + provider: "nvidia-prod", + dockerDriverGateway: true, + verifyDirectSandboxGpu: vi.fn(), + runCaptureOpenshell: vi.fn(), + }, + dashboard: { + chatUiUrl: "http://127.0.0.1:8643", + initialHermesState: { config: null, enabled: false }, + releasePort: vi.fn(), + ensureForward, + getForwardPort: vi.fn(), + resolveHermesState: vi.fn(), + ensureHermesForward: vi.fn(), + }, + workload: {}, + } as never, + { + revalidateSandboxIdentity: vi.fn(), + discoverFreshOpenClawImagePluginInstalls: vi.fn(), + restoreRecreatedSandboxState: vi.fn(), + getDcodeSelectionDrift: vi.fn(), + note: vi.fn(), + error: vi.fn(), + exitProcess: vi.fn() as never, + registerCreatedSandbox, + }, + ); + const confirmManagedRuntimeCommitReadiness = vi.fn(() => { + throw new Error("managed runtime did not return to Ready"); + }); + const created = { + origin: "created", + createResult: { status: 0, output: "", sawProgress: true }, + route: "native", + firstCreateOutput: "", + registryImageRef: null, + lifecycleRegistrationFields: { lifecycleGeneration: "generation-1" }, + runtimePatch: {}, + confirmManagedRuntimeCommitReadiness, + } as unknown as SandboxGpuCreateFlowResult; + const lifecycle = { + generation: "generation-1", + recordExactIdentity: vi.fn(), + capture: vi.fn(() => lifecycleRegistration), + revalidate: vi.fn(() => lifecycleRegistration), + }; + + await expect( + completion.complete( + created, + null, + "created", + true, + () => ({ lifecycleGeneration: "generation-1" }), + lifecycle, + ), + ).rejects.toThrow("managed runtime did not return to Ready"); + + expect(confirmManagedRuntimeCommitReadiness).toHaveBeenCalledOnce(); + expect(ensureForward).not.toHaveBeenCalled(); + expect(registerCreatedSandbox).not.toHaveBeenCalled(); + }); + it.each([ ["ordinary", true, false], ["schema-5", false, true], diff --git a/src/lib/onboard/created-sandbox-finalization.ts b/src/lib/onboard/created-sandbox-finalization.ts index bc023627988..bded5e16830 100644 --- a/src/lib/onboard/created-sandbox-finalization.ts +++ b/src/lib/onboard/created-sandbox-finalization.ts @@ -372,6 +372,7 @@ export function createCreatedSandboxCompletionActions( `committing GPU capability for sandbox '${options.finalization.sandboxName}'`, ), ); + await created.confirmManagedRuntimeCommitReadiness(); } function recordHermesGpuProof(): void { options.gpu.config.sandboxGpuProof = options.gpu.verifyDirectSandboxGpu( diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding-ready-timeout.test.ts b/src/lib/onboard/experimental/hermes-portable-onboarding-ready-timeout.test.ts new file mode 100644 index 00000000000..8132a777519 --- /dev/null +++ b/src/lib/onboard/experimental/hermes-portable-onboarding-ready-timeout.test.ts @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + createHermesPortableTestInput, + createHermesPortableTransactionFixture, + HERMES_PORTABLE_TEST_LIVE_IDENTITY, + HERMES_PORTABLE_TEST_POLICY, +} from "../../../../test/helpers/hermes-portable-onboarding-fixture"; +import { runHermesPortableOnboardingTransaction } from "./hermes-portable-onboarding"; + +let stateDir: string; +let policyPath: string; + +beforeEach(() => { + stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-ready-timeout-")); + policyPath = path.join(stateDir, "create.yaml"); + fs.writeFileSync(policyPath, HERMES_PORTABLE_TEST_POLICY, { mode: 0o600 }); +}); + +afterEach(() => fs.rmSync(stateDir, { recursive: true, force: true })); + +describe("Hermes portable onboarding readiness timeout", () => { + it("uses the configured timeout to settle identity after the old Ready deadline (#9211)", async () => { + const currentInput = { + ...createHermesPortableTestInput(stateDir, policyPath), + sandboxReadyTimeoutSecs: 90, + }; + const present = { + kind: "present" as const, + sandboxId: "sandbox-id-1", + liveIdentityFingerprint: HERMES_PORTABLE_TEST_LIVE_IDENTITY, + }; + let nowMs = 0; + let created = false; + let classificationObservations = 0; + const boundedBudgets: Array<{ budgetMs: number; remainingMs: number }> = []; + const observeSandbox = vi.fn((timeoutBudgetMs?: number) => { + const budgetMs = timeoutBudgetMs ?? 0; + const classification = timeoutBudgetMs === undefined; + classificationObservations += Number(classification); + boundedBudgets.push({ budgetMs, remainingMs: 90_000 - nowMs }); + const observed = !created + ? { kind: "absent" as const } + : classification || nowMs >= 61_000 + ? present + : { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }; + nowMs += !created || classification ? 0 : Math.min(budgetMs, Math.max(0, 61_000 - nowMs)); + return observed; + }); + const delaySandboxReadyPublicationPoll = async (milliseconds: number) => { + nowMs += milliseconds; + }; + const fixture = createHermesPortableTransactionFixture(currentInput, { + observeSandbox, + createSandbox: async () => { + created = true; + return { ready: true }; + }, + delaySandboxReadyPublicationPoll, + readSandboxReadyPublicationClockMs: () => nowMs, + }); + + const completed = await runHermesPortableOnboardingTransaction(currentInput, fixture.value); + + expect(completed.active.receipt.phase).toBe("active"); + expect(completed.created).toBe(true); + expect(fixture.events.filter((event) => event === "create")).toHaveLength(1); + expect(classificationObservations).toBeGreaterThan(0); + expect(nowMs).toBeGreaterThan(60_000); + expect(boundedBudgets.length).toBeGreaterThan(0); + expect(boundedBudgets.find(({ budgetMs }) => budgetMs > 0)?.budgetMs).toBe(90_000); + expect(boundedBudgets.every(({ budgetMs, remainingMs }) => budgetMs <= remainingMs)).toBe(true); + expect(fixture.events[0]).toBe("lock-enter"); + expect(fixture.events.at(-1)).toBe("lock-exit"); + }); +}); diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts b/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts index 9eb38f7c1e3..4d66b883e2f 100644 --- a/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts +++ b/src/lib/onboard/experimental/hermes-portable-onboarding.test.ts @@ -333,49 +333,6 @@ describe("Hermes portable onboarding transaction", () => { ); }); - it("settles the exact post-create sandbox identity after the old Ready deadline (#9211)", async () => { - const present = { - kind: "present" as const, - sandboxId: "sandbox-id-1", - liveIdentityFingerprint: HERMES_PORTABLE_TEST_LIVE_IDENTITY, - }; - const observations = [ - { kind: "absent" as const }, - { kind: "absent" as const }, - { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }, - { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }, - present, - ]; - let nowMs = 0; - let boundedObservations = 0; - const observeSandbox = vi.fn((timeoutBudgetMs?: number) => { - const observation = observations.shift() ?? present; - nowMs += timeoutBudgetMs === undefined ? 0 : ([61_000][boundedObservations++] ?? 0); - return observation; - }); - const delaySandboxReadyPublicationPoll = vi.fn(async (milliseconds: number) => { - nowMs += milliseconds; - }); - const fixture = deps({ - observeSandbox, - delaySandboxReadyPublicationPoll, - readSandboxReadyPublicationClockMs: () => nowMs, - }); - - const completed = await runHermesPortableOnboardingTransaction(input(), fixture.value); - - expect(completed.active.receipt.phase).toBe("active"); - expect(completed.created).toBe(true); - expect(fixture.events.filter((event) => event === "create")).toHaveLength(1); - expect(delaySandboxReadyPublicationPoll).toHaveBeenCalledTimes(2); - expect(delaySandboxReadyPublicationPoll).toHaveBeenCalledWith(1_000); - expect( - observeSandbox.mock.calls.filter(([timeoutBudgetMs]) => timeoutBudgetMs !== undefined), - ).toEqual([[180_000], [118_000], [117_000]]); - expect(fixture.events[0]).toBe("lock-enter"); - expect(fixture.events.at(-1)).toBe("lock-exit"); - }); - it("resumes a pending post-create receipt while Ready publication lags (#9203)", async () => { let firstNowMs = 0; let firstObservations = 0; @@ -403,13 +360,17 @@ describe("Hermes portable onboarding transaction", () => { sandboxId: "sandbox-id-1", liveIdentityFingerprint: HERMES_PORTABLE_TEST_LIVE_IDENTITY, }; + let resumeNowMs = 0; const resumeObservations = [ { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }, { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }, present, ]; - const resumeObserveSandbox = vi.fn(() => resumeObservations.shift() ?? present); - let resumeNowMs = 0; + const resumeObservationElapsedMs = [30_000, 0, 0]; + const resumeObserveSandbox = vi.fn(() => { + resumeNowMs += resumeObservationElapsedMs.shift() ?? 0; + return resumeObservations.shift() ?? present; + }); const delayResumePoll = vi.fn(async (milliseconds: number) => { resumeNowMs += milliseconds; }); @@ -427,23 +388,29 @@ describe("Hermes portable onboarding transaction", () => { expect(delayResumePoll).toHaveBeenCalledTimes(1); expect(delayResumePoll).toHaveBeenCalledWith(1_000); expect(resumeObserveSandbox.mock.calls.slice(0, 3)).toEqual([ - [undefined], [180_000], - [179_000], + [150_000], + [149_000], ]); + expect(resumeNowMs).toBe(31_000); }); it("fails closed when exact post-create Ready publication exceeds its bound (#9211)", async () => { - let observations = 0; - const observeSandbox = vi.fn(() => - observations++ < 2 - ? { kind: "absent" as const } - : { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }, - ); let nowMs = 0; - const delaySandboxReadyPublicationPoll = vi.fn(async (milliseconds: number) => { - nowMs += milliseconds; + const observationElapsedMs = [0, 0, 180_000]; + const boundedBudgets: Array<{ budgetMs: number; remainingMs: number }> = []; + const observeSandbox = vi.fn((timeoutBudgetMs?: number) => { + const budgetMs = timeoutBudgetMs ?? 0; + boundedBudgets.push({ budgetMs, remainingMs: 180_000 - nowMs }); + const elapsedMs = Math.min(budgetMs, observationElapsedMs.shift() ?? 0); + nowMs += elapsedMs; + return elapsedMs === 0 + ? { kind: "absent" as const } + : { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }; }); + const delaySandboxReadyPublicationPoll = async (milliseconds: number) => { + nowMs += milliseconds; + }; const fixture = deps({ observeSandbox, delaySandboxReadyPublicationPoll, @@ -455,32 +422,33 @@ describe("Hermes portable onboarding transaction", () => { ); expect(fixture.events.filter((event) => event === "create")).toHaveLength(1); - expect(delaySandboxReadyPublicationPoll).toHaveBeenCalledTimes(180); - expect(observeSandbox.mock.calls.slice(2)).toEqual( - Array.from({ length: 180 }, (_value, index) => [180_000 - index * 1_000]), - ); + expect(nowMs).toBe(180_000); + expect(boundedBudgets.length).toBeGreaterThan(0); + expect(boundedBudgets.every(({ budgetMs, remainingMs }) => budgetMs <= remainingMs)).toBe(true); expect(fixture.events).not.toContain("registry"); expect(fixture.events.at(-1)).toBe("lock-exit"); }); it("counts OpenShell observation time against the total Ready publication deadline (#9211)", async () => { let nowMs = 0; - let observationIndex = 0; - const observationDurationsMs = [0, 0, 166_000, 13_000] as const; - const observations = [ - { kind: "absent" as const }, - { kind: "absent" as const }, - { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }, - { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }, - ]; - const observeSandbox = vi.fn((_timeoutBudgetMs?: number) => { - const currentIndex = observationIndex++; - nowMs += observationDurationsMs[currentIndex] ?? 0; - return observations[currentIndex] ?? observations.at(-1)!; + let observedMs = 0; + let delayedMs = 0; + const observationElapsedMs = [0, 0, 179_000]; + const boundedBudgets: Array<{ budgetMs: number; remainingMs: number }> = []; + const observeSandbox = vi.fn((timeoutBudgetMs?: number) => { + const budgetMs = timeoutBudgetMs ?? 0; + boundedBudgets.push({ budgetMs, remainingMs: 180_000 - nowMs }); + const elapsedMs = Math.min(budgetMs, observationElapsedMs.shift() ?? 0); + observedMs += elapsedMs; + nowMs += elapsedMs; + return elapsedMs === 0 + ? { kind: "absent" as const } + : { kind: "ambiguous" as const, detail: "exact OpenShell sandbox is not Ready" }; }); - const delaySandboxReadyPublicationPoll = vi.fn(async (milliseconds: number) => { + const delaySandboxReadyPublicationPoll = async (milliseconds: number) => { + delayedMs += milliseconds; nowMs += milliseconds; - }); + }; const fixture = deps({ observeSandbox, delaySandboxReadyPublicationPoll, @@ -491,10 +459,10 @@ describe("Hermes portable onboarding transaction", () => { "cannot classify create result: exact OpenShell sandbox is not Ready", ); - expect(observeSandbox.mock.calls.slice(2)).toEqual([[180_000], [13_000]]); - expect(delaySandboxReadyPublicationPoll).toHaveBeenCalledTimes(1); - expect(delaySandboxReadyPublicationPoll).toHaveBeenCalledWith(1_000); expect(nowMs).toBe(180_000); + expect(observedMs).toBeGreaterThan(delayedMs); + expect(boundedBudgets.length).toBeGreaterThan(0); + expect(boundedBudgets.every(({ budgetMs, remainingMs }) => budgetMs <= remainingMs)).toBe(true); expect(fixture.events.filter((event) => event === "create")).toHaveLength(1); expect(fixture.events).not.toContain("registry"); }); @@ -636,7 +604,16 @@ network_policies: ...input(), createPolicySourceBytes: Buffer.from(regeneratedPolicy), }; - const second = createHermesPortableTransactionFixture(resumedInput); + let consumedPolicyPath: string | null = null; + let consumedPolicyBytes: Buffer | null = null; + const second = createHermesPortableTransactionFixture(resumedInput, { + createSandbox: async (argv, _buildContextPath, effectivePolicySourcePath) => { + consumedPolicyPath = effectivePolicySourcePath; + consumedPolicyBytes = fs.readFileSync(effectivePolicySourcePath); + expect(argv[argv.indexOf("--policy") + 1]).toBe(effectivePolicySourcePath); + return { ready: true }; + }, + }); const resumed = await runHermesPortableOnboardingTransaction(resumedInput, second.value); @@ -648,6 +625,11 @@ network_policies: ).toBe(false); expect(resumed.created).toBe(true); expect(second.events.filter((event) => event === "create")).toHaveLength(1); + expect(consumedPolicyPath).toBe( + hermesPortablePolicySourcePath("alpha", resumed.active.receipt.transactionId, stateDir), + ); + expect(consumedPolicyBytes).toEqual(Buffer.from(POLICY)); + expect(consumedPolicyBytes).not.toEqual(Buffer.from(regeneratedPolicy)); }); it("rejects changed non-policy create intent on pending reentry before effects (#9203)", async () => { @@ -1346,19 +1328,39 @@ network_policies: const run = createHermesPortableReadyRunner("alpha", "nemoclaw", capture); expect(run(["sandbox", "get", "alpha"]).status).toBe(0); - expect(run(["sandbox", "delete", "alpha"]).status).toBe(0); expect(run(["sandbox", "exec", "--name", "alpha", "--", "true"]).status).toBe(0); expect(capture.mock.calls).toEqual([ [["sandbox", "get", "-g", "nemoclaw", "alpha"]], - [["sandbox", "delete", "-g", "nemoclaw", "alpha"]], [["sandbox", "exec", "-g", "nemoclaw", "--name", "alpha", "--", "true"]], ]); + expect(() => run(["sandbox", "delete", "alpha"])).toThrow("unsupported OpenShell command"); expect(() => run(["sandbox", "get", "beta"])).toThrow("unsupported OpenShell command"); expect(() => run(["sandbox", "exec", "--name", "beta", "--", "true"])).toThrow( "unsupported OpenShell command", ); }); + it("accepts readiness commands already scoped to the receipt gateway", () => { + const capture = vi.fn(() => ({ + status: 0, + stdout: Buffer.from("ready"), + stderr: Buffer.alloc(0), + })); + const run = createHermesPortableReadyRunner("alpha", "nemoclaw", capture); + + expect(run(["sandbox", "list", "-g", "nemoclaw"]).status).toBe(0); + expect(run(["sandbox", "exec", "-g", "nemoclaw", "--name", "alpha", "--", "true"]).status).toBe( + 0, + ); + expect(capture.mock.calls).toEqual([ + [["sandbox", "list", "-g", "nemoclaw"]], + [["sandbox", "exec", "-g", "nemoclaw", "--name", "alpha", "--", "true"]], + ]); + expect(() => run(["sandbox", "list", "-g", "other-gateway"])).toThrow( + "unsupported OpenShell command", + ); + }); + it("rejects exact-gateway identity that has not reached Ready (#9203)", () => { const capture = vi .fn() diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding.ts b/src/lib/onboard/experimental/hermes-portable-onboarding.ts index b3b3662024a..413f3d976ef 100644 --- a/src/lib/onboard/experimental/hermes-portable-onboarding.ts +++ b/src/lib/onboard/experimental/hermes-portable-onboarding.ts @@ -109,6 +109,7 @@ export interface HermesPortableOnboardingInput { readonly sandboxName: string; readonly gatewayName: string; readonly lifecycleGeneration: string; + readonly sandboxReadyTimeoutSecs: number; readonly runtimeAuthority: CheckpointPortableRuntimeAuthority; readonly openshellExecutableAuthority: HermesPortableOpenShellExecutableAuthority; readonly stateDir: string; @@ -354,7 +355,7 @@ function scopeHermesPortableReadyExecArgs( return null; } -/** Route create readiness and failed-create cleanup through exact schema-7 authority. */ +/** Route create readiness through exact schema-7 authority. */ export function createHermesPortableReadyRunner( sandboxName: string, gatewayName: string, @@ -365,10 +366,7 @@ export function createHermesPortableReadyRunner( scopeHermesPortableCreatedIdentityArgs(args, gatewayName) ?? scopeHermesPortableReadyGetArgs(args, sandboxName, gatewayName) ?? scopeHermesPortableReadyListArgs(args, gatewayName) ?? - scopeHermesPortableReadyExecArgs(args, sandboxName, gatewayName) ?? - (args[0] === "sandbox" && args[1] === "delete" && args.length === 3 && args[2] === sandboxName - ? ["sandbox", "delete", "-g", gatewayName, args[2]!] - : null); + scopeHermesPortableReadyExecArgs(args, sandboxName, gatewayName); if (!scoped) fail("create lifecycle attempted an unsupported OpenShell command"); return capture(scoped); }; @@ -708,10 +706,6 @@ function parseHermesPortableSandboxJson( } const HERMES_PORTABLE_READY_PUBLICATION_POLL_INTERVAL_MS = 1_000; -const HERMES_PORTABLE_READY_PUBLICATION_TIMEOUT_MS = 180_000; -const HERMES_PORTABLE_READY_PUBLICATION_MAX_POLLS = Math.ceil( - HERMES_PORTABLE_READY_PUBLICATION_TIMEOUT_MS / HERMES_PORTABLE_READY_PUBLICATION_POLL_INTERVAL_MS, -); const HERMES_PORTABLE_NOT_READY_DETAIL = "exact OpenShell sandbox is not Ready"; const HERMES_PORTABLE_READY_PUBLICATION_TIMEOUT_DETAIL = "exact OpenShell sandbox Ready publication exceeded its total deadline"; @@ -794,13 +788,17 @@ async function settleCreatedHermesPortableSandboxReadyPublication( observeSandbox: (timeoutBudgetMs?: number) => HermesPortableSandboxObservation, delayPoll: (milliseconds: number) => Promise, readClockMs: () => number, + deadlineMs: number, ): Promise { - const deadlineMs = readClockMs() + HERMES_PORTABLE_READY_PUBLICATION_TIMEOUT_MS; + const maxPolls = Math.ceil( + Math.max(1, Math.round(deadlineMs - readClockMs())) / + HERMES_PORTABLE_READY_PUBLICATION_POLL_INTERVAL_MS, + ); let observation: HermesPortableSandboxObservation = { kind: "ambiguous", detail: HERMES_PORTABLE_READY_PUBLICATION_TIMEOUT_DETAIL, }; - for (let poll = 0; poll <= HERMES_PORTABLE_READY_PUBLICATION_MAX_POLLS; poll += 1) { + for (let poll = 0; poll <= maxPolls; poll += 1) { const observationBudgetMs = Math.floor(deadlineMs - readClockMs()); if (observationBudgetMs < 1) return observation; observation = observeSandbox(observationBudgetMs); @@ -810,7 +808,7 @@ async function settleCreatedHermesPortableSandboxReadyPublication( ) { return observation; } - if (poll === HERMES_PORTABLE_READY_PUBLICATION_MAX_POLLS) return observation; + if (poll === maxPolls) return observation; const delayBudgetMs = Math.floor(deadlineMs - readClockMs()); if (delayBudgetMs < 1) return observation; await delayPoll(Math.min(HERMES_PORTABLE_READY_PUBLICATION_POLL_INTERVAL_MS, delayBudgetMs)); @@ -1086,6 +1084,7 @@ export async function runHermesPortableOnboardingTransaction( deps: HermesPortableOnboardingDeps, ): Promise> { return await deps.withLifecycleLock(input.sandboxName, async () => { + const readyPublicationTimeoutMs = input.sandboxReadyTimeoutSecs * 1_000; assertHermesPortableUninstallCompleteForOnboarding(input.stateDir); const assertOpenShellExecutableAuthority = (): void => deps.assertOpenShellExecutableAuthority(input.openshellExecutableAuthority); @@ -1412,6 +1411,12 @@ export async function runHermesPortableOnboardingTransaction( } if (snapshot.receipt.phase === "pending") { + const readReadyPublicationClockMs = + deps.readSandboxReadyPublicationClockMs ?? performance.now.bind(performance); + const delayReadyPublicationPoll = + deps.delaySandboxReadyPublicationPoll ?? delayHermesPortableReadyPublicationPoll; + const createReadyPublicationDeadline = () => + readReadyPublicationClockMs() + Math.max(1, Math.round(readyPublicationTimeoutMs)); const createPolicySourcePath = snapshot.receipt.policy.sourcePath; assertRegistryMissingBeforeConfiguration( snapshot.receipt, @@ -1423,15 +1428,26 @@ export async function runHermesPortableOnboardingTransaction( createIntentSha256: snapshot.receipt.createIntentSha256, stateDir: input.stateDir, }); - let observation = observeSandbox(); + const pendingObservationDeadlineMs = createReadyPublicationDeadline(); + const pendingObservationBudgetMs = Math.floor( + pendingObservationDeadlineMs - readReadyPublicationClockMs(), + ); + let observation = + pendingObservationBudgetMs < 1 + ? { + kind: "ambiguous" as const, + detail: HERMES_PORTABLE_READY_PUBLICATION_TIMEOUT_DETAIL, + } + : observeSandbox(pendingObservationBudgetMs); if ( observation.kind === "ambiguous" && observation.detail === HERMES_PORTABLE_NOT_READY_DETAIL ) { observation = await settleCreatedHermesPortableSandboxReadyPublication( observeSandbox, - deps.delaySandboxReadyPublicationPoll ?? delayHermesPortableReadyPublicationPoll, - deps.readSandboxReadyPublicationClockMs ?? performance.now.bind(performance), + delayReadyPublicationPoll, + readReadyPublicationClockMs, + pendingObservationDeadlineMs, ); } if (observation.kind === "ambiguous") @@ -1471,8 +1487,9 @@ export async function runHermesPortableOnboardingTransaction( created = true; observation = await settleCreatedHermesPortableSandboxReadyPublication( observeSandbox, - deps.delaySandboxReadyPublicationPoll ?? delayHermesPortableReadyPublicationPoll, - deps.readSandboxReadyPublicationClockMs ?? performance.now.bind(performance), + delayReadyPublicationPoll, + readReadyPublicationClockMs, + createReadyPublicationDeadline(), ); buildContext.assertCurrent(); input.buildContext.assertCurrentSource(); @@ -1620,6 +1637,7 @@ export interface HermesPortableOnboardingFromOnboardInput { readonly sandboxName: string; readonly gatewayName: string; readonly lifecycleGeneration: string; + readonly sandboxReadyTimeoutSecs: number; readonly portableRuntime: PortableOnboardRuntimeContext; readonly createArgv: readonly string[]; readonly createPolicyPath: string; @@ -1682,6 +1700,7 @@ export async function runHermesPortableOnboardingFromOnboard( sandboxName, gatewayName, lifecycleGeneration, + sandboxReadyTimeoutSecs, portableRuntime, createArgv, createPolicyPath, @@ -1741,6 +1760,7 @@ export async function runHermesPortableOnboardingFromOnboard( sandboxName, gatewayName, lifecycleGeneration, + sandboxReadyTimeoutSecs, runtimeAuthority, openshellExecutableAuthority, stateDir: defaultPortableDemoStateDir(process.env), diff --git a/src/lib/onboard/gateway-health-wait.ts b/src/lib/onboard/gateway-health-wait.ts index db1d0f88e20..4befa7e54a7 100644 --- a/src/lib/onboard/gateway-health-wait.ts +++ b/src/lib/onboard/gateway-health-wait.ts @@ -1,9 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { type WaitUntilOptions, waitUntilAsync } from "../core/wait"; import { envInt } from "./env"; -import { createReadinessWaitOptions, getLegacyPollDeadlineBudgetMs } from "./readiness-wait"; +import { + createReadinessWaitOptions, + getLegacyPollDeadlineBudgetMs, + type WaitUntilOptions, + waitUntilAsync, +} from "../core/readiness-wait"; type RunCaptureOpenshell = (args: string[], opts?: { ignoreError?: boolean }) => string; diff --git a/src/lib/onboard/gateway-recovery.test.ts b/src/lib/onboard/gateway-recovery.test.ts index 1bdf58ea7e5..9983fd192b2 100644 --- a/src/lib/onboard/gateway-recovery.test.ts +++ b/src/lib/onboard/gateway-recovery.test.ts @@ -12,15 +12,15 @@ import { type GatewayRecoveryDeps, startGatewayForRecovery } from "./gateway-rec // actually sleeps. Tests get deterministic deadline expiration without any // real wall-clock waits or global timer state. // -// `advance` is exposed so a test can also advance the clock from inside a -// mocked probe. This is how the timeout test proves the loop is truly -// deadline-driven: if each probe advances the clock, then a maxAttempts=N -// cap would exit at a different observable count than a pure deadline -// would, so the assertions can only be satisfied by the deadline path. +// `advance` is exposed so a test can also account for time spent inside a +// mocked probe. `elapsedMs` lets the timeout test observe that the complete +// configured deadline was consumed without depending on an internal call +// count or the number of OpenShell observations in one recovery probe. function makeVirtualClock(startMs = 1_000_000_000_000) { let now = startMs; return { now: () => now, + elapsedMs: () => now - startMs, advance: (seconds: number) => { now += Math.max(0, seconds) * 1000; }, @@ -100,52 +100,31 @@ describe("gateway recovery", () => { }); it("polls until the configured recovery deadline and reports it in the timeout (#3768)", async () => { - // #3768: prove the loop is DEADLINE-driven, not just attempt-capped. - // Design: with count=10 and interval=1s the wait budget is 10s. Make - // each subprocess-probe advance the clock by ~1s so probes are the - // primary time-consumer, then sleeps at 1s add another second per - // iteration. Under a pure deadline: iterations run until ~2s per - // iteration cumulatively hits 10s -> ~5 probes. Under a hidden - // maxAttempts=count cap, the loop would exit at exactly 10 probes - // (attempt cap hits first because probes and sleeps take equal time), - // which is a different observable count from the deadline path. The - // strict upper bound `probeCount < 10` therefore only passes when the - // deadline (not an attempt cap) terminates the loop. + // #3768: prove the loop consumes its configured wall-clock deadline. + // Account for one second of work only when the externally visible + // status probe begins; the other OpenShell observations remain free to + // change without changing this test's oracle. vi.stubEnv("NEMOCLAW_HEALTH_POLL_COUNT", "10"); vi.stubEnv("NEMOCLAW_HEALTH_POLL_INTERVAL", "1"); const clock = makeVirtualClock(); - // Only advance the clock ONCE per probe iteration (three subprocess - // calls per probe): status is the first call, gateway-info-g the - // second, gateway-info the third. Use a modulo counter so the test - // body stays linear (per repo growth guardrail on if statements in - // changed test files). - let mockCallIndex = 0; - const advanceOnStatusCall = (index: number) => (index % 3 === 0 ? clock.advance(1) : undefined); + const statusProbe = vi.fn(() => { + clock.advance(1); + return "Disconnected"; + }); const deps = createDeps({ sleepSeconds: clock.sleeper, now: clock.now, - runCaptureOpenshell: vi.fn(() => { - advanceOnStatusCall(mockCallIndex); - mockCallIndex += 1; - return "Disconnected"; - }), + runCaptureOpenshell: vi.fn((argv) => + argv[0] === "status" ? statusProbe() : "Disconnected", + ), }); await expect(startGatewayForRecovery({ gatewayPort: 8091 }, deps)).rejects.toThrow( "configured 10s recovery deadline (1s poll interval)", ); - const runCaptureCalls = (deps.runCaptureOpenshell as ReturnType).mock.calls - .length; - const probeCount = runCaptureCalls / 3; - // The deadline (not an attempt cap) MUST have terminated the loop: - // probe advances 1s + sleep advances 1s = 2s per iteration, so under - // a 10s budget the loop runs ~5 iterations and cannot reach the 10 - // attempts a hidden attempt cap would permit. - expect(probeCount).toBeGreaterThan(0); - expect(probeCount).toBeLessThan(10); - // Sleeps happen after every probe except the last one (deadline check - // after the final probe short-circuits before an extra sleep). + expect(statusProbe).toHaveBeenCalled(); + expect(clock.elapsedMs()).toBe(10_000); expect(clock.sleeper).toHaveBeenCalled(); expect(clock.sleeper).toHaveBeenNthCalledWith(1, 0.25); expect(clock.sleeper.mock.calls.every(([s]) => s <= 1)).toBe(true); @@ -169,9 +148,6 @@ describe("gateway recovery", () => { expect(process.env.OPENSHELL_GATEWAY).toBe("nemoclaw-8091"); expect(deps.sleepSeconds).not.toHaveBeenCalled(); - // First iteration only: 3 subprocess calls (status + gateway info -g + - // gateway info); loop returns before the next iteration would start. - expect(deps.runCaptureOpenshell).toHaveBeenCalledTimes(3); }); it("succeeds after retrying past unhealthy probes and still sets OPENSHELL_GATEWAY (#3768)", async () => { @@ -195,11 +171,8 @@ describe("gateway recovery", () => { await startGatewayForRecovery({ gatewayPort: 8091 }, deps); expect(process.env.OPENSHELL_GATEWAY).toBe("nemoclaw-8091"); - // Exactly one inter-attempt sleep between the unhealthy first probe - // and the healthy second probe. expect(deps.sleepSeconds).toHaveBeenCalledTimes(1); expect(deps.sleepSeconds).toHaveBeenNthCalledWith(1, 0.25); - expect(deps.runCaptureOpenshell).toHaveBeenCalledTimes(6); }); it("with NEMOCLAW_HEALTH_POLL_COUNT=0 fails fast without silently claiming healthy (#3768)", async () => { @@ -235,6 +208,21 @@ describe("gateway recovery", () => { expect(deps.sleepSeconds).toHaveBeenNthCalledWith(2, 0); }); + it("uses the shared extended health configuration for an existing gateway container (#10652)", async () => { + vi.stubEnv("NEMOCLAW_HEALTH_POLL_COUNT", "1"); + vi.stubEnv("NEMOCLAW_HEALTH_POLL_INTERVAL", "0"); + vi.stubEnv("NEMOCLAW_GATEWAY_START_POLL_COUNT", "3"); + vi.stubEnv("NEMOCLAW_GATEWAY_START_POLL_INTERVAL", "0"); + const deps = createDeps({ getGatewayClusterContainerState: () => "running starting" }); + + await expect(startGatewayForRecovery({ gatewayPort: 8091 }, deps)).rejects.toThrow( + "did not become ready within the configured 3 immediate health probes", + ); + + expect(deps.runCaptureOpenshell).toHaveBeenCalledTimes(9); + expect(deps.sleepSeconds).toHaveBeenCalledTimes(2); + }); + it("rejects non-canonical gateway recovery names before invoking OpenShell", async () => { const deps = createDeps(); diff --git a/src/lib/onboard/gateway-recovery.ts b/src/lib/onboard/gateway-recovery.ts index c48a2a535b8..ee0898c3ebe 100644 --- a/src/lib/onboard/gateway-recovery.ts +++ b/src/lib/onboard/gateway-recovery.ts @@ -27,14 +27,14 @@ import { isGatewayHealthy } from "../state/gateway"; import { isLinuxDockerDriverGatewayEnabled } from "./docker-driver-platform"; import { envInt } from "./env"; import { resolveGatewayName, resolveGatewayPortFromName } from "./gateway-binding"; -import { formatGatewayHealthWaitLimit } from "./gateway-health-wait"; +import { formatGatewayHealthWaitLimit, getGatewayHealthWaitConfig } from "./gateway-health-wait"; import { isGatewayHttpReady } from "./gateway-http-readiness"; import { getContainerRuntime } from "./local-inference-topology"; import { createReadinessWaitOptions, formatReadinessDeadline, getLegacyPollDeadlineBudgetMs, -} from "./readiness-wait"; +} from "../core/readiness-wait"; export type StartGatewayForRecoveryOptions = { gatewayName?: string; @@ -135,26 +135,6 @@ function getDefaultGatewayClusterContainerState(gatewayName: string): string { return state || "missing"; } -function getGatewayHealthWaitConfig(_startStatus = 0, containerState = "") { - const isArm64 = process.arch === "arm64"; - const standardCount = envInt("NEMOCLAW_HEALTH_POLL_COUNT", isArm64 ? 30 : 12); - const standardInterval = envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", isArm64 ? 10 : 5); - const extendedCount = envInt("NEMOCLAW_GATEWAY_START_POLL_COUNT", standardCount); - const extendedInterval = envInt("NEMOCLAW_GATEWAY_START_POLL_INTERVAL", standardInterval); - const normalizedState = String(containerState || "") - .trim() - .toLowerCase(); - const normalizedContainerState = normalizedState || "missing"; - const useExtendedWait = normalizedContainerState !== "missing"; - - return { - count: useExtendedWait ? extendedCount : standardCount, - interval: useExtendedWait ? extendedInterval : standardInterval, - extended: useExtendedWait, - containerState: normalizedContainerState, - }; -} - function getGatewayRecoveryWaitBudgetMs(pollCount: number, pollIntervalSeconds: number): number { return getLegacyPollDeadlineBudgetMs(pollCount, pollIntervalSeconds); } diff --git a/src/lib/onboard/sandbox-create-step.test.ts b/src/lib/onboard/sandbox-create-step.test.ts deleted file mode 100644 index c4ed01887ca..00000000000 --- a/src/lib/onboard/sandbox-create-step.test.ts +++ /dev/null @@ -1,456 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { afterEach, describe, expect, it, vi } from "vitest"; - -import { streamSandboxCreate } from "../sandbox/create-stream"; -import { - dockerEnv, - FakeChild, - makePollingOptions, - vmEnv, -} from "../sandbox/create-stream-test-fixtures"; -import { - runSandboxCreateStep, - type SandboxCreateStepContext, - type SandboxCreateStepDeps, -} from "./sandbox-create-step"; - -afterEach(() => { - vi.unstubAllEnvs(); -}); - -function makeLaunch(overrides: Record = {}) { - return { - createCommand: "openshell sandbox create alpha", - effectiveDashboardPort: "18789", - createArgv: ["openshell", "sandbox", "create", "alpha"], - envArgs: [], - sandboxEnv: { FOO: "bar" }, - sandboxStartupCommand: ["run", "alpha"], - prebuild: { imageRef: "img:tag", createArgs: ["sandbox", "create", "alpha"] }, - ...overrides, - }; -} - -function makePatch() { - return { - maybeApplyDuringCreate: vi.fn(), - createFailureMessage: vi.fn(() => null), - ensureApplied: vi.fn(), - }; -} - -function makeContext(overrides: Partial = {}): SandboxCreateStepContext { - // Cast once at the boundary: hermesDashboardState / openshellShellCommand / - // prebuild are structural seams this orchestration test does not exercise. - const base = { - agent: null, - observabilityEnabled: false, - chatUiUrl: "", - createArgs: ["sandbox", "create", "alpha"], - sandboxName: "alpha", - env: {}, - extraPlaceholderKeys: [], - getDashboardForwardPort: () => "18789", - hermesDashboardState: null, - manageDashboard: false, - openshellShellCommand: null, - prebuild: { buildCtx: "/tmp/ctx", buildId: "b1", dockerDriverGateway: null, origin: "local" }, - useDockerGpuPatch: false, - gpuDevice: null, - gpuBackend: "generic" as const, - timeoutSecs: 300, - }; - return { ...base, ...overrides } as unknown as SandboxCreateStepContext; -} - -function makeDeps( - launch: ReturnType, - patch: ReturnType, - createResult: { status: number; output: string }, - overrides: Partial = {}, -): SandboxCreateStepDeps { - return { - prepareCreateLaunch: vi.fn(async () => launch), - createDockerGpuPatch: vi.fn(() => patch), - streamCreate: vi.fn(async () => createResult), - isSandboxReady: vi.fn(() => false), - isTerminalAgent: vi.fn(() => false), - addTraceEvent: vi.fn(), - runOpenshell: vi.fn(() => ({ status: 0, output: "" })), - runCaptureOpenshell: vi.fn(() => "sandbox-list"), - sleepSeconds: vi.fn(), - ...overrides, - } as unknown as SandboxCreateStepDeps; -} - -describe("runSandboxCreateStep", () => { - it("threads the prebuild handoff into launch, GPU patch, and stream, and returns the handles", async () => { - const launch = makeLaunch(); - const patch = makePatch(); - const createResult = { status: 0, output: "created" }; - const deps = makeDeps(launch, patch, createResult); - - const result = await runSandboxCreateStep( - makeContext({ - useDockerGpuPatch: true, - gpuDevice: "nvidia.com/gpu=all", - gpuBackend: "jetson", - }), - deps, - ); - - // prepareCreateLaunch receives the assembled launch input incl. the prebuild handoff. - expect(deps.prepareCreateLaunch).toHaveBeenCalledWith( - expect.objectContaining({ - sandboxName: "alpha", - prebuild: { - buildCtx: "/tmp/ctx", - buildId: "b1", - dockerDriverGateway: null, - origin: "local", - }, - }), - ); - // GPU patch is created with the startup command from the launch result + backend/device. - expect(deps.createDockerGpuPatch).toHaveBeenCalledWith( - expect.objectContaining({ - route: "compatibility", - openshellSandboxCommand: ["run", "alpha"], - gpuDevice: "nvidia.com/gpu=all", - backend: "jetson", - }), - ); - // stream is fed the launch command + env. - expect(deps.streamCreate).toHaveBeenCalledWith( - "openshell", - ["sandbox", "create", "alpha"], - { FOO: "bar" }, - expect.objectContaining({ traceEvent: deps.addTraceEvent }), - ); - // Handles returned for downstream consumers. - expect(result).toEqual({ - createResult, - prebuild: launch.prebuild, - effectiveDashboardPort: "18789", - dockerGpuCreatePatch: patch, - }); - }); - - it.each([ - { label: "OpenClaw", agent: null }, - { label: "Hermes", agent: { name: "hermes" } as SandboxCreateStepContext["agent"] }, - ])("persists the $label startup command for Docker-driver container restarts", async ({ - agent, - }) => { - const launch = makeLaunch({ - sandboxStartupCommand: ["env", "CHAT_UI_URL=http://127.0.0.1:8642", "nemoclaw-start"], - }); - const patch = makePatch(); - const deps = makeDeps(launch, patch, { status: 0, output: "created" }); - - await runSandboxCreateStep( - makeContext({ - agent, - prebuild: { - buildCtx: "/tmp/ctx", - buildId: "b1", - dockerDriverGateway: true, - origin: "generated", - }, - }), - deps, - ); - - expect(deps.createDockerGpuPatch).toHaveBeenCalledWith( - expect.objectContaining({ - route: "native", - persistStartupCommand: true, - openshellSandboxCommand: ["env", "CHAT_UI_URL=http://127.0.0.1:8642", "nemoclaw-start"], - }), - ); - }); - - it("gates restart-safe persistence on the step's own portable env, not process.env (#9462)", async () => { - vi.stubEnv("NEMOCLAW_EXPERIMENTAL_PROFILE", "default"); - const launch = makeLaunch({ - sandboxStartupCommand: ["env", "nemoclaw-start"], - }); - const patch = makePatch(); - const deps = makeDeps(launch, patch, { status: 0, output: "created" }); - - await runSandboxCreateStep( - makeContext({ - agent: { name: "hermes" } as SandboxCreateStepContext["agent"], - env: { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" }, - prebuild: { - buildCtx: "/tmp/ctx", - buildId: "b1", - dockerDriverGateway: true, - origin: "generated", - }, - }), - deps, - ); - - expect(deps.createDockerGpuPatch).toHaveBeenCalledWith( - expect.objectContaining({ - persistStartupCommand: false, - }), - ); - }); - - it("persists DCode startup with its exact Docker resource limits", async () => { - const launch = makeLaunch({ - sandboxStartupCommand: ["env", "nemoclaw-start"], - }); - const patch = makePatch(); - const deps = makeDeps(launch, patch, { status: 0, output: "created" }); - - await runSandboxCreateStep( - makeContext({ - agent: { - name: "langchain-deepagents-code", - } as SandboxCreateStepContext["agent"], - prebuild: { - buildCtx: "/tmp/ctx", - buildId: "b1", - dockerDriverGateway: true, - origin: "generated", - }, - }), - deps, - ); - - expect(deps.createDockerGpuPatch).toHaveBeenCalledWith( - expect.objectContaining({ - persistStartupCommand: true, - requiredUlimits: [ - { name: "nproc", soft: 512, hard: 512 }, - { name: "nofile", soft: 65_536, hard: 65_536 }, - ], - }), - ); - }); - - it("separates readiness detection from GPU patch polling", async () => { - const launch = makeLaunch(); - const patch = makePatch(); - const deps = makeDeps( - launch, - patch, - { status: 0, output: "" }, - { isSandboxReady: vi.fn(() => true) }, - ); - - await runSandboxCreateStep(makeContext(), deps); - const streamOpts = (deps.streamCreate as unknown as { mock: { calls: unknown[][] } }).mock - .calls[0][3] as { readyCheck: () => boolean; onPoll: () => void }; - - expect(streamOpts.readyCheck()).toBe(true); - expect(patch.maybeApplyDuringCreate).not.toHaveBeenCalled(); - - (deps.isSandboxReady as unknown as ReturnType).mockReturnValue(false); - expect(streamOpts.readyCheck()).toBe(false); - expect(patch.maybeApplyDuringCreate).not.toHaveBeenCalled(); - - streamOpts.onPoll(); - expect(patch.maybeApplyDuringCreate).toHaveBeenCalledTimes(1); - }); - - it("waits for the create ownership handoff before restart-safe recreation (#8720)", async () => { - vi.useFakeTimers(); - const child = new FakeChild(); - const patch = makePatch(); - let ready = false; - let resolved = false; - const deps = makeDeps( - makeLaunch({ sandboxEnv: dockerEnv }), - patch, - { status: 0, output: "" }, - { - streamCreate: ((command, args, sandboxEnv, options) => - streamSandboxCreate(command, args, sandboxEnv, { - ...options, - ...makePollingOptions(child), - })) as SandboxCreateStepDeps["streamCreate"], - isSandboxReady: vi.fn(() => ready), - }, - ); - - const create = runSandboxCreateStep( - makeContext({ - prebuild: { - buildCtx: "/tmp/ctx", - buildId: "b1", - dockerDriverGateway: true, - origin: "generated", - }, - }), - deps, - ).then((result) => { - resolved = true; - return result; - }); - - child.stdout.emit("data", Buffer.from("Created sandbox: alpha\n")); - ready = true; - await vi.advanceTimersByTimeAsync(6); - - expect(resolved).toBe(false); - expect(child.kill).not.toHaveBeenCalled(); - child.stderr.emit("data", Buffer.from("Setting up NemoClaw...\n")); - await vi.advanceTimersByTimeAsync(6); - - expect(child.kill).toHaveBeenCalledWith("SIGTERM"); - expect(patch.maybeApplyDuringCreate).not.toHaveBeenCalled(); - - child.emit("close", 143); - await expect(create).resolves.toMatchObject({ - createResult: { status: 0, forcedReady: true }, - }); - }); - - it("threads the terminal-agent early-ready gate into stream options", async () => { - const terminalDeps = makeDeps( - makeLaunch(), - makePatch(), - { status: 0, output: "" }, - { - isTerminalAgent: vi.fn(() => true), - }, - ); - await runSandboxCreateStep(makeContext(), terminalDeps); - expect( - (terminalDeps.streamCreate as unknown as { mock: { calls: unknown[][] } }).mock.calls[0][3], - ).toMatchObject({ readyCheckOutputPatterns: [] }); - - const nonTerminalDeps = makeDeps(makeLaunch({ sandboxEnv: vmEnv }), makePatch(), { - status: 0, - output: "", - }); - await runSandboxCreateStep(makeContext(), nonTerminalDeps); - expect( - (nonTerminalDeps.streamCreate as unknown as { mock: { calls: unknown[][] } }).mock - .calls[0][3], - ).toMatchObject({ readyCheckOutputPatterns: [expect.any(RegExp)] }); - }); - - it.each([ - ["terminal VM", true, vmEnv], - ["terminal Docker", true, dockerEnv], - ])("detaches immediately for %s", async (_label, isTerminalAgent, env) => { - vi.useFakeTimers(); - - const child = new FakeChild(); - const logLine = vi.fn(); - const streamOptions = makePollingOptions(child, { logLine }); - const deps = makeDeps( - makeLaunch({ sandboxEnv: env }), - makePatch(), - { status: 0, output: "" }, - { - streamCreate: ((command, args, sandboxEnv, options) => - streamSandboxCreate(command, args, sandboxEnv, { - ...options, - ...streamOptions, - })) as SandboxCreateStepDeps["streamCreate"], - isTerminalAgent: vi.fn(() => isTerminalAgent), - }, - ); - let ready = false; - deps.isSandboxReady = vi.fn(() => ready); - deps.addTraceEvent = vi.fn(); - - const promise = runSandboxCreateStep(makeContext(), deps); - child.stdout.emit("data", Buffer.from("Created sandbox: alpha\n")); - ready = true; - await vi.advanceTimersByTimeAsync(6); - - expect(logLine).not.toHaveBeenCalledWith( - " Sandbox reported Ready; waiting for startup command output before detaching.", - ); - await expect(promise).resolves.toMatchObject({ - createResult: expect.objectContaining({ status: 0, forcedReady: true }), - }); - expect(child.kill).toHaveBeenCalledWith("SIGTERM"); - vi.useRealTimers(); - }); - - it.each([ - ["VM", vmEnv], - ["Docker", dockerEnv], - ])("waits for startup output for non-terminal %s creates", async (_label, env) => { - vi.useFakeTimers(); - - const child = new FakeChild(); - const logLine = vi.fn(); - const streamOptions = makePollingOptions(child, { logLine }); - const deps = makeDeps( - makeLaunch({ sandboxEnv: env }), - makePatch(), - { status: 0, output: "" }, - { - streamCreate: ((command, args, sandboxEnv, options) => - streamSandboxCreate(command, args, sandboxEnv, { - ...options, - ...streamOptions, - })) as SandboxCreateStepDeps["streamCreate"], - }, - ); - let ready = false; - deps.isSandboxReady = vi.fn(() => ready); - deps.addTraceEvent = vi.fn(); - - const promise = runSandboxCreateStep(makeContext(), deps); - child.stdout.emit("data", Buffer.from("Created sandbox: alpha\n")); - ready = true; - await vi.advanceTimersByTimeAsync(6); - - expect(child.kill).not.toHaveBeenCalled(); - expect(logLine).toHaveBeenCalledWith( - " Sandbox reported Ready; waiting for startup command output before detaching.", - ); - - child.stderr.emit("data", Buffer.from("Setting up NemoClaw (Hermes)...\n")); - await vi.advanceTimersByTimeAsync(6); - - await expect(promise).resolves.toMatchObject({ - createResult: expect.objectContaining({ status: 0, forcedReady: true }), - }); - expect(child.kill).toHaveBeenCalledWith("SIGTERM"); - vi.useRealTimers(); - }); - - it("recovers SSH 255 exits when the sandbox is ready", async () => { - vi.useFakeTimers(); - - const child = new FakeChild(); - const streamOptions = makePollingOptions(child, { pollIntervalMs: 60_000 }); - const deps = makeDeps( - makeLaunch({ sandboxEnv: dockerEnv }), - makePatch(), - { status: 0, output: "" }, - { - streamCreate: ((command, args, sandboxEnv, options) => - streamSandboxCreate(command, args, sandboxEnv, { - ...options, - ...streamOptions, - })) as SandboxCreateStepDeps["streamCreate"], - isSandboxReady: vi.fn(() => true), - }, - ); - - const promise = runSandboxCreateStep(makeContext(), deps); - await vi.advanceTimersByTimeAsync(0); - child.stdout.emit("data", Buffer.from("Created sandbox: alpha\n")); - child.stderr.emit("data", Buffer.from("Setting up NemoClaw...\n")); - child.emit("close", 255); - - await expect(promise).resolves.toMatchObject({ - createResult: expect.objectContaining({ status: 0, forcedReady: true }), - }); - vi.useRealTimers(); - }); -}); diff --git a/src/lib/onboard/sandbox-create-step.ts b/src/lib/onboard/sandbox-create-step.ts deleted file mode 100644 index 0cef3edfc7a..00000000000 --- a/src/lib/onboard/sandbox-create-step.ts +++ /dev/null @@ -1,143 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { AgentDefinition } from "../agent/defs"; -import type { - StreamSandboxCreateOptions, - StreamSandboxCreateResult, - streamSandboxCreate, -} from "../sandbox/create-stream"; -import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-ready-gate"; -import type { - createDockerGpuSandboxCreatePatch, - DockerGpuSandboxCreatePatch, -} from "./docker-gpu-sandbox-create"; -import { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; -import type { - prepareSandboxCreateLaunchWithPrebuild, - SandboxCreateLaunchWithPrebuild, - SandboxCreateLaunchWithPrebuildInput, -} from "./sandbox-create-launch"; - -type LaunchInput = SandboxCreateLaunchWithPrebuildInput; -type GpuPatchDeps = Parameters[0]["deps"]; - -export type SandboxCreateStepContext = { - agent: LaunchInput["agent"]; - observabilityEnabled: boolean; - chatUiUrl: string; - createArgs: LaunchInput["createArgs"]; - sandboxName: string; - env: NodeJS.ProcessEnv; - extraPlaceholderKeys: LaunchInput["extraPlaceholderKeys"]; - getDashboardForwardPort: LaunchInput["getDashboardForwardPort"]; - hermesDashboardState: LaunchInput["hermesDashboardState"]; - manageDashboard: boolean; - openshellShellCommand: LaunchInput["openshellShellCommand"]; - openshellArgv?: LaunchInput["openshellArgv"]; - prebuild: LaunchInput["prebuild"]; - useDockerGpuPatch: boolean; - gpuDevice: string | null | undefined; - gpuBackend: "jetson" | "generic"; - timeoutSecs: number; -}; - -export type SandboxCreateStepDeps = { - prepareCreateLaunch: typeof prepareSandboxCreateLaunchWithPrebuild; - createDockerGpuPatch: typeof createDockerGpuSandboxCreatePatch; - streamCreate: typeof streamSandboxCreate; - isSandboxReady(output: string, sandboxName: string): boolean; - isTerminalAgent(agent: AgentDefinition | null | undefined): boolean; - addTraceEvent: NonNullable; - runOpenshell: GpuPatchDeps["runOpenshell"]; - runCaptureOpenshell: NonNullable; - sleepSeconds: GpuPatchDeps["sleep"]; -}; - -export type SandboxCreateStepResult = { - createResult: StreamSandboxCreateResult; - prebuild: SandboxCreateLaunchWithPrebuild["prebuild"]; - effectiveDashboardPort: string; - dockerGpuCreatePatch: DockerGpuSandboxCreatePatch; -}; - -/** - * Resolve the BuildKit prebuild handoff into the launch command, provision the - * Docker-GPU create patch, and stream the sandbox create. Returns the create - * result plus the handles the caller needs downstream (prebuild identity, the - * dashboard port, and the GPU patch for its ready/verify hooks). Build-context - * and exit-listener cleanup stay with the caller that armed them. - */ -export async function runSandboxCreateStep( - context: SandboxCreateStepContext, - deps: SandboxCreateStepDeps, -): Promise { - const { - createCommand, - createArgv, - effectiveDashboardPort, - prebuild, - sandboxEnv, - sandboxStartupCommand, - } = await deps.prepareCreateLaunch({ - agent: context.agent, - observabilityEnabled: context.observabilityEnabled, - chatUiUrl: context.chatUiUrl, - createArgs: context.createArgs, - sandboxName: context.sandboxName, - env: context.env, - extraPlaceholderKeys: context.extraPlaceholderKeys, - getDashboardForwardPort: context.getDashboardForwardPort, - hermesDashboardState: context.hermesDashboardState, - manageDashboard: context.manageDashboard, - openshellShellCommand: context.openshellShellCommand, - openshellArgv: context.openshellArgv, - prebuild: context.prebuild, - }); - const startupCommandPatch = resolveDockerStartupCommandPatch( - context.agent, - context.prebuild.dockerDriverGateway, - context.env, - ); - const deferRestartSafeCutover = - startupCommandPatch.persistStartupCommand && !context.useDockerGpuPatch; - const dockerGpuCreatePatch = deps.createDockerGpuPatch({ - route: context.useDockerGpuPatch ? "compatibility" : "native", - persistStartupCommand: startupCommandPatch.persistStartupCommand, - requiredUlimits: startupCommandPatch.requiredUlimits, - sandboxName: context.sandboxName, - gpuDevice: context.gpuDevice, - openshellSandboxCommand: sandboxStartupCommand, - timeoutSecs: context.timeoutSecs, - backend: context.gpuBackend, - deps: { - runOpenshell: deps.runOpenshell, - runCaptureOpenshell: deps.runCaptureOpenshell, - sleep: deps.sleepSeconds, - }, - }); - const [createExecutable, ...createExecutableArgs] = createArgv; - const createResult = await deps.streamCreate( - createExecutable ?? createCommand, - createExecutableArgs, - sandboxEnv, - { - readyCheck: () => { - const list = deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); - return deps.isSandboxReady(list, context.sandboxName); - }, - onPoll: () => { - if (!deferRestartSafeCutover) dockerGpuCreatePatch.maybeApplyDuringCreate(); - }, - readyCheckOutputPatterns: getReadyCheckOutputPatternsForAgent({ - isTerminalAgent: deps.isTerminalAgent(context.agent), - startupRunsDuringCreate: true, - env: sandboxEnv, - }), - failureCheck: dockerGpuCreatePatch.createFailureMessage, - traceEvent: deps.addTraceEvent, - waitForReadyTermination: deferRestartSafeCutover, - }, - ); - return { createResult, prebuild, effectiveDashboardPort, dockerGpuCreatePatch }; -} diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 153518c1117..839d14bb5ea 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -409,15 +409,60 @@ describe("retained create recovery persistence", () => { expect(caught).toBeInstanceOf(AggregateError); expect(operation).toHaveBeenCalledOnce(); expect(recordRecovery).toHaveBeenCalledOnce(); + expect(exitHandlers).toHaveLength(1); - exitHandlers[0](); + exitHandlers[0]!(); expect(recordRecovery).toHaveBeenCalledTimes(2); expect(operation).toHaveBeenCalledOnce(); - exitHandlers[0](); + exitHandlers[0]!(); expect(recordRecovery).toHaveBeenCalledTimes(2); }, ); + + it("does not install a retry handler until a recovery write fails (#10652)", () => { + const registerExitHandler = vi.fn(); + const owner = installPostCreateRecoveryRetryOwner({ registerExitHandler }); + const firstWriter = vi.fn(); + const secondWriter = vi.fn(); + + owner.record(firstWriter); + expect(firstWriter).toHaveBeenCalledOnce(); + expect(registerExitHandler).not.toHaveBeenCalled(); + + owner.record(secondWriter); + + expect(secondWriter).toHaveBeenCalledOnce(); + expect(registerExitHandler).not.toHaveBeenCalled(); + }); + + it("retries both failed writers held by independent operation owners (#10652)", () => { + const exitHandlers: Array<() => void> = []; + const owners = [0, 1].map(() => + installPostCreateRecoveryRetryOwner({ + log: vi.fn(), + registerExitHandler: (handler) => exitHandlers.push(handler), + }), + ); + const writers = [0, 1].map((index) => + vi + .fn() + .mockImplementationOnce(() => { + throw new Error(`operation ${index + 1} recovery write failed`); + }) + .mockImplementationOnce(() => undefined), + ); + + expect(() => owners[0]?.record(writers[0]!)).toThrow(/operation 1/u); + expect(() => owners[1]?.record(writers[1]!)).toThrow(/operation 2/u); + expect(writers.map((writer) => writer.mock.calls.length)).toEqual([1, 1]); + expect(exitHandlers).toHaveLength(2); + expect(exitHandlers[0]).not.toBe(exitHandlers[1]); + + exitHandlers.forEach((handler) => handler()); + + expect(writers.map((writer) => writer.mock.calls.length)).toEqual([2, 2]); + }); }); describe("APF create policy selection", () => { diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index c33f37f034e..c1f2191d824 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -29,6 +29,14 @@ import type { HermesAuthMethod } from "../hermes-auth"; import type { PreparedSandboxBuildContext } from "../build-context-stage"; import type { DcodeSelectionDriftReader } from "../dcode-selection-drift"; import { assertProviderlessInterceptorEnvironment } from "../entry-options"; +import { + createHermesPortableReadyCapture, + createHermesPortableReadyRunner, + defaultHermesPortableStateDir, + runHermesPortableOnboardingFromOnboard, + shouldManageHermesPortableDashboard, +} from "../experimental/hermes-portable-onboarding"; +import { inspectPortableAgentReceiptAuthorityForClassification } from "../experimental/hermes-portable-receipt"; import type { ManagedHermesStateVolumeCleanupResult, ManagedHermesStateVolumeContext, @@ -307,20 +315,32 @@ export function installPostCreateRecoveryRetryOwner( } = {}, ): PostCreateRecoveryRetryOwner { let pending: (() => void) | null = null; + let exitHandlerRegistered = false; const log = options.log ?? ((message: string) => console.error(message)); - const attemptPending = (propagateFailure: boolean): void => { + const register = + options.registerExitHandler ?? + ((handler: () => void) => { + process.on("exit", handler); + }); + const ensureExitHandler = (): void => { + if (exitHandlerRegistered) return; + register(() => attemptPending(false)); + exitHandlerRegistered = true; + }; + function attemptPending(propagateFailure: boolean): void { if (pending === null) return; const attempt = pending; try { attempt(); if (pending === attempt) pending = null; } catch (error) { + ensureExitHandler(); if (propagateFailure) throw error; log( " NemoClaw still could not save the retained sandbox recovery record; the recovery-only session remains blocked for administrator recovery.", ); } - }; + } const owner: PostCreateRecoveryRetryOwner = { record(recordRecovery): void { attemptPending(true); @@ -328,12 +348,6 @@ export function installPostCreateRecoveryRetryOwner( attemptPending(true); }, }; - const register = - options.registerExitHandler ?? - ((handler: () => void) => { - process.on("exit", handler); - }); - register(() => attemptPending(false)); return owner; } @@ -1149,6 +1163,22 @@ type PortableAgentReceiptGenerationObservation = readonly lifecycleGeneration: string; }; +function inspectPortableAgentReceiptGeneration( + sandboxName: string, +): PortableAgentReceiptGenerationObservation { + const authority = inspectPortableAgentReceiptAuthorityForClassification( + sandboxName, + defaultHermesPortableStateDir(process.env), + ); + if (authority.kind === "none") return { kind: "absent" }; + if (authority.kind === "openclaw") return { kind: "openclaw" }; + return { + kind: "hermes", + gatewayName: authority.snapshot.receipt.gatewayName, + lifecycleGeneration: authority.snapshot.receipt.lifecycleGeneration, + }; +} + function readHermesPortableLifecycleGeneration(input: { readonly enabled: boolean; readonly sandboxName: string; @@ -1163,7 +1193,6 @@ function readHermesPortableLifecycleGeneration(input: { } export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrchestrationRuntime) { - const postCreateRecoveryRetryOwner = installPostCreateRecoveryRetryOwner(); return async function createSandboxWithBaseImageResolution( baseImageResolutionContext: import("../base-image-resolution-flow").BaseImageResolutionContext, portableRuntimeContext: PortableOnboardRuntimeContext | null, @@ -1192,6 +1221,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche runVerifiedSandboxCreateEffects: import("../types").VerifiedSandboxCreateEffects | null = null, preparedBuildContext: PreparedSandboxBuildContext | null = null, ) { + const postCreateRecoveryRetryOwner = installPostCreateRecoveryRetryOwner(); const portableRuntimeAuthority = portableRuntimeContext?.authority ?? null; const { DASHBOARD_PORT, @@ -1233,6 +1263,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche hasSandboxGpuDrift, inferenceConfig, inspectSandboxForCreate, + isSandboxReady, isLinuxDockerDriverGatewayEnabled, isNonInteractive, isRecreateSandbox, @@ -1251,6 +1282,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche path, planRegisteredExtraProviders, preparedDcodeRebuild, + printSandboxCreateRecoveryHints, promptValidatedSandboxName, promptYesNoOrDefault, providerExistsInGateway, @@ -1351,7 +1383,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche ); const resolvedCreateIntent = preparedCreateIntent.intent; const messagingCapabilities = preparedCreateIntent.messagingCapabilities; - const manageDashboard = sandboxGpuCreateFlow.shouldManageHermesPortableDashboard( + const manageDashboard = shouldManageHermesPortableDashboard( dashboardRuntime.shouldManageDashboardForAgent(agent), agent, ); @@ -2220,7 +2252,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche enabled: agentCreateInput.hermesPortableLifecycle, sandboxName, gatewayName: GATEWAY_NAME, - inspect: sandboxGpuCreateFlow.inspectPortableAgentReceiptDisposition, + inspect: inspectPortableAgentReceiptGeneration, }); const createdSandboxLifecycle = sandboxRecreateTransaction.createCreatedSandboxLifecycle( recreateRuntime, @@ -2429,8 +2461,8 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche ); const runCreateFlow = async ( attemptCreateArgv: string[], - hermesPortableReadyCapture?: import("../sandbox-gpu-create-flow").HermesPortableReadyCapture, - hermesPortableReadyRunner?: import("../sandbox-gpu-create-flow").HermesPortableReadyRunner, + hermesPortableReadyCapture?: ReturnType, + hermesPortableReadyRunner?: ReturnType, createWorkingDirectory?: string, effectivePolicySourcePath?: string, runDeferredProviderEffects?: (context: VerifiedSandboxCreateEffectsContext) => Promise, @@ -2571,6 +2603,8 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche ), sleep: sleepSeconds, openshellArgv, + isSandboxReady, + printCreateRecoveryHints: printSandboxCreateRecoveryHints, verifyDirectSandboxGpu: createGpuVerifier, }, ); @@ -2579,12 +2613,30 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche }); }; - const cleanupBuildContext = - sandboxGpuCreateFlow.createSandboxBuildContextCleanup(legacyBuildContext); - const cleanupInitialCreateSource = sandboxGpuCreateFlow.createSandboxCreateSourceCleanup( - initialSandboxPolicy, - agentCreateInput.hermesPortableLifecycle, - ); + let buildContextCleanupCompleted = false; + const cleanupBuildContext = (): boolean => { + if (buildContextCleanupCompleted || !legacyBuildContext?.cleanupBuildCtx) return true; + buildContextCleanupCompleted = legacyBuildContext.cleanupBuildCtx(); + if (buildContextCleanupCompleted) { + process.removeListener("exit", legacyBuildContext.cleanupBuildCtx); + } + return buildContextCleanupCompleted; + }; + let initialCreateSourceCleanupCompleted = false; + const cleanupInitialCreateSource = (): boolean => { + if (initialCreateSourceCleanupCompleted) return true; + const cleanup = initialSandboxPolicy.cleanup; + const exactCleanup = initialSandboxPolicy.cleanupExact; + if (agentCreateInput.hermesPortableLifecycle && cleanup && !exactCleanup) { + throw new Error("Hermes portable temporary policy source has no exact cleanup authority."); + } + const selectedCleanup = exactCleanup ?? cleanup; + initialCreateSourceCleanupCompleted = selectedCleanup?.() ?? true; + if (initialCreateSourceCleanupCompleted && cleanup) { + process.removeListener("exit", cleanup); + } + return initialCreateSourceCleanupCompleted; + }; const cleanupSandboxCreateSources = (): void => { const cleanupErrors: Error[] = []; try { @@ -2736,12 +2788,13 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche sessionId: inferenceRouteReservationAuthority.sessionId, selection: inferenceRouteReservationAuthority.selection, }; - await sandboxGpuCreateFlow.runHermesPortableOnboardingFromOnboard< + await runHermesPortableOnboardingFromOnboard< import("../sandbox-gpu-create-flow").SandboxGpuCreateFlowResult >({ sandboxName, gatewayName: GATEWAY_NAME, lifecycleGeneration: createdSandboxLifecycle.generation, + sandboxReadyTimeoutSecs, portableRuntime: portableRuntimeContext, createArgv, createPolicyPath: initialSandboxPolicy.policyPath, diff --git a/src/lib/onboard/sandbox-gpu-create-apf-policyless.test.ts b/src/lib/onboard/sandbox-gpu-create-apf-policyless.test.ts index 8840f896230..e6667096fa8 100644 --- a/src/lib/onboard/sandbox-gpu-create-apf-policyless.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-apf-policyless.test.ts @@ -1,11 +1,113 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { refuseApfMutableNameFallbackCleanup } from "./sandbox-gpu-create-flow"; +const mocks = vi.hoisted(() => ({ + streamSandboxCreate: vi.fn(), + waitForCreatedSandboxReadyWithTrace: vi.fn(), + printReadinessFailure: vi.fn(), + enforceDockerGpuPatchPreserveNetwork: vi.fn(), + verifyGpuSandboxAccessAfterReady: vi.fn(), + createDockerGpuSandboxCreatePatch: vi.fn(), + printSandboxCreateFailureDiagnostics: vi.fn(), + collectDockerGpuPatchDiagnostics: vi.fn(), + queryOpenShellDockerSandboxContainers: vi.fn(), + queryOpenShellDockerSandboxRuntimeSnapshot: vi.fn(), +})); + +vi.mock("../sandbox/create-stream", () => ({ + streamSandboxCreate: mocks.streamSandboxCreate, +})); +vi.mock("./sandbox-readiness-tracing", async (importOriginal) => ({ + ...(await importOriginal()), + waitForCreatedSandboxReadyWithTrace: mocks.waitForCreatedSandboxReadyWithTrace, + printReadinessFailure: mocks.printReadinessFailure, +})); +vi.mock("./docker-gpu-local-inference", () => ({ + enforceDockerGpuPatchPreserveNetwork: mocks.enforceDockerGpuPatchPreserveNetwork, + verifyGpuSandboxAccessAfterReady: mocks.verifyGpuSandboxAccessAfterReady, +})); +vi.mock("./docker-gpu-sandbox-create", () => ({ + createDockerGpuSandboxCreatePatch: mocks.createDockerGpuSandboxCreatePatch, +})); +vi.mock("./sandbox-create-failure", () => ({ + printSandboxCreateFailureDiagnostics: mocks.printSandboxCreateFailureDiagnostics, +})); +vi.mock("./docker-gpu-patch", async (importOriginal) => ({ + ...(await importOriginal()), + collectDockerGpuPatchDiagnostics: mocks.collectDockerGpuPatchDiagnostics, +})); +vi.mock("./openshell-docker-sandbox-containers", async (importOriginal) => ({ + ...(await importOriginal()), + queryOpenShellDockerSandboxContainers: mocks.queryOpenShellDockerSandboxContainers, + queryOpenShellDockerSandboxRuntimeSnapshot: mocks.queryOpenShellDockerSandboxRuntimeSnapshot, +})); + +import { NEMOCLAW_CREATE_ATTEMPT_LABEL } from "../adapters/openshell/sandbox-identity"; +import { + ALPHA_SANDBOX_IDENTITY_FINGERPRINT, + createAttemptNonce, + createGpuFlowDeps, + createGpuFlowInput, + resetGpuFlowMocks, + sandboxListJson, + setupGpuFlowMocks, +} from "./__test-helpers__/sandbox-gpu-create-flow"; + +import { + refuseApfMutableNameFallbackCleanup, + runSandboxGpuCreateFlow, +} from "./sandbox-gpu-create-flow"; import { assertPolicylessSandboxCreateArgv } from "./sandbox-gpu-create-run-attempt"; +function expectNoSandboxDelete(deps: ReturnType): void { + expect( + vi + .mocked(deps.runOpenshell) + .mock.calls.some(([args]) => args[0] === "sandbox" && args[1] === "delete"), + ).toBe(false); +} + +function createApfFallbackRecoveryFixture(captureExactIdentity = true) { + let nonce = ""; + const input = createGpuFlowInput(); + input.requirePolicylessCreate = true; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + input.persistRetainedSandboxRecovery = vi.fn(() => true); + mocks.streamSandboxCreate.mockImplementationOnce(async (_command, args) => { + nonce = createAttemptNonce(args); + return { + status: 1, + output: "native runtime failed after sandbox creation", + sawProgress: true, + }; + }); + mocks.queryOpenShellDockerSandboxRuntimeSnapshot.mockReturnValue({ + ok: true, + imageId: "sha256:" + "a".repeat(64), + bookkeepingImageRef: "openshell/sandbox-from:test", + stateError: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + deviceRequests: null, + devices: null, + runtime: "runc", + nvidiaVisibleDevices: null, + nativeGpuAttachmentState: "absent", + containerId: "container-a", + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementation(() => + captureExactIdentity + ? sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }) + : "[]", + ); + return { deps, input, readNonce: () => nonce }; +} + +beforeEach(() => setupGpuFlowMocks(mocks)); +afterEach(resetGpuFlowMocks); + describe("APF policyless sandbox create attempts", () => { it("accepts create arguments without a caller policy (#9833)", () => { expect(() => @@ -56,4 +158,90 @@ describe("APF policyless sandbox create attempts", () => { containerIds: null, }); }); + + it("persists exact APF recovery evidence before refusing native fallback (#9833)", async () => { + const { deps, input, readNonce } = createApfFallbackRecoveryFixture(); + const exit = vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("process.exit:1"); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1"); + + const nonce = readNonce(); + const fingerprint = ALPHA_SANDBOX_IDENTITY_FINGERPRINT; + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringMatching( + new RegExp( + `^Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}\\. Durable sandbox identity fingerprint: ${fingerprint}\\.`, + "u", + ), + ), + fingerprint, + nonce, + ); + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledBefore(exit); + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); + expect(output).toContain(`Durable sandbox identity fingerprint: ${fingerprint}`); + expect(output).not.toContain("alpha-sandbox-id"); + expectNoSandboxDelete(deps); + expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); + }); + + it("persists the APF create-attempt label when exact recovery identity is unavailable (#9833)", async () => { + const { deps, input, readNonce } = createApfFallbackRecoveryFixture(false); + vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("process.exit:1"); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1"); + + const nonce = readNonce(); + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringMatching( + new RegExp( + `^Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}\\..*Recovery is blocked until an OpenShell administrator resolves the create-attempt label`, + "u", + ), + ), + undefined, + nonce, + ); + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); + expect(output).toContain("Recovery is blocked"); + expectNoSandboxDelete(deps); + }); + + it.each([ + ["returns false", (): boolean => false], + [ + "throws", + (): boolean => { + throw new Error("durable writer failed"); + }, + ], + ] as const)( + "blocks APF fallback when durable recovery persistence %s (#9833)", + async (_name, writer) => { + const { deps, input, readNonce } = createApfFallbackRecoveryFixture(); + const persist = vi.fn(writer); + input.persistRetainedSandboxRecovery = persist; + const exit = vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("process.exit:1"); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "The APF recovery-only session remains blocked until its durable recovery record can be saved.", + ); + + expect(persist).toHaveBeenCalledOnce(); + expect(exit).not.toHaveBeenCalled(); + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${readNonce()}`); + expect(output).toContain("APF recovery is blocked because NemoClaw could not save"); + expectNoSandboxDelete(deps); + expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); + }, + ); }); diff --git a/src/lib/onboard/sandbox-gpu-create-flow-hermes-portable.test.ts b/src/lib/onboard/sandbox-gpu-create-flow-hermes-portable.test.ts index 861333626c5..febb7d773ac 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow-hermes-portable.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow-hermes-portable.test.ts @@ -41,18 +41,23 @@ vi.mock("./openshell-docker-sandbox-containers", async (importOriginal) => ({ queryOpenShellDockerSandboxRuntimeSnapshot: mocks.queryOpenShellDockerSandboxRuntimeSnapshot, })); +import { NEMOCLAW_CREATE_ATTEMPT_LABEL } from "../adapters/openshell/sandbox-identity"; import type { CheckpointPortableRuntimeAuthority } from "../state/onboard-checkpoint-types"; import { + createAttemptNonce, createGpuFlowDeps as createDeps, createGpuFlowInput as createInput, + createGpuPatchFixture, + createNoGpuFlowInput as createNoGpuInput, resetGpuFlowMocks, + sandboxListJson, setupGpuFlowMocks, } from "./__test-helpers__/sandbox-gpu-create-flow"; import { - cleanupSandboxCreateSource, - runSandboxGpuCreateFlow, - type SandboxGpuCreateFlowInput, -} from "./sandbox-gpu-create-flow"; + createHermesPortableReadyCapture, + createHermesPortableReadyRunner, +} from "./experimental/hermes-portable-onboarding"; +import { runSandboxGpuCreateFlow, type SandboxGpuCreateFlowInput } from "./sandbox-gpu-create-flow"; import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; const PORTABLE_RUNTIME_AUTHORITY: CheckpointPortableRuntimeAuthority = { @@ -70,45 +75,6 @@ beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); describe("Hermes portable sandbox create flow", () => { - it("releases exit cleanup ownership only after successful retirement (#9203)", () => { - const cleanup = vi.fn(() => true); - process.on("exit", cleanup); - try { - expect(cleanupSandboxCreateSource(cleanup)).toBe(true); - expect(process.listeners("exit")).not.toContain(cleanup); - } finally { - process.removeListener("exit", cleanup); - } - }); - - it("preserves exit cleanup ownership when retirement is incomplete (#9203)", () => { - const cleanup = vi.fn(() => false); - process.on("exit", cleanup); - try { - expect(cleanupSandboxCreateSource(cleanup)).toBe(false); - expect(process.listeners("exit")).toContain(cleanup); - } finally { - process.removeListener("exit", cleanup); - } - }); - - it("requires and uses exact source cleanup for Hermes portable custody (#9203)", () => { - const cleanup = vi.fn(() => true); - const exactCleanup = vi.fn(() => true); - process.on("exit", cleanup); - try { - expect(cleanupSandboxCreateSource(cleanup, { exactCleanup, requireExact: true })).toBe(true); - expect(exactCleanup).toHaveBeenCalledOnce(); - expect(cleanup).not.toHaveBeenCalled(); - expect(process.listeners("exit")).not.toContain(cleanup); - expect(() => cleanupSandboxCreateSource(cleanup, { requireExact: true })).toThrow( - "has no exact cleanup authority", - ); - } finally { - process.removeListener("exit", cleanup); - } - }); - it("keeps non-OpenClaw portable creation on the existing runtime patch (#9068)", async () => { const input = createInput(); input.hostEnv = { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" }; @@ -225,4 +191,83 @@ describe("Hermes portable sandbox create flow", () => { expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); expect(mocks.createDockerGpuSandboxCreatePatch).not.toHaveBeenCalled(); }); + + it("carries Hermes receipt authority from selector settlement through publication lookup (#10423)", async () => { + let nonce = ""; + const input = createNoGpuInput(); + const patch = createGpuPatchFixture(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { + nonce = createAttemptNonce(args); + expect(options.readyCheck?.()).toBe(true); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValue({ + ready: true, + reason: "ready", + failurePhase: null, + }); + const capture = vi.fn((args: readonly string[]) => { + const results = { + [["sandbox", "list", "-g", "nemoclaw"].join("\0")]: () => ({ + status: 0, + stdout: Buffer.from("alpha Ready"), + stderr: Buffer.alloc(0), + }), + [[ + "sandbox", + "list", + "-g", + "nemoclaw", + "--selector", + `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`, + "--output", + "json", + "--limit", + "2", + ].join("\0")]: () => ({ + status: 0, + stdout: Buffer.from( + sandboxListJson("alpha-sandbox-id", { + [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, + }), + ), + stderr: Buffer.alloc(0), + }), + [["sandbox", "get", "-g", "nemoclaw", "alpha"].join("\0")]: () => ({ + status: 0, + stdout: Buffer.from("ID: alpha-sandbox-id\n"), + stderr: Buffer.alloc(0), + }), + } satisfies Readonly< + Record { status: number; stdout: Buffer; stderr: Buffer }> + >; + return ( + results[args.join("\0") as keyof typeof results] ?? + (() => ({ status: 1, stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) })) + )(); + }); + const deps = createDeps(); + deps.runOpenshell = createHermesPortableReadyRunner("alpha", "nemoclaw", capture); + deps.runCaptureOpenshell = createHermesPortableReadyCapture("alpha", "nemoclaw", capture); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); + + expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); + expect(capture).toHaveBeenCalledWith([ + "sandbox", + "list", + "-g", + "nemoclaw", + "--selector", + `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`, + "--output", + "json", + "--limit", + "2", + ]); + expect(capture).toHaveBeenCalledWith(["sandbox", "get", "-g", "nemoclaw", "alpha"]); + }); }); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index fbfe616dde5..4cb35f3fe0a 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -401,12 +401,12 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { mocks.waitForCreatedSandboxReadyWithTrace.mock.calls.map( ([options]) => options.stableReadyPolls, ), - ).toEqual([2, 2]); + ).toEqual([2, 2, 2]); vi.mocked(deps.runCaptureOpenshell).mockClear(); await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); expect(deps.runCaptureOpenshell).toHaveBeenCalledWith( - ["sandbox", "list", "-g", "nemoclaw"], - READY_CHECK_OPTIONS, + ["sandbox", "get", "-g", "nemoclaw", "alpha"], + { ignoreError: false }, ); expect(vi.mocked(console.warn).mock.calls.flat().join("\n")).toContain( "unrelated sandbox 'bravo'", @@ -942,10 +942,15 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { expect(deps.runOpenshell).not.toHaveBeenCalled(); }); - it("redacts create errors and preserves their exact nonzero status (#6110)", async () => { + it("redacts hard create errors and preserves diagnostics, hints, and status (#6110)", async () => { mocks.streamSandboxCreate.mockResolvedValueOnce({ status: 19, - output: "provider failed with NVIDIA_API_KEY=super-secret-create-value", + output: [ + "provider failed with NVIDIA_API_KEY=super-secret-create-value", + "github ghp_abcdefghijklmnopqrstuvwxyz1234567890", + "openai sk-abcdefghijklmnopqrstuvwxyz1234567890", + "aws AKIAABCDEFGHIJKLMNOP", // gitleaks:allow + ].join("\n"), sawProgress: true, }); const exit = mockExit(19); @@ -957,7 +962,13 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { const output = vi.mocked(console.error).mock.calls.flat().join("\n"); expect(exit).toHaveBeenCalledWith(19); expect(output).toMatch(/NVIDIA_API_KEY=[^\n]*\*+/); - expect(output).not.toContain("super-secret-create-value"); + expect(output).not.toMatch( + /super-secret-create-value|ghp_abcdefghijklmnopqrstuvwxyz1234567890|sk-abcdefghijklmnopqrstuvwxyz1234567890|AKIAABCDEFGHIJKLMNOP/u, // gitleaks:allow + ); + expect(output).toContain("Try: openshell sandbox list"); + expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { + backupPath: null, + }); }); it("does not retry compatibility for a non-GPU native readiness failure (#6110)", async () => { @@ -1059,22 +1070,6 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { ); }); - it("keeps a created sandbox when portable lifecycle setup fails (#8441)", async () => { - const deps = createDeps(); - deps.installPortableDemoLifecycle = vi.fn(() => { - throw new Error("Authorization: Bearer portable-secret"); - }); - - await expect(runSandboxGpuCreateFlow(createInput(), deps)).resolves.toMatchObject({ - route: "native", - }); - - const warning = vi.mocked(console.warn).mock.calls.flat().join("\n"); - expect(warning).toContain("Portable demo lifecycle setup did not complete"); - expect(warning).toContain("Authorization: Bearer "); - expect(warning).not.toContain("portable-secret"); - }); - it("uses the exact portable lifecycle without Docker container substitution (#9068)", async () => { const input = createInput(); input.gpuRoutePlan = "native-only"; @@ -1265,7 +1260,7 @@ describe("runSandboxGpuCreateFlow fallback ordering", () => { expect(mocks.streamSandboxCreate.mock.calls.flat()).not.toContain("-lc"); }); - it("discloses the compatibility container-swap confinement tradeoff and native-only opt-out", async () => { + it("discloses the compatibility container-swap tradeoff and fallback authorization", async () => { failNativeCreate(); const deps = createDeps(); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index c7a05598366..069e73aecdc 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -17,25 +17,11 @@ import { renderCompatibilityFallbackCreateArgs } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; import { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; import { - classifyHermesPortableRegistry, createHermesPortableChildEnvironment, - createHermesPortableContainerDeps, - createHermesPortableOpenShellCapture, - createHermesPortableReadyCapture, - createHermesPortableReadyRunner, - defaultHermesPortableStateDir, isHermesPortableLifecycleMode, - observeHermesPortableSandbox, - runHermesPortableOnboardingFromOnboard, - runHermesPortableOnboardingTransaction, - shouldManageHermesPortableDashboard, } from "./experimental/hermes-portable-onboarding"; import { installPortableDemoSandboxLifecycle } from "./experimental/portable-demo-lifecycle"; -import { - buildHermesPortableCommandAuthority, - buildHermesPortableOnboardingCommandAuthority, - inspectPortableAgentReceiptDisposition, -} from "./experimental/portable-agent-lifecycle"; +import { buildHermesPortableOnboardingCommandAuthority } from "./experimental/portable-agent-lifecycle"; import { isPortableExperimentalProfile } from "./experimental/portable-profile"; import { createManagedBootstrapIdentity, @@ -55,7 +41,10 @@ import type { RuntimeProviderManagedImageBootstrapSurface, } from "./runtime-provider/contract"; import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; -import { createSandboxGpuCreateAttemptRunner } from "./sandbox-gpu-create-run-attempt"; +import { + createSandboxGpuCreateAttemptRunner, + persistRetainedSandboxRecoveryOrBlock, +} from "./sandbox-gpu-create-run-attempt"; import { managedBootstrapCreateArgs } from "./sandbox-create-launch"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; import { @@ -67,68 +56,6 @@ import type { SandboxPrebuildResult } from "./sandbox-prebuild"; import { addTraceEvent } from "./tracing"; export { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; -export { - classifyHermesPortableRegistry, - createHermesPortableChildEnvironment, - createHermesPortableContainerDeps, - createHermesPortableOpenShellCapture, - createHermesPortableReadyCapture, - createHermesPortableReadyRunner, - defaultHermesPortableStateDir, - observeHermesPortableSandbox, - runHermesPortableOnboardingFromOnboard, - runHermesPortableOnboardingTransaction, - shouldManageHermesPortableDashboard, - buildHermesPortableCommandAuthority, - inspectPortableAgentReceiptDisposition, -}; -export type HermesPortableReadyCapture = ReturnType; -export type HermesPortableReadyRunner = ReturnType; - -/** Release the exit cleanup listener only after its exact create source was retired. */ -export function cleanupSandboxCreateSource( - cleanup: (() => boolean) | undefined, - options: { readonly exactCleanup?: () => boolean; readonly requireExact?: boolean } = {}, -): boolean { - if (options.requireExact && cleanup && !options.exactCleanup) { - throw new Error("Hermes portable temporary policy source has no exact cleanup authority."); - } - const selected = options.exactCleanup ?? cleanup; - if (!selected) return true; - const completed = selected(); - if (completed && cleanup) process.removeListener("exit", cleanup); - return completed; -} - -/** Bind the exact create-source retirement decision without moving its execution point. */ -export function createSandboxCreateSourceCleanup( - source: { readonly cleanup?: () => boolean; readonly cleanupExact?: () => boolean }, - requireExact: boolean, -): () => boolean { - let completed = false; - return () => { - if (completed) return true; - completed = cleanupSandboxCreateSource(source.cleanup, { - exactCleanup: source.cleanupExact, - requireExact, - }); - return completed; - }; -} - -/** Bind cleanup for the one staged build context owned by this create attempt. */ -export function createSandboxBuildContextCleanup( - context: { readonly cleanupBuildCtx?: () => boolean } | null, -): () => boolean { - let completed = false; - return () => { - if (completed) return true; - if (!context?.cleanupBuildCtx) return true; - completed = context.cleanupBuildCtx(); - if (completed) process.removeListener("exit", context.cleanupBuildCtx); - return completed; - }; -} export function resolvePortableLifecycleMode( agent: AgentDefinition | null, @@ -295,8 +222,15 @@ export interface SandboxGpuCreateFlowDeps { runCaptureOpenshell: RunCaptureOpenshell; sandboxObserver: OpenShellSandboxObserver; sleep: Sleep; + /** Production callers use a monotonic clock; tests may inject post-create deadline time. */ + publicationNow?: () => number; openshellArgv(args: string[]): string[]; + isSandboxReady(output: string, sandboxName: string): boolean; verifyDirectSandboxGpu(sandboxName: string): SandboxGpuProofResult; + printCreateRecoveryHints( + output: string, + options: { readonly createArgs: readonly string[] }, + ): void; printCreateFailureDiagnostics?: ( sandboxName: string, options: { readonly backupPath?: string | null }, @@ -309,6 +243,8 @@ export interface SandboxGpuCreateFlowDeps { interface SandboxGpuCreateFlowResultCommon { runtimePatch: ManagedBootstrapRuntimePatch; + /** Confirm executable Ready state after the managed runtime commit. */ + confirmManagedRuntimeCommitReadiness(): Promise; route: SelectedDockerGpuRoute; /** Mutable tag/reference retained only for registry and image-GC bookkeeping. */ registryImageRef: string | null; @@ -410,129 +346,131 @@ export async function runSandboxGpuCreateFlow( } : deps, ); - const gpuCreateOutcome = await (input.resumeVerifiedCreate - ? attemptRunner.runAttempt(input.resumeVerifiedCreate.route) - : sandboxGpuCreateAttempt.executeSandboxGpuCreatePlan(input.gpuRoutePlan, { - runAttempt: attemptRunner.runAttempt, - captureNativeFailure: (failure) => { - const routeAdapter = adaptDockerGpuRouteForPatch(failure.route); - const diagnostics = collectDockerGpuPatchDiagnostics( - input.sandboxName, - { - error: failure.error, - additionalSummaryLines: routeAdapter.additionalSummaryLines, + const gpuCreateOutcome = await ( + input.resumeVerifiedCreate + ? attemptRunner.runAttempt(input.resumeVerifiedCreate.route) + : sandboxGpuCreateAttempt.executeSandboxGpuCreatePlan(input.gpuRoutePlan, { + runAttempt: attemptRunner.runAttempt, + captureNativeFailure: (failure) => { + const routeAdapter = adaptDockerGpuRouteForPatch(failure.route); + const diagnostics = collectDockerGpuPatchDiagnostics( + input.sandboxName, + { + error: failure.error, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }, + { runCaptureOpenshell: deps.runCaptureOpenshell }, + ); + if (diagnostics) console.error(` Native GPU diagnostics saved: ${diagnostics.dir}`); }, - { runCaptureOpenshell: deps.runCaptureOpenshell }, - ); - if (diagnostics) console.error(` Native GPU diagnostics saved: ${diagnostics.dir}`); - }, - cleanupNativeFailure: (failure) => { - if (input.requirePolicylessCreate) { - return refuseApfMutableNameFallbackCleanup(input.sandboxName); - } - return sandboxGpuCreateAttempt.cleanupNativeGpuFailureForFallback( - input.sandboxName, - failure, - { - runOpenshell: deps.runOpenshell, - sleep: deps.sleep, + cleanupNativeFailure: (failure) => { + if (input.requirePolicylessCreate) { + return refuseApfMutableNameFallbackCleanup(input.sandboxName); + } + return sandboxGpuCreateAttempt.cleanupNativeGpuFailureForFallback( + input.sandboxName, + failure, + { + runOpenshell: deps.runOpenshell, + sleep: deps.sleep, + }, + ); }, - ); - }, - prepareCompatibilityAttempt: async () => { - if (!input.compatibilityPolicyPath) { - throw new Error("Compatibility retry policy was not materialized."); - } - const nativeRuntimeSnapshot = attemptRunner.state.nativeRuntimeSnapshot; - if (attemptRunner.managedRouting) { - const managedBootstrap = input.managedBootstrap; - if (!managedBootstrap) { - throw new Error("Managed compatibility routing is missing bootstrap authority."); - } - const bootstrapIdentity = createManagedBootstrapIdentity(); - const heldWorkloadArgv = [ - ...renderManagedBootstrapHeldCommand( - managedBootstrap.request, - bootstrapIdentity, - managedBootstrap.intendedWorkloadArgv, - ), - ]; - const prepared = attemptRunner.managedRouting.prepareCompatibilityLaunch({ - createArgs: managedBootstrapCreateArgs(input.prebuild.createArgs, bootstrapIdentity), - currentRegistryImageRef: registryImageRef, - managedImageReference: `${managedBootstrap.image.repository}@${managedBootstrap.image.manifestDigest}`, - prebuildImageId: input.prebuild.imageId, - allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, - compatibilityPolicyPath: input.compatibilityPolicyPath, - startupCommand: heldWorkloadArgv, - runtimeSnapshot: nativeRuntimeSnapshot, - }); - attemptRunner.state.compatibilityArgv = [...prepared.createArgv]; - attemptRunner.state.compatibilityBootstrapIdentity = bootstrapIdentity; - attemptRunner.state.compatibilityHeldWorkloadArgv = heldWorkloadArgv; - registryImageRef = prepared.registryImageRef; - } else { - const prebuildImageId = input.prebuild.imageId; - const imageId = - nativeRuntimeSnapshot?.imageId ?? - (prebuildImageId && isImmutableDockerImageId(prebuildImageId) - ? prebuildImageId.toLowerCase() - : null); - if ( - !registryImageRef && - nativeRuntimeSnapshot?.bookkeepingImageRef && - !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) - ) { - registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; - } - const compatibilityArgs = renderCompatibilityFallbackCreateArgs( - input.prebuild.createArgs, - { - imageRef: imageId, - allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, - compatibilityPolicyPath: input.compatibilityPolicyPath, - }, - ); - attemptRunner.state.compatibilityArgv = deps.openshellArgv([ - "sandbox", - "create", - ...compatibilityArgs, - "--", - ...input.sandboxStartupCommand, - ]); - } - if (attemptRunner.state.compatibilityArgv.length === 0) { - throw new Error("Compatibility sandbox create executable is missing."); - } - }, - activateCompatibilityAttempt: async () => { - if (!input.managedBootstrap) { - await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( - input.provider, - input.sandboxGpuConfig, - { - dockerDriverGateway: input.dockerDriverGateway, - selectedRoute: "compatibility", - gatewayPort: input.gatewayPort, - log: console.log, - }, - ); - } - input.sandboxGpuConfig.sandboxGpuProof = null; - }, - traceEvent: addTraceEvent, - })) - .catch((error: unknown) => { - if (error instanceof ManagedBootstrapRecoveryBlockedError) { - exitForManagedBootstrapRecovery(error); - } - throw error; - }); + prepareCompatibilityAttempt: async () => { + if (!input.compatibilityPolicyPath) { + throw new Error("Compatibility retry policy was not materialized."); + } + const nativeRuntimeSnapshot = attemptRunner.state.nativeRuntimeSnapshot; + if (attemptRunner.managedRouting) { + const managedBootstrap = input.managedBootstrap; + if (!managedBootstrap) { + throw new Error("Managed compatibility routing is missing bootstrap authority."); + } + const bootstrapIdentity = createManagedBootstrapIdentity(); + const heldWorkloadArgv = [ + ...renderManagedBootstrapHeldCommand( + managedBootstrap.request, + bootstrapIdentity, + managedBootstrap.intendedWorkloadArgv, + ), + ]; + const prepared = attemptRunner.managedRouting.prepareCompatibilityLaunch({ + createArgs: managedBootstrapCreateArgs( + input.prebuild.createArgs, + bootstrapIdentity, + ), + currentRegistryImageRef: registryImageRef, + managedImageReference: `${managedBootstrap.image.repository}@${managedBootstrap.image.manifestDigest}`, + prebuildImageId: input.prebuild.imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + startupCommand: heldWorkloadArgv, + runtimeSnapshot: nativeRuntimeSnapshot, + }); + attemptRunner.state.compatibilityArgv = [...prepared.createArgv]; + attemptRunner.state.compatibilityBootstrapIdentity = bootstrapIdentity; + attemptRunner.state.compatibilityHeldWorkloadArgv = heldWorkloadArgv; + registryImageRef = prepared.registryImageRef; + } else { + const prebuildImageId = input.prebuild.imageId; + const imageId = + nativeRuntimeSnapshot?.imageId ?? + (prebuildImageId && isImmutableDockerImageId(prebuildImageId) + ? prebuildImageId.toLowerCase() + : null); + if ( + !registryImageRef && + nativeRuntimeSnapshot?.bookkeepingImageRef && + !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) + ) { + registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; + } + const compatibilityArgs = renderCompatibilityFallbackCreateArgs( + input.prebuild.createArgs, + { + imageRef: imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + }, + ); + attemptRunner.state.compatibilityArgv = deps.openshellArgv([ + "sandbox", + "create", + ...compatibilityArgs, + "--", + ...input.sandboxStartupCommand, + ]); + } + if (attemptRunner.state.compatibilityArgv.length === 0) { + throw new Error("Compatibility sandbox create executable is missing."); + } + }, + activateCompatibilityAttempt: async () => { + if (!input.managedBootstrap) { + await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( + input.provider, + input.sandboxGpuConfig, + { + dockerDriverGateway: input.dockerDriverGateway, + selectedRoute: "compatibility", + gatewayPort: input.gatewayPort, + log: console.log, + }, + ); + } + input.sandboxGpuConfig.sandboxGpuProof = null; + }, + traceEvent: addTraceEvent, + }) + ).catch((error: unknown) => { + if (error instanceof ManagedBootstrapRecoveryBlockedError) { + exitForManagedBootstrapRecovery(error); + } + throw error; + }); if (!gpuCreateOutcome.ok) { const preparationRefused = - "preparationRefused" in gpuCreateOutcome - ? gpuCreateOutcome.preparationRefused - : undefined; + "preparationRefused" in gpuCreateOutcome ? gpuCreateOutcome.preparationRefused : undefined; const cleanupRefused = "cleanupRefused" in gpuCreateOutcome ? gpuCreateOutcome.cleanupRefused : undefined; const nativeCleanupHandoff = @@ -574,24 +512,18 @@ export async function runSandboxGpuCreateFlow( `APF sandbox '${input.sandboxName}' may have been retained after native GPU fallback stopped. ` + `Gateway '${input.gatewayName}'. ${identityGuidance} ` + "Do not delete a sandbox by mutable name; use an identity-bound administrator recovery procedure."; - let persisted = false; - try { - persisted = evidence.liveIdentityFingerprint - ? persistRetainedSandboxRecovery( - message, - evidence.liveIdentityFingerprint, - evidence.createAttemptNonce, - ) - : persistRetainedSandboxRecovery(message, undefined, evidence.createAttemptNonce); - } catch { - persisted = false; - } - console.error(` ${message}`); - if (!persisted) { - console.error( + persistRetainedSandboxRecoveryOrBlock({ + persist: persistRetainedSandboxRecovery, + message, + createAttemptNonce: evidence.createAttemptNonce, + ...(evidence.liveIdentityFingerprint + ? { sandboxIdentityFingerprint: evidence.liveIdentityFingerprint } + : {}), + persistenceFailureDiagnostic: " APF recovery is blocked because NemoClaw could not save this create-attempt evidence. Preserve the terminal output for an OpenShell administrator.", - ); - } + persistenceFailureMessage: + "The APF recovery-only session remains blocked until its durable recovery record can be saved.", + }); } } process.exit(1); @@ -622,12 +554,33 @@ export async function runSandboxGpuCreateFlow( 0, 500, ); - console.warn(` Portable demo lifecycle setup did not complete: ${detail}`); + const identity = attemptRunner.state.verifiedCreatedSandboxIdentity; + const persist = input.persistRetainedSandboxRecovery; + if (!identity || !persist) { + throw new Error( + `Portable demo lifecycle setup failed after sandbox creation without exact durable recovery authority: ${detail}`, + ); + } + const message = + `Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${identity.createAttemptNonce}. ` + + `Durable sandbox identity fingerprint: ${identity.liveIdentityFingerprint}. ` + + `Portable lifecycle receipt setup did not complete for sandbox '${input.sandboxName}' on gateway '${input.gatewayName}'. ` + + "NemoClaw stopped before registry publication and success output. " + + `Run the retained identity-bound destroy action for sandbox '${input.sandboxName}'; stop if destroy cannot prove that identity.`; + persistRetainedSandboxRecoveryOrBlock({ + persist, + message, + createAttemptNonce: identity.createAttemptNonce, + sandboxIdentityFingerprint: identity.liveIdentityFingerprint, + }); + throw new Error(`Portable demo lifecycle setup did not complete: ${detail}`); } } const common = { runtimePatch: gpuCreateOutcome.value.runtimePatch, + confirmManagedRuntimeCommitReadiness: + gpuCreateOutcome.value.confirmManagedRuntimeCommitReadiness, route: gpuCreateOutcome.route, registryImageRef, lifecycleRegistrationFields: { diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 737c5967a52..386578ba1c4 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -1,8 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { performance } from "node:perf_hooks"; - import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ @@ -57,61 +55,69 @@ import { NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH, } from "../adapters/openshell/sandbox-identity"; import { + ALPHA_SANDBOX_IDENTITY_FINGERPRINT, + createAttemptNonce, createGpuFlowDeps, createGpuFlowInput, createGpuPatchFixture, + createNoGpuFlowInput as noGpuInput, + createTimedOutCreateResult, + expectNoPostCreateEffects as expectNoEffects, resetGpuFlowMocks, + sandboxListJson, setupGpuFlowMocks, } from "./__test-helpers__/sandbox-gpu-create-flow"; -import { - createHermesPortableReadyCapture, - createHermesPortableReadyRunner, -} from "./experimental/hermes-portable-onboarding"; import { runSandboxGpuCreateFlow } from "./sandbox-gpu-create-flow"; import { fingerprintSandboxRecreateValue } from "./sandbox-recreate-transaction"; -const ALPHA_SANDBOX_ID_FINGERPRINT = - "8174fa2a5d65755138d8339e086c03d736633130b22dca10952e80e74750c01d"; - -function sandboxListJson( - sandboxId: string, - labels: Readonly>, - overrides: Readonly> = {}, -): string { - return JSON.stringify([ - { - id: sandboxId, - name: "alpha", - labels, - resource_version: 1, - created_at: "2026-08-25T00:00:00Z", - phase: "Ready", - current_policy_version: 1, - ...overrides, +const durableRecoveryWriterFailures = [ + ["returns false", () => false], + [ + "throws", + () => { + throw new Error("durable writer failed"); }, - ]); -} - -function createAttemptNonce(args: readonly string[]): string { - const labelIndex = args.indexOf("--label"); - return (args[labelIndex + 1] ?? "").slice(NEMOCLAW_CREATE_ATTEMPT_LABEL.length + 1); + ], +] as const; + +function expectNoSandboxDelete(deps: ReturnType): void { + expect( + vi + .mocked(deps.runOpenshell) + .mock.calls.some(([args]) => args[0] === "sandbox" && args[1] === "delete"), + ).toBe(false); } -function noGpuInput() { - const input = createGpuFlowInput(); - input.sandboxGpuConfig = { - mode: "0", - hostGpuDetected: false, - hostGpuPlatform: null, - sandboxGpuEnabled: false, - sandboxGpuDevice: null, - errors: [], - }; - input.gpuRoutePlan = "none"; - input.initialGpuRoute = "none"; - input.createArgv = ["openshell", "sandbox", "create", "--name", "alpha", "--", "agent"]; - input.persistRetainedSandboxRecovery = vi.fn(() => true); - return input; +function attachManagedBootstrap( + input: ReturnType, + patch: ReturnType, + mode: { freshCreate?: boolean } = {}, +): void { + input.managedBootstrap = { + bootstrapIdentity: "b".repeat(64), + stateRoot: "/tmp/nemoclaw-managed-bootstrap", + runtimeProvider: { + identity: { id: "mxc" }, + bootstrap: { + createOnboardRouting: () => ({ nativeFallbackHasCleanBaseline: false }), + createLifecycle: (lifecycleOptions: { + launchArgv: readonly string[]; + heldWorkloadArgv: readonly string[]; + bootstrapIdentity: string; + }) => ({ + launchArgv: lifecycleOptions.launchArgv, + patch, + recoverUnfinished: async () => null, + prepareNetwork: async () => undefined, + runCreate: mode.freshCreate + ? async ( + start: (held: typeof lifecycleOptions) => Promise<{ readonly value: T }>, + ): Promise => (await start(lifecycleOptions)).value + : async () => Promise.reject(new Error("resumed create must not launch")), + }), + }, + }, + } as never; } function refuseEffectStartingWith(prefix: string): (operation: string) => void { @@ -120,6 +126,54 @@ function refuseEffectStartingWith(prefix: string): (operation: string) => void { }; } +function createCommittedReadinessPersistenceFixture() { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + const input = noGpuInput(); + input.resumeVerifiedCreate = { + route: "none", + liveIdentityFingerprint: ALPHA_SANDBOX_IDENTITY_FINGERPRINT, + createAttemptNonce: "a".repeat(62), + }; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + attachManagedBootstrap(input, patch); + const deps = createGpuFlowDeps("alpha-sandbox-id"); + mocks.waitForCreatedSandboxReadyWithTrace + .mockResolvedValueOnce({ ready: true, reason: "ready", failurePhase: null }) + .mockResolvedValue({ ready: false, reason: "timeout", failurePhase: null }); + return { deps, error, input, patch }; +} + +function createPostVerificationReadinessFailureFixture(writer: () => boolean = () => true) { + const events: string[] = []; + let nonce = ""; + const input = noGpuInput(); + input.persistRetainedSandboxRecovery = vi.fn(() => { + events.push("persist"); + return writer(); + }); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + patch.rollbackManagedStartupAfterCreateFailure.mockImplementation(() => events.push("rollback")); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValue({ + ready: false, + reason: "timeout", + failurePhase: null, + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + return { deps, events, input, nonce: () => nonce }; +} + beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); @@ -128,7 +182,7 @@ describe("created sandbox identity gate", () => { const gatewayName = "nemoclaw-18080"; const input = createGpuFlowInput(); input.gatewayName = gatewayName; - const deps = createGpuFlowDeps(gatewayName, true); + const deps = createGpuFlowDeps(); mocks.streamSandboxCreate.mockImplementationOnce(async (...args) => { expect(args[3].readyCheck()).toBe(true); return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; @@ -160,6 +214,260 @@ describe("created sandbox identity gate", () => { ); }); + it("reconfirms exact managed sandbox readiness after the runtime commit (#9211)", async () => { + const sandboxId = "alpha-sandbox-id"; + const input = noGpuInput(); + input.gatewayName = "owner-gateway"; + input.resumeVerifiedCreate = { + route: "none", + liveIdentityFingerprint: fingerprintSandboxRecreateValue(sandboxId), + createAttemptNonce: "a".repeat(62), + }; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + attachManagedBootstrap(input, patch); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runOpenshell) + .mockReturnValueOnce({ + status: 0, + stdout: `Name: alpha\nId: ${sandboxId}\nState: Ready\n`, + stderr: "", + }) + .mockReturnValueOnce({ + status: 1, + stdout: "", + stderr: + "Error: × code: 'The system is not in a state required for the operation's\n" + + ' │ execution\', message: "sandbox is not ready"\n', + }) + .mockReturnValueOnce({ + status: 0, + stdout: `Name: alpha\nId: ${sandboxId}\nState: Ready\n`, + stderr: "", + }) + .mockReturnValue({ status: 0, stdout: "", stderr: "" }); + mocks.waitForCreatedSandboxReadyWithTrace + .mockResolvedValueOnce({ ready: true, reason: "ready", failurePhase: null }) + .mockImplementationOnce(async (options) => { + expect(patch.commitAfterReady).toHaveBeenCalledOnce(); + expect(options.target).toEqual({ kind: "named", gatewayName: "owner-gateway" }); + expect(options.checkReadyIdentity?.()).toBe("not_ready"); + expect(options.checkReadyIdentity?.()).toBe("ready"); + return { ready: true, reason: "ready", failurePhase: null }; + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ + origin: "resumed", + route: "none", + }); + + expect(mocks.waitForCreatedSandboxReadyWithTrace).toHaveBeenCalledTimes(2); + expect(deps.runOpenshell).toHaveBeenCalledWith( + ["sandbox", "get", "-g", "owner-gateway", "alpha"], + expect.objectContaining({ suppressOutput: true }), + ); + expect(deps.runOpenshell).toHaveBeenCalledWith( + ["sandbox", "exec", "-g", "owner-gateway", "--name", "alpha", "--", "true"], + expect.objectContaining({ suppressOutput: true }), + ); + expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + expectNoSandboxDelete(deps); + }); + + it("waits through managed bootstrap publication beyond the former five-second probe (#10652)", async () => { + const actualTracing = await vi.importActual( + "./sandbox-readiness-tracing", + ); + let nonce = ""; + let nowMs = 1_000; + const input = noGpuInput(); + input.sandboxReadyTimeoutSecs = 20; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + attachManagedBootstrap(input, createGpuPatchFixture(), { freshCreate: true }); + mocks.streamSandboxCreate.mockImplementationOnce(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + deps.publicationNow = () => nowMs; + vi.mocked(deps.sleep).mockImplementation((seconds) => { + nowMs += seconds * 1_000; + }); + let readyObservations = 0; + vi.mocked(deps.runCaptureOpenshell).mockImplementation((args) => + args[1] !== "list" + ? "Name: alpha\nId: alpha-sandbox-id\nState: Ready\n" + : nowMs < 7_000 + ? "alpha Pending" + : readyObservations++ === 0 + ? "alpha Ready" + : sandboxListJson("alpha-sandbox-id", { + [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, + }), + ); + mocks.waitForCreatedSandboxReadyWithTrace + .mockImplementationOnce((options) => + actualTracing.waitForCreatedSandboxReadyWithTrace(options), + ) + .mockResolvedValue({ ready: true, reason: "ready", failurePhase: null }); + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); + expect(nowMs).toBeGreaterThanOrEqual(7_000); + const firstReadiness = mocks.waitForCreatedSandboxReadyWithTrace.mock.calls[0]?.[0]; + expect(firstReadiness?.timeoutSecs).toBe(20); + expect(firstReadiness?.now).toBe(deps.publicationNow); + expect(firstReadiness?.stableReadyPolls).toBe(1); + }); + + it("starts a fresh readiness deadline after the managed runtime commit (#10652)", async () => { + let nonce = ""; + let nowMs = 0; + const input = noGpuInput(); + input.sandboxReadyTimeoutSecs = 10; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + attachManagedBootstrap(input, patch, { freshCreate: true }); + mocks.streamSandboxCreate.mockImplementationOnce(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + deps.publicationNow = () => nowMs; + vi.mocked(deps.runCaptureOpenshell).mockImplementation((args) => + args[1] === "list" + ? sandboxListJson("alpha-sandbox-id", { + [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, + }) + : "Name: alpha\nId: alpha-sandbox-id\nState: Ready\n", + ); + const readinessTimeouts: number[] = []; + mocks.waitForCreatedSandboxReadyWithTrace.mockImplementation(async (options) => { + readinessTimeouts.push(options.timeoutSecs); + nowMs += [3_000, 2_000, 0][readinessTimeouts.length - 1] ?? 0; + return { ready: true, reason: "ready", failurePhase: null }; + }); + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); + expect(readinessTimeouts).toEqual([10, 7, 10]); + expect( + mocks.waitForCreatedSandboxReadyWithTrace.mock.calls.map(([options]) => options.now), + ).toEqual([deps.publicationNow, deps.publicationNow, deps.publicationNow]); + expect(patch.commitAfterReady).toHaveBeenCalledOnce(); + }); + + it("does not charge slow GPU inference validation to post-commit readiness (#10652)", async () => { + let nonce = ""; + let nowMs = 0; + const input = noGpuInput(); + input.sandboxGpuConfig = { + mode: "1", + hostGpuDetected: true, + hostGpuPlatform: "linux", + sandboxGpuEnabled: true, + sandboxGpuDevice: null, + errors: [], + }; + input.gpuRoutePlan = "native-only"; + input.initialGpuRoute = "native"; + input.sandboxReadyTimeoutSecs = 10; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + attachManagedBootstrap(input, patch, { freshCreate: true }); + mocks.streamSandboxCreate.mockImplementationOnce(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + deps.publicationNow = () => nowMs; + vi.mocked(deps.runCaptureOpenshell).mockImplementation((args) => + args[1] === "list" + ? sandboxListJson("alpha-sandbox-id", { + [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, + }) + : "Name: alpha\nId: alpha-sandbox-id\nState: Ready\n", + ); + const readinessTimeouts: number[] = []; + mocks.waitForCreatedSandboxReadyWithTrace.mockImplementation(async (options) => { + readinessTimeouts.push(options.timeoutSecs); + return { ready: true, reason: "ready", failurePhase: null }; + }); + + const created = await runSandboxGpuCreateFlow(input, deps); + expect(patch.commitAfterReady).not.toHaveBeenCalled(); + nowMs += 9_500; + await created.runtimePatch.commitAfterReady(); + await created.confirmManagedRuntimeCommitReadiness(); + + expect(readinessTimeouts.at(-1)).toBe(10); + expect(patch.commitAfterReady).toHaveBeenCalledOnce(); + }); + + it("retains exact recovery when committed managed readiness does not return (#9211)", async () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + const sandboxId = "alpha-sandbox-id"; + const sandboxIdentityFingerprint = ALPHA_SANDBOX_IDENTITY_FINGERPRINT; + const input = noGpuInput(); + input.resumeVerifiedCreate = { + route: "none", + liveIdentityFingerprint: sandboxIdentityFingerprint, + createAttemptNonce: "a".repeat(62), + }; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + attachManagedBootstrap(input, patch); + const deps = createGpuFlowDeps(sandboxId); + mocks.waitForCreatedSandboxReadyWithTrace + .mockResolvedValueOnce({ ready: true, reason: "ready", failurePhase: null }) + .mockResolvedValue({ ready: false, reason: "timeout", failurePhase: null }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "did not return to Ready after its managed runtime commit", + ); + + expect(patch.commitAfterReady).toHaveBeenCalledOnce(); + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining("did not return to executable Ready state"), + sandboxIdentityFingerprint, + "a".repeat(62), + ); + expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + expectNoSandboxDelete(deps); + expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { + backupPath: null, + }); + const recoveryOutput = error.mock.calls.flat().join("\n"); + expect(recoveryOutput).toContain("Do not delete sandbox 'alpha' by name."); + expect(recoveryOutput).toContain( + "Give the create-attempt label above to an OpenShell administrator", + ); + expect(recoveryOutput).not.toContain("destroy --yes"); + expect(recoveryOutput).not.toContain("After OpenShell confirms the sandbox is absent"); + }); + + it.each(durableRecoveryWriterFailures)( + "blocks committed-readiness recovery when durable persistence %s (#9211)", + async (_name, writer) => { + const { deps, error, input, patch } = createCommittedReadinessPersistenceFixture(); + const persist = vi.fn(writer); + input.persistRetainedSandboxRecovery = persist; + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "the recovery-only session remains blocked", + ); + + expect(persist).toHaveBeenCalledOnce(); + expect(error.mock.calls.flat().join("\n")).toContain( + "The recovery-only session remains blocked until its durable recovery record can be saved.", + ); + expect(error.mock.calls.flat().join("\n")).not.toContain("Preserve the terminal output"); + expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + expectNoSandboxDelete(deps); + }, + ); + it("resumes the exact verified sandbox without issuing another create (#9833)", async () => { const events: string[] = []; const sandboxId = "alpha-sandbox-id"; @@ -214,6 +522,7 @@ describe("created sandbox identity gate", () => { ["sandbox", "get", "-g", gatewayName, "alpha"], expect.objectContaining({ ignoreError: true, suppressOutput: true }), ); + expect(deps.installPortableDemoLifecycle).toHaveBeenCalledOnce(); expect(events).toEqual([ "verify-created", "revalidate:activate managed sandbox network for 'alpha'", @@ -231,6 +540,41 @@ describe("created sandbox identity gate", () => { ]); }); + it("retains exact recovery when portable lifecycle setup fails after resume (#8441)", async () => { + const sandboxId = "alpha-sandbox-id"; + const createAttemptNonce = "a".repeat(62); + const liveIdentityFingerprint = fingerprintSandboxRecreateValue(sandboxId); + const input = noGpuInput(); + input.resumeVerifiedCreate = { + route: "none", + liveIdentityFingerprint, + createAttemptNonce, + }; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const deps = createGpuFlowDeps(sandboxId); + deps.installPortableDemoLifecycle = vi.fn(() => { + throw new Error("Authorization: Bearer portable-secret"); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "Portable demo lifecycle setup did not complete: Authorization: Bearer ", + ); + + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining("Run the retained identity-bound destroy action for sandbox 'alpha'"), + liveIdentityFingerprint, + createAttemptNonce, + ); + expect(deps.installPortableDemoLifecycle).toHaveBeenCalledOnce(); + expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); + expectNoSandboxDelete(deps); + expect(vi.mocked(console.error).mock.calls.flat().join("\n")).not.toContain("portable-secret"); + expect(vi.mocked(console.log).mock.calls.flat().join("\n")).not.toContain( + "Sandbox 'alpha' created", + ); + }); + it("refuses a changed live identity before resumed effects (#9833)", async () => { const input = noGpuInput(); input.resumeVerifiedCreate = { @@ -286,420 +630,62 @@ describe("created sandbox identity gate", () => { expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); }); - it("ends the create-client handoff after a nonce-owned ID appears and settles metadata before effects (#10769)", async () => { - const events: string[] = []; + it("allows identity-bound post-create effects after the former 30-second cap (#10652)", async () => { let nonce = ""; + let nowMs = 0; const input = noGpuInput(); + input.sandboxReadyTimeoutSecs = 90; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); - input.verifyCreatedSandboxBeforeEffects = vi.fn(async (identity) => { - events.push("verify-created"); - expect(identity).toEqual({ - sandboxId: "alpha-sandbox-id", - liveIdentityFingerprint: expect.stringMatching(/^[0-9a-f]{64}$/u), - createAttemptNonce: expect.stringMatching(/^[0-9a-f]{62}$/u), - route: "none", - }); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); - }); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn((operation) => - events.push(`revalidate:${operation}`), - ); - patch.exitOnPatchError.mockImplementation(() => events.push("runtime-check")); - patch.ensureApplied.mockImplementation(() => events.push("runtime-patch")); - patch.waitForSupervisorReconnectIfNeeded.mockImplementation(() => events.push("reconnect")); - patch.commitAfterReady.mockImplementation(() => events.push("commit")); mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { - events.push("create"); - expect(options.onPoll).toBeUndefined(); - expect(options.waitForReadyTermination).toBe(true); - expect(args.indexOf("--label")).toBeGreaterThan(0); - expect(args.indexOf("--label")).toBeLessThan(args.indexOf("--")); nonce = createAttemptNonce(args); expect(nonce).toMatch(/^[0-9a-f]{62}$/u); expect(nonce).toHaveLength(NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH); - expect(nonce.length).toBeLessThanOrEqual(63); expect(options.readyCheck?.()).toBe(true); return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; }); - mocks.waitForCreatedSandboxReadyWithTrace.mockImplementation(() => { - events.push("readiness"); - return { ready: true, reason: "ready", failurePhase: null }; - }); + const deps = createGpuFlowDeps(); - vi.mocked(deps.sleep).mockImplementation(() => { - events.push("identity-settle"); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - }); - deps.installPortableDemoLifecycle = vi.fn(() => { - events.push("portable-lifecycle"); - return "generation-1"; + deps.publicationNow = () => nowMs; + vi.mocked(deps.sleep).mockImplementation((seconds) => { + nowMs += seconds * 1_000; }); + deps.installPortableDemoLifecycle = vi.fn(() => "generation-1"); + const identityLabels = () => ({ [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }); + const pendingMetadata = { + resource_version: null, + created_at: null, + phase: null, + current_policy_version: null, + }; vi.mocked(deps.runCaptureOpenshell) - .mockImplementationOnce((args) => { - expect(args).not.toContain("--selector"); - events.push("ready-visible"); - return "alpha Ready"; - }) - .mockImplementationOnce((args) => { - expect(args).toContain("--selector"); - events.push("identity-metadata-pending"); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - return sandboxListJson( - "alpha-sandbox-id", - { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }, - { - resource_version: null, - created_at: null, - phase: null, - current_policy_version: null, - }, - ); + .mockReturnValueOnce("alpha Ready") + .mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", identityLabels(), pendingMetadata), + ) + .mockImplementationOnce(() => { + nowMs += 31_000; + return sandboxListJson("alpha-sandbox-id", identityLabels(), pendingMetadata); }) - .mockImplementationOnce((args) => { - expect(args).toContain("--selector"); - events.push("identity-matched"); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - return sandboxListJson("alpha-sandbox-id", { - [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, - }); - }); + .mockImplementationOnce(() => sandboxListJson("alpha-sandbox-id", identityLabels())); await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); - expect(events).toEqual([ - "create", - "ready-visible", - "identity-metadata-pending", - "identity-matched", - "verify-created", - "revalidate:validate runtime patch for sandbox 'alpha'", - "runtime-check", - "revalidate:apply runtime patch for sandbox 'alpha'", - "runtime-patch", - "reconnect", - "revalidate:reconnect sandbox supervisor for 'alpha'", - "readiness", - "revalidate:commit runtime readiness for sandbox 'alpha'", - "commit", - "revalidate:record portable lifecycle for sandbox 'alpha'", - "portable-lifecycle", - ]); - expect(deps.runCaptureOpenshell).toHaveBeenNthCalledWith( - 2, - [ - "sandbox", - "list", - "-g", - "nemoclaw", - "--selector", - `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`, - "--output", - "json", - "--limit", - "2", - ], - { - ignoreError: false, - timeout: expect.any(Number), - maxBuffer: 1024 * 1024, - killSignal: "SIGKILL", - killProcessTreeOnTimeout: true, - }, + expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledExactlyOnceWith({ + sandboxId: "alpha-sandbox-id", + liveIdentityFingerprint: expect.stringMatching(/^[0-9a-f]{64}$/u), + createAttemptNonce: nonce, + route: "none", + }); + expect(input.revalidateVerifiedSandboxBeforeEffect).toHaveBeenCalledWith( + "apply runtime patch for sandbox 'alpha'", ); - const firstIdentityTimeout = vi.mocked(deps.runCaptureOpenshell).mock.calls[1]?.[1]?.timeout; - expect(firstIdentityTimeout).toEqual(expect.any(Number)); - expect(firstIdentityTimeout as number).toBeGreaterThan(0); - expect(firstIdentityTimeout as number).toBeLessThanOrEqual(30_000); - expect(deps.runCaptureOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "get", "-g", "nemoclaw", "alpha"], - expect.anything(), - ); - expect(deps.sleep).not.toHaveBeenCalled(); - }); - - it("returns false and blocks effects when the create-attempt selector returns no sandbox ID (#10769)", async () => { - const input = noGpuInput(); - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - const patch = createGpuPatchFixture(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, _args, _env, options) => { - expect(options.readyCheck?.()).toBe(false); - return { status: 0, output: "", sawProgress: true }; - }); - const deps = createGpuFlowDeps(); - deps.installPortableDemoLifecycle = vi.fn(); - vi.mocked(deps.runCaptureOpenshell) - .mockReturnValueOnce("alpha Ready") - .mockReturnValueOnce("[]"); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "did not return one exact durable sandbox identity before post-create effects", - ); - - expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledOnce(); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(input.revalidateVerifiedSandboxBeforeEffect).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - expect(patch.waitForSupervisorReconnectIfNeeded).not.toHaveBeenCalled(); - expect(patch.commitAfterReady).not.toHaveBeenCalled(); - expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); - expect(deps.installPortableDemoLifecycle).not.toHaveBeenCalled(); - }); - - it("persists recovery before reporting a handoff timeout that looks like an incomplete create (#10769)", async () => { - const events: string[] = []; - let nonce = ""; - const input = noGpuInput(); - input.persistRetainedSandboxRecovery = vi.fn(() => { - events.push("persist-recovery"); - return true; - }); - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - const patch = createGpuPatchFixture(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { - nonce = createAttemptNonce(args); - expect(options.readyCheck?.()).toBe(true); - return { - status: 1, - output: - "Created sandbox: alpha\nOpenShell create client did not exit after Ready; aborting cutover.", - sawProgress: true, - readyTerminationTimedOut: true, - }; - }); - const deps = createGpuFlowDeps(); - deps.installPortableDemoLifecycle = vi.fn(); - vi.mocked(deps.runCaptureOpenshell) - .mockReturnValueOnce("alpha Ready") - .mockImplementationOnce(() => - sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), - ); - vi.mocked(console.error).mockImplementation(() => { - events.push("report-recovery"); - }); - const exit = vi.spyOn(process, "exit").mockImplementation(() => { - throw new Error("process.exit:1"); - }); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "OpenShell create client did not exit after Ready for sandbox 'alpha'", - ); - - const fingerprint = ALPHA_SANDBOX_ID_FINGERPRINT; - expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( - expect.stringMatching( - new RegExp( - `^Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}\\. Durable sandbox identity fingerprint: ${fingerprint}\\.`, - "u", - ), - ), - fingerprint, - nonce, - ); - expect(events.slice(0, 2)).toEqual(["persist-recovery", "report-recovery"]); - expect(exit).not.toHaveBeenCalled(); - const output = vi.mocked(console.error).mock.calls.flat().join("\n"); - expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); - expect(output).toContain(`Durable sandbox identity fingerprint: ${fingerprint}`); - expect(output).toContain("Run 'nemoclaw alpha destroy'"); - expect(output).toContain("the command removes nothing and preserves the recovery record"); - expect(output).toContain("Give the create-attempt label to an OpenShell administrator"); - expect(output).toContain("After OpenShell confirms removal"); - expect(output).toContain("run 'nemoclaw alpha destroy --yes'"); - expect(output).not.toContain("alpha-sandbox-id"); - expect(output).not.toContain("Recovery:"); - expect(output).not.toContain("Or: nemoclaw onboard"); - expect(output).not.toContain("onboard --resume"); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(input.revalidateVerifiedSandboxBeforeEffect).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - expect(patch.waitForSupervisorReconnectIfNeeded).not.toHaveBeenCalled(); - expect(patch.commitAfterReady).not.toHaveBeenCalled(); - expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); - expect(deps.installPortableDemoLifecycle).not.toHaveBeenCalled(); - }); - - it("blocks a restart-safe handoff timeout without create-attempt identity (#10769)", async () => { - const input = noGpuInput(); - input.persistStartupCommand = true; - const patch = createGpuPatchFixture(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, _args, _env, options) => { - expect(options.waitForReadyTermination).toBe(true); - expect(options.readyCheck?.()).toBe(true); - return { - status: 1, - output: - "Created sandbox: alpha\nOpenShell create client did not exit after Ready; aborting cutover.", - sawProgress: true, - readyTerminationTimedOut: true, - }; - }); - const deps = createGpuFlowDeps(); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "No create-attempt identity was available for retained recovery", - ); - - expect(input.persistRetainedSandboxRecovery).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - expect(patch.waitForSupervisorReconnectIfNeeded).not.toHaveBeenCalled(); - expect(patch.commitAfterReady).not.toHaveBeenCalled(); - expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); - }); - - it.each([ - ["returns false", (): boolean => false], - ["throws", (): boolean => { - throw new Error("recovery writer failed"); - }], - ] as const)( - "blocks the create after retained recovery persistence %s (#10769)", - async (_failureMode, persistRecovery) => { - let nonce = ""; - const input = noGpuInput(); - input.persistRetainedSandboxRecovery = vi.fn(persistRecovery); - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - const patch = createGpuPatchFixture(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { - nonce = createAttemptNonce(args); - expect(options.readyCheck?.()).toBe(true); - return { - status: 1, - output: "OpenShell create client did not exit after Ready; aborting cutover.", - sawProgress: true, - readyTerminationTimedOut: true, - }; - }); - const deps = createGpuFlowDeps(); - vi.mocked(deps.runCaptureOpenshell) - .mockReturnValueOnce("alpha Ready") - .mockImplementationOnce(() => - sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), - ); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "NemoClaw could not save the retained sandbox recovery record for this create attempt", - ); - - expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( - expect.stringContaining( - `Durable sandbox identity fingerprint: ${ALPHA_SANDBOX_ID_FINGERPRINT}`, - ), - ALPHA_SANDBOX_ID_FINGERPRINT, - nonce, - ); - const output = vi.mocked(console.error).mock.calls.flat().join("\n"); - expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); - expect(output).toContain( - "NemoClaw could not save the retained sandbox recovery record for this create attempt", - ); - expect(output).not.toContain("alpha-sandbox-id"); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(input.revalidateVerifiedSandboxBeforeEffect).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - expect(patch.waitForSupervisorReconnectIfNeeded).not.toHaveBeenCalled(); - expect(patch.commitAfterReady).not.toHaveBeenCalled(); - }, - ); - - it("carries Hermes receipt authority from selector settlement through publication lookup (#10423)", async () => { - const events: string[] = []; - let nonce = ""; - const input = noGpuInput(); - const patch = createGpuPatchFixture(); - input.verifyCreatedSandboxBeforeEffects = vi.fn(async () => { - events.push("verify-policy"); - }); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { - events.push("create"); - nonce = createAttemptNonce(args); - expect(options.readyCheck?.()).toBe(true); - return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; - }); - mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValue({ - ready: true, - reason: "ready", - failurePhase: null, - }); - const capture = vi.fn((args: readonly string[]) => { - const results = { - [["sandbox", "list", "-g", "nemoclaw"].join("\0")]: () => { - events.push("ready-visible"); - return { status: 0, stdout: Buffer.from("alpha Ready"), stderr: Buffer.alloc(0) }; - }, - [[ - "sandbox", - "list", - "-g", - "nemoclaw", - "--selector", - `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`, - "--output", - "json", - "--limit", - "2", - ].join("\0")]: () => { - events.push("selector-settled"); - return { - status: 0, - stdout: Buffer.from( - sandboxListJson("alpha-sandbox-id", { - [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, - }), - ), - stderr: Buffer.alloc(0), - }; - }, - [["sandbox", "get", "-g", "nemoclaw", "alpha"].join("\0")]: () => { - events.push("publication-get"); - return { - status: 0, - stdout: Buffer.from("ID: alpha-sandbox-id\n"), - stderr: Buffer.alloc(0), - }; - }, - } satisfies Readonly< - Record { status: number; stdout: Buffer; stderr: Buffer }> - >; - return ( - results[args.join("\0") as keyof typeof results] ?? - (() => ({ status: 1, stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) })) - )(); - }); - const deps = createGpuFlowDeps(); - deps.runOpenshell = createHermesPortableReadyRunner("alpha", "nemoclaw", capture); - deps.runCaptureOpenshell = createHermesPortableReadyCapture("alpha", "nemoclaw", capture); - - await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); - - expect(events.slice(0, 6)).toEqual([ - "create", - "ready-visible", - "selector-settled", - "selector-settled", - "publication-get", - "verify-policy", - ]); - expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); - expect(capture).toHaveBeenCalledWith(["sandbox", "get", "-g", "nemoclaw", "alpha"]); + expect(patch.ensureApplied).toHaveBeenCalledOnce(); + expect(patch.commitAfterReady).toHaveBeenCalledOnce(); + expect(deps.sleep).toHaveBeenCalledExactlyOnceWith(0.25); }); it.each([ @@ -818,9 +804,11 @@ describe("created sandbox identity gate", () => { expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); }); - it("rejects a different owner-scoped sandbox identity before post-create effects (#9833)", async () => { + it("shares publication time with the final post-create readiness deadline (#10652)", async () => { let nonce = ""; + let nowMs = 1_000; const input = noGpuInput(); + input.sandboxReadyTimeoutSecs = 10; input.verifyCreatedSandboxBeforeEffects = vi.fn(); input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); const patch = createGpuPatchFixture(); @@ -830,36 +818,55 @@ describe("created sandbox identity gate", () => { return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; }); const deps = createGpuFlowDeps(); + deps.publicationNow = () => nowMs; + vi.mocked(deps.sleep).mockImplementation((seconds) => { + nowMs += seconds * 1_000; + }); vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), ); - vi.mocked(deps.runOpenshell).mockReturnValue({ - status: 0, - stdout: "Name: alpha\nId: replacement-sandbox-id\nState: Ready\n", - stderr: "", + const missingSandbox = { + status: 1, + stdout: "", + stderr: + "Error: × code: 'Some requested entity was not found', message: \"sandbox not found\"", + }; + vi.mocked(deps.runOpenshell) + .mockReturnValueOnce(missingSandbox) + .mockReturnValueOnce(missingSandbox) + .mockReturnValueOnce(missingSandbox) + .mockReturnValueOnce(missingSandbox) + .mockReturnValue({ + status: 0, + stdout: "Name: alpha\nId: alpha-sandbox-id\nState: Ready\n", + stderr: "", + }); + let finalReadinessTimeoutSecs = Number.NaN; + mocks.waitForCreatedSandboxReadyWithTrace.mockImplementationOnce(async (options) => { + finalReadinessTimeoutSecs = options.timeoutSecs; + nowMs += options.timeoutSecs * 1_000; + return { ready: false, reason: "timeout", failurePhase: null }; }); await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "changed identity before identity verification completed", + "did not become ready after verified creation", ); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( - expect.stringContaining( - `Durable sandbox identity fingerprint: ${fingerprintSandboxRecreateValue("alpha-sandbox-id")}`, - ), - fingerprintSandboxRecreateValue("alpha-sandbox-id"), - nonce, + expect(deps.sleep).toHaveBeenCalledTimes(4); + expect(finalReadinessTimeoutSecs).toBe(6); + expect(nowMs).toBe(11_000); + expect(mocks.waitForCreatedSandboxReadyWithTrace).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + timeoutSecs: 6, + now: deps.publicationNow, + }), ); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); + expect(patch.commitAfterReady).not.toHaveBeenCalled(); }); - it("stops when owner-scoped sandbox publication exceeds the deadline (#9833)", async () => { + it("stops publication probing on a non-transient OpenShell failure (#9833)", async () => { let nonce = ""; const input = noGpuInput(); - input.sandboxReadyTimeoutSecs = 0.001; input.verifyCreatedSandboxBeforeEffects = vi.fn(); input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); const patch = createGpuPatchFixture(); @@ -875,32 +882,30 @@ describe("created sandbox identity gate", () => { vi.mocked(deps.runOpenshell).mockReturnValue({ status: 1, stdout: "", - stderr: - "Error: × code: 'Some requested entity was not found', message: \"sandbox not found\"", + stderr: "permission denied: NVIDIA_API_KEY=nvapi-publication-secret", }); await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "did not become visible through its owning gateway before identity verification completed", + "OpenShell could not verify publication", ); + expect(deps.sleep).not.toHaveBeenCalled(); expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( - expect.stringContaining( - `Durable sandbox identity fingerprint: ${fingerprintSandboxRecreateValue("alpha-sandbox-id")}`, - ), - fingerprintSandboxRecreateValue("alpha-sandbox-id"), + expect.stringMatching(/OpenShell detail: .*permission denied: NVIDIA_API_KEY=/u), + ALPHA_SANDBOX_IDENTITY_FINGERPRINT, nonce, ); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); + const recoveryMessage = + vi.mocked(input.persistRetainedSandboxRecovery!).mock.calls[0]?.[0] ?? ""; + expect(recoveryMessage).not.toContain("nvapi-publication-secret"); }); - it("rejects a same-name replacement before post-create effects (#9833)", async () => { + it("rejects a different owner-scoped sandbox identity before post-create effects (#9833)", async () => { let nonce = ""; - const outputCanary = "replacement-output-must-not-be-reported"; const input = noGpuInput(); input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); const patch = createGpuPatchFixture(); mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { @@ -909,15 +914,98 @@ describe("created sandbox identity gate", () => { }); const deps = createGpuFlowDeps(); vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => - sandboxListJson("replacement-id", { - [NEMOCLAW_CREATE_ATTEMPT_LABEL]: "0".repeat(NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH), - untrusted: outputCanary, - }), + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), ); + vi.mocked(deps.runOpenshell).mockReturnValue({ + status: 0, + stdout: "Name: alpha\nId: replacement-sandbox-id\nState: Ready\n", + stderr: "", + }); - const error = await runSandboxGpuCreateFlow(input, deps).catch((caught: unknown) => caught); - - expect(error).toBeInstanceOf(Error); + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "changed identity before identity verification completed", + ); + + expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining( + `Durable sandbox identity fingerprint: ${ALPHA_SANDBOX_IDENTITY_FINGERPRINT}`, + ), + ALPHA_SANDBOX_IDENTITY_FINGERPRINT, + nonce, + ); + expect(patch.exitOnPatchError).not.toHaveBeenCalled(); + expect(patch.ensureApplied).not.toHaveBeenCalled(); + expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); + }); + + it("stops when owner-scoped sandbox publication exceeds the deadline (#9833)", async () => { + let nonce = ""; + const input = noGpuInput(); + input.sandboxReadyTimeoutSecs = 0.001; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + let nowMs = 0; + deps.publicationNow = () => nowMs; + vi.mocked(deps.sleep).mockImplementation((seconds) => { + nowMs += seconds * 1_000; + }); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + vi.mocked(deps.runOpenshell).mockReturnValue({ + status: 1, + stdout: "", + stderr: + "Error: × code: 'Some requested entity was not found', message: \"sandbox not found\"", + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "did not become visible through its owning gateway before identity verification completed", + ); + + expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining( + `Durable sandbox identity fingerprint: ${ALPHA_SANDBOX_IDENTITY_FINGERPRINT}`, + ), + ALPHA_SANDBOX_IDENTITY_FINGERPRINT, + nonce, + ); + expect(patch.exitOnPatchError).not.toHaveBeenCalled(); + expect(patch.ensureApplied).not.toHaveBeenCalled(); + expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); + }); + + it("rejects a same-name replacement before post-create effects (#9833)", async () => { + let nonce = ""; + const outputCanary = "replacement-output-must-not-be-reported"; + const input = noGpuInput(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("replacement-id", { + [NEMOCLAW_CREATE_ATTEMPT_LABEL]: "0".repeat(NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH), + untrusted: outputCanary, + }), + ); + + const error = await runSandboxGpuCreateFlow(input, deps).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); expect(String(error)).toContain( "did not return one exact durable sandbox identity before post-create effects", ); @@ -928,10 +1016,11 @@ describe("created sandbox identity gate", () => { expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); }); - it("persists create-attempt recovery when Ready identity settlement reaches its deadline (#9211)", async () => { + it("persists create-attempt recovery when Ready identity settlement reaches its configured deadline (#9211)", async () => { const events: string[] = []; let nonce = ""; const input = noGpuInput(); + input.sandboxReadyTimeoutSecs = 60; input.verifyCreatedSandboxBeforeEffects = vi.fn(); input.persistRetainedSandboxRecovery = vi.fn(() => { events.push("persist-recovery"); @@ -944,11 +1033,12 @@ describe("created sandbox identity gate", () => { return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; }); const deps = createGpuFlowDeps(); - vi.mocked(deps.runCaptureOpenshell).mockReturnValue("[]"); - vi.spyOn(performance, "now") - .mockReturnValueOnce(0) - .mockReturnValueOnce(0) - .mockReturnValueOnce(30_000); + let nowMs = 0; + deps.publicationNow = () => nowMs; + vi.mocked(deps.runCaptureOpenshell).mockImplementation(() => { + nowMs = 60_000; + return "[]"; + }); const error = await runSandboxGpuCreateFlow(input, deps).catch((caught: unknown) => { events.push("rejected"); @@ -1002,37 +1092,30 @@ describe("created sandbox identity gate", () => { expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); }); - it("returns a post-verification readiness failure to the recovery owner (#9833)", async () => { - let nonce = ""; - const input = noGpuInput(); - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - const patch = createGpuPatchFixture(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { - nonce = createAttemptNonce(args); - return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; - }); - mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValue({ - ready: false, - reason: "timeout", - failurePhase: null, - }); - const deps = createGpuFlowDeps(); - vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => - sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), - ); - const exit = vi.spyOn(process, "exit").mockImplementation(() => { - throw new Error("direct process exit bypassed the recovery owner"); - }); - + it("persists recovery before rollback when post-verification readiness fails (#9833)", async () => { + const { deps, events, input, nonce } = createPostVerificationReadinessFailureFixture(); await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( "Sandbox 'alpha' did not become ready after verified creation", ); - - expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); - expect(exit).not.toHaveBeenCalled(); + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringMatching(/Post-verification readiness detail: .*within 60s\./u), + ALPHA_SANDBOX_IDENTITY_FINGERPRINT, + nonce(), + ); + expect(events).toEqual(["persist", "rollback"]); }); + it.each(durableRecoveryWriterFailures)( + "blocks rollback when post-verification recovery persistence %s (#9833)", + async (_failureMode, writer) => { + const { deps, events, input } = createPostVerificationReadinessFailureFixture(writer); + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "could not save the retained sandbox recovery record", + ); + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledOnce(); + expect(events).toEqual(["persist"]); + expect(mocks.printReadinessFailure).not.toHaveBeenCalled(); + }, + ); it("uses a distinct identity label for each create attempt (#9833)", async () => { const input = createGpuFlowInput(); @@ -1071,120 +1154,6 @@ describe("created sandbox identity gate", () => { expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); }); - it("persists exact APF recovery evidence before refusing native fallback (#9833)", async () => { - let nonce = ""; - const input = createGpuFlowInput(); - input.requirePolicylessCreate = true; - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - input.persistRetainedSandboxRecovery = vi.fn(() => true); - mocks.streamSandboxCreate.mockImplementationOnce(async (_command, args) => { - nonce = createAttemptNonce(args); - return { - status: 1, - output: "native runtime failed after sandbox creation", - sawProgress: true, - }; - }); - mocks.queryOpenShellDockerSandboxRuntimeSnapshot.mockReturnValue({ - ok: true, - imageId: "sha256:" + "a".repeat(64), - bookkeepingImageRef: "openshell/sandbox-from:test", - stateError: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", - deviceRequests: null, - devices: null, - runtime: "runc", - nvidiaVisibleDevices: null, - nativeGpuAttachmentState: "absent", - containerId: "container-a", - }); - const deps = createGpuFlowDeps(); - vi.mocked(deps.runCaptureOpenshell).mockImplementation(() => - sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), - ); - const exit = vi.spyOn(process, "exit").mockImplementation(() => { - throw new Error("process.exit:1"); - }); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1"); - - const fingerprint = fingerprintSandboxRecreateValue("alpha-sandbox-id"); - expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( - expect.stringMatching( - new RegExp( - `^Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}\\. Durable sandbox identity fingerprint: ${fingerprint}\\.`, - "u", - ), - ), - fingerprint, - nonce, - ); - expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledBefore(exit); - const output = vi.mocked(console.error).mock.calls.flat().join("\n"); - expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); - expect(output).toContain(`Durable sandbox identity fingerprint: ${fingerprint}`); - expect(output).not.toContain("alpha-sandbox-id"); - expect(deps.runOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), - ); - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - }); - - it("persists the APF create-attempt label when exact recovery identity is unavailable (#9833)", async () => { - let nonce = ""; - const input = createGpuFlowInput(); - input.requirePolicylessCreate = true; - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - input.persistRetainedSandboxRecovery = vi.fn(() => true); - mocks.streamSandboxCreate.mockImplementationOnce(async (_command, args) => { - nonce = createAttemptNonce(args); - return { - status: 1, - output: "native runtime failed after sandbox creation", - sawProgress: true, - }; - }); - mocks.queryOpenShellDockerSandboxRuntimeSnapshot.mockReturnValue({ - ok: true, - imageId: "sha256:" + "a".repeat(64), - bookkeepingImageRef: "openshell/sandbox-from:test", - stateError: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", - deviceRequests: null, - devices: null, - runtime: "runc", - nvidiaVisibleDevices: null, - nativeGpuAttachmentState: "absent", - containerId: "container-a", - }); - const deps = createGpuFlowDeps(); - vi.mocked(deps.runCaptureOpenshell).mockReturnValue("[]"); - vi.spyOn(process, "exit").mockImplementation(() => { - throw new Error("process.exit:1"); - }); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1"); - - expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( - expect.stringMatching( - new RegExp( - `^Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}\\..*Recovery is blocked until an OpenShell administrator resolves the create-attempt label`, - "u", - ), - ), - undefined, - nonce, - ); - const output = vi.mocked(console.error).mock.calls.flat().join("\n"); - expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); - expect(output).toContain("Recovery is blocked"); - expect(deps.runOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), - ); - }); - it("stops before a runtime patch when the durable checkpoint drifts (#9833)", async () => { let nonce = ""; const input = noGpuInput(); @@ -1263,4 +1232,269 @@ describe("created sandbox identity gate", () => { expect(mocks.verifyGpuSandboxAccessAfterReady).not.toHaveBeenCalled(); expect(patch.commitAfterReady).not.toHaveBeenCalled(); }); + + it("ends the create-client handoff after a nonce-owned ID appears and settles metadata before effects (#10769)", async () => { + const metadataStates: string[] = []; + let nonce = ""; + const input = noGpuInput(); + const patch = createGpuPatchFixture(); + const deps = createGpuFlowDeps(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(async (identity) => { + expect(metadataStates).toEqual(["pending", "complete"]); + expect(identity).toEqual({ + sandboxId: "alpha-sandbox-id", + liveIdentityFingerprint: expect.stringMatching(/^[0-9a-f]{64}$/u), + createAttemptNonce: expect.stringMatching(/^[0-9a-f]{62}$/u), + route: "none", + }); + expect(patch.exitOnPatchError).not.toHaveBeenCalled(); + expect(patch.ensureApplied).not.toHaveBeenCalled(); + expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); + expect(deps.installPortableDemoLifecycle).not.toHaveBeenCalled(); + }); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { + expect(options.waitForReadyTermination).toBe(true); + expect(args.indexOf("--label")).toBeLessThan(args.indexOf("--")); + nonce = createAttemptNonce(args); + expect(nonce).toMatch(/^[0-9a-f]{62}$/u); + expect(options.readyCheck?.()).toBe(true); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValue({ + ready: true, + reason: "ready", + failurePhase: null, + }); + deps.installPortableDemoLifecycle = vi.fn(() => "generation-1"); + vi.mocked(deps.runCaptureOpenshell) + .mockImplementationOnce((args) => { + expect(args).not.toContain("--selector"); + return "alpha Ready"; + }) + .mockImplementationOnce((args) => { + expect(args).toContain("--selector"); + metadataStates.push("pending"); + expectNoEffects(input, patch, deps, mocks.waitForCreatedSandboxReadyWithTrace); + return sandboxListJson( + "alpha-sandbox-id", + { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }, + { + resource_version: null, + created_at: null, + phase: null, + current_policy_version: null, + }, + ); + }) + .mockImplementationOnce((args) => { + expect(args).toContain("--selector"); + metadataStates.push("complete"); + expectNoEffects(input, patch, deps, mocks.waitForCreatedSandboxReadyWithTrace); + return sandboxListJson("alpha-sandbox-id", { + [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce, + }); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); + + expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); + expect(patch.exitOnPatchError).toHaveBeenCalledOnce(); + expect(patch.ensureApplied).toHaveBeenCalledOnce(); + expect(patch.waitForSupervisorReconnectIfNeeded).toHaveBeenCalledOnce(); + expect(mocks.waitForCreatedSandboxReadyWithTrace).toHaveBeenCalledOnce(); + expect(patch.commitAfterReady).toHaveBeenCalledOnce(); + expect(deps.installPortableDemoLifecycle).toHaveBeenCalledOnce(); + expect(deps.runCaptureOpenshell).toHaveBeenNthCalledWith( + 2, + [ + "sandbox", + "list", + "-g", + "nemoclaw", + "--selector", + `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`, + "--output", + "json", + "--limit", + "2", + ], + { + ignoreError: false, + timeout: expect.any(Number), + maxBuffer: 1024 * 1024, + killSignal: "SIGKILL", + killProcessTreeOnTimeout: true, + }, + ); + const firstIdentityTimeout = vi.mocked(deps.runCaptureOpenshell).mock.calls[1]?.[1]?.timeout; + expect(firstIdentityTimeout as number).toBeGreaterThan(0); + expect(firstIdentityTimeout as number).toBeLessThanOrEqual(30_000); + expect(deps.runCaptureOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "get", "-g", "nemoclaw", "alpha"], + expect.anything(), + ); + expect(deps.sleep).not.toHaveBeenCalled(); + }); + + it("returns false and blocks effects when the create-attempt selector returns no sandbox ID (#10769)", async () => { + const input = noGpuInput(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, _args, _env, options) => { + expect(options.readyCheck?.()).toBe(false); + return { status: 0, output: "", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell) + .mockReturnValueOnce("alpha Ready") + .mockReturnValueOnce("[]"); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "did not return one exact durable sandbox identity before post-create effects", + ); + + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledOnce(); + expectNoEffects(input, patch, deps, mocks.waitForCreatedSandboxReadyWithTrace); + }); + + it("persists recovery before reporting a handoff timeout that looks like an incomplete create (#10769)", async () => { + const events: string[] = []; + let nonce = ""; + const input = noGpuInput(); + input.persistRetainedSandboxRecovery = vi.fn(() => { + events.push("persist-recovery"); + return true; + }); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { + nonce = createAttemptNonce(args); + expect(options.readyCheck?.()).toBe(true); + return createTimedOutCreateResult( + "Created sandbox: alpha\nOpenShell create client did not exit after Ready; aborting cutover.", + ); + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell) + .mockReturnValueOnce("alpha Ready") + .mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + vi.mocked(console.error).mockImplementation(() => events.push("report-recovery")); + const exit = vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("process.exit:1"); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "OpenShell create client did not exit after Ready for sandbox 'alpha'", + ); + + const fingerprint = ALPHA_SANDBOX_IDENTITY_FINGERPRINT; + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringMatching( + new RegExp( + `^Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}\\. Durable sandbox identity fingerprint: ${fingerprint}\\.`, + "u", + ), + ), + fingerprint, + nonce, + ); + expect(events.slice(0, 2)).toEqual(["persist-recovery", "report-recovery"]); + expect(exit).not.toHaveBeenCalled(); + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); + expect(output).toContain(`Durable sandbox identity fingerprint: ${fingerprint}`); + expect(output).toContain("Run 'nemoclaw alpha destroy'"); + expect(output).toContain("the command removes nothing and preserves the recovery record"); + expect(output).toContain("Give the create-attempt label to an OpenShell administrator"); + expect(output).toContain("After OpenShell confirms removal"); + expect(output).toContain("run 'nemoclaw alpha destroy --yes'"); + expect(output).not.toContain("alpha-sandbox-id"); + expect(output).not.toContain("Recovery:"); + expect(output).not.toContain("Or: nemoclaw onboard"); + expect(output).not.toContain("onboard --resume"); + expectNoEffects(input, patch, deps, mocks.waitForCreatedSandboxReadyWithTrace); + }); + + it("blocks a restart-safe handoff timeout without create-attempt identity (#10769)", async () => { + const input = noGpuInput(); + input.persistStartupCommand = true; + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, _args, _env, options) => { + expect(options.waitForReadyTermination).toBe(true); + expect(options.readyCheck?.()).toBe(true); + return createTimedOutCreateResult( + "Created sandbox: alpha\nOpenShell create client did not exit after Ready; aborting cutover.", + ); + }); + const deps = createGpuFlowDeps(); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "No create-attempt identity was available for retained recovery", + ); + + expect(input.persistRetainedSandboxRecovery).not.toHaveBeenCalled(); + expectNoEffects(input, patch, deps, mocks.waitForCreatedSandboxReadyWithTrace); + }); + + it.each([ + ["returns false", (): boolean => false], + [ + "throws", + (): boolean => { + throw new Error("recovery writer failed"); + }, + ], + ] as const)( + "blocks the create after retained recovery persistence %s (#10769)", + async (_failureMode, persistRecovery) => { + let nonce = ""; + const input = noGpuInput(); + input.persistRetainedSandboxRecovery = vi.fn(persistRecovery); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { + nonce = createAttemptNonce(args); + expect(options.readyCheck?.()).toBe(true); + return createTimedOutCreateResult( + "OpenShell create client did not exit after Ready; aborting cutover.", + ); + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell) + .mockReturnValueOnce("alpha Ready") + .mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "NemoClaw could not save the retained sandbox recovery record for this create attempt", + ); + + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining( + `Durable sandbox identity fingerprint: ${ALPHA_SANDBOX_IDENTITY_FINGERPRINT}`, + ), + ALPHA_SANDBOX_IDENTITY_FINGERPRINT, + nonce, + ); + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(output).toContain(`${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`); + expect(output).toContain( + "NemoClaw could not save the retained sandbox recovery record for this create attempt", + ); + expect(output).not.toContain("alpha-sandbox-id"); + expectNoEffects(input, patch, deps, mocks.waitForCreatedSandboxReadyWithTrace); + }, + ); }); diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index a136730c190..2db434675a5 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { randomBytes } from "node:crypto"; +import { performance } from "node:perf_hooks"; import { mergeIsolatedDockerClientEnv, @@ -17,16 +18,13 @@ import { resolveOpenShellSandboxId, settleCreatedOpenShellSandboxId, } from "../adapters/openshell/sandbox-identity"; -import { printSandboxCreateRecoveryHints } from "../build-context"; +import { createReadinessWaitOptions } from "../core/readiness-wait"; +import { waitUntil } from "../core/wait"; import { streamSandboxCreate, type StreamSandboxCreateResult } from "../sandbox/create-stream"; import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-ready-gate"; -import { isSandboxReady } from "../state/gateway"; +import { redact, redactFullWithUrls } from "../security/redact"; import type { SandboxGpuProofResult } from "../state/registry"; import { classifySandboxCreateFailure } from "../validation"; -import { - formatRetainedSandboxRecoveryMessage, - reportSandboxCreateFailure, -} from "./created-sandbox-failure"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; @@ -45,13 +43,23 @@ import { import { printSandboxCreateFailureDiagnostics } from "./sandbox-create-failure"; import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; import type { + CreatedSandboxIdentity, SandboxGpuCreateFlowDeps, SandboxGpuCreateFlowInput, } from "./sandbox-gpu-create-flow"; -import { fingerprintSandboxRecreateValue } from "./sandbox-recreate-transaction"; +import { + fingerprintSandboxRecreateValue, + formatRetainedSandboxRecoveryDetail, +} from "./sandbox-recreate-transaction"; import * as sandboxGpuPreflight from "./sandbox-gpu-preflight"; -import { SANDBOX_RECREATE_PROBE_TIMEOUT_MS } from "./sandbox-recreate-probe"; -import type { CreatedSandboxReadyIdentityCheck } from "./sandbox-readiness-tracing"; +import { + isExplicitMissingSandboxGatewayOutput, + SANDBOX_RECREATE_PROBE_TIMEOUT_MS, +} from "./sandbox-recreate-probe"; +import type { + CreatedSandboxReadinessResult, + CreatedSandboxReadyIdentityCheck, +} from "./sandbox-readiness-tracing"; import * as sandboxReadinessTracing from "./sandbox-readiness-tracing"; import { addTraceEvent } from "./tracing"; @@ -65,6 +73,7 @@ export type SandboxGpuCreateAttemptState = { allowUnbuiltCompatibilitySource: boolean; nativeRuntimeSnapshot: NativeRuntimeSnapshot | null; portableLifecycleGeneration: string | null; + verifiedCreatedSandboxIdentity: CreatedSandboxIdentity | null; }; // A runtime-managed container replacement can briefly observe the original @@ -72,7 +81,62 @@ export type SandboxGpuCreateAttemptState = { // to live validation or the GPU proof. const REPLACEMENT_STABLE_READY_POLLS = 2; const SANDBOX_READY_PROBE_TIMEOUT_MS = 5_000; -const CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_SECONDS = 1; +const CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_MS = 1_000; +const CREATED_SANDBOX_PUBLICATION_DIAGNOSTIC_LIMIT = 1_000; + +type PostCreateReadinessDeadline = Readonly<{ + deadlineMs: number; + now: () => number; +}>; + +function createPostCreateReadinessDeadline( + input: SandboxGpuCreateFlowInput, + deps: SandboxGpuCreateFlowDeps, +): PostCreateReadinessDeadline { + const now = deps.publicationNow ?? (() => performance.now()); + return { + deadlineMs: now() + Math.max(1, Math.round(input.sandboxReadyTimeoutSecs * 1_000)), + now, + }; +} + +function remainingPostCreateReadinessMs(deadline: PostCreateReadinessDeadline): number { + return Math.max(0, deadline.deadlineMs - deadline.now()); +} + +function redactCreatedSandboxFailureDiagnostic(value: string, limit: number): string { + return redactFullWithUrls(value).replace(/\s+/gu, " ").trim().slice(0, limit); +} + +function reportSandboxCreateFailure(options: { + readonly sandboxName: string; + readonly createStatus: number; + readonly createOutput: string; + readonly restoreBackupPath: string | null; + readonly createArgs: readonly string[]; + readonly printCreateFailureDiagnostics: ( + sandboxName: string, + options: { backupPath: string | null }, + ) => void; + readonly printCreateRecoveryHints: ( + output: string, + options: { readonly createArgs: readonly string[] }, + ) => void; +}): never { + const redactedCreateOutput = redact(options.createOutput); + console.error(""); + console.error(` Sandbox creation failed (exit ${options.createStatus}).`); + if (options.createOutput) { + console.error(""); + console.error(redactedCreateOutput); + } + options.printCreateFailureDiagnostics(options.sandboxName, { + backupPath: options.restoreBackupPath, + }); + console.error(" Try: openshell sandbox list # check gateway state"); + options.printCreateRecoveryHints(redactedCreateOutput, { createArgs: options.createArgs }); + return process.exit(options.createStatus === 0 ? 1 : options.createStatus); +} async function streamSandboxCreateWithPublicImageCredentialIsolation( isolate: boolean, @@ -159,6 +223,19 @@ type NativeFallbackCleanupEvidence = Readonly<{ nativeCleanupHandoff?: ManagedBootstrapNativeGpuFallbackOwnerCleanupHandoff; }>; +function warnForAuthorizedCompatibilityRetry( + route: SelectedDockerGpuRoute, + initialGpuRoute: SelectedDockerGpuRoute, +): void { + if (route !== "compatibility" || initialGpuRoute !== "native") return; + console.warn( + " Native OpenShell GPU onboarding did not complete; retrying once by recreating the OpenShell-managed Docker container with the legacy GPU compatibility envelope.", + ); + console.warn( + " This compatibility container swap may relax container confinement compared with native injection. The retry is running only because NEMOCLAW_DOCKER_GPU_PATCH=fallback explicitly authorized it.", + ); +} + async function rollbackNativeGpuFailureForFallback( managedLifecycle: ManagedBootstrapRuntimeCreateLifecycle | null, runtimePatch: ManagedBootstrapRuntimePatch, @@ -182,6 +259,20 @@ function normalizedOpenShellCommandOutput(result: OpenShellCommandResult): strin .trim(); } +function boundedPublicationDiagnostic(value: string): string { + return redactCreatedSandboxFailureDiagnostic(value, CREATED_SANDBOX_PUBLICATION_DIAGNOSTIC_LIMIT); +} + +function publicationFailureDiagnostic(result: OpenShellCommandResult): string { + const commandOutput = normalizedOpenShellCommandOutput(result); + const processError = + result.error instanceof Error ? result.error.message : String(result.error ?? ""); + const status = result.status === null ? "no exit status" : `exit ${result.status}`; + return boundedPublicationDiagnostic( + [status, processError, commandOutput].filter((value) => value.length > 0).join(": "), + ); +} + type OpenShellSandboxIdentityProbe = | { state: "identified"; sandboxId: string } | { state: "not_ready" } @@ -244,16 +335,215 @@ async function verifyCreatedSandboxBeforeEffects( createAttemptNonce: string, route: SelectedDockerGpuRoute, input: SandboxGpuCreateFlowInput, -): Promise { - if (!input.verifyCreatedSandboxBeforeEffects) return; - await input.verifyCreatedSandboxBeforeEffects({ +): Promise { + const identity = { sandboxId, liveIdentityFingerprint: fingerprintSandboxRecreateValue(sandboxId), createAttemptNonce, route, + }; + await input.verifyCreatedSandboxBeforeEffects?.(identity); + return identity; +} + +export function persistRetainedSandboxRecoveryOrBlock(options: { + readonly persist: NonNullable; + readonly message: string; + readonly createAttemptNonce: string; + readonly sandboxIdentityFingerprint?: string; + readonly persistenceFailureDiagnostic?: string; + readonly persistenceFailureMessage?: string; +}): void { + const { persist, message, createAttemptNonce, sandboxIdentityFingerprint } = options; + let persistenceFailure: unknown = null; + try { + if (!persist(message, sandboxIdentityFingerprint, createAttemptNonce)) { + persistenceFailure = new Error( + "The retained sandbox recovery writer did not confirm durable persistence.", + ); + } + } catch (error) { + persistenceFailure = error; + } + console.error(` ${message}`); + if (!persistenceFailure) return; + console.error( + options.persistenceFailureDiagnostic ?? + " NemoClaw could not save this create-attempt evidence. The recovery-only session remains blocked until its durable recovery record can be saved.", + ); + throw new Error( + options.persistenceFailureMessage ?? + "NemoClaw could not save the retained sandbox recovery record; the recovery-only session remains blocked.", + { cause: persistenceFailure }, + ); +} + +function persistCreateAttemptRecovery(options: { + readonly input: SandboxGpuCreateFlowInput; + readonly createAttemptNonce: string | null; + readonly detail: string; + readonly sandboxIdentityFingerprint?: string; + readonly persistenceFailureDiagnostic?: string; + readonly persistenceFailureMessage?: string; +}): void { + const { input, createAttemptNonce, detail, sandboxIdentityFingerprint } = options; + if (!createAttemptNonce) { + throw new Error("Sandbox create-attempt identity was not generated."); + } + const persist = input.persistRetainedSandboxRecovery; + if (!persist) { + throw new Error("Verified sandbox creation has no durable recovery evidence owner."); + } + const message = + `Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${createAttemptNonce}. ` + + (sandboxIdentityFingerprint + ? `Durable sandbox identity fingerprint: ${sandboxIdentityFingerprint}. ` + : "") + + detail; + persistRetainedSandboxRecoveryOrBlock({ + persist, + message, + createAttemptNonce, + ...(sandboxIdentityFingerprint ? { sandboxIdentityFingerprint } : {}), + ...(options.persistenceFailureDiagnostic + ? { persistenceFailureDiagnostic: options.persistenceFailureDiagnostic } + : {}), + ...(options.persistenceFailureMessage + ? { persistenceFailureMessage: options.persistenceFailureMessage } + : {}), }); } +function persistIdentitySettlementRecoveryEvidence(options: { + readonly input: SandboxGpuCreateFlowInput; + readonly createAttemptNonce: string | null; + readonly sandboxIdentityFingerprint: string | null; + readonly failureDiagnostic?: string; +}): void { + const { input, createAttemptNonce, sandboxIdentityFingerprint } = options; + const identityEvidence = formatRetainedSandboxRecoveryDetail({ + sandboxName: input.sandboxName, + gatewayName: input.gatewayName, + sandboxIdentityFingerprint, + }); + persistCreateAttemptRecovery({ + input, + createAttemptNonce, + detail: + `${identityEvidence} ` + + (options.failureDiagnostic ? `OpenShell detail: ${options.failureDiagnostic}.` : ""), + sandboxIdentityFingerprint: sandboxIdentityFingerprint ?? undefined, + persistenceFailureDiagnostic: + " NemoClaw could not save the retained sandbox recovery record for this create attempt. Preserve the terminal output for an OpenShell administrator.", + persistenceFailureMessage: + "NemoClaw could not save the retained sandbox recovery record for this create attempt.", + }); +} + +function persistPostVerificationReadinessRecovery(options: { + readonly input: SandboxGpuCreateFlowInput; + readonly createAttemptNonce: string | null; + readonly verifiedIdentity: CreatedSandboxIdentity | null; + readonly readiness: CreatedSandboxReadinessResult; +}): void { + if (!options.verifiedIdentity) return; + const { input } = options; + persistCreateAttemptRecovery({ + input, + createAttemptNonce: options.createAttemptNonce, + detail: `Post-verification readiness detail: ${boundedPublicationDiagnostic(sandboxReadinessTracing.formatCreatedSandboxReadinessFailureMessage(input.sandboxName, options.readiness, input.sandboxReadyTimeoutSecs).trim())}`, + sandboxIdentityFingerprint: options.verifiedIdentity.liveIdentityFingerprint, + }); +} + +async function confirmManagedRuntimeCommitReadiness(options: { + readonly input: SandboxGpuCreateFlowInput; + readonly deps: SandboxGpuCreateFlowDeps; + readonly sandboxId: string | null; + readonly createAttemptNonce: string | null; + readonly deadline: PostCreateReadinessDeadline; +}): Promise { + const { input, deps, sandboxId } = options; + if (!sandboxId) return; + input.revalidateVerifiedSandboxBeforeEffect?.( + `confirm committed runtime readiness for sandbox '${input.sandboxName}'`, + ); + const committedReadiness = await sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ + sandboxName: input.sandboxName, + timeoutSecs: remainingPostCreateReadinessMs(options.deadline) / 1_000, + observer: deps.sandboxObserver, + target: { kind: "named", gatewayName: input.gatewayName }, + stableReadyPolls: REPLACEMENT_STABLE_READY_POLLS, + checkReadyIdentity: (getRemainingMs = () => SANDBOX_RECREATE_PROBE_TIMEOUT_MS) => + checkRecreatedSandboxReadyIdentity( + input.sandboxName, + input.gatewayName, + sandboxId, + deps, + getRemainingMs, + ), + sleep: deps.sleep, + now: options.deadline.now, + }); + if (committedReadiness.ready) return; + console.error(""); + sandboxReadinessTracing.printReadinessFailure( + committedReadiness, + input.sandboxName, + input.sandboxReadyTimeoutSecs, + ); + const sandboxIdentityFingerprint = fingerprintSandboxRecreateValue(sandboxId); + persistCreateAttemptRecovery({ + input, + createAttemptNonce: options.createAttemptNonce, + sandboxIdentityFingerprint, + detail: + `Managed runtime commit completed for sandbox '${input.sandboxName}', but the same sandbox did not return to executable Ready state through owning gateway '${input.gatewayName}'. ` + + "Do not delete a sandbox by mutable name; preserve it for identity-bound recovery.", + }); + (deps.printCreateFailureDiagnostics ?? printSandboxCreateFailureDiagnostics)(input.sandboxName, { + backupPath: input.restoreBackupPath, + }); + console.error( + " NemoClaw did not start dashboard forwarding. NemoClaw left the sandbox in place for identity-bound recovery.", + ); + console.error( + ` Do not delete sandbox '${input.sandboxName}' by name. Give the create-attempt label above to an OpenShell administrator and ask them to remove that exact sandbox and reconcile its retained recovery state through an identity-bound procedure.`, + ); + throw new Error( + `Sandbox '${input.sandboxName}' did not return to Ready after its managed runtime commit.`, + ); +} + +async function requireManagedBootstrapCreatedSandboxReady(options: { + readonly input: SandboxGpuCreateFlowInput; + readonly deps: SandboxGpuCreateFlowDeps; + readonly createAttemptNonce: string | null; + readonly persistIdentitySettlementRecovery: () => void; + readonly deadline: PostCreateReadinessDeadline; +}): Promise { + const readiness = await sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ + sandboxName: options.input.sandboxName, + timeoutSecs: remainingPostCreateReadinessMs(options.deadline) / 1_000, + observer: options.deps.sandboxObserver, + target: { kind: "named", gatewayName: options.input.gatewayName }, + stableReadyPolls: 1, + sleep: options.deps.sleep, + now: options.deadline.now, + }); + if (readiness.ready) return; + if (options.createAttemptNonce) options.persistIdentitySettlementRecovery(); + throw new Error( + sandboxReadinessTracing + .formatCreatedSandboxReadinessFailureMessage( + options.input.sandboxName, + readiness, + options.input.sandboxReadyTimeoutSecs, + ) + .trimStart(), + ); +} + function resolveCreateAttemptNonce( input: SandboxGpuCreateFlowInput, deferPostCreateEffects: boolean, @@ -280,47 +570,89 @@ function waitForCreatedOpenShellSandboxPublication( sandboxId: string, input: SandboxGpuCreateFlowInput, deps: SandboxGpuCreateFlowDeps, + deadline: PostCreateReadinessDeadline, ): void { - const timeoutMs = Math.max(1, Math.round(input.sandboxReadyTimeoutSecs * 1_000)); - const deadlineMs = Date.now() + timeoutMs; - const maxPolls = - Math.ceil(timeoutMs / (CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_SECONDS * 1_000)) + 1; - for (let poll = 0; poll < maxPolls; poll += 1) { - const remainingMs = Math.max(1, deadlineMs - Date.now()); - const result = deps.runOpenshell( - ["sandbox", "get", "-g", input.gatewayName, input.sandboxName], - { - ignoreError: true, - suppressOutput: true, - timeout: Math.min(SANDBOX_READY_PROBE_TIMEOUT_MS, remainingMs), - killSignal: "SIGKILL", - }, + const waitOptions = createReadinessWaitOptions({ + budgetMs: remainingPostCreateReadinessMs(deadline), + initialIntervalMs: CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_MS, + maxIntervalMs: CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_MS, + now: deadline.now, + sleep: (milliseconds) => deps.sleep(milliseconds / 1_000), + }); + const deadlineMs = waitOptions?.deadlineMs; + const now = waitOptions?.now; + const published = + waitOptions && deadlineMs !== undefined && now + ? waitUntil(() => { + const getRemainingMs = () => Math.max(1, deadlineMs - now()); + const result = deps.runOpenshell( + ["sandbox", "get", "-g", input.gatewayName, input.sandboxName], + { + ignoreError: true, + suppressOutput: true, + timeout: Math.min(SANDBOX_READY_PROBE_TIMEOUT_MS, getRemainingMs()), + killSignal: "SIGKILL", + }, + ); + if (result.status === 0 && !result.error) { + const publishedSandboxId = parseOpenShellSandboxId(String(result.stdout ?? "")); + if (!publishedSandboxId) { + throw new Error( + `OpenShell returned no exact durable ID for created sandbox '${input.sandboxName}'.`, + ); + } + if (publishedSandboxId !== sandboxId) { + throw new Error( + `Created sandbox '${input.sandboxName}' changed identity before identity verification completed.`, + ); + } + return true; + } + const output = normalizedOpenShellCommandOutput(result); + const failedCleanly = + !result.error && + result.status !== null && + !("signal" in result && result.signal) && + result.status !== 0; + if ( + failedCleanly && + (OPENSHELL_SANDBOX_NOT_READY.test(output) || + isExplicitMissingSandboxGatewayOutput(output, input.sandboxName)) + ) { + return false; + } + const diagnostic = publicationFailureDiagnostic(result); + throw new Error( + `OpenShell could not verify publication of created sandbox '${input.sandboxName}'${diagnostic ? `: ${diagnostic}` : "."}`, + ); + }, waitOptions) + : false; + if (!published) { + throw new Error( + `Created sandbox '${input.sandboxName}' did not become visible through its owning gateway before identity verification completed.`, ); - if (result.status === 0 && !result.error) { - const publishedSandboxId = parseOpenShellSandboxId(String(result.stdout ?? "")); - if (!publishedSandboxId) { - throw new Error( - `OpenShell returned no exact durable ID for created sandbox '${input.sandboxName}'.`, - ); - } - if (publishedSandboxId !== sandboxId) { - throw new Error( - `Created sandbox '${input.sandboxName}' changed identity before identity verification completed.`, - ); - } - return; - } - if (poll + 1 >= maxPolls || Date.now() >= deadlineMs) break; - deps.sleep( - Math.min( - CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_SECONDS, - Math.max(0, (deadlineMs - Date.now()) / 1_000), - ), + } +} + +function waitForCreatedSandboxPublicationOrPersist( + sandboxId: string, + input: SandboxGpuCreateFlowInput, + deps: SandboxGpuCreateFlowDeps, + deadline: PostCreateReadinessDeadline, + persistIdentitySettlementRecovery: ( + sandboxIdentityFingerprint: string, + failureDiagnostic?: string, + ) => void, +): void { + try { + waitForCreatedOpenShellSandboxPublication(sandboxId, input, deps, deadline); + } catch (error) { + persistIdentitySettlementRecovery( + fingerprintSandboxRecreateValue(sandboxId), + boundedPublicationDiagnostic(error instanceof Error ? error.message : String(error)), ); + throw error; } - throw new Error( - `Created sandbox '${input.sandboxName}' did not become visible through its owning gateway before identity verification completed.`, - ); } function checkRecreatedSandboxReadyIdentity( @@ -412,6 +744,7 @@ export function createSandboxGpuCreateAttemptRunner( allowUnbuiltCompatibilitySource: false, nativeRuntimeSnapshot: null, portableLifecycleGeneration: null, + verifiedCreatedSandboxIdentity: null, }; const revalidatePostCreateEffect = (operation: string): void => { if (!input.verifyCreatedSandboxBeforeEffects) return; @@ -440,62 +773,38 @@ export function createSandboxGpuCreateAttemptRunner( const runAttempt = async (route: SelectedDockerGpuRoute) => { const deferPostCreateEffects = input.verifyCreatedSandboxBeforeEffects !== undefined; const compatibility = route === "compatibility"; - if (compatibility && input.initialGpuRoute === "native") { - console.warn( - " Native OpenShell GPU onboarding did not complete; retrying once by recreating the OpenShell-managed Docker container with the legacy GPU compatibility envelope.", - ); - console.warn( - " This compatibility container swap may relax container confinement compared with native injection. The retry is running only because NEMOCLAW_DOCKER_GPU_PATCH=fallback explicitly authorized it.", - ); - } + warnForAuthorizedCompatibilityRetry(route, input.initialGpuRoute); const hasRequiredUlimits = (input.requiredUlimits?.length ?? 0) > 0; const managedBootstrap = input.managedBootstrap ?? null; const unboundAttemptArgv = state.compatibilityArgv ?? input.createArgv; if (input.requirePolicylessCreate) assertPolicylessSandboxCreateArgv(unboundAttemptArgv); const createAttemptNonce = resolveCreateAttemptNonce(input, deferPostCreateEffects); + let postCreateReadinessDeadline: PostCreateReadinessDeadline | null = null; + const requirePostCreateReadinessDeadline = (): PostCreateReadinessDeadline => { + postCreateReadinessDeadline ??= createPostCreateReadinessDeadline(input, deps); + return postCreateReadinessDeadline; + }; + const remainingPostCreateReadinessSecs = (): number => + remainingPostCreateReadinessMs(requirePostCreateReadinessDeadline()) / 1_000; const persistIdentitySettlementRecovery = ( sandboxIdentityFingerprint: string | null = null, + failureDiagnostic?: string, ): void => { - if (!createAttemptNonce) { - throw new Error("Sandbox create-attempt identity was not generated."); - } - const persist = input.persistRetainedSandboxRecovery; - if (!persist) { - throw new Error("Verified sandbox creation has no durable recovery evidence owner."); - } - const message = formatRetainedSandboxRecoveryMessage({ - sandboxName: input.sandboxName, - gatewayName: input.gatewayName, - createAttemptLabel: `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${createAttemptNonce}`, + persistIdentitySettlementRecoveryEvidence({ + input, + createAttemptNonce, sandboxIdentityFingerprint, + ...(failureDiagnostic ? { failureDiagnostic } : {}), }); - let persisted = false; - let persistenceCause: unknown; - try { - persisted = sandboxIdentityFingerprint - ? persist(message, sandboxIdentityFingerprint, createAttemptNonce) - : persist(message, undefined, createAttemptNonce); - } catch (error) { - persistenceCause = error; - } - console.error(` ${message}`); - if (!persisted) { - const persistenceFailureMessage = - "NemoClaw could not save the retained sandbox recovery record for this create attempt."; - console.error( - ` ${persistenceFailureMessage} Preserve the terminal output for an OpenShell administrator.`, - ); - throw new Error(persistenceFailureMessage, { cause: persistenceCause }); - } - }; - const waitForCreatedSandboxPublication = (sandboxId: string): void => { - try { - waitForCreatedOpenShellSandboxPublication(sandboxId, input, deps); - } catch (error) { - persistIdentitySettlementRecovery(fingerprintSandboxRecreateValue(sandboxId)); - throw error; - } }; + const waitForCreatedSandboxPublication = (sandboxId: string): void => + waitForCreatedSandboxPublicationOrPersist( + sandboxId, + input, + deps, + requirePostCreateReadinessDeadline(), + persistIdentitySettlementRecovery, + ); const captureRetainedSandboxRecovery = () => { if (!input.requirePolicylessCreate || !createAttemptNonce) return {}; let liveIdentityFingerprint: string | null = null; @@ -618,12 +927,15 @@ export function createSandboxGpuCreateAttemptRunner( }; const settleCreatedIdentity = (): string => { if (readyCheckCreatedIdentityFailure !== null) throw readyCheckCreatedIdentityFailure; + const deadline = requirePostCreateReadinessDeadline(); const sandboxId = settleCreatedOpenShellSandboxId({ sandboxName: input.sandboxName, gatewayName: input.gatewayName, createAttemptNonce: createAttemptNonce!, runCaptureOpenshell: deps.runCaptureOpenshell, priorSandboxId: readyCheckCreatedSandboxId, + now: deadline.now, + timeoutMs: remainingPostCreateReadinessMs(deadline), sleep: (milliseconds) => deps.sleep(milliseconds / 1000), }); if (readyCheckCreatedSandboxId && sandboxId !== readyCheckCreatedSandboxId) { @@ -644,7 +956,7 @@ export function createSandboxGpuCreateAttemptRunner( ignoreError: true, timeout: SANDBOX_READY_PROBE_TIMEOUT_MS, }); - const ready = isSandboxReady(list, input.sandboxName); + const ready = deps.isSandboxReady(list, input.sandboxName); if (!ready || !createAttemptNonce) return ready; const observation = observeCreatedOpenShellSandboxId( { @@ -713,6 +1025,7 @@ export function createSandboxGpuCreateAttemptRunner( }; let createResult: Awaited> | null = null; let resumedSandboxId: string | null = null; + let verifiedCreatedSandboxId: string | null = null; let managedIncompleteCreateRecovered = false; let createdSandboxVerified = false; const failAfterCreatedSandboxVerification = (message: string, status: number): never => { @@ -720,6 +1033,7 @@ export function createSandboxGpuCreateAttemptRunner( return process.exit(status); }; if (input.resumeVerifiedCreate) { + requirePostCreateReadinessDeadline(); if (route !== input.resumeVerifiedCreate.route) { throw new Error("Verified sandbox recovery route changed before continuation."); } @@ -736,7 +1050,8 @@ export function createSandboxGpuCreateAttemptRunner( ); } resumedSandboxId = identity.sandboxId; - await verifyCreatedSandboxBeforeEffects( + verifiedCreatedSandboxId = identity.sandboxId; + state.verifiedCreatedSandboxIdentity = await verifyCreatedSandboxBeforeEffects( identity.sandboxId, createAttemptNonce!, route, @@ -761,6 +1076,7 @@ export function createSandboxGpuCreateAttemptRunner( ); } const result = await streamCreate(); + requirePostCreateReadinessDeadline(); const createFailure = result.status === 0 ? null : classifySandboxCreateFailure(result.output); if (result.status !== 0 && createFailure?.kind !== "sandbox_create_incomplete") { @@ -769,11 +1085,12 @@ export function createSandboxGpuCreateAttemptRunner( if (createFailure?.kind === "sandbox_create_incomplete") { const readiness = await sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ sandboxName: input.sandboxName, - timeoutSecs: input.sandboxReadyTimeoutSecs, + timeoutSecs: remainingPostCreateReadinessSecs(), observer: deps.sandboxObserver, target: { kind: "named", gatewayName: input.gatewayName }, stableReadyPolls: REPLACEMENT_STABLE_READY_POLLS, sleep: deps.sleep, + now: requirePostCreateReadinessDeadline().now, }); if (!readiness.ready) { if (createAttemptNonce) persistIdentitySettlementRecovery(); @@ -788,27 +1105,13 @@ export function createSandboxGpuCreateAttemptRunner( ); } } else { - const observation = await sandboxReadinessTracing.observeOpenShellSandbox( - deps.sandboxObserver, - { kind: "named", gatewayName: input.gatewayName }, - input.sandboxName, - SANDBOX_READY_PROBE_TIMEOUT_MS, - ); - if (!observation.ok) { - if (createAttemptNonce) persistIdentitySettlementRecovery(); - throw new Error( - `Managed bootstrap create completed, but NemoClaw could not observe the sandbox. ${observation.error.message}`, - ); - } - if ( - observation.value.state !== "present" || - observation.value.sandbox.readiness !== "ready" - ) { - if (createAttemptNonce) persistIdentitySettlementRecovery(); - throw new Error( - "Managed bootstrap create completed without an authoritative Ready sandbox.", - ); - } + await requireManagedBootstrapCreatedSandboxReady({ + input, + deps, + createAttemptNonce, + persistIdentitySettlementRecovery, + deadline: requirePostCreateReadinessDeadline(), + }); } let sandboxId: string; try { @@ -831,7 +1134,13 @@ export function createSandboxGpuCreateAttemptRunner( ); } waitForCreatedSandboxPublication(sandboxId); - await verifyCreatedSandboxBeforeEffects(sandboxId, createAttemptNonce!, route, input); + verifiedCreatedSandboxId = sandboxId; + state.verifiedCreatedSandboxIdentity = await verifyCreatedSandboxBeforeEffects( + sandboxId, + createAttemptNonce!, + route, + input, + ); createdSandboxVerified = true; if (deferPostCreateEffects) { revalidatePostCreateEffect( @@ -930,23 +1239,15 @@ export function createSandboxGpuCreateAttemptRunner( } as const; } else { await runtimePatch.rollbackManagedStartupAfterCreateFailure(); - reportSandboxCreateFailure( - { - sandboxName: input.sandboxName, - createStatus: createResult.status, - createOutput: createResult.output, - restoreBackupPath: input.restoreBackupPath, - createArgs: input.prebuild.createArgs, - }, - { - classifyCreateFailure: classifySandboxCreateFailure, - printCreateFailureDiagnostics, - printRecoveryHints: printSandboxCreateRecoveryHints, - warn: (message) => console.warn(message), - error: (message) => console.error(message), - exitProcess: (code) => process.exit(code), - }, - ); + reportSandboxCreateFailure({ + sandboxName: input.sandboxName, + createStatus: createResult.status, + createOutput: createResult.output, + restoreBackupPath: input.restoreBackupPath, + createArgs: input.prebuild.createArgs, + printCreateFailureDiagnostics, + printCreateRecoveryHints: deps.printCreateRecoveryHints, + }); } } if (!createdSandboxVerified && deferPostCreateEffects) { @@ -964,7 +1265,13 @@ export function createSandboxGpuCreateAttemptRunner( ); } waitForCreatedSandboxPublication(sandboxId); - await verifyCreatedSandboxBeforeEffects(sandboxId, createAttemptNonce!, route, input); + verifiedCreatedSandboxId = sandboxId; + state.verifiedCreatedSandboxIdentity = await verifyCreatedSandboxBeforeEffects( + sandboxId, + createAttemptNonce!, + route, + input, + ); createdSandboxVerified = true; } if (deferPostCreateEffects) { @@ -1000,7 +1307,7 @@ export function createSandboxGpuCreateAttemptRunner( console.log(" Waiting for sandbox to become ready..."); const readiness = await sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ sandboxName: input.sandboxName, - timeoutSecs: input.sandboxReadyTimeoutSecs, + timeoutSecs: remainingPostCreateReadinessSecs(), observer: deps.sandboxObserver, target: { kind: "named", gatewayName: input.gatewayName }, stableReadyPolls: @@ -1026,14 +1333,9 @@ export function createSandboxGpuCreateAttemptRunner( getRemainingMs, ), sleep: deps.sleep, + now: requirePostCreateReadinessDeadline().now, }); if (!readiness.ready) { - console.error(""); - sandboxReadinessTracing.printReadinessFailure( - readiness, - input.sandboxName, - input.sandboxReadyTimeoutSecs, - ); const canClassifyNativeReadiness = route === "native" && input.gpuRoutePlan === "native-with-fallback" && @@ -1069,6 +1371,18 @@ export function createSandboxGpuCreateAttemptRunner( ...nativeCleanup, } as const; } + persistPostVerificationReadinessRecovery({ + input, + createAttemptNonce, + verifiedIdentity: state.verifiedCreatedSandboxIdentity, + readiness, + }); + console.error(""); + sandboxReadinessTracing.printReadinessFailure( + readiness, + input.sandboxName, + input.sandboxReadyTimeoutSecs, + ); await runtimePatch.rollbackManagedStartupAfterCreateFailure(); printCreateFailureDiagnostics(input.sandboxName, { backupPath: input.restoreBackupPath, @@ -1171,10 +1485,36 @@ export function createSandboxGpuCreateAttemptRunner( revalidatePostCreateEffect(`commit runtime readiness for sandbox '${input.sandboxName}'`); await runtimePatch.commitAfterReady(); } + let committedRuntimeReadinessDeadline: PostCreateReadinessDeadline | null = null; + const confirmCommittedRuntimeReadiness = () => { + // GPU local-inference validation has its own timeout and runs before + // commit. Start this budget only when the caller has committed the + // runtime so slow inference validation cannot consume Ready recovery. + committedRuntimeReadinessDeadline ??= createPostCreateReadinessDeadline(input, deps); + return confirmManagedRuntimeCommitReadiness({ + input, + deps, + sandboxId: managedBootstrap ? verifiedCreatedSandboxId : null, + createAttemptNonce, + deadline: committedRuntimeReadinessDeadline, + }); + }; + if (!input.sandboxGpuConfig.sandboxGpuEnabled) { + await confirmCommittedRuntimeReadiness(); + } return { ok: true, route, - value: createResult ? { createResult, runtimePatch } : { runtimePatch }, + value: createResult + ? { + createResult, + runtimePatch, + confirmManagedRuntimeCommitReadiness: confirmCommittedRuntimeReadiness, + } + : { + runtimePatch, + confirmManagedRuntimeCommitReadiness: confirmCommittedRuntimeReadiness, + }, } as const; }; diff --git a/src/lib/onboard/sandbox-readiness-tracing.test.ts b/src/lib/onboard/sandbox-readiness-tracing.test.ts index 0701b040248..aa49d4ae27f 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.test.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.test.ts @@ -81,7 +81,6 @@ describe("createSandboxReadyWaiter", () => { isLinuxDockerDriverGatewayEnabled: () => true, now: () => 0, sleep, - now: () => 0, }); await expect(waitForSandboxReady(NAME, 2, 3)).resolves.toEqual({ diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index 6225eca9a82..a99ccbe859e 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -15,13 +15,14 @@ import { createCliOpenShellSandboxObserver, type CliOpenShellSandboxObserverDeps, } from "../adapters/openshell/sandbox-observer-cli"; -import { waitUntil, waitUntilAsync } from "../core/wait"; -import { envInt } from "./env"; import { createReadinessWaitOptions, formatReadinessDeadline, getLegacyPollDeadlineBudgetMs, -} from "./readiness-wait"; + waitUntil, + waitUntilAsync, +} from "../core/readiness-wait"; +import { envInt } from "./env"; import { addTraceEvent, withDashboardReadinessTrace, withSandboxReadinessTrace } from "./tracing"; type RunCaptureOpenshell = (args: string[], options?: { ignoreError?: boolean }) => string; diff --git a/src/lib/onboard/sandbox-recreate-transaction.ts b/src/lib/onboard/sandbox-recreate-transaction.ts index 33a22cd2777..525572ae82e 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.ts @@ -19,6 +19,7 @@ import { type RuntimeProviderWorkloadCleanupResult, requireRuntimeProviderDestructiveCleanupAuthority, } from "./runtime-provider/access"; +import { cliName } from "./branding"; import type { SandboxCreateIntent } from "./types"; const ORDERED_PHASES: readonly CheckpointSandboxRecreatePhase[] = [ @@ -46,6 +47,28 @@ export type ReplacedSandboxSourceEntry = Omit & { }; }; +/** Format recovery detail without authorizing mutable-name deletion or an unsafe retry. */ +export function formatRetainedSandboxRecoveryDetail(input: { + readonly sandboxName: string; + readonly gatewayName: string; + readonly sandboxIdentityFingerprint: string | null; +}): string { + if (!input.sandboxIdentityFingerprint) { + return ( + `Sandbox '${input.sandboxName}' reached Ready before OpenShell returned one exact durable create identity. Gateway '${input.gatewayName}'. ` + + "OpenShell did not return one exact durable sandbox identity for this create attempt. " + + "Do not delete a sandbox by mutable name; preserve it until an OpenShell administrator resolves the create-attempt label to one sandbox." + ); + } + return ( + `NemoClaw stopped before owning-gateway publication and identity verification completed for sandbox '${input.sandboxName}' through gateway '${input.gatewayName}'. ` + + `Do not delete the sandbox by mutable name. Run '${cliName()} ${input.sandboxName} destroy'. ` + + "If OpenShell reports the sandbox present, the command removes nothing and preserves the recovery record. " + + "Give the create-attempt label to an OpenShell administrator for identity-bound removal. " + + `After OpenShell confirms removal, run '${cliName()} ${input.sandboxName} destroy --yes' to reconcile the recovery record.` + ); +} + interface ReplacedSandboxWorkloadCleanupDeps { readonly runtimeProviders?: RuntimeProviderBundleRegistry; } diff --git a/test/automation/pull-requests/pr-risk-plan.test.ts b/test/automation/pull-requests/pr-risk-plan.test.ts index 694a91b9c67..ea106420aff 100644 --- a/test/automation/pull-requests/pr-risk-plan.test.ts +++ b/test/automation/pull-requests/pr-risk-plan.test.ts @@ -869,7 +869,6 @@ describe("deterministic PR risk plan", () => { "src/lib/actions/sandbox/status-snapshot.ts", "src/lib/onboard/docker-driver-sandbox-recovery.ts", "src/lib/onboard/docker-startup-command-agent.ts", - "src/lib/onboard/sandbox-create-step.ts", ])("selects post-reboot recovery for Docker delivery changes in %s (#7824)", (changedFile) => { const result = plan(changedFile); const adjacentStatusFile = plan("src/lib/actions/sandbox/status-text.ts"); diff --git a/test/helpers/hermes-portable-onboarding-fixture.ts b/test/helpers/hermes-portable-onboarding-fixture.ts index 4f7a34f008e..585bb730fdb 100644 --- a/test/helpers/hermes-portable-onboarding-fixture.ts +++ b/test/helpers/hermes-portable-onboarding-fixture.ts @@ -204,6 +204,7 @@ export function createHermesPortableTestInput(stateDir: string, policyPath: stri sandboxName: "alpha", gatewayName: "nemoclaw", lifecycleGeneration: "generation-1", + sandboxReadyTimeoutSecs: 180, stateDir, createPolicyPath: policyPath, createArgv: [ diff --git a/test/helpers/managed-image-buildless-e2e.ts b/test/helpers/managed-image-buildless-e2e.ts index b9e73149d87..1d13a2ea000 100644 --- a/test/helpers/managed-image-buildless-e2e.ts +++ b/test/helpers/managed-image-buildless-e2e.ts @@ -849,6 +849,27 @@ function assertManagedLaunch( ), ), ).toBe(true); + const sandboxGetCommands = result.payload.runnerCommands.filter( + (command) => + command.includes("sandbox get") && command.includes(bootstrapRequest?.sandboxName ?? ""), + ); + expect( + sandboxGetCommands.filter( + (command) => + !command.includes("sandbox get -g nemoclaw") && + !command.includes("sandbox get --gateway nemoclaw"), + ), + ).toEqual([]); + const sandboxExecCommands = result.payload.runnerCommands.filter( + (command) => + command.includes("sandbox exec") && command.includes(bootstrapRequest?.sandboxName ?? ""), + ); + expect( + sandboxExecCommands.filter( + (command) => + !command.includes("sandbox exec -g nemoclaw") && !command.includes("--gateway nemoclaw"), + ), + ).toEqual([]); } expect(createArgs.filter((arg) => arg.startsWith("NEMOCLAW_CORPORATE_CA_B64="))).toEqual([]); expect(profile.proxy).toMatchObject({ diff --git a/test/inference/managed/managed-image-protected-runtime-contract.test.ts b/test/inference/managed/managed-image-protected-runtime-contract.test.ts index d338b17447a..805de218a94 100644 --- a/test/inference/managed/managed-image-protected-runtime-contract.test.ts +++ b/test/inference/managed/managed-image-protected-runtime-contract.test.ts @@ -30,14 +30,21 @@ import { managedImageOpenShellBasePolicyPath, managedImageOpenShellCommittedProbe, managedImageOpenShellProbe, + managedImageRetainedSandboxRecoveryPath, + managedImageSandboxCleanupOwnershipError, parseManagedImageOpenShellE2eInputs, + persistManagedImageRetainedSandboxRecovery, + readManagedImageRetainedSandboxRecovery, removeManagedImageGatewayStateIfSafe, resolveManagedImageOnboardModule, + waitForCommittedSandboxProbe, } from "../../../scripts/checks/run-managed-image-openshell-e2e.ts"; +import { persistRetainedSandboxRecoveryOrBlock } from "../../../src/lib/onboard/sandbox-gpu-create-run-attempt.ts"; import { resolveOnboardManagedBootstrapLaunch } from "../../../src/lib/onboard/managed-workload/onboard-orchestration.js"; const IMAGE = `localhost:5000/nemoclaw-managed-protected/openclaw@sha256:${"a".repeat(64)}`; const VALID_SANDBOX = "managed-openclaw"; +const LOCAL_MODEL = "nvidia/nemotron-3-nano"; const MANAGED_IMAGE_ONBOARD = resolveManagedImageOnboardModule( await import("../../../src/lib/onboard.ts"), ); @@ -48,6 +55,93 @@ const SUCCESS_WITHOUT_OUTPUT: ManagedImageCommandResult = { stderr: "", }; +function managedProbeFixturePath(rootPath: string, absolutePath: string): string { + return path.join(rootPath, absolutePath.replace(/^\/+/, "")); +} + +function writeManagedProbeFixture( + rootPath: string, + absolutePath: string, + contents: string, + mode: number, +): void { + const fixturePath = managedProbeFixturePath(rootPath, absolutePath); + fs.mkdirSync(path.dirname(fixturePath), { recursive: true }); + fs.writeFileSync(fixturePath, contents, { mode }); +} + +function materializeManagedProbeFixture(rootPath: string): void { + const rootOwnedStatStub = `#!/usr/bin/env node +const fs = require("node:fs"); +const stat = fs.statSync(process.argv.at(-1)); +process.stdout.write(\`0:0:\${(stat.mode & 0o777).toString(8)}\\n\`); +`; + const curlStub = `#!/bin/sh +case "$*" in + *http_code*) printf '200' ;; +esac +exit 0 +`; + const successStub = "#!/bin/sh\nexit 0\n"; + const caMaterial = "managed corporate CA\n"; + + writeManagedProbeFixture( + rootPath, + "/sandbox/.openclaw/openclaw.json", + `${LOCAL_MODEL}\n`, + 0o444, + ); + writeManagedProbeFixture(rootPath, "/sandbox/.hermes/config.yaml", `${LOCAL_MODEL}\n`, 0o444); + writeManagedProbeFixture( + rootPath, + "/sandbox/.deepagents/config.toml", + `${LOCAL_MODEL}\n`, + 0o444, + ); + writeManagedProbeFixture(rootPath, "/usr/local/bin/openclaw", successStub, 0o555); + writeManagedProbeFixture(rootPath, "/usr/local/bin/hermes", successStub, 0o555); + writeManagedProbeFixture(rootPath, "/usr/local/bin/dcode", successStub, 0o555); + writeManagedProbeFixture( + rootPath, + "/run/nemoclaw/managed-startup-runtime.env", + "MODEL=managed\n", + 0o444, + ); + writeManagedProbeFixture( + rootPath, + "/run/nemoclaw/managed-startup-complete.json", + '{"status":"ready"}\n', + 0o444, + ); + writeManagedProbeFixture( + rootPath, + "/usr/local/share/nemoclaw/corporate-ca.pem", + caMaterial, + 0o444, + ); + writeManagedProbeFixture( + rootPath, + "/usr/local/share/ca-certificates/nemoclaw-corporate-ca-01.crt", + caMaterial, + 0o444, + ); + writeManagedProbeFixture( + rootPath, + "/etc/ssl/certs/ca-certificates.crt", + caMaterial, + 0o444, + ); + writeManagedProbeFixture( + rootPath, + "/run/nemoclaw/managed-startup-ca-bundle.pem", + caMaterial, + 0o444, + ); + writeManagedProbeFixture(rootPath, "/usr/bin/stat", rootOwnedStatStub, 0o555); + writeManagedProbeFixture(rootPath, "/usr/bin/curl", curlStub, 0o555); + writeManagedProbeFixture(rootPath, "/usr/bin/openssl", successStub, 0o555); +} + function managedContainerInspectResult( contentId: string, running: boolean, @@ -156,25 +250,14 @@ describe("protected managed-image runtime contract", () => { expect(Object.isFrozen(protectedLaunch.expectedSupervisorArgv)).toBe(true); }); - it.each([ - "openshellArgv", - "runOpenshell", - "runCaptureOpenshell", - "sleepSeconds", - "startGatewayForRecovery", - ] as const)( - "loads every OpenShell operation required before protected image launch [%s] (#7744)", - (operation) => { - expect(MANAGED_IMAGE_ONBOARD[operation], operation).toBeTypeOf("function"); - }, - ); - it("rejects a missing protected OpenShell operation with a precise contract error (#8759)", () => { expect(() => resolveManagedImageOnboardModule({ default: { openshellArgv: () => [], runCaptureOpenshell: () => "", + isSandboxReady: () => false, + printSandboxCreateRecoveryHints: () => undefined, sleepSeconds: () => undefined, startGatewayForRecovery: async () => undefined, }, @@ -182,6 +265,170 @@ describe("protected managed-image runtime contract", () => { ).toThrow("managed-image onboard module is missing required operation(s): runOpenshell"); }); + it("uses the shared wait contract for an immediate probe and fixed retries (#10652)", async () => { + const input = parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + VALID_SANDBOX, + ]); + const onboard = { + openshellArgv: (argv: string[]) => ["openshell", ...argv], + } as never; + let nowMs = 0; + let healthAttempt = 0; + const healthProbeTimes: number[] = []; + const sleeps: number[] = []; + const healthResult = () => { + healthProbeTimes.push(nowMs); + healthAttempt += 1; + return healthAttempt === 1 + ? { status: 1, stdout: "", stderr: "sandbox warming" } + : SUCCESS_WITHOUT_OUTPUT; + }; + const runCommand = vi.fn((argv: readonly string[]) => + String(argv.at(-1)).includes("managed-startup-shared-state-transaction-v1") + ? SUCCESS_WITHOUT_OUTPUT + : healthResult(), + ); + + await waitForCommittedSandboxProbe(onboard, input, {}, true, { + budgetMs: 6_000, + now: () => nowMs, + runCommand, + sleep: (milliseconds) => { + sleeps.push(milliseconds); + nowMs += milliseconds; + }, + }); + + expect(healthProbeTimes).toEqual([0, 2_000]); + expect(sleeps).toEqual([2_000]); + }); + + it("reports the final managed-image probe diagnostic at its deadline (#10652)", async () => { + const input = parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + VALID_SANDBOX, + ]); + const onboard = { + openshellArgv: (argv: string[]) => ["openshell", ...argv], + } as never; + let nowMs = 0; + const sleeps: number[] = []; + + await expect( + waitForCommittedSandboxProbe(onboard, input, {}, true, { + budgetMs: 4_000, + now: () => nowMs, + runCommand: () => ({ status: 1, stdout: "", stderr: "last sandbox diagnostic" }), + sleep: (milliseconds) => { + sleeps.push(milliseconds); + nowMs += milliseconds; + }, + }), + ).rejects.toThrow( + "OpenShell sandbox did not pass the exact-image managed-bootstrap probe within 240s: last sandbox diagnostic", + ); + expect(nowMs).toBe(4_000); + expect(sleeps).toEqual([2_000, 2_000]); + }); + + it("fails immediately when the healthy sandbox still has transaction state (#10652)", async () => { + const input = parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + VALID_SANDBOX, + ]); + const onboard = { + openshellArgv: (argv: string[]) => ["openshell", ...argv], + } as never; + const sleeps: number[] = []; + const runCommand = vi + .fn() + .mockReturnValueOnce(SUCCESS_WITHOUT_OUTPUT) + .mockReturnValueOnce({ status: 1, stdout: "", stderr: "transaction remains" }); + + await expect( + waitForCommittedSandboxProbe(onboard, input, {}, true, { + budgetMs: 4_000, + now: () => 0, + runCommand, + sleep: (milliseconds) => { + sleeps.push(milliseconds); + }, + }), + ).rejects.toThrow( + "managed bootstrap committed, but transaction cleanup was not observable through the exact sandbox: transaction remains", + ); + expect(runCommand).toHaveBeenCalledTimes(2); + expect(sleeps).toEqual([]); + }); + + it("durably preserves harness recovery evidence when acknowledgement blocks (#10652)", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-recovery-")); + const createAttemptNonce = "a".repeat(62); + const sandboxIdentityFingerprint = "b".repeat(64); + const message = "Retain this exact managed-image sandbox for recovery."; + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + const persist = vi.fn( + ( + persistedMessage: string, + persistedFingerprint?: string, + persistedNonce?: string, + ) => { + expect( + persistManagedImageRetainedSandboxRecovery({ + stateDir, + sandboxName: VALID_SANDBOX, + message: persistedMessage, + ...(persistedFingerprint + ? { sandboxIdentityFingerprint: persistedFingerprint } + : {}), + ...(persistedNonce ? { createAttemptNonce: persistedNonce } : {}), + }), + ).toBe(true); + return false; + }, + ); + + try { + expect(() => + persistRetainedSandboxRecoveryOrBlock({ + persist, + message, + createAttemptNonce, + sandboxIdentityFingerprint, + }), + ).toThrow("the recovery-only session remains blocked"); + expect(readManagedImageRetainedSandboxRecovery(stateDir)).toEqual({ + schemaVersion: 1, + sandboxName: VALID_SANDBOX, + createAttemptLabel: expect.stringMatching(new RegExp(`=${createAttemptNonce}$`, "u")), + sandboxIdentityFingerprint, + message, + }); + expect( + fs.statSync(path.join(stateDir, "retained-sandbox-recovery.json")).mode & 0o777, + ).toBe(0o600); + expect(persist).toHaveBeenCalledOnce(); + expect(error.mock.calls.flat().join("\n")).toContain( + "recovery-only session remains blocked", + ); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } + }); + it.each([ ["unknown ownership", { failed: [], ownershipFailures: ["status cannot be proven"] }, 0], ["denied signal", { failed: [9_999_601], ownershipFailures: [] }, 0], @@ -201,9 +448,53 @@ describe("protected managed-image runtime contract", () => { } }); + it("retains exact recovery evidence when managed-image cleanup is unresolved (#10652)", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-state-recovery-")); + const createAttemptNonce = "c".repeat(62); + const sandboxIdentityFingerprint = "d".repeat(64); + expect( + persistManagedImageRetainedSandboxRecovery({ + stateDir, + sandboxName: VALID_SANDBOX, + message: "Retained after incomplete cleanup.", + createAttemptNonce, + sandboxIdentityFingerprint, + }), + ).toBe(true); + + try { + expect( + removeManagedImageGatewayStateIfSafe( + stateDir, + { failed: [], ownershipFailures: [] }, + 0, + false, + ), + ).toBe(false); + expect(managedImageRetainedSandboxRecoveryPath(stateDir)).toBe( + path.join(stateDir, "retained-sandbox-recovery.json"), + ); + expect(readManagedImageRetainedSandboxRecovery(stateDir)).toMatchObject({ + createAttemptLabel: expect.stringMatching(new RegExp(`=${createAttemptNonce}$`, "u")), + sandboxIdentityFingerprint, + }); + } finally { + fs.rmSync(stateDir, { force: true, recursive: true }); + } + }); + it("removes gateway state only after scoped stop and gateway removal succeed (#7744)", () => { const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-state-remove-")); fs.writeFileSync(path.join(stateDir, "openshell-gateway.pid"), "9999601\n"); + expect( + persistManagedImageRetainedSandboxRecovery({ + stateDir, + sandboxName: VALID_SANDBOX, + message: "Retire after exact cleanup.", + createAttemptNonce: "e".repeat(62), + sandboxIdentityFingerprint: "f".repeat(64), + }), + ).toBe(true); expect( removeManagedImageGatewayStateIfSafe(stateDir, { failed: [], ownershipFailures: [] }, 0), @@ -211,6 +502,75 @@ describe("protected managed-image runtime contract", () => { expect(fs.existsSync(stateDir)).toBe(false); }); + it("refuses to delete a same-name replacement during managed-image cleanup (#10652)", () => { + const runOpenshell = vi.fn(() => ({ + status: 0, + stdout: "Name: managed-openclaw\nId: sandbox-replacement\nPhase: Ready\n", + stderr: "", + })); + const input = parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + VALID_SANDBOX, + ]); + + expect( + managedImageSandboxCleanupOwnershipError( + { runOpenshell } as never, + input, + "sandbox-created-by-harness", + {}, + ), + ).toBe("refusing managed-image sandbox cleanup because its durable identity changed"); + expect(runOpenshell).toHaveBeenCalledWith( + ["sandbox", "get", "-g", "nemoclaw", VALID_SANDBOX], + expect.objectContaining({ ignoreError: true }), + ); + expect(runOpenshell).toHaveBeenCalledTimes(1); + }); + + it("retains a same-name replacement that appears at the delete boundary (#10652)", () => { + const runOpenshell = vi + .fn() + .mockReturnValueOnce({ + status: 0, + stdout: "Name: managed-openclaw\nId: sandbox-created-by-harness\nPhase: Ready\n", + stderr: "", + }) + .mockReturnValueOnce({ + status: 0, + stdout: "Name: managed-openclaw\nId: sandbox-replacement\nPhase: Ready\n", + stderr: "", + }); + const input = parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + VALID_SANDBOX, + ]); + + expect( + managedImageSandboxCleanupOwnershipError( + { runOpenshell } as never, + input, + "sandbox-created-by-harness", + {}, + ), + ).toBe( + "refusing managed-image sandbox cleanup because its durable identity changed at the delete boundary", + ); + expect(runOpenshell).toHaveBeenCalledTimes(2); + expect(runOpenshell.mock.calls.map(([argv]) => argv)).toEqual([ + ["sandbox", "get", "-g", "nemoclaw", VALID_SANDBOX], + ["sandbox", "get", "-g", "nemoclaw", VALID_SANDBOX], + ]); + }); + it("distinguishes the running image from exact quiescent rollback retention (#7744)", () => { const calls: string[][] = []; const contentId = `sha256:${"b".repeat(64)}`; @@ -308,12 +668,22 @@ describe("protected managed-image runtime contract", () => { expect(() => assertFailedSandboxOwnerCleanupRetention( - { runOpenshell } as never, + { ...MANAGED_IMAGE_ONBOARD, runOpenshell }, input, expectedSandboxId, {}, ), ).not.toThrow(); + expect(runOpenshell).toHaveBeenNthCalledWith( + 1, + ["sandbox", "get", "-g", "nemoclaw", VALID_SANDBOX], + expect.objectContaining({ ignoreError: true }), + ); + expect(runOpenshell).toHaveBeenNthCalledWith( + 2, + ["sandbox", "list", "-g", "nemoclaw"], + expect.objectContaining({ ignoreError: true }), + ); }); it("rejects a containing sandbox name and an exact name mentioned only in stderr", () => { @@ -343,7 +713,7 @@ describe("protected managed-image runtime contract", () => { ); const assertion = () => assertFailedSandboxOwnerCleanupRetention( - { runOpenshell } as never, + { ...MANAGED_IMAGE_ONBOARD, runOpenshell }, input, expectedSandboxId, {}, @@ -352,7 +722,51 @@ describe("protected managed-image runtime contract", () => { expect(assertion).toThrow("exact OpenShell owner-cleanup state"); expect(runOpenshell).toHaveBeenNthCalledWith( 2, - ["sandbox", "list"], + ["sandbox", "list", "-g", "nemoclaw"], + expect.objectContaining({ ignoreError: true }), + ); + }); + + it("rejects foreign-gateway rollback retention evidence (#10652)", () => { + const expectedSandboxId = "sandbox-id-123"; + const input = parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + VALID_SANDBOX, + ]); + const valid = { status: 0, stdout: `Id: ${expectedSandboxId}\n`, stderr: "" }; + const missing = { status: 1, stdout: "", stderr: "sandbox not found" }; + const responses = new Map([ + [JSON.stringify(["sandbox", "get", VALID_SANDBOX]), valid], + [JSON.stringify(["sandbox", "list"]), { + status: 0, + stdout: `NAME STATUS\n${VALID_SANDBOX} Ready\n`, + stderr: "", + }], + ]); + const runOpenshell = vi.fn( + (argv: readonly string[]) => responses.get(JSON.stringify(argv)) ?? missing, + ); + + expect(() => + assertFailedSandboxOwnerCleanupRetention( + { runOpenshell } as never, + input, + expectedSandboxId, + {}, + ), + ).toThrow("exact OpenShell owner-cleanup state"); + expect(runOpenshell).toHaveBeenNthCalledWith( + 1, + ["sandbox", "get", "-g", "nemoclaw", VALID_SANDBOX], + expect.objectContaining({ ignoreError: true }), + ); + expect(runOpenshell).toHaveBeenNthCalledWith( + 2, + ["sandbox", "list", "-g", "nemoclaw"], expect.objectContaining({ ignoreError: true }), ); }); @@ -362,30 +776,15 @@ describe("protected managed-image runtime contract", () => { const qualifications = PROTECTED_MANAGED_IMAGE_AGENTS.flatMap((agent) => routeKinds.map((routeKind) => ({ agent, + routeKind, sandbox: managedImageProtectedSandboxName(agent, routeKind), })), ); const names = qualifications.map(({ sandbox }) => sandbox); - expect(names).toEqual([ - "nmc-mi-oc-lc", - "nmc-mi-oc-ol", - "nmc-mi-oc-ni", - "nmc-mi-oc-vl", - "nmc-mi-oc-rb", - "nmc-mi-he-lc", - "nmc-mi-he-ol", - "nmc-mi-he-ni", - "nmc-mi-he-vl", - "nmc-mi-he-rb", - "nmc-mi-dc-lc", - "nmc-mi-dc-ol", - "nmc-mi-dc-ni", - "nmc-mi-dc-vl", - "nmc-mi-dc-rb", - ]); expect(new Set(names).size).toBe(names.length); - qualifications.forEach(({ agent, sandbox: name }) => { + qualifications.forEach(({ agent, routeKind, sandbox: name }) => { + expect(managedImageProtectedSandboxName(agent, routeKind)).toBe(name); expect(name.startsWith(MANAGED_IMAGE_PROTECTED_SANDBOX_PREFIX)).toBe(true); expect(name.length).toBeLessThanOrEqual(19); expect(name).not.toContain("--"); @@ -457,7 +856,7 @@ describe("protected managed-image runtime contract", () => { }); it.each(["openclaw", "hermes", "langchain-deepagents-code"] as const)( - "binds %s to an exact GPU/local-inference launch", + "binds %s to an exact GPU/local-inference launch and verified startup receipt", (agent) => { const parsed = parseManagedImageOpenShellE2eInputs([ "--agent", @@ -470,7 +869,7 @@ describe("protected managed-image runtime contract", () => { "--local-provider", "nim", "--model", - "nvidia/nemotron-3-nano", + LOCAL_MODEL, ]); expect(parsed).toEqual({ @@ -478,26 +877,34 @@ describe("protected managed-image runtime contract", () => { gpu: true, image: IMAGE, localProvider: "nim", - model: "nvidia/nemotron-3-nano", + model: LOCAL_MODEL, sandbox: managedImageProtectedSandboxName(agent, "nim"), }); expect(path.isAbsolute(managedImageOpenShellBasePolicyPath(agent))).toBe(true); - const probe = managedImageOpenShellProbe(agent); - const syntax = spawnSync("/bin/sh", ["-n", "-c", probe], { encoding: "utf8" }); - expect(syntax.status, syntax.stderr).toBe(0); - expect(probe).toContain("managed-startup-complete.json"); - expect(probe).toContain( - `managed-image startup probe failed: ${ - agent === "openclaw" - ? "OpenClaw health endpoint" - : agent === "hermes" - ? "Hermes health endpoint" - : "LangChain Deep Agents Code version command" - }`, - ); - expect(probe).toContain( - "managed-image startup probe failed: managed startup completion owner, group, and mode must equal 0:0:444", - ); + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-probe-")); + try { + materializeManagedProbeFixture(fixtureRoot); + const probe = managedImageOpenShellProbe(agent, LOCAL_MODEL, { rootPath: fixtureRoot }); + const syntax = spawnSync("/bin/sh", ["-n", "-c", probe], { encoding: "utf8" }); + expect(syntax.status, syntax.stderr).toBe(0); + + const verified = spawnSync("/bin/sh", ["-eu", "-c", probe], { encoding: "utf8" }); + expect(verified.status, verified.stderr).toBe(0); + + fs.chmodSync( + managedProbeFixturePath(fixtureRoot, "/run/nemoclaw/managed-startup-complete.json"), + 0o644, + ); + const mutableReceipt = spawnSync("/bin/sh", ["-eu", "-c", probe], { + encoding: "utf8", + }); + expect(mutableReceipt.status).not.toBe(0); + expect(mutableReceipt.stderr).toContain( + "managed startup completion owner, group, and mode must equal 0:0:444", + ); + } finally { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + } }, ); @@ -625,10 +1032,7 @@ describe("protected managed-image runtime contract", () => { ).toThrow(/cannot be combined/u); }); - it("keeps rollback cleanup distinct from initial readiness", () => { - expect(managedImageOpenShellCommittedProbe()).toContain( - "managed-startup-shared-state-transaction-v1", - ); + it("keeps rollback cleanup distinct from initial readiness (#10652)", () => { expect( parseManagedImageOpenShellE2eInputs([ "--agent", @@ -640,5 +1044,22 @@ describe("protected managed-image runtime contract", () => { "--inject-bootstrap-completion-failure", ]), ).toMatchObject({ failureInjection: "bootstrap-completion" }); + + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-committed-probe-")); + try { + const probe = managedImageOpenShellCommittedProbe({ rootPath: fixtureRoot }); + const completed = spawnSync("/bin/sh", ["-eu", "-c", probe], { encoding: "utf8" }); + expect(completed.status, completed.stderr).toBe(0); + + const retainedTransaction = managedProbeFixturePath( + fixtureRoot, + "/var/lib/nemoclaw/managed-startup-shared-state-transaction-v1", + ); + fs.mkdirSync(retainedTransaction, { recursive: true }); + const incomplete = spawnSync("/bin/sh", ["-eu", "-c", probe], { encoding: "utf8" }); + expect(incomplete.status).not.toBe(0); + } finally { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + } }); }); diff --git a/test/installer-integration/install-hermes-portable-active.test.ts b/test/installer-integration/install-hermes-portable-active.test.ts index 7dedcdf040c..19ca8c889ef 100644 --- a/test/installer-integration/install-hermes-portable-active.test.ts +++ b/test/installer-integration/install-hermes-portable-active.test.ts @@ -258,6 +258,7 @@ describe("Hermes portable installer admission", testTimeoutOptions(60_000), () = sandboxName, gatewayName, lifecycleGeneration, + sandboxReadyTimeoutSecs: 180, runtimeAuthority, openshellExecutableAuthority: hermesPortableTestOpenShellAuthority(), stateDir, diff --git a/test/onboarding/onboard-prepared-build-context.test.ts b/test/onboarding/onboard-prepared-build-context.test.ts index 9224fc8e4be..9c747860418 100644 --- a/test/onboarding/onboard-prepared-build-context.test.ts +++ b/test/onboarding/onboard-prepared-build-context.test.ts @@ -9,11 +9,12 @@ import path from "node:path"; import { describe, it } from "vitest"; import { writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; -type PreparedContextScenario = "create" | "custom-dockerfile"; +type PreparedContextScenario = "create" | "custom-dockerfile" | "cleanup-incomplete"; type PreparedContextResult = { buildCtx: string; buildId: string; + buildCleanupListenerRegistered: boolean; cleanupCalls: number; commands: string[]; errorMessage: string | null; @@ -200,16 +201,19 @@ childProcess.spawn = (...args) => { return child; }; +const cleanupBuildCtx = () => { + cleanupCalls += 1; + if (scenario === "cleanup-incomplete") return false; + fs.rmSync(buildCtx, { recursive: true, force: true }); + return true; +}; +process.on("exit", cleanupBuildCtx); const preparedBuildContext = { buildCtx, stagedDockerfile: buildCtx + "/Dockerfile", buildId, origin: "generated", - cleanupBuildCtx: () => { - cleanupCalls += 1; - fs.rmSync(buildCtx, { recursive: true, force: true }); - return true; - }, + cleanupBuildCtx, }; const { createSandbox } = require(${onboardPath}); @@ -251,6 +255,7 @@ const { createSandbox } = require(${onboardPath}); console.log(JSON.stringify({ buildCtx, buildId, + buildCleanupListenerRegistered: process.listeners("exit").includes(cleanupBuildCtx), cleanupCalls, commands, errorMessage, @@ -305,6 +310,7 @@ describe("onboard prepared DCode build context", () => { assert.deepEqual(result.planFromRefs, [`${result.buildCtx}/Dockerfile`]); assert.deepEqual(result.resolvedBuildIds, [result.buildId]); assert.equal(result.cleanupCalls, 1); + assert.equal(result.buildCleanupListenerRegistered, false); assert.ok( result.commands.some((command) => command.includes(`sandbox create --from ${result.buildCtx}/Dockerfile`), @@ -320,6 +326,20 @@ describe("onboard prepared DCode build context", () => { }, ); + it( + "retains exit cleanup ownership when build-context retirement is incomplete (#10652)", + { + timeout: 90_000, + }, + () => { + const result = runPreparedContextScenario("cleanup-incomplete"); + + assert.equal(result.errorMessage, null); + assert.equal(result.cleanupCalls, 1); + assert.equal(result.buildCleanupListenerRegistered, true); + }, + ); + it( "passes the seconds-based sleep helper to the Docker GPU patch during prepared-context onboarding (#9218)", { diff --git a/test/security/shellquote-sandbox.test.ts b/test/security/shellquote-sandbox.test.ts index 2f2d41ed642..b0a12be936d 100644 --- a/test/security/shellquote-sandbox.test.ts +++ b/test/security/shellquote-sandbox.test.ts @@ -2,40 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "child_process"; +import { createHash } from "node:crypto"; // Verify sandbox names stay validated and out of raw shell command strings. import fs from "fs"; import os from "os"; import path from "path"; import { describe, expect, it } from "vitest"; +import { completeOrdinaryOnboardSandboxCreation } from "../../src/lib/onboard/created-sandbox-finalization"; import { writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; describe("sandboxName command hardening in onboard.js", () => { - it("rejects a marker-only security inventory fixture probe", async () => { - const helper = (await import("../helpers/onboard-script-mocks.cjs")) as { - isOpenClawSecurityInventoryProbe: (command: unknown) => boolean; - }; - - expect( - helper.isOpenClawSecurityInventoryProbe([ - "run", - "--rm", - "--network", - "none", - "--cap-drop", - "ALL", - "--security-opt", - "no-new-privileges", - "--read-only", - "--entrypoint", - "/bin/sh", - "nemoclaw:test", - "-c", - "echo nemoclaw-security-inventory-ok", - ]), - ).toBe(false); - }); - it("re-validates sandboxName at the createSandbox boundary", async () => { const onboardModule = await import("../../src/lib/onboard.js"); const { createSandbox } = onboardModule as unknown as { @@ -53,7 +30,56 @@ describe("sandboxName command hardening in onboard.js", () => { ).rejects.toThrow(/Invalid sandbox name/); }); - it("runs setup-dns-proxy.sh through the argv helper instead of bash -c interpolation", () => { + it("passes DNS proxy gateway values as one literal argument", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dns-literal-")); + const argsFile = path.join(tmpDir, "dns-args.txt"); + const sideEffectFile = path.join(tmpDir, "shell-expanded"); + const gatewayName = `nemoclaw; touch ${sideEffectFile}; #`; + fs.writeFileSync( + path.join(tmpDir, "setup-dns-proxy.sh"), + '#!/usr/bin/env bash\nset -eu\nprintf \'%s\\n\' "$1" "$2" > "$NEMOCLAW_DNS_ARGS_FILE"\n', + ); + + try { + completeOrdinaryOnboardSandboxCreation( + { + sandboxName: "my-assistant", + sandboxWasLiveDefault: false, + gatewayPort: 8080, + runtimeFields: { openshellDriver: "kubernetes" }, + messagingProviders: [], + liveExists: true, + } as never, + { + setDefault: () => undefined, + runFile: (command: string, args: string[]) => + spawnSync(command, args, { + encoding: "utf-8", + env: { ...process.env, NEMOCLAW_DNS_ARGS_FILE: argsFile }, + }), + scriptsDir: tmpDir, + gatewayName, + providerExistsInGateway: () => true, + armCancelRollback: () => undefined, + markCancellationRecovery: () => undefined, + dockerInfoFormat: () => "", + runCapture: () => "", + revalidateSandboxIdentity: () => undefined, + applyVmDnsMonkeypatch: () => undefined, + } as never, + ); + + expect(fs.readFileSync(argsFile, "utf-8").trim().split("\n")).toEqual([ + gatewayName, + "my-assistant", + ]); + expect(fs.existsSync(sideEffectFile)).toBe(false); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("scopes created sandbox probes to the owning gateway", () => { const repoRoot = path.join(import.meta.dirname, "../.."); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dns-argv-")); const fakeBin = path.join(tmpDir, "bin"); @@ -89,7 +115,43 @@ for (const key of Object.keys(process.env)) { process.env.NEMOCLAW_OPENSHELL_BIN = ${JSON.stringify(path.join(fakeBin, "openshell"))}; const commands = []; const asText = (command) => Array.isArray(command) ? command.join(" ") : String(command); -const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); +const createdSandbox = fixtureMocks.createCreatedSandboxFixture({ + gatewayName: "nemoclaw", + sandboxId: "sandbox-owning-gateway", +}); +const foreignSandbox = fixtureMocks.createCreatedSandboxFixture({ + gatewayName: "foreign-gateway", + sandboxId: "sandbox-foreign-gateway", + lifecycleState: "created", +}); +const probeEffects = []; +const runCreatedSandboxProbe = (command) => { + const args = Array.isArray(command) ? command.map(String) : []; + const sandboxIndex = args.indexOf("sandbox"); + const action = sandboxIndex < 0 ? null : args[sandboxIndex + 1]; + if (action !== "get" && action !== "exec") return null; + const name = action === "get" ? args.at(-1) : args[args.indexOf("--name") + 1]; + if (name !== "my-assistant") return null; + const gatewayIndex = args.findIndex((arg) => arg === "-g" || arg === "--gateway"); + const gateway = gatewayIndex < 0 ? null : args[gatewayIndex + 1] ?? null; + const target = gateway === "nemoclaw" ? "owning" : "foreign"; + probeEffects.push({ action, gateway, target }); + if (action === "get") { + const result = (target === "owning" ? createdSandbox : foreignSandbox).run(command); + return result ?? { + status: 1, + stdout: Buffer.alloc(0), + stderr: Buffer.from("gateway-scoped fixture rejected sandbox get\n"), + }; + } + return target === "owning" + ? { status: 0, stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) } + : { + status: 1, + stdout: Buffer.alloc(0), + stderr: Buffer.from("foreign gateway cannot satisfy owning sandbox exec\n"), + }; +}; createdSandbox.installRuntimeObservation(); runner.run = (command, opts = {}) => { const text = asText(command); @@ -107,6 +169,8 @@ runner.run = (command, opts = {}) => { stderr: Buffer.alloc(0), }; } + const sandboxProbe = runCreatedSandboxProbe(command); + if (sandboxProbe !== null) return sandboxProbe; return createdSandbox.run(command) ?? { status: 0 }; }; runner.runFile = (file, args = [], opts = {}) => { @@ -126,13 +190,14 @@ runner.runCapture = (command) => { }; registry.getSandbox = () => null; registry.getDisabledChannels = () => []; -registry.registerSandbox = () => true; registry.removeSandbox = () => true; registry.updateSandbox = () => true; +let registeredSandbox = null; const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", provider: "nvidia-prod", model: "gpt-5.4", + registerSandbox: (entry) => { registeredSandbox = entry; }, }); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; @@ -172,7 +237,14 @@ try { createFixture, ), ); - console.log(JSON.stringify({ sandboxName, commands })); + console.log(JSON.stringify({ + sandboxName, + commands, + probeEffects, + registeredSandbox, + owningSandboxId: createdSandbox.state.sandboxId, + foreignSandbox: foreignSandbox.state, + })); } catch (error) { console.error(error && error.stack ? error.stack : String(error)); process.exit(1); @@ -208,28 +280,27 @@ try { .find((line) => line.startsWith("{") && line.endsWith("}")); expect(payloadLine).toBeTruthy(); const payload = JSON.parse(payloadLine!); - const dnsCommand = payload.commands.find( - (entry: { type: string; args: string[] }) => - entry.type === "runFile" && entry.args[0]?.endsWith("setup-dns-proxy.sh"), + expect(payload.sandboxName).toBe("my-assistant"); + expect(payload.registeredSandbox.lifecycleLiveIdentityFingerprint).toBe( + createHash("sha256").update(payload.owningSandboxId).digest("hex"), + ); + expect(payload.probeEffects).toEqual( + expect.arrayContaining([ + expect.objectContaining({ action: "get", target: "owning" }), + expect.objectContaining({ action: "exec", target: "owning" }), + ]), ); - expect(dnsCommand).toBeTruthy(); - expect(dnsCommand.file).toBe("bash"); - expect(dnsCommand.args).toEqual([ - expect.stringMatching(/setup-dns-proxy\.sh$/), - "nemoclaw", - "my-assistant", - ]); - expect(dnsCommand.command).not.toContain("bash -c"); - expect( - payload.commands.some((entry: { command: string }) => - entry.command.includes("sandbox get -g nemoclaw my-assistant"), - ), - ).toBe(true); expect( - payload.commands.some((entry: { command: string }) => - entry.command.includes("sandbox exec -g nemoclaw --name my-assistant -- true"), + payload.probeEffects.every( + (effect: { target: string }) => effect.target === "owning", ), ).toBe(true); + expect(payload.foreignSandbox).toMatchObject({ + sandboxName: "my-assistant", + sandboxId: "sandbox-foreign-gateway", + gatewayName: "foreign-gateway", + lifecycleState: "created", + }); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index 9d32687c94e..449fa4b7438 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -46,7 +46,6 @@ const POST_REBOOT_DELIVERY_RUNTIME_FILES = new Set([ "src/lib/actions/sandbox/status-snapshot.ts", "src/lib/onboard/docker-driver-sandbox-recovery.ts", "src/lib/onboard/docker-startup-command-agent.ts", - "src/lib/onboard/sandbox-create-step.ts", "tools/e2e/onboard-timeout-contract.mts", ]); export const GATEWAY_TOPOLOGY_FILES = [