From d52faaa64439a38a5e11a9c03b0b490b7a8a38ab Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 07:40:26 -0700 Subject: [PATCH 1/7] feat(runtime): complete host-local inference lifecycle Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/destroy-execution.ts | 40 ++++ .../destroy-host-local-inference.test.ts | 133 +++++++++++ src/lib/actions/sandbox/destroy.ts | 10 + src/lib/actions/sandbox/snapshot.ts | 190 ++++++++++++--- .../sandbox/snapshot/backup-authority.test.ts | 100 ++++++++ .../sandbox/snapshot/backup-authority.ts | 58 ++++- .../actions/sandbox/snapshot/dependencies.ts | 8 + .../sandbox/snapshot/restore-authority.ts | 119 +++++---- .../restore-host-local-authority.test.ts | 225 ++++++++++++++++++ .../host-local-inference-lifecycle.test.ts | 138 +++++++++++ .../host-local-inference-lifecycle.ts | 170 +++++++++++++ .../host-local-inference-routing.test.ts | 2 + .../runtime-provider/host-local-inference.ts | 15 ++ .../podman-host-local-inference.test.ts | 38 +++ .../podman-host-local-inference.ts | 42 ++++ .../onboard/runtime-provider/podman.test.ts | 2 + src/lib/onboard/runtime-provider/registry.ts | 2 + .../runtime-provider-contract.test.ts | 2 + src/lib/state/sandbox.ts | 38 ++- ...odman-host-local-inference-test-harness.ts | 7 + ...board-host-local-inference-routing.test.ts | 2 + test/runtime-provider-source-shape.test.ts | 1 + 22 files changed, 1251 insertions(+), 91 deletions(-) create mode 100644 src/lib/actions/sandbox/destroy-host-local-inference.test.ts create mode 100644 src/lib/actions/sandbox/snapshot/restore-host-local-authority.test.ts create mode 100644 src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts create mode 100644 src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts diff --git a/src/lib/actions/sandbox/destroy-execution.ts b/src/lib/actions/sandbox/destroy-execution.ts index b5b0c94b66d..71165659f6a 100644 --- a/src/lib/actions/sandbox/destroy-execution.ts +++ b/src/lib/actions/sandbox/destroy-execution.ts @@ -9,6 +9,11 @@ import { type RuntimeProviderBundleRegistry, requireRuntimeProviderDestructiveCleanupAuthority, } from "../../onboard/runtime-provider/access"; +import { + type PreparedHostLocalInferenceAuthority, + prepareSandboxHostLocalInferenceDestroyAuthority, + retirePreparedHostLocalInferenceAuthority, +} from "../../onboard/runtime-provider/host-local-inference-lifecycle"; import { type DetachSandboxProvidersResult, runSandboxProviderPreDeleteCleanup, @@ -34,6 +39,8 @@ export function redactDestroyError(error: unknown): string { type SandboxDestroyExecutionInput = { cleanupShieldsArtifacts: (sandboxName: string) => void; force: boolean; + getSandbox: (sandboxName: string) => SandboxEntry | null; + listSandboxes: () => { sandboxes: SandboxEntry[] }; runOpenshell: DestroyRunOpenshell; sandbox: SandboxEntry | null; sandboxConfirmedAbsent: boolean; @@ -61,6 +68,8 @@ export type SandboxDestroyExecutionResult = gatewayUnreachable: boolean; mcpOwnershipRequiresGateway: boolean; mcpRecoveryFailure?: string; + hostLocalInferenceCleanupFailure?: string; + deleteConfirmed?: boolean; }; type HardenedDeleteState = { @@ -191,6 +200,8 @@ async function finalizeMcpDestroy( export async function executeSandboxDestroy({ cleanupShieldsArtifacts, force, + getSandbox, + listSandboxes, runOpenshell, sandbox, sandboxConfirmedAbsent, @@ -200,6 +211,7 @@ export async function executeSandboxDestroy({ }: SandboxDestroyExecutionInput): Promise { return withTimerBoundShieldsMutationLockAsync(sandboxName, "destroy sandbox", async () => { let runtimeProvider: RuntimeProviderBundle | null = null; + let hostLocalInferenceAuthority: PreparedHostLocalInferenceAuthority | null = null; if (sandbox) { try { runtimeProvider = requireRuntimeProviderDestructiveCleanupAuthority( @@ -207,6 +219,10 @@ export async function executeSandboxDestroy({ sandbox, runtimeProviders, ).provider; + hostLocalInferenceAuthority = prepareSandboxHostLocalInferenceDestroyAuthority( + runtimeProvider, + sandbox, + ); } catch (error) { return { ok: false as const, @@ -268,6 +284,30 @@ export async function executeSandboxDestroy({ if (!forcedLocalCleanup) { await finalizeMcpDestroy(sandboxName, mcpPreparation, force); } + if (!forcedLocalCleanup && runtimeProvider && sandbox && hostLocalInferenceAuthority) { + try { + const current = getSandbox(sandboxName); + if (!current) { + throw new Error(`sandbox '${sandboxName}' is no longer registered`); + } + retirePreparedHostLocalInferenceAuthority( + runtimeProvider, + current, + hostLocalInferenceAuthority, + listSandboxes().sandboxes, + ); + } catch (error) { + return { + ok: false as const, + deleteOutput, + exitCode: 1, + gatewayUnreachable: false, + mcpOwnershipRequiresGateway: false, + hostLocalInferenceCleanupFailure: redactDestroyError(error), + deleteConfirmed: true, + }; + } + } return { ok: true as const, detachOutcome, diff --git a/src/lib/actions/sandbox/destroy-host-local-inference.test.ts b/src/lib/actions/sandbox/destroy-host-local-inference.test.ts new file mode 100644 index 00000000000..2ec812df39e --- /dev/null +++ b/src/lib/actions/sandbox/destroy-host-local-inference.test.ts @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { createInMemoryRuntimeProviderBundle } from "../../../../test/helpers/runtime-provider-bundle"; +import { + type HostLocalInferenceReceipt, + serializeHostLocalInferenceReceipt, +} from "../../onboard/runtime-provider/host-local-inference"; +import type { SandboxEntry } from "../../state/registry"; +import { executeSandboxDestroy } from "./destroy-execution"; + +function receipt(): HostLocalInferenceReceipt { + return { + schemaVersion: 1, + providerId: "mxc", + service: "vllm", + engineAuthority: { + schemaVersion: 1, + providerId: "mxc", + operation: "host-local-inference", + engineId: "mxc", + authorityId: "mxc:host-local", + bindingSha256: "a".repeat(64), + }, + endpoint: { host: "mxc.internal", port: 8000, networkName: "mxc-network" }, + runtime: { + kind: "container", + runtimeId: "mxc-vllm", + name: "nemoclaw-vllm", + imageRef: `nvcr.io/nvidia/vllm@sha256:${"b".repeat(64)}`, + specSha256: "c".repeat(64), + gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, + }, + }; +} + +function sandbox(name = "alpha"): SandboxEntry { + return { + name, + agent: "openclaw", + openshellDriver: "mxc", + hostLocalInferenceReceipt: serializeHostLocalInferenceReceipt(receipt()), + }; +} + +function provider(destroyError?: string) { + const preserveForRebuild = vi.fn((value: HostLocalInferenceReceipt) => value); + const prepareDestroy = vi.fn((value: HostLocalInferenceReceipt) => value); + const destroy = vi.fn((value: HostLocalInferenceReceipt) => { + if (destroyError) throw new Error(destroyError); + return { status: "removed" as const, receipt: value }; + }); + const bundle = createInMemoryRuntimeProviderBundle({ + providerId: "mxc", + workloadProfile: { + support: null, + hostArchitectures: ["amd64"], + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: false, + }, + hostLocalInferenceRuntime: { + providerId: "mxc", + authorityId: "mxc:host-local", + services: ["ollama", "nim", "vllm"], + translateContainerArgs: (args: readonly string[]) => args, + qualifyOllama: vi.fn(), + startManaged: vi.fn(), + inspectManaged: vi.fn(), + stopManaged: vi.fn(), + preserveForRebuild, + prepareDestroy, + destroy, + }, + }); + return { bundle, destroy, prepareDestroy, preserveForRebuild }; +} + +async function runDestroy(runtimeProvider: ReturnType, peers: SandboxEntry[]) { + const entry = sandbox(); + const events: string[] = []; + const result = await executeSandboxDestroy({ + cleanupShieldsArtifacts: () => events.push("cleanup"), + force: false, + getSandbox: () => entry, + listSandboxes: () => ({ sandboxes: [entry, ...peers] }), + runOpenshell: (args) => { + if (args[0] === "sandbox" && args[1] === "delete") events.push("delete"); + return { status: 0, stdout: "", stderr: "" }; + }, + sandbox: entry, + sandboxConfirmedAbsent: false, + sandboxName: "alpha", + runtimeProviders: { mxc: runtimeProvider.bundle }, + deps: { + readTimerMarker: () => null, + wipeSandboxState: () => undefined, + }, + }); + return { events, result }; +} + +describe("sandbox destroy host-local inference transaction", () => { + it("deletes the sandbox before retiring the exact unshared runtime", async () => { + const runtimeProvider = provider(); + const { events, result } = await runDestroy(runtimeProvider, []); + + expect(result).toMatchObject({ ok: true }); + expect(events).toEqual(["delete", "cleanup"]); + expect(runtimeProvider.prepareDestroy).toHaveBeenCalledTimes(2); + expect(runtimeProvider.destroy).toHaveBeenCalledOnce(); + }); + + it("keeps a runtime referenced by another sandbox", async () => { + const runtimeProvider = provider(); + const { result } = await runDestroy(runtimeProvider, [sandbox("beta")]); + + expect(result).toMatchObject({ ok: true }); + expect(runtimeProvider.destroy).not.toHaveBeenCalled(); + }); + + it("preserves local ownership when exact runtime retirement fails", async () => { + const runtimeProvider = provider("injected runtime removal failure"); + const { events, result } = await runDestroy(runtimeProvider, []); + + expect(result).toMatchObject({ + ok: false, + deleteConfirmed: true, + hostLocalInferenceCleanupFailure: "injected runtime removal failure", + }); + expect(events).toEqual(["delete", "cleanup"]); + }); +}); diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 04ecd956ec0..ea5d8203dea 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -416,12 +416,22 @@ async function destroySandboxUnlocked( const destructiveResult = await executeSandboxDestroy({ cleanupShieldsArtifacts: cleanupShieldsDestroyArtifacts, force: normalized.force === true, + getSandbox: registry.getSandbox, + listSandboxes: registry.listSandboxes, runOpenshell, sandbox, sandboxConfirmedAbsent, sandboxName, }); if (!destructiveResult.ok) { + if (destructiveResult.hostLocalInferenceCleanupFailure) { + console.error( + ` Sandbox '${sandboxName}' is gone, but its exact host-local inference cleanup failed: ${destructiveResult.hostLocalInferenceCleanupFailure}`, + ); + console.error( + ` Local ownership state was preserved. Re-run '${CLI_NAME} ${sandboxName} destroy --yes' to reconcile only the recorded provider runtime.`, + ); + } if (destructiveResult.deleteOutput) { console.error(` ${destructiveResult.deleteOutput}`); } diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 2a1f1562a80..9ab371ad9ec 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -81,13 +81,19 @@ import { } from "./sandbox-gateway-routing"; import { backupSandboxStateWithManagedAuthority, + confirmHostLocalInferenceAuthority, confirmSandboxRuntimeRestore, + type PreparedHostLocalInferenceAuthority, type PreparedSandboxRuntimeRestore, + prepareHostLocalInferenceAuthority, prepareManagedSnapshotProfileRestore, + prepareSandboxHostLocalInferenceDestroyAuthority, prepareSandboxRuntimeRestore, + type RuntimeProviderBundle, readManagedSnapshotProfileAuthority, rejectManagedSnapshotCloneUntilRebind, requireCurrentSnapshotRuntimeProvider, + retirePreparedHostLocalInferenceAuthority, } from "./snapshot/dependencies"; import { formatSnapshotBaselineExclusionSummary } from "./snapshot-baseline-exclusion-summary"; import { printHermesGatewayRestoreHint } from "./snapshot-hermes-gateway-hint"; @@ -489,8 +495,14 @@ function deleteSandboxForRestore(name: string): void { ); snapshotExit(1); } + let runtimeProvider: RuntimeProviderBundle; + let hostLocalInferenceAuthority: PreparedHostLocalInferenceAuthority | null; try { - requireSandboxDestructiveCleanupAuthority(name, sbMeta); + runtimeProvider = requireSandboxDestructiveCleanupAuthority(name, sbMeta).provider; + hostLocalInferenceAuthority = prepareSandboxHostLocalInferenceDestroyAuthority( + runtimeProvider, + sbMeta, + ); } catch (error) { const detail = error instanceof Error ? error.message : String(error); console.error( @@ -526,6 +538,25 @@ function deleteSandboxForRestore(name: string): void { ); snapshotExit(1); } + if (hostLocalInferenceAuthority) { + try { + const current = registry.getSandbox(name); + if (!current) throw new Error(`sandbox '${name}' is no longer registered`); + retirePreparedHostLocalInferenceAuthority( + runtimeProvider, + current, + hostLocalInferenceAuthority, + registry.listSandboxes().sandboxes, + ); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + console.error( + ` Destination '${name}' is gone, but its host-local inference cleanup failed: ${detail}`, + ); + console.error(" Local ownership state was preserved; retry the restore to reconcile it."); + snapshotExit(1); + } + } // Destination-only cleanup so the recreated sandbox does not inherit stale // host-side state or hit provider-name conflicts (Codex #3796 P2): // - /tmp/nemoclaw-services-: PID dir for this sandbox's services @@ -1059,6 +1090,7 @@ async function runSnapshotRestoreUnlocked( }; const currentSourceEntry = registry.getSandbox(sandboxName); let hasManagedProfileAuthority = false; + const hostLocalInferenceReceipt = resolvedSnapshot.hostLocalInferenceReceipt; let snapshotRestoreAuthority: sandboxState.SnapshotRestoreAuthority | null = null; try { const snapshotAuthority = readManagedSnapshotProfileAuthority(snapshotProfileSource); @@ -1079,7 +1111,18 @@ async function runSnapshotRestoreUnlocked( if (isCrossSandboxRestore && hasManagedProfileAuthority) { rejectManagedSnapshotCloneUntilRebind(snapshotProfileSource, targetSandbox); } - if (hasManagedProfileAuthority) { + if (typeof hostLocalInferenceReceipt === "string") { + if (!currentSourceEntry) { + throw new Error("host-local inference snapshot source is no longer registered"); + } + const sourceProvider = requireCurrentSnapshotRuntimeProvider(currentSourceEntry); + prepareHostLocalInferenceAuthority( + sourceProvider, + currentSourceEntry, + hostLocalInferenceReceipt, + ); + } + if (hasManagedProfileAuthority || typeof hostLocalInferenceReceipt === "string") { snapshotRestoreAuthority = sandboxState.captureSnapshotRestoreAuthority( backupPath, resolvedSnapshot, @@ -1090,7 +1133,7 @@ async function runSnapshotRestoreUnlocked( } } catch (error) { console.error( - ` Cannot restore managed snapshot authority: ${ + ` Cannot restore provider snapshot authority: ${ error instanceof Error ? error.message : String(error) }.`, ); @@ -1099,6 +1142,7 @@ async function runSnapshotRestoreUnlocked( } let preparedRuntimeRestore: PreparedSandboxRuntimeRestore | null = null; + let preparedHostLocalInferenceRestore: PreparedHostLocalInferenceAuthority | null = null; if (!isCrossSandboxRestore) { // Self-restore: target is `sandboxName`. Cannot auto-create; the // source pod is the target, so it must already be live. @@ -1139,6 +1183,30 @@ async function runSnapshotRestoreUnlocked( snapshotExit(1); } } + if (typeof hostLocalInferenceReceipt === "string") { + const currentTarget = registry.getSandbox(targetSandbox); + if (!currentTarget) { + console.error( + ` Cannot restore host-local inference snapshot '${sandboxName}': target authority is missing.`, + ); + snapshotExit(1); + } + try { + const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); + preparedHostLocalInferenceRestore = prepareHostLocalInferenceAuthority( + provider, + currentTarget, + hostLocalInferenceReceipt, + ); + } catch (error) { + console.error( + ` Cannot preflight host-local inference snapshot restore: ${ + error instanceof Error ? error.message : String(error) + }.`, + ); + snapshotExit(1); + } + } } else { // #3756: cross-sandbox restore into a destination that already exists // used to overlay onto the live filesystem silently. Refuse by default @@ -1279,73 +1347,119 @@ async function runSnapshotRestoreUnlocked( await withDashboardPortReservationLock(() => withGatewayRouteMutationLock(sourceGatewayName, createAndRegisterClone), ); + if (typeof hostLocalInferenceReceipt === "string") { + const currentTarget = registry.getSandbox(targetSandbox); + if (!currentTarget) { + console.error( + ` Clone '${targetSandbox}' was created without durable host-local inference authority.`, + ); + snapshotExit(1); + } + try { + const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); + preparedHostLocalInferenceRestore = prepareHostLocalInferenceAuthority( + provider, + currentTarget, + hostLocalInferenceReceipt, + ); + } catch (error) { + console.error( + ` Cannot bind clone '${targetSandbox}' to snapshot inference authority: ${ + error instanceof Error ? error.message : String(error) + }.`, + ); + snapshotExit(1); + } + } } withTimerBoundShieldsMutationLock(targetSandbox, "restore sandbox snapshot", () => { // Serialize filesystem restore, mutable-permission repair, and policy // reconciliation under the active timer generation. Normal auto-restore // waits; the absolute deadline may preempt this process and reclaim the // token, preventing policy/config mutation after lockdown resumes. - const validateManagedRestoreBeforeMutation = preparedRuntimeRestore - ? () => { - const currentTarget = registry.getSandbox(targetSandbox); - if (!currentTarget) { - throw new Error(`target '${targetSandbox}' is no longer registered`); + const validateProviderRestoreBeforeMutation = + preparedRuntimeRestore || preparedHostLocalInferenceRestore + ? () => { + const currentTarget = registry.getSandbox(targetSandbox); + if (!currentTarget) { + throw new Error(`target '${targetSandbox}' is no longer registered`); + } + const provider = preparedRuntimeRestore + ? requireCurrentSnapshotRuntimeProvider(currentTarget) + : requireCurrentSnapshotRuntimeProvider(currentTarget); + if (preparedRuntimeRestore) { + const profileRestore = prepareManagedSnapshotProfileRestore( + snapshotProfileSource, + currentTarget, + provider, + ); + if (!profileRestore) { + throw new Error("managed profile restore authority is missing"); + } + const prepared = preparedRuntimeRestore; + // The state layer invokes this after local tar staging and + // immediately before its first remote filesystem mutation. + preparedRuntimeRestore = prepareSandboxRuntimeRestore( + provider, + currentTarget, + prepared.source, + profileRestore.providerRestoreAuthority, + ); + } + if (typeof hostLocalInferenceReceipt === "string") { + preparedHostLocalInferenceRestore = prepareHostLocalInferenceAuthority( + provider, + currentTarget, + hostLocalInferenceReceipt, + ); + } } - const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); - const profileRestore = prepareManagedSnapshotProfileRestore( - snapshotProfileSource, - currentTarget, - provider, - ); - if (!profileRestore) { - throw new Error("managed profile restore authority is missing"); - } - const prepared = preparedRuntimeRestore; - if (!prepared) throw new Error("managed runtime restore authority is missing"); - // The state layer invokes this after local tar staging and - // immediately before its first remote filesystem mutation. - preparedRuntimeRestore = prepareSandboxRuntimeRestore( - provider, - currentTarget, - prepared.source, - profileRestore.providerRestoreAuthority, - ); - } - : null; + : null; if (targetSandbox !== sandboxName) { console.log(` Restoring snapshot from '${sandboxName}' into '${targetSandbox}'...`); } else { console.log(` Restoring snapshot into '${sandboxName}'...`); } - if (Boolean(snapshotRestoreAuthority) !== Boolean(validateManagedRestoreBeforeMutation)) { + if (Boolean(snapshotRestoreAuthority) !== Boolean(validateProviderRestoreBeforeMutation)) { console.error( - ` Cannot restore managed snapshot '${sandboxName}': content authority and the runtime mutation fence must both be present.`, + ` Cannot restore provider snapshot '${sandboxName}': content authority and the runtime mutation fence must both be present.`, ); console.error(` Destination '${targetSandbox}' was not changed.`); snapshotExit(1); } const result = - snapshotRestoreAuthority && validateManagedRestoreBeforeMutation + snapshotRestoreAuthority && validateProviderRestoreBeforeMutation ? sandboxState.restoreSandboxState(targetSandbox, backupPath, { authority: snapshotRestoreAuthority, - validateBeforeMutation: validateManagedRestoreBeforeMutation, + validateBeforeMutation: validateProviderRestoreBeforeMutation, }) : sandboxState.restoreSandboxState(targetSandbox, backupPath); if (result.success) { - if (preparedRuntimeRestore) { + if (preparedRuntimeRestore || preparedHostLocalInferenceRestore) { const currentTarget = registry.getSandbox(targetSandbox); if (!currentTarget) { console.error( - ` Managed snapshot state was restored, but target '${targetSandbox}' is no longer registered.`, + ` Provider snapshot state was restored, but target '${targetSandbox}' is no longer registered.`, ); snapshotExit(1); } try { - const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); - confirmSandboxRuntimeRestore(provider, currentTarget, preparedRuntimeRestore); + const provider = preparedRuntimeRestore + ? requireCurrentSnapshotRuntimeProvider(currentTarget) + : requireCurrentSnapshotRuntimeProvider(currentTarget); + if (preparedRuntimeRestore) { + confirmSandboxRuntimeRestore(provider, currentTarget, preparedRuntimeRestore); + } + if (preparedHostLocalInferenceRestore) { + confirmHostLocalInferenceAuthority( + provider, + currentTarget, + preparedHostLocalInferenceRestore, + ); + } } catch (error) { console.error( - ` Managed snapshot state was restored, but provider restore proof failed: ${ + ` Provider snapshot state was restored, but provider restore proof failed: ${ error instanceof Error ? error.message : String(error) }.`, ); diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts index 8221b1535bc..67067ac46c6 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts @@ -12,6 +12,7 @@ import { } from "../../../onboard/managed-image/contract"; import { encodeManagedStartupProfile } from "../../../onboard/managed-startup/profile"; import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; +import { serializeHostLocalInferenceReceipt } from "../../../onboard/runtime-provider/host-local-inference"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../../state/registry/types"; import type { BackupOptions, BackupResult } from "../../../state/sandbox"; import { backupSandboxStateWithManagedAuthority } from "./backup-authority"; @@ -121,6 +122,31 @@ function successfulBackup(options: BackupOptions): BackupResult { }; } +function hostLocalReceipt(port = 8000): string { + return serializeHostLocalInferenceReceipt({ + schemaVersion: 1, + providerId: "mxc", + service: "vllm", + engineAuthority: { + schemaVersion: 1, + providerId: "mxc", + operation: "host-local-inference", + engineId: "mxc", + authorityId: "mxc:host-local", + bindingSha256: "c".repeat(64), + }, + endpoint: { host: "mxc.internal", port, networkName: "mxc-network" }, + runtime: { + kind: "container", + runtimeId: "mxc-vllm-runtime", + name: "nemoclaw-vllm", + imageRef: `nvcr.io/nvidia/vllm@sha256:${"d".repeat(64)}`, + specSha256: "e".repeat(64), + gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, + }, + }); +} + describe("managed snapshot backup authority", () => { it.each([ "openclaw", @@ -182,6 +208,80 @@ describe("managed snapshot backup authority", () => { expect(captureRuntime).not.toHaveBeenCalled(); }); + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("captures and republishes exact %s host-local inference authority", (agent) => { + const receipt = hostLocalReceipt(); + const entry = { + name: "alpha", + agent, + openshellDriver: "mxc", + hostLocalInferenceReceipt: receipt, + } satisfies SandboxEntry; + const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); + const reproveHostLocalInference = vi.fn((_provider, serialized: string) => serialized); + + const result = backupSandboxStateWithManagedAuthority( + "alpha", + { name: "host-local" }, + { + getSandbox: () => entry, + requireProvider: () => provider(), + captureRuntime: vi.fn() as never, + reproveHostLocalInference, + backup, + }, + ); + + expect(result.success).toBe(true); + expect(backup).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + name: "host-local", + hostLocalInferenceReceipt: receipt, + validateBeforePublish: expect.any(Function), + }), + ); + expect(reproveHostLocalInference).toHaveBeenCalledTimes(2); + }); + + it("rejects host-local route drift before manifest publication", () => { + const receipts = [hostLocalReceipt(), hostLocalReceipt(8001)]; + const entries = receipts.map( + (hostLocalInferenceReceipt) => + ({ + name: "alpha", + agent: "hermes", + openshellDriver: "mxc", + hostLocalInferenceReceipt, + }) satisfies SandboxEntry, + ); + const getSandbox = vi + .fn<() => SandboxEntry | null>() + .mockReturnValueOnce(entries[0]) + .mockReturnValueOnce(entries[1]); + const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); + + const result = backupSandboxStateWithManagedAuthority( + "alpha", + {}, + { + getSandbox, + requireProvider: () => provider(), + captureRuntime: vi.fn() as never, + reproveHostLocalInference: (_provider, serialized) => serialized, + backup, + }, + ); + + expect(result).toMatchObject({ + success: false, + error: expect.stringContaining("host-local inference changed during backup"), + }); + }); + it("fails before filesystem capture when the provider rejects managed authority", () => { const entry = sandbox("openclaw"); const backup = vi.fn(); diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.ts b/src/lib/actions/sandbox/snapshot/backup-authority.ts index bc8ccb371b3..3b374e4b514 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority.ts @@ -5,6 +5,7 @@ import { isDeepStrictEqual } from "node:util"; import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provider/current"; +import { reproveHostLocalInferenceReceipt } from "../../../onboard/runtime-provider/host-local-inference-lifecycle"; import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime-provider/registry"; import type { SandboxEntry } from "../../../state/registry/types"; import * as sandboxState from "../../../state/sandbox"; @@ -13,13 +14,14 @@ import { captureSandboxRuntimeSnapshot } from "./provider-lifecycle"; type SnapshotBackupAuthority = Pick< sandboxState.BackupOptions, - "runtimeSnapshot" | "workload" | "validateBeforePublish" + "runtimeSnapshot" | "workload" | "hostLocalInferenceReceipt" | "validateBeforePublish" >; interface SnapshotBackupAuthorityDependencies { readonly getSandbox: (sandboxName: string) => SandboxEntry | null; readonly requireProvider: (sandbox: SandboxEntry) => RuntimeProviderBundle; readonly captureRuntime: typeof captureSandboxRuntimeSnapshot; + readonly reproveHostLocalInference: typeof reproveHostLocalInferenceReceipt; readonly backup: typeof sandboxState.backupSandboxState; } @@ -27,6 +29,7 @@ const defaultDependencies: Omit requireRuntimeProviderBundleForSandbox(sandbox, CURRENT_RUNTIME_PROVIDER_BUNDLES), captureRuntime: captureSandboxRuntimeSnapshot, + reproveHostLocalInference: reproveHostLocalInferenceReceipt, // Keep the call late-bound so tests and alternative state stores can replace // the module export without this adapter retaining an import-time reference. backup: (...args) => sandboxState.backupSandboxState(...args), @@ -40,7 +43,7 @@ function failure(error: unknown): sandboxState.BackupResult { failedDirs: [], backedUpFiles: [], failedFiles: [], - error: `Cannot capture managed snapshot authority: ${detail}.`, + error: `Cannot capture provider snapshot authority: ${detail}.`, }; } @@ -106,6 +109,55 @@ function captureManagedAuthority( }; } +function captureHostLocalInferenceAuthority( + entry: SandboxEntry, + dependencies: SnapshotBackupAuthorityDependencies, +): Pick | null { + const receipt = entry.hostLocalInferenceReceipt; + if (typeof receipt !== "string") return null; + const provider = dependencies.requireProvider(entry); + if (dependencies.reproveHostLocalInference(provider, receipt) !== receipt) { + throw new Error("host-local inference authority changed before backup"); + } + return { + hostLocalInferenceReceipt: receipt, + validateBeforePublish: () => { + const current = dependencies.getSandbox(entry.name); + if (!current) throw new Error(`sandbox '${entry.name}' is no longer registered`); + if (current.hostLocalInferenceReceipt !== receipt) { + throw new Error(`sandbox '${entry.name}' host-local inference changed during backup`); + } + const currentProvider = dependencies.requireProvider(current); + if (currentProvider.identity.id !== provider.identity.id) { + throw new Error(`sandbox '${entry.name}' runtime provider changed during backup`); + } + if (dependencies.reproveHostLocalInference(currentProvider, receipt) !== receipt) { + throw new Error(`sandbox '${entry.name}' host-local inference changed during backup`); + } + }, + }; +} + +function captureSnapshotAuthority( + entry: SandboxEntry, + dependencies: SnapshotBackupAuthorityDependencies, +): SnapshotBackupAuthority | null { + const managed = captureManagedAuthority(entry, dependencies); + const hostLocal = captureHostLocalInferenceAuthority(entry, dependencies); + if (!managed && !hostLocal) return null; + return { + ...(managed?.runtimeSnapshot === undefined ? {} : { runtimeSnapshot: managed.runtimeSnapshot }), + ...(managed?.workload === undefined ? {} : { workload: managed.workload }), + ...(hostLocal?.hostLocalInferenceReceipt === undefined + ? {} + : { hostLocalInferenceReceipt: hostLocal.hostLocalInferenceReceipt }), + validateBeforePublish: () => { + managed?.validateBeforePublish?.(); + hostLocal?.validateBeforePublish?.(); + }, + }; +} + /** * Capture one managed workload and runtime authority pair around the complete * filesystem copy. The state layer publishes the manifest only after the @@ -123,7 +175,7 @@ export function backupSandboxStateWithManagedAuthority( let authority: SnapshotBackupAuthority | null; try { - authority = captureManagedAuthority(entry, dependencies); + authority = captureSnapshotAuthority(entry, dependencies); } catch (error) { return failure(error); } diff --git a/src/lib/actions/sandbox/snapshot/dependencies.ts b/src/lib/actions/sandbox/snapshot/dependencies.ts index 957d60dd625..079ed824eaf 100644 --- a/src/lib/actions/sandbox/snapshot/dependencies.ts +++ b/src/lib/actions/sandbox/snapshot/dependencies.ts @@ -6,6 +6,13 @@ import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provi import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime-provider/registry"; import type { SandboxEntry } from "../../../state/registry/types"; +export { + confirmHostLocalInferenceAuthority, + type PreparedHostLocalInferenceAuthority, + prepareHostLocalInferenceAuthority, + prepareSandboxHostLocalInferenceDestroyAuthority, + retirePreparedHostLocalInferenceAuthority, +} from "../../../onboard/runtime-provider/host-local-inference-lifecycle"; export type { ManagedWorkloadCloneSnapshot, PreparedManagedWorkloadCloneHandoff, @@ -38,6 +45,7 @@ export { prepareSandboxRuntimeRestore, SandboxSnapshotProviderError, } from "./provider-lifecycle"; +export type { RuntimeProviderBundle }; /** * Resolve the one already-registered provider bundle for a durable sandbox. diff --git a/src/lib/actions/sandbox/snapshot/restore-authority.ts b/src/lib/actions/sandbox/snapshot/restore-authority.ts index 8d5b2198e58..3a81212500b 100644 --- a/src/lib/actions/sandbox/snapshot/restore-authority.ts +++ b/src/lib/actions/sandbox/snapshot/restore-authority.ts @@ -3,6 +3,11 @@ import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provider/current"; +import { + confirmHostLocalInferenceAuthority, + type PreparedHostLocalInferenceAuthority, + prepareHostLocalInferenceAuthority, +} from "../../../onboard/runtime-provider/host-local-inference-lifecycle"; import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime-provider/registry"; import type { SandboxEntry } from "../../../state/registry/types"; import * as sandboxState from "../../../state/sandbox"; @@ -16,14 +21,14 @@ import { prepareSandboxRuntimeRestore, } from "./provider-lifecycle"; -interface ManagedRestoreAuthorityDependencies { +interface ProviderRestoreAuthorityDependencies { readonly getSandbox: (sandboxName: string) => SandboxEntry | null; readonly requireProvider: (sandbox: SandboxEntry) => RuntimeProviderBundle; readonly captureContentAuthority: typeof sandboxState.captureSnapshotRestoreAuthority; readonly restore: typeof sandboxState.restoreRecreatedSandboxState; } -const defaultDependencies: Omit = { +const defaultDependencies: Omit = { requireProvider: (sandbox) => requireRuntimeProviderBundleForSandbox(sandbox, CURRENT_RUNTIME_PROVIDER_BUNDLES), captureContentAuthority: (...args) => sandboxState.captureSnapshotRestoreAuthority(...args), @@ -38,21 +43,21 @@ function failure(error: unknown): sandboxState.RestoreResult { failedDirs: ["manifest"], restoredFiles: [], failedFiles: [], - error: `Cannot restore managed snapshot authority: ${detail}.`, + error: `Cannot restore provider snapshot authority: ${detail}.`, }; } /** - * Restore a rebuild backup through the same provider and content authority - * boundary as an explicit snapshot restore. Legacy/custom-image manifests - * retain their existing state-only path. + * Restore a rebuild backup through its provider runtime and content authority. + * Legacy/custom-image manifests without provider-backed state retain the + * existing state-only path. */ export function restoreRecreatedSandboxStateWithManagedAuthority( sandboxName: string, manifest: sandboxState.RebuildManifest, options: sandboxState.RecreatedSandboxRestoreOptions, - overrides: Pick & - Partial>, + overrides: Pick & + Partial>, ): sandboxState.RestoreResult { const dependencies = { ...defaultDependencies, ...overrides }; let snapshotProfile; @@ -65,14 +70,16 @@ export function restoreRecreatedSandboxStateWithManagedAuthority( } catch (error) { return failure(error); } - if (!snapshotProfile) { + const hostLocalInferenceReceipt = manifest.hostLocalInferenceReceipt; + if (!snapshotProfile && typeof hostLocalInferenceReceipt !== "string") { return dependencies.restore(sandboxName, manifest.backupPath, options); } - if (!manifest.runtimeSnapshot) { + if (snapshotProfile && !manifest.runtimeSnapshot) { return failure("managed snapshot is missing provider runtime authority"); } - let prepared: PreparedSandboxRuntimeRestore; + let preparedRuntime: PreparedSandboxRuntimeRestore | null = null; + let preparedHostLocal: PreparedHostLocalInferenceAuthority | null = null; let providerId: string; let contentAuthority: sandboxState.SnapshotRestoreAuthority; try { @@ -80,25 +87,34 @@ export function restoreRecreatedSandboxStateWithManagedAuthority( if (!target) throw new Error(`target '${sandboxName}' is not registered`); const provider = dependencies.requireProvider(target); providerId = provider.identity.id; - const profileRestore = prepareManagedSnapshotProfileRestore( - { - sandboxName: manifest.sandboxName, - agentType: manifest.agentType, - workload: manifest.workload, - }, - target, - provider, - ); - if (!profileRestore) throw new Error("managed profile restore authority is missing"); + if (snapshotProfile) { + const profileRestore = prepareManagedSnapshotProfileRestore( + { + sandboxName: manifest.sandboxName, + agentType: manifest.agentType, + workload: manifest.workload, + }, + target, + provider, + ); + if (!profileRestore) throw new Error("managed profile restore authority is missing"); + preparedRuntime = prepareSandboxRuntimeRestore( + provider, + target, + manifest.runtimeSnapshot!, + profileRestore.providerRestoreAuthority, + ); + } + if (typeof hostLocalInferenceReceipt === "string") { + preparedHostLocal = prepareHostLocalInferenceAuthority( + provider, + target, + hostLocalInferenceReceipt, + ); + } const captured = dependencies.captureContentAuthority(manifest.backupPath, manifest); if (!captured) throw new Error("selected snapshot content changed during restore preflight"); contentAuthority = captured; - prepared = prepareSandboxRuntimeRestore( - provider, - target, - manifest.runtimeSnapshot, - profileRestore.providerRestoreAuthority, - ); } catch (error) { return failure(error); } @@ -113,22 +129,32 @@ export function restoreRecreatedSandboxStateWithManagedAuthority( if (provider.identity.id !== providerId) { throw new Error(`target '${sandboxName}' runtime provider changed before restore`); } - const profileRestore = prepareManagedSnapshotProfileRestore( - { - sandboxName: manifest.sandboxName, - agentType: manifest.agentType, - workload: manifest.workload, - }, - current, - provider, - ); - if (!profileRestore) throw new Error("managed profile restore authority is missing"); - prepared = prepareSandboxRuntimeRestore( - provider, - current, - prepared.source, - profileRestore.providerRestoreAuthority, - ); + if (snapshotProfile) { + const profileRestore = prepareManagedSnapshotProfileRestore( + { + sandboxName: manifest.sandboxName, + agentType: manifest.agentType, + workload: manifest.workload, + }, + current, + provider, + ); + if (!profileRestore) throw new Error("managed profile restore authority is missing"); + if (!preparedRuntime) throw new Error("managed runtime restore authority is missing"); + preparedRuntime = prepareSandboxRuntimeRestore( + provider, + current, + preparedRuntime.source, + profileRestore.providerRestoreAuthority, + ); + } + if (typeof hostLocalInferenceReceipt === "string") { + preparedHostLocal = prepareHostLocalInferenceAuthority( + provider, + current, + hostLocalInferenceReceipt, + ); + } }, }); if (!restore.success) return restore; @@ -140,7 +166,10 @@ export function restoreRecreatedSandboxStateWithManagedAuthority( if (provider.identity.id !== providerId) { throw new Error(`target '${sandboxName}' runtime provider changed during restore`); } - confirmSandboxRuntimeRestore(provider, current, prepared); + if (preparedRuntime) confirmSandboxRuntimeRestore(provider, current, preparedRuntime); + if (preparedHostLocal) { + confirmHostLocalInferenceAuthority(provider, current, preparedHostLocal); + } return restore; } catch (error) { const detail = error instanceof Error ? error.message : String(error); @@ -148,7 +177,7 @@ export function restoreRecreatedSandboxStateWithManagedAuthority( ...restore, success: false, error: - `State was restored, but managed runtime proof failed: ${detail}. ` + + `State was restored, but provider runtime proof failed: ${detail}. ` + `Retry this exact snapshot after the runtime stabilizes.`, }; } diff --git a/src/lib/actions/sandbox/snapshot/restore-host-local-authority.test.ts b/src/lib/actions/sandbox/snapshot/restore-host-local-authority.test.ts new file mode 100644 index 00000000000..1d756439f92 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/restore-host-local-authority.test.ts @@ -0,0 +1,225 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; +import { + type HostLocalInferenceReceipt, + serializeHostLocalInferenceReceipt, +} from "../../../onboard/runtime-provider/host-local-inference"; +import type { SandboxEntry } from "../../../state/registry/types"; +import type { + RebuildManifest, + RecreatedSandboxRestoreOptions, + RestoreResult, +} from "../../../state/sandbox"; +import { restoreRecreatedSandboxStateWithManagedAuthority } from "./restore-authority"; + +type Agent = "openclaw" | "hermes" | "langchain-deepagents-code"; +type Service = "ollama" | "nim" | "vllm"; + +function receipt(service: Service, port = 8000): HostLocalInferenceReceipt { + return { + schemaVersion: 1, + providerId: "mxc", + service, + engineAuthority: { + schemaVersion: 1, + providerId: "mxc", + operation: "host-local-inference", + engineId: "mxc", + authorityId: "mxc:host-local", + bindingSha256: "a".repeat(64), + }, + endpoint: { host: "mxc.internal", port, networkName: "mxc-network" }, + runtime: + service === "ollama" + ? { + kind: "host", + probeImageRef: `quay.io/curl/curl@sha256:${"b".repeat(64)}`, + } + : { + kind: "container", + runtimeId: `mxc-${service}-runtime`, + name: `nemoclaw-${service}`, + imageRef: `nvcr.io/nvidia/${service}@sha256:${"c".repeat(64)}`, + specSha256: "d".repeat(64), + gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, + }, + }; +} + +function manifest(agent: Agent, service: Service, port = 8000): RebuildManifest { + return { + version: 1, + sandboxName: "alpha", + timestamp: "2026-08-02T00-00-00-000Z", + agentType: agent, + agentVersion: null, + expectedVersion: null, + stateDirs: [], + dir: "/sandbox", + backupPath: "/tmp/alpha", + blueprintDigest: null, + hostLocalInferenceReceipt: serializeHostLocalInferenceReceipt(receipt(service, port)), + }; +} + +function sandbox(agent: Agent, service: Service, port = 8000): SandboxEntry { + return { + name: "alpha", + agent, + openshellDriver: "mxc", + hostLocalInferenceReceipt: serializeHostLocalInferenceReceipt(receipt(service, port)), + }; +} + +function provider() { + const preserveForRebuild = vi.fn((value: HostLocalInferenceReceipt) => value); + const bundle = { + identity: { contractVersion: 1, id: "mxc", displayName: "MXC" }, + hostLocalInference: { + providerId: "mxc", + supported: true, + runtime: { + providerId: "mxc", + authorityId: "mxc:host-local", + services: ["ollama", "nim", "vllm"], + translateContainerArgs: (args: readonly string[]) => args, + qualifyOllama: vi.fn(), + startManaged: vi.fn(), + inspectManaged: vi.fn(), + stopManaged: vi.fn(), + preserveForRebuild, + prepareDestroy: vi.fn((value: HostLocalInferenceReceipt) => value), + destroy: vi.fn((value: HostLocalInferenceReceipt) => ({ + status: "removed" as const, + receipt: value, + })), + }, + }, + } as unknown as RuntimeProviderBundle; + return { bundle, preserveForRebuild }; +} + +function successfulRestore(options: RecreatedSandboxRestoreOptions): RestoreResult { + try { + options.validateBeforeMutation?.(); + return { + success: true, + restoredDirs: ["workspace"], + failedDirs: [], + restoredFiles: [], + failedFiles: [], + }; + } catch (error) { + return { + success: false, + restoredDirs: [], + failedDirs: ["workspace"], + restoredFiles: [], + failedFiles: [], + error: error instanceof Error ? error.message : String(error), + }; + } +} + +describe("host-local inference snapshot restore authority", () => { + it.each([ + ["openclaw", "ollama"], + ["openclaw", "nim"], + ["openclaw", "vllm"], + ["hermes", "ollama"], + ["hermes", "nim"], + ["hermes", "vllm"], + ["langchain-deepagents-code", "ollama"], + ["langchain-deepagents-code", "nim"], + ["langchain-deepagents-code", "vllm"], + ] as const)("re-proves exact %s %s authority before, at, and after restore", (agent, service) => { + const target = sandbox(agent, service); + const runtimeProvider = provider(); + const restore = vi.fn((_name, _path, options: RecreatedSandboxRestoreOptions) => + successfulRestore(options), + ); + + const result = restoreRecreatedSandboxStateWithManagedAuthority( + "alpha", + manifest(agent, service), + { targetAgentType: agent }, + { + getSandbox: () => target, + requireProvider: () => runtimeProvider.bundle, + captureContentAuthority: () => ({ + schemaVersion: 1, + backupPath: "/tmp/alpha", + contentSha256: "e".repeat(64), + }), + restore, + }, + ); + + expect(result.success).toBe(true); + expect(runtimeProvider.preserveForRebuild).toHaveBeenCalledTimes(3); + expect(restore).toHaveBeenCalledWith( + "alpha", + "/tmp/alpha", + expect.objectContaining({ + authority: expect.objectContaining({ contentSha256: "e".repeat(64) }), + validateBeforeMutation: expect.any(Function), + }), + ); + }); + + it("fails before mutation when the target route differs from the manifest", () => { + const runtimeProvider = provider(); + const restore = vi.fn(); + const result = restoreRecreatedSandboxStateWithManagedAuthority( + "alpha", + manifest("hermes", "vllm"), + { targetAgentType: "hermes" }, + { + getSandbox: () => sandbox("hermes", "vllm", 8001), + requireProvider: () => runtimeProvider.bundle, + captureContentAuthority: vi.fn(), + restore, + }, + ); + + expect(result).toMatchObject({ + success: false, + error: expect.stringContaining("different host-local inference authority"), + }); + expect(restore).not.toHaveBeenCalled(); + }); + + it("fails closed when the route changes at the filesystem mutation fence", () => { + const runtimeProvider = provider(); + const entries = [sandbox("openclaw", "ollama"), sandbox("openclaw", "ollama", 11435)]; + const getSandbox = vi + .fn<() => SandboxEntry | null>() + .mockReturnValueOnce(entries[0]) + .mockReturnValueOnce(entries[1]); + + const result = restoreRecreatedSandboxStateWithManagedAuthority( + "alpha", + manifest("openclaw", "ollama"), + { targetAgentType: "openclaw" }, + { + getSandbox, + requireProvider: () => runtimeProvider.bundle, + captureContentAuthority: () => ({ + schemaVersion: 1, + backupPath: "/tmp/alpha", + contentSha256: "f".repeat(64), + }), + restore: (_name, _path, options) => successfulRestore(options), + }, + ); + + expect(result).toMatchObject({ + success: false, + error: expect.stringContaining("different host-local inference authority"), + }); + }); +}); diff --git a/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts new file mode 100644 index 00000000000..470829b6707 --- /dev/null +++ b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { SandboxEntry } from "../../state/registry/types"; +import type { RuntimeProviderBundle } from "./contract"; +import { + type HostLocalInferenceReceipt, + serializeHostLocalInferenceReceipt, +} from "./host-local-inference"; +import { + prepareSandboxHostLocalInferenceDestroyAuthority, + retirePreparedHostLocalInferenceAuthority, +} from "./host-local-inference-lifecycle"; + +function serializedReceipt(service: "ollama" | "nim" | "vllm" = "vllm"): string { + return serializeHostLocalInferenceReceipt({ + schemaVersion: 1, + providerId: "mxc", + service, + engineAuthority: { + schemaVersion: 1, + providerId: "mxc", + operation: "host-local-inference", + engineId: "mxc", + authorityId: "mxc:host-local", + bindingSha256: "a".repeat(64), + }, + endpoint: { host: "mxc.internal", port: 8000, networkName: "mxc-network" }, + runtime: + service === "ollama" + ? { kind: "host", probeImageRef: `quay.io/curl/curl@sha256:${"b".repeat(64)}` } + : { + kind: "container", + runtimeId: `mxc-${service}`, + name: `nemoclaw-${service}`, + imageRef: `nvcr.io/nvidia/${service}@sha256:${"c".repeat(64)}`, + specSha256: "d".repeat(64), + gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, + }, + }); +} + +function sandbox(name: string, receipt = serializedReceipt()): SandboxEntry { + return { name, agent: "openclaw", openshellDriver: "mxc", hostLocalInferenceReceipt: receipt }; +} + +function provider() { + const preserveForRebuild = vi.fn((receipt: HostLocalInferenceReceipt) => receipt); + const prepareDestroy = vi.fn((receipt: HostLocalInferenceReceipt) => receipt); + const destroy = vi.fn((receipt: HostLocalInferenceReceipt) => ({ + status: receipt.runtime.kind === "host" ? ("retained" as const) : ("removed" as const), + ...(receipt.runtime.kind === "host" ? { reason: "host-process" as const } : {}), + receipt, + })); + const bundle = { + identity: { contractVersion: 1, id: "mxc", displayName: "MXC" }, + hostLocalInference: { + providerId: "mxc", + supported: true, + runtime: { + providerId: "mxc", + authorityId: "mxc:host-local", + services: ["ollama", "nim", "vllm"], + translateContainerArgs: (args: readonly string[]) => args, + qualifyOllama: vi.fn(), + startManaged: vi.fn(), + inspectManaged: vi.fn(), + stopManaged: vi.fn(), + preserveForRebuild, + prepareDestroy, + destroy, + }, + }, + } as unknown as RuntimeProviderBundle; + return { bundle, destroy, prepareDestroy, preserveForRebuild }; +} + +describe("host-local inference lifecycle authority", () => { + it.each([ + "ollama", + "nim", + "vllm", + ] as const)("retires exact unshared %s authority through an MXC-style provider", (service) => { + const runtimeProvider = provider(); + const entry = sandbox("alpha", serializedReceipt(service)); + const prepared = prepareSandboxHostLocalInferenceDestroyAuthority( + runtimeProvider.bundle, + entry, + ); + + expect(prepared).not.toBeNull(); + expect( + retirePreparedHostLocalInferenceAuthority(runtimeProvider.bundle, entry, prepared!, [entry]) + .status, + ).toBe(service === "ollama" ? "retained" : "removed"); + expect(runtimeProvider.destroy).toHaveBeenCalledOnce(); + expect(runtimeProvider.prepareDestroy).toHaveBeenCalledTimes(2); + }); + + it("keeps a managed runtime while another sandbox owns the exact receipt", () => { + const runtimeProvider = provider(); + const alpha = sandbox("alpha"); + const beta = sandbox("beta"); + const prepared = prepareSandboxHostLocalInferenceDestroyAuthority( + runtimeProvider.bundle, + alpha, + ); + + expect( + retirePreparedHostLocalInferenceAuthority(runtimeProvider.bundle, alpha, prepared!, [ + alpha, + beta, + ]).status, + ).toBe("shared"); + expect(runtimeProvider.destroy).not.toHaveBeenCalled(); + }); + + it("rejects registry drift before provider mutation", () => { + const runtimeProvider = provider(); + const alpha = sandbox("alpha"); + const prepared = prepareSandboxHostLocalInferenceDestroyAuthority( + runtimeProvider.bundle, + alpha, + ); + + expect(() => + retirePreparedHostLocalInferenceAuthority( + runtimeProvider.bundle, + sandbox("alpha", serializedReceipt("nim")), + prepared!, + [], + ), + ).toThrow("destroy target changed runtime identity"); + expect(runtimeProvider.destroy).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts new file mode 100644 index 00000000000..06af9fe63a8 --- /dev/null +++ b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxEntry } from "../../state/registry/types"; +import type { RuntimeProviderBundle } from "./contract"; +import { + type HostLocalInferenceDestroyResult, + type HostLocalInferenceReceipt, + type HostLocalInferenceRuntime, + parseHostLocalInferenceReceipt, + serializeHostLocalInferenceReceipt, +} from "./host-local-inference"; + +export interface PreparedHostLocalInferenceAuthority { + readonly providerId: string; + readonly sandboxName: string; + readonly serializedReceipt: string; + readonly receipt: HostLocalInferenceReceipt; +} + +export type HostLocalInferenceRetirementResult = + | HostLocalInferenceDestroyResult + | { + readonly status: "shared"; + readonly receipt: HostLocalInferenceReceipt; + }; + +function requireRuntime( + provider: RuntimeProviderBundle, + receipt: HostLocalInferenceReceipt, +): HostLocalInferenceRuntime { + const surface = provider.hostLocalInference; + if (!surface.supported) { + throw new Error( + `Runtime provider '${provider.identity.id}' does not support host-local inference.`, + ); + } + if ( + surface.providerId !== provider.identity.id || + surface.runtime.providerId !== provider.identity.id || + receipt.providerId !== provider.identity.id + ) { + throw new Error("Host-local inference receipt belongs to a different runtime provider."); + } + if (!surface.runtime.services.includes(receipt.service)) { + throw new Error( + `Runtime provider '${provider.identity.id}' does not support host-local ${receipt.service}.`, + ); + } + return surface.runtime; +} + +/** Re-prove an exact durable route through its owning provider. */ +export function reproveHostLocalInferenceReceipt( + provider: RuntimeProviderBundle, + serialized: string, +): string { + const receipt = parseHostLocalInferenceReceipt(serialized); + const runtime = requireRuntime(provider, receipt); + const reproved = serializeHostLocalInferenceReceipt(runtime.preserveForRebuild(receipt)); + if (reproved !== serialized) { + throw new Error("Host-local inference authority changed while it was being preserved."); + } + return reproved; +} + +export function prepareHostLocalInferenceAuthority( + provider: RuntimeProviderBundle, + sandbox: Pick, + serialized: string, +): PreparedHostLocalInferenceAuthority { + if (sandbox.hostLocalInferenceReceipt !== serialized) { + throw new Error(`target '${sandbox.name}' has different host-local inference authority`); + } + reproveHostLocalInferenceReceipt(provider, serialized); + return Object.freeze({ + providerId: provider.identity.id, + sandboxName: sandbox.name, + serializedReceipt: serialized, + receipt: parseHostLocalInferenceReceipt(serialized), + }); +} + +export function prepareSandboxHostLocalInferenceAuthority( + provider: RuntimeProviderBundle, + sandbox: Pick, +): PreparedHostLocalInferenceAuthority | null { + return typeof sandbox.hostLocalInferenceReceipt === "string" + ? prepareHostLocalInferenceAuthority(provider, sandbox, sandbox.hostLocalInferenceReceipt) + : null; +} + +export function prepareSandboxHostLocalInferenceDestroyAuthority( + provider: RuntimeProviderBundle, + sandbox: Pick, +): PreparedHostLocalInferenceAuthority | null { + const serialized = sandbox.hostLocalInferenceReceipt; + if (typeof serialized !== "string") return null; + const receipt = parseHostLocalInferenceReceipt(serialized); + const runtime = requireRuntime(provider, receipt); + if (serializeHostLocalInferenceReceipt(runtime.prepareDestroy(receipt)) !== serialized) { + throw new Error("Host-local inference authority changed during destroy preflight."); + } + return Object.freeze({ + providerId: provider.identity.id, + sandboxName: sandbox.name, + serializedReceipt: serialized, + receipt, + }); +} + +export function confirmHostLocalInferenceAuthority( + provider: RuntimeProviderBundle, + sandbox: Pick, + prepared: PreparedHostLocalInferenceAuthority, +): void { + if (provider.identity.id !== prepared.providerId || sandbox.name !== prepared.sandboxName) { + throw new Error("Host-local inference restore target changed runtime identity."); + } + prepareHostLocalInferenceAuthority(provider, sandbox, prepared.serializedReceipt); +} + +function confirmHostLocalInferenceDestroyAuthority( + provider: RuntimeProviderBundle, + sandbox: Pick, + prepared: PreparedHostLocalInferenceAuthority, +): void { + if ( + provider.identity.id !== prepared.providerId || + sandbox.name !== prepared.sandboxName || + sandbox.hostLocalInferenceReceipt !== prepared.serializedReceipt + ) { + throw new Error("Host-local inference destroy target changed runtime identity."); + } + const runtime = requireRuntime(provider, prepared.receipt); + if ( + serializeHostLocalInferenceReceipt(runtime.prepareDestroy(prepared.receipt)) !== + prepared.serializedReceipt + ) { + throw new Error("Host-local inference authority changed before destroy mutation."); + } +} + +/** + * Retire an exact managed runtime only after the caller has confirmed sandbox + * deletion. Shared receipts remain live for their peer sandboxes, and host + * Ollama is always retained because NemoClaw does not own that process. + */ +export function retirePreparedHostLocalInferenceAuthority( + provider: RuntimeProviderBundle, + sandbox: Pick, + prepared: PreparedHostLocalInferenceAuthority, + peers: readonly Pick[], +): HostLocalInferenceRetirementResult { + confirmHostLocalInferenceDestroyAuthority(provider, sandbox, prepared); + if ( + peers.some( + (peer) => + peer.name !== sandbox.name && peer.hostLocalInferenceReceipt === prepared.serializedReceipt, + ) + ) { + return Object.freeze({ status: "shared" as const, receipt: prepared.receipt }); + } + const runtime = requireRuntime(provider, prepared.receipt); + const result = runtime.destroy(prepared.receipt); + if (serializeHostLocalInferenceReceipt(result.receipt) !== prepared.serializedReceipt) { + throw new Error("Host-local inference destroy returned different runtime authority."); + } + return result; +} diff --git a/src/lib/onboard/runtime-provider/host-local-inference-routing.test.ts b/src/lib/onboard/runtime-provider/host-local-inference-routing.test.ts index 1cfe0a60ec4..45509c51623 100644 --- a/src/lib/onboard/runtime-provider/host-local-inference-routing.test.ts +++ b/src/lib/onboard/runtime-provider/host-local-inference-routing.test.ts @@ -57,6 +57,8 @@ function runtime(): HostLocalInferenceRuntime { inspectManaged: vi.fn((value) => ({ running: true, receipt: value })), stopManaged: vi.fn((value) => ({ running: false, receipt: value })), preserveForRebuild: vi.fn((value) => value), + prepareDestroy: vi.fn((value) => value), + destroy: vi.fn((value) => ({ status: "removed" as const, receipt: value })), }; } diff --git a/src/lib/onboard/runtime-provider/host-local-inference.ts b/src/lib/onboard/runtime-provider/host-local-inference.ts index 41c62496554..327e2991e49 100644 --- a/src/lib/onboard/runtime-provider/host-local-inference.ts +++ b/src/lib/onboard/runtime-provider/host-local-inference.ts @@ -80,6 +80,17 @@ export interface HostLocalManagedInferenceInspection { readonly receipt: HostLocalInferenceReceipt; } +export type HostLocalInferenceDestroyResult = + | { + readonly status: "retained"; + readonly reason: "host-process"; + readonly receipt: HostLocalInferenceReceipt; + } + | { + readonly status: "removed" | "already-absent"; + readonly receipt: HostLocalInferenceReceipt; + }; + export interface HostLocalInferenceRouteAuthority { readonly schemaVersion: 1; readonly providerId: string; @@ -113,6 +124,10 @@ export interface HostLocalInferenceRuntime { stopManaged(receipt: HostLocalInferenceReceipt): HostLocalManagedInferenceInspection; /** Re-prove the same out-of-sandbox service before carrying it into a rebuild. */ preserveForRebuild(receipt: HostLocalInferenceReceipt): HostLocalInferenceReceipt; + /** Prove exact ownership for teardown without requiring the service to be healthy. */ + prepareDestroy(receipt: HostLocalInferenceReceipt): HostLocalInferenceReceipt; + /** Retire only the exact provider-owned runtime; host processes remain externally owned. */ + destroy(receipt: HostLocalInferenceReceipt): HostLocalInferenceDestroyResult; } const PROVIDER_ID = /^[a-z][a-z0-9-]{0,62}$/u; diff --git a/src/lib/onboard/runtime-provider/podman-host-local-inference.test.ts b/src/lib/onboard/runtime-provider/podman-host-local-inference.test.ts index 022a075b948..d4e7c2ff69e 100644 --- a/src/lib/onboard/runtime-provider/podman-host-local-inference.test.ts +++ b/src/lib/onboard/runtime-provider/podman-host-local-inference.test.ts @@ -68,6 +68,25 @@ describe("Podman host-local inference runtime", () => { ]); }); + it("retains externally owned Ollama while proving the exact route during destroy", () => { + const host = runtimeHarness(); + const receipt = host.runtime.qualifyOllama({ + networkName: "openshell", + hostPort: 11434, + probeImageRef: PROBE_IMAGE_REF, + }); + host.setProbeFailure("host process is temporarily unavailable"); + + expect(host.runtime.destroy(receipt)).toEqual({ + status: "retained", + reason: "host-process", + receipt, + }); + expect( + host.capture.mock.calls.map(([args]) => args).some(([operation]) => operation === "rm"), + ).toBe(false); + }); + it.each([ "nim", "vllm", @@ -115,6 +134,25 @@ describe("Podman host-local inference runtime", () => { ]); }); + it.each([ + "nim", + "vllm", + ] as const)("removes exact managed %s authority and makes destroy idempotent", (service) => { + const host = runtimeHarness(); + const receipt = host.runtime.startManaged(managedInput(service)); + const runtimeId = + receipt.runtime.kind === "container" ? receipt.runtime.runtimeId : "unreachable"; + + expect(host.runtime.destroy(receipt)).toEqual({ status: "removed", receipt }); + expect(host.containers.has(runtimeId)).toBe(false); + expect(host.runtime.destroy(receipt)).toEqual({ status: "already-absent", receipt }); + expect(host.capture.mock.calls.map(([args]) => args)).toContainEqual([ + "rm", + "--force", + runtimeId, + ]); + }); + it("removes only a newly created container when exact post-start inspection fails", () => { const host = runtimeHarness(); host.setInspectImageOverride(`nvcr.io/nvidia/other@sha256:${"d".repeat(64)}`); diff --git a/src/lib/onboard/runtime-provider/podman-host-local-inference.ts b/src/lib/onboard/runtime-provider/podman-host-local-inference.ts index 5846ae0a6cb..c13b27fdd68 100644 --- a/src/lib/onboard/runtime-provider/podman-host-local-inference.ts +++ b/src/lib/onboard/runtime-provider/podman-host-local-inference.ts @@ -376,6 +376,16 @@ function inspectContainer(engine: ContainerEngine, runtimeId: string): ManagedCo }); } +function exactContainerExists(engine: ContainerEngine, runtimeId: string): boolean { + const result = engine.capture(["container", "exists", runtimeId], PROBE_TIMEOUT_MS); + if (result.error) { + throw new Error(`Podman inference container existence check failed: ${commandDetail(result)}`); + } + if (result.status === 0) return true; + if (result.status === 1) return false; + throw new Error(`Podman inference container existence check failed: ${commandDetail(result)}`); +} + function requireManagedIdentity( container: ManagedContainer, expected: { @@ -753,5 +763,37 @@ export function createPodmanHostLocalInferenceRuntime( } return normalized; }, + prepareDestroy(receipt: HostLocalInferenceReceipt) { + const normalized = authorizeReceipt(receipt); + if ( + normalized.runtime.kind === "container" && + exactContainerExists(engine, normalized.runtime.runtimeId) + ) { + inspectReceipt(normalized); + } + return normalized; + }, + destroy(receipt: HostLocalInferenceReceipt) { + const normalized = authorizeReceipt(receipt); + if (normalized.runtime.kind === "host") { + return Object.freeze({ + status: "retained" as const, + reason: "host-process" as const, + receipt: normalized, + }); + } + if (!exactContainerExists(engine, normalized.runtime.runtimeId)) { + return Object.freeze({ status: "already-absent" as const, receipt: normalized }); + } + const inspected = inspectReceipt(normalized); + requireSuccess( + "container removal", + engine.capture(["rm", "--force", inspected.container.runtimeId], MUTATION_TIMEOUT_MS), + ); + if (exactContainerExists(engine, inspected.container.runtimeId)) { + throw new Error("Podman inference removal left the exact managed container present."); + } + return Object.freeze({ status: "removed" as const, receipt: inspected.receipt }); + }, }); } diff --git a/src/lib/onboard/runtime-provider/podman.test.ts b/src/lib/onboard/runtime-provider/podman.test.ts index 287c425f553..4ae91bee139 100644 --- a/src/lib/onboard/runtime-provider/podman.test.ts +++ b/src/lib/onboard/runtime-provider/podman.test.ts @@ -143,6 +143,8 @@ function inferenceRuntime(authorityId = AUTHORITY_ID): HostLocalInferenceRuntime inspectManaged: unavailable, stopManaged: unavailable, preserveForRebuild: unavailable, + prepareDestroy: unavailable, + destroy: unavailable, }; } diff --git a/src/lib/onboard/runtime-provider/registry.ts b/src/lib/onboard/runtime-provider/registry.ts index 9948d910014..048d46b1396 100644 --- a/src/lib/onboard/runtime-provider/registry.ts +++ b/src/lib/onboard/runtime-provider/registry.ts @@ -335,6 +335,8 @@ function validateHostLocalInferenceSurface( "inspectManaged", "stopManaged", "preserveForRebuild", + "prepareDestroy", + "destroy", ] as const) { requireFunction(runtime, operation, "hostLocalInference runtime"); } diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index f006ad65730..28ee169cb8a 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -109,6 +109,8 @@ function mxcInferenceRuntime(): HostLocalInferenceRuntime { inspectManaged: unavailable, stopManaged: unavailable, preserveForRebuild: unavailable, + prepareDestroy: unavailable, + destroy: unavailable, }; } diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index bcbb710d867..302d8eb0c4f 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -60,6 +60,7 @@ import { parseOpenClawImagePluginInstalls, planOpenClawPluginRestore, } from "./openclaw-plugin-restore.js"; +import { cloneSandboxHostLocalInferenceReceipt } from "./registry/host-local-inference.js"; import { cloneSandboxRuntimeSnapshot, type SandboxRuntimeSnapshot, @@ -81,6 +82,8 @@ export const OPENCLAW_IMAGE_PLUGIN_PROVENANCE_RESTORE_ERROR = "custom-image OpenClaw plugin provenance is missing or invalid"; export const MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR = "managed snapshot restore requires exact content and runtime authority"; +export const HOST_LOCAL_INFERENCE_SNAPSHOT_RESTORE_AUTHORITY_ERROR = + "host-local inference snapshot restore requires exact content and runtime authority"; function parseJson(text: string): T { return JSON.parse(text); @@ -131,6 +134,8 @@ export interface RebuildManifest { * snapshot. Older and explicit Dockerfile snapshots omit this field. */ workload?: SandboxWorkloadReceipt; + /** Exact provider-neutral authority for out-of-sandbox inference. */ + hostLocalInferenceReceipt?: string; instances?: InstanceBackup[]; // Optional user-provided label for `snapshot restore `. name?: string; @@ -145,6 +150,7 @@ export interface BackupOptions { name?: string | null; runtimeSnapshot?: SandboxRuntimeSnapshot; workload?: SandboxWorkloadReceipt; + hostLocalInferenceReceipt?: string; /** * Internal publication fence for provider-backed backups. The callback * runs after data capture and sanitization but before the manifest becomes @@ -320,6 +326,9 @@ function isRebuildManifest(value: unknown): value is RebuildManifest { : cloneSandboxRuntimeSnapshot(value.runtimeSnapshot); const workload = value.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(value.workload as never); + const hostLocalInferenceReceipt = cloneSandboxHostLocalInferenceReceipt( + value.hostLocalInferenceReceipt as string | null | undefined, + ); return ( typeof value.version === "number" && typeof value.sandboxName === "string" && @@ -347,6 +356,8 @@ function isRebuildManifest(value: unknown): value is RebuildManifest { (value.customPolicies === undefined || isCustomPolicyEntryArray(value.customPolicies)) && (value.runtimeSnapshot === undefined || runtimeSnapshot !== undefined) && (value.workload === undefined || workload !== undefined) && + (value.hostLocalInferenceReceipt === undefined || + (typeof hostLocalInferenceReceipt === "string" && hostLocalInferenceReceipt.length > 0)) && (workload?.kind !== "managed-image" || runtimeSnapshot !== undefined) && (value.instances === undefined || (Array.isArray(value.instances) && @@ -944,6 +955,7 @@ export { isSshTransportFailure }; function normalizeSnapshotBackupAuthority(options: BackupOptions): { readonly runtimeSnapshot?: SandboxRuntimeSnapshot; readonly workload?: SandboxWorkloadReceipt; + readonly hostLocalInferenceReceipt?: string; readonly error?: string; } { const runtimeSnapshot = @@ -952,18 +964,28 @@ function normalizeSnapshotBackupAuthority(options: BackupOptions): { : cloneSandboxRuntimeSnapshot(options.runtimeSnapshot); const workload = options.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(options.workload); + const hostLocalInferenceReceipt = cloneSandboxHostLocalInferenceReceipt( + options.hostLocalInferenceReceipt, + ); if (options.runtimeSnapshot !== undefined && runtimeSnapshot === undefined) { return { error: "snapshot runtime state is invalid or cannot be represented" }; } if (options.workload !== undefined && workload === undefined) { return { error: "snapshot workload authority is invalid" }; } + if ( + options.hostLocalInferenceReceipt !== undefined && + typeof hostLocalInferenceReceipt !== "string" + ) { + return { error: "snapshot host-local inference authority is invalid" }; + } if (workload?.kind === "managed-image" && runtimeSnapshot === undefined) { return { error: "managed snapshot is missing provider runtime state" }; } return { ...(runtimeSnapshot === undefined ? {} : { runtimeSnapshot }), ...(workload === undefined ? {} : { workload }), + ...(typeof hostLocalInferenceReceipt === "string" ? { hostLocalInferenceReceipt } : {}), }; } @@ -1780,11 +1802,13 @@ function restoreSandboxStateInternal( error, }; }; - if ( - manifest.workload?.kind === "managed-image" && - (!options.authority || !options.validateBeforeMutation) - ) { - return failRestoreContract(MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR); + if (!options.authority || !options.validateBeforeMutation) { + if (manifest.workload?.kind === "managed-image") { + return failRestoreContract(MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR); + } + if (typeof manifest.hostLocalInferenceReceipt === "string") { + return failRestoreContract(HOST_LOCAL_INFERENCE_SNAPSHOT_RESTORE_AUTHORITY_ERROR); + } } if (options.targetAgentType !== manifest.agentType) { return failRestoreContract( @@ -2203,6 +2227,9 @@ function readManifest(backupPath: string): RebuildManifest | null { : cloneSandboxRuntimeSnapshot(manifest.runtimeSnapshot); const workload = manifest.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(manifest.workload); + const hostLocalInferenceReceipt = cloneSandboxHostLocalInferenceReceipt( + manifest.hostLocalInferenceReceipt, + ); return { ...manifest, dir, @@ -2212,6 +2239,7 @@ function readManifest(backupPath: string): RebuildManifest | null { blueprintDigest: manifest.blueprintDigest ?? null, ...(runtimeSnapshot === undefined ? {} : { runtimeSnapshot }), ...(workload === undefined ? {} : { workload }), + ...(typeof hostLocalInferenceReceipt === "string" ? { hostLocalInferenceReceipt } : {}), }; } catch { return null; diff --git a/test/helpers/podman-host-local-inference-test-harness.ts b/test/helpers/podman-host-local-inference-test-harness.ts index 7798df86f89..26f9e684abe 100644 --- a/test/helpers/podman-host-local-inference-test-harness.ts +++ b/test/helpers/podman-host-local-inference-test-harness.ts @@ -152,6 +152,13 @@ function engineHarness(authorityId = AUTHORITY_ID) { }; } case "container": { + if (args[1] === "exists") { + return { + status: containers.has(String(args[2])) ? 0 : 1, + stdout: "", + stderr: "", + }; + } assert.equal(args[1], "inspect", "unexpected container operation"); const container = requireContainer(containers, String(args[2])); return { diff --git a/test/onboard-host-local-inference-routing.test.ts b/test/onboard-host-local-inference-routing.test.ts index 02a85ed45a6..3c02390e1af 100644 --- a/test/onboard-host-local-inference-routing.test.ts +++ b/test/onboard-host-local-inference-routing.test.ts @@ -58,6 +58,8 @@ function runtime(): HostLocalInferenceRuntime { inspectManaged: vi.fn((receipt) => ({ running: true, receipt })), stopManaged: vi.fn((receipt) => ({ running: false, receipt })), preserveForRebuild: vi.fn((receipt) => receipt), + prepareDestroy: vi.fn((receipt) => receipt), + destroy: vi.fn((receipt) => ({ status: "removed" as const, receipt })), }; } diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index dec1b7981fc..517ca368bf8 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -165,6 +165,7 @@ describe("runtime provider central source boundary", () => { "src/lib/onboard/runtime-provider/contract.ts", "src/lib/onboard/runtime-provider/current.ts", "src/lib/onboard/runtime-provider/docker.ts", + "src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts", "src/lib/onboard/runtime-provider/host-local-inference-routing.ts", "src/lib/onboard/runtime-provider/host-local-inference.ts", "src/lib/onboard/runtime-provider/persisted-engine-authority.ts", From 5ca986c2c73e0b8c305129d1a847ae87cbf2a1dc Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 07:45:34 -0700 Subject: [PATCH 2/7] test(runtime): bind destroy registry reads Signed-off-by: Aaron Erickson --- .../onboard/runtime-provider/runtime-provider-contract.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index 28ee169cb8a..1ca70b03142 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -909,6 +909,8 @@ describe("socket-free MXC action contract", () => { executeSandboxDestroy({ cleanupShieldsArtifacts, force: false, + getSandbox, + listSandboxes: () => ({ sandboxes: [entry] }), runOpenshell, sandbox: entry, sandboxConfirmedAbsent: false, From 57c269a6f8b53ef21467d0bdebc22b6bc8385a02 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 07:55:06 -0700 Subject: [PATCH 3/7] test(runtime): linearize host-local destroy harness Signed-off-by: Aaron Erickson --- .../destroy-host-local-inference.test.ts | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/lib/actions/sandbox/destroy-host-local-inference.test.ts b/src/lib/actions/sandbox/destroy-host-local-inference.test.ts index 2ec812df39e..cd8075d2a60 100644 --- a/src/lib/actions/sandbox/destroy-host-local-inference.test.ts +++ b/src/lib/actions/sandbox/destroy-host-local-inference.test.ts @@ -44,13 +44,25 @@ function sandbox(name = "alpha"): SandboxEntry { }; } -function provider(destroyError?: string) { +function destroySuccessfully(value: HostLocalInferenceReceipt) { + return { status: "removed" as const, receipt: value }; +} + +function failDestroy(message: string) { + return (_value: HostLocalInferenceReceipt): never => { + throw new Error(message); + }; +} + +function provider( + destroyRuntime: (value: HostLocalInferenceReceipt) => { + status: "removed"; + receipt: HostLocalInferenceReceipt; + } = destroySuccessfully, +) { const preserveForRebuild = vi.fn((value: HostLocalInferenceReceipt) => value); const prepareDestroy = vi.fn((value: HostLocalInferenceReceipt) => value); - const destroy = vi.fn((value: HostLocalInferenceReceipt) => { - if (destroyError) throw new Error(destroyError); - return { status: "removed" as const, receipt: value }; - }); + const destroy = vi.fn(destroyRuntime); const bundle = createInMemoryRuntimeProviderBundle({ providerId: "mxc", workloadProfile: { @@ -85,7 +97,7 @@ async function runDestroy(runtimeProvider: ReturnType, peers: S getSandbox: () => entry, listSandboxes: () => ({ sandboxes: [entry, ...peers] }), runOpenshell: (args) => { - if (args[0] === "sandbox" && args[1] === "delete") events.push("delete"); + events.push(args.join(" ")); return { status: 0, stdout: "", stderr: "" }; }, sandbox: entry, @@ -106,7 +118,7 @@ describe("sandbox destroy host-local inference transaction", () => { const { events, result } = await runDestroy(runtimeProvider, []); expect(result).toMatchObject({ ok: true }); - expect(events).toEqual(["delete", "cleanup"]); + expect(events.slice(-2)).toEqual(["sandbox delete alpha", "cleanup"]); expect(runtimeProvider.prepareDestroy).toHaveBeenCalledTimes(2); expect(runtimeProvider.destroy).toHaveBeenCalledOnce(); }); @@ -120,7 +132,7 @@ describe("sandbox destroy host-local inference transaction", () => { }); it("preserves local ownership when exact runtime retirement fails", async () => { - const runtimeProvider = provider("injected runtime removal failure"); + const runtimeProvider = provider(failDestroy("injected runtime removal failure")); const { events, result } = await runDestroy(runtimeProvider, []); expect(result).toMatchObject({ @@ -128,6 +140,6 @@ describe("sandbox destroy host-local inference transaction", () => { deleteConfirmed: true, hostLocalInferenceCleanupFailure: "injected runtime removal failure", }); - expect(events).toEqual(["delete", "cleanup"]); + expect(events.slice(-2)).toEqual(["sandbox delete alpha", "cleanup"]); }); }); From 7b1567ad78d51256ef2f197e089eb16b9bb27bf6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 08:21:26 -0700 Subject: [PATCH 4/7] test(runtime): carry managed probe authority Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/destroy-host-local-inference.test.ts | 1 + src/lib/actions/sandbox/snapshot/backup-authority.test.ts | 1 + .../sandbox/snapshot/restore-host-local-authority.test.ts | 1 + .../runtime-provider/host-local-inference-lifecycle.test.ts | 1 + 4 files changed, 4 insertions(+) diff --git a/src/lib/actions/sandbox/destroy-host-local-inference.test.ts b/src/lib/actions/sandbox/destroy-host-local-inference.test.ts index cd8075d2a60..8e5600d2f2b 100644 --- a/src/lib/actions/sandbox/destroy-host-local-inference.test.ts +++ b/src/lib/actions/sandbox/destroy-host-local-inference.test.ts @@ -29,6 +29,7 @@ function receipt(): HostLocalInferenceReceipt { runtimeId: "mxc-vllm", name: "nemoclaw-vllm", imageRef: `nvcr.io/nvidia/vllm@sha256:${"b".repeat(64)}`, + probeImageRef: `quay.io/curl/curl@sha256:${"d".repeat(64)}`, specSha256: "c".repeat(64), gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, }, diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts index 67067ac46c6..fda55c091a1 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts @@ -141,6 +141,7 @@ function hostLocalReceipt(port = 8000): string { runtimeId: "mxc-vllm-runtime", name: "nemoclaw-vllm", imageRef: `nvcr.io/nvidia/vllm@sha256:${"d".repeat(64)}`, + probeImageRef: `quay.io/curl/curl@sha256:${"f".repeat(64)}`, specSha256: "e".repeat(64), gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, }, diff --git a/src/lib/actions/sandbox/snapshot/restore-host-local-authority.test.ts b/src/lib/actions/sandbox/snapshot/restore-host-local-authority.test.ts index 1d756439f92..f2c2af57329 100644 --- a/src/lib/actions/sandbox/snapshot/restore-host-local-authority.test.ts +++ b/src/lib/actions/sandbox/snapshot/restore-host-local-authority.test.ts @@ -44,6 +44,7 @@ function receipt(service: Service, port = 8000): HostLocalInferenceReceipt { runtimeId: `mxc-${service}-runtime`, name: `nemoclaw-${service}`, imageRef: `nvcr.io/nvidia/${service}@sha256:${"c".repeat(64)}`, + probeImageRef: `quay.io/curl/curl@sha256:${"b".repeat(64)}`, specSha256: "d".repeat(64), gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, }, diff --git a/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts index 470829b6707..8d29fe3b76a 100644 --- a/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts +++ b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts @@ -36,6 +36,7 @@ function serializedReceipt(service: "ollama" | "nim" | "vllm" = "vllm"): string runtimeId: `mxc-${service}`, name: `nemoclaw-${service}`, imageRef: `nvcr.io/nvidia/${service}@sha256:${"c".repeat(64)}`, + probeImageRef: `quay.io/curl/curl@sha256:${"b".repeat(64)}`, specSha256: "d".repeat(64), gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, }, From ee978db778e691b0fe8f160913d323254bb61d60 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 10:24:19 -0700 Subject: [PATCH 5/7] test(snapshot): return runtime cleanup authority Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/snapshot.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 19cf6428de0..9575f19b995 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -91,7 +91,6 @@ const lifecycleMock = vi.hoisted(() => { ), }; }); - const backupSandboxStateMock = vi.fn(); const captureOpenshellMock = vi.fn< (args: string[], opts?: Record) => OpenshellCaptureResult @@ -239,12 +238,13 @@ vi.mock("./restore-gateway-pairing", () => ({ establishRestoredSandboxGatewayPairing: vi.fn(), waitForRestoredSandboxGatewaySupervisor: vi.fn(() => true), })); - vi.mock("./destroy", () => ({ cleanupShieldsDestroyArtifacts: lifecycleMock.cleanupShieldsDestroyArtifactsMock, removeSandboxRegistryEntry: vi.fn(), removeSandboxRegistryEntryOutcome: vi.fn(() => ({ status: "complete", removed: true })), - requireSandboxDestructiveCleanupAuthority: vi.fn(), + requireSandboxDestructiveCleanupAuthority: vi.fn(() => ({ + provider: { identity: { id: "docker" } }, + })), })); describe("runSandboxSnapshot", () => { beforeEach(() => { From 90238a1efdfd3b8929fb527ee7585a4fc0664b39 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 12:24:54 -0700 Subject: [PATCH 6/7] test(snapshot): prove command restore inference authority Signed-off-by: Aaron Erickson --- ...pshot-command-host-local-authority.test.ts | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 src/lib/actions/sandbox/snapshot-command-host-local-authority.test.ts diff --git a/src/lib/actions/sandbox/snapshot-command-host-local-authority.test.ts b/src/lib/actions/sandbox/snapshot-command-host-local-authority.test.ts new file mode 100644 index 00000000000..92a59c683d8 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot-command-host-local-authority.test.ts @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + hostLocalInferenceReceipt, + serializedHostLocalInferenceReceipt, +} from "../../../../test/helpers/host-local-inference-receipt"; +import type { RuntimeProviderBundle } from "../../onboard/runtime-provider/contract"; +import { serializeHostLocalInferenceReceipt } from "../../onboard/runtime-provider/host-local-inference"; +import type { SandboxEntry } from "../../state/registry/types"; +import type { RebuildManifest, RestoreResult, SnapshotRestoreOptions } from "../../state/sandbox"; +import { runSandboxSnapshot } from "./snapshot"; + +const harness = vi.hoisted(() => { + let registryEntry: unknown = null; + const events: string[] = []; + return { + events, + getSandbox: vi.fn(() => registryEntry), + setRegistryEntry: (entry: unknown) => { + registryEntry = entry; + }, + getLatestBackup: vi.fn(), + captureSnapshotRestoreAuthority: vi.fn(), + restoreSandboxState: vi.fn(), + preserveForRebuild: vi.fn((receipt: unknown) => { + events.push("reprove"); + return receipt; + }), + }; +}); + +const provider = { + identity: { contractVersion: 1, id: "docker", displayName: "Docker" }, + hostLocalInference: { + providerId: "docker", + supported: true, + runtime: { + providerId: "docker", + authorityId: "docker:host-local", + services: ["vllm"], + preserveForRebuild: harness.preserveForRebuild, + }, + }, +} as unknown as RuntimeProviderBundle; + +vi.mock("../../adapters/openshell/runtime", () => ({ + captureOpenshell: vi.fn(() => ({ status: 0, output: "alpha Ready\n" })), + getOpenshellBinary: vi.fn(() => "openshell"), + runOpenshell: vi.fn(() => ({ status: 0, output: "" })), +})); + +vi.mock("../../policy", () => ({ + applyPreset: vi.fn(() => true), + applyPresetContent: vi.fn(() => true), + getAppliedPresets: vi.fn(() => []), + getPresetContentGatewayState: vi.fn(() => "absent"), + loadPresetForSandbox: vi.fn(() => null), + removePreset: vi.fn(() => true), +})); + +vi.mock("../../runtime-recovery", () => ({ + parseLiveSandboxNames: vi.fn(() => new Set(["alpha"])), +})); + +vi.mock("../../shields", () => ({ + isShieldsDown: vi.fn(() => true), + repairMutableConfigPerms: vi.fn(() => ({ applied: true, verified: true, errors: [] })), +})); + +vi.mock("../../shields/timer-bound-lock", () => ({ + withTimerBoundShieldsMutationLock: vi.fn( + (_name: string, _operation: string, callback: () => unknown) => callback(), + ), +})); + +vi.mock("../../state/mcp-lifecycle-lock", () => ({ + withSandboxMutationLock: vi.fn((_name: string, callback: () => Promise) => callback()), +})); + +vi.mock("../../state/registry", () => ({ + getBaselineExclusions: vi.fn(() => []), + getCustomPolicies: vi.fn(() => []), + getSandbox: harness.getSandbox, + listSandboxes: vi.fn(() => ({ + sandboxes: [harness.getSandbox()].filter(Boolean), + defaultSandbox: "alpha", + })), + updateSandbox: vi.fn(), +})); + +vi.mock("../../state/sandbox", () => ({ + captureSnapshotRestoreAuthority: harness.captureSnapshotRestoreAuthority, + findBackup: vi.fn(() => ({ match: null })), + getLatestBackup: harness.getLatestBackup, + listBackups: vi.fn(() => []), + restoreSandboxState: harness.restoreSandboxState, +})); + +vi.mock("./sandbox-gateway-routing", () => ({ + probeGatewayRunning: vi.fn(() => true), + selectSandboxGatewayIfRegistered: vi.fn(() => true), + usesGatewayMetadataProbe: vi.fn(() => false), +})); + +vi.mock("./snapshot/dependencies", async (importOriginal) => ({ + ...(await importOriginal()), + requireCurrentSnapshotRuntimeProvider: vi.fn(() => provider), +})); + +function receiptAtPort(port: number): string { + const receipt = hostLocalInferenceReceipt("docker"); + return serializeHostLocalInferenceReceipt({ + ...receipt, + endpoint: { ...receipt.endpoint, port }, + }); +} + +function manifest(receipt: string): RebuildManifest { + return { + version: 1, + sandboxName: "alpha", + timestamp: "2026-08-02T00-00-00-000Z", + agentType: "openclaw", + agentVersion: null, + expectedVersion: null, + stateDirs: [], + dir: "/sandbox", + backupPath: "/tmp/backup-alpha", + blueprintDigest: null, + hostLocalInferenceReceipt: receipt, + }; +} + +function sandbox(receipt: string): SandboxEntry { + return { + name: "alpha", + agent: "openclaw", + openshellDriver: "docker", + hostLocalInferenceReceipt: receipt, + }; +} + +function successfulRestore( + _name: string, + _path: string, + options: SnapshotRestoreOptions = {}, +): RestoreResult { + harness.events.push("restore-start"); + options.validateBeforeMutation?.(); + harness.events.push("restore-complete"); + return { + success: true, + restoredDirs: [], + failedDirs: [], + restoredFiles: [], + failedFiles: [], + }; +} + +describe("snapshot command host-local inference authority", () => { + beforeEach(() => { + vi.clearAllMocks(); + harness.events.length = 0; + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("re-proves authority before restore, at the mutation fence, and after success", async () => { + const receipt = serializedHostLocalInferenceReceipt("docker"); + harness.setRegistryEntry(sandbox(receipt)); + harness.getLatestBackup.mockReturnValue(manifest(receipt)); + harness.captureSnapshotRestoreAuthority.mockReturnValue({ + schemaVersion: 1, + backupPath: "/tmp/backup-alpha", + contentSha256: "e".repeat(64), + }); + harness.restoreSandboxState.mockImplementation(successfulRestore); + vi.spyOn(console, "log").mockImplementation(() => {}); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(harness.preserveForRebuild).toHaveBeenCalledTimes(4); + expect(harness.events).toEqual([ + "reprove", + "reprove", + "restore-start", + "reprove", + "restore-complete", + "reprove", + ]); + expect(harness.restoreSandboxState).toHaveBeenCalledWith( + "alpha", + "/tmp/backup-alpha", + expect.objectContaining({ + authority: expect.objectContaining({ contentSha256: "e".repeat(64) }), + validateBeforeMutation: expect.any(Function), + }), + ); + }); + + it("rejects mismatched target authority before filesystem restore", async () => { + const snapshotReceipt = receiptAtPort(8000); + harness.setRegistryEntry(sandbox(receiptAtPort(8001))); + harness.getLatestBackup.mockReturnValue(manifest(snapshotReceipt)); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + + await expect(runSandboxSnapshot("alpha", { kind: "restore" })).rejects.toMatchObject({ + exitCode: 1, + }); + + expect(harness.restoreSandboxState).not.toHaveBeenCalled(); + expect(consoleError.mock.calls.flat().join("\n")).toContain( + "different host-local inference authority", + ); + }); +}); From 75cd0af36c2be335f24603c7e612237ed0f10c58 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 12:56:40 -0700 Subject: [PATCH 7/7] fix: close host-local inference recovery gaps Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/destroy-execution.ts | 5 + .../destroy-host-local-inference.test.ts | 35 ++++- .../snapshot-auto-create-failure.test.ts | 144 +++++++++++++++--- src/lib/actions/sandbox/snapshot.ts | 6 +- .../runtime-provider/host-local-inference.ts | 14 +- 5 files changed, 176 insertions(+), 28 deletions(-) diff --git a/src/lib/actions/sandbox/destroy-execution.ts b/src/lib/actions/sandbox/destroy-execution.ts index 71165659f6a..63f91cd65ab 100644 --- a/src/lib/actions/sandbox/destroy-execution.ts +++ b/src/lib/actions/sandbox/destroy-execution.ts @@ -285,6 +285,11 @@ export async function executeSandboxDestroy({ await finalizeMcpDestroy(sandboxName, mcpPreparation, force); } if (!forcedLocalCleanup && runtimeProvider && sandbox && hostLocalInferenceAuthority) { + // Keep retirement after confirmed sandbox deletion: retiring first could + // leave a still-live sandbox without inference when its delete fails. + // The registry row is the durable cleanup journal. A retirement failure + // returns before that row is removed, and a retry takes the already-gone + // path to converge the provider's idempotent exact-runtime teardown. try { const current = getSandbox(sandboxName); if (!current) { diff --git a/src/lib/actions/sandbox/destroy-host-local-inference.test.ts b/src/lib/actions/sandbox/destroy-host-local-inference.test.ts index 8e5600d2f2b..2b6041515d7 100644 --- a/src/lib/actions/sandbox/destroy-host-local-inference.test.ts +++ b/src/lib/actions/sandbox/destroy-host-local-inference.test.ts @@ -89,7 +89,16 @@ function provider( return { bundle, destroy, prepareDestroy, preserveForRebuild }; } -async function runDestroy(runtimeProvider: ReturnType, peers: SandboxEntry[]) { +async function runDestroy( + runtimeProvider: ReturnType, + peers: SandboxEntry[], + deleteResult: { status: number; stdout: string; stderr: string } = { + status: 0, + stdout: "", + stderr: "", + }, + sandboxConfirmedAbsent = false, +) { const entry = sandbox(); const events: string[] = []; const result = await executeSandboxDestroy({ @@ -99,10 +108,10 @@ async function runDestroy(runtimeProvider: ReturnType, peers: S listSandboxes: () => ({ sandboxes: [entry, ...peers] }), runOpenshell: (args) => { events.push(args.join(" ")); - return { status: 0, stdout: "", stderr: "" }; + return deleteResult; }, sandbox: entry, - sandboxConfirmedAbsent: false, + sandboxConfirmedAbsent, sandboxName: "alpha", runtimeProviders: { mxc: runtimeProvider.bundle }, deps: { @@ -143,4 +152,24 @@ describe("sandbox destroy host-local inference transaction", () => { }); expect(events.slice(-2)).toEqual(["sandbox delete alpha", "cleanup"]); }); + + it("reconciles retained ownership when destroy is retried after confirmed deletion", async () => { + const destroyRuntime = vi + .fn(destroySuccessfully) + .mockImplementationOnce(failDestroy("injected runtime removal failure")); + const runtimeProvider = provider(destroyRuntime); + + const first = await runDestroy(runtimeProvider, []); + const retry = await runDestroy( + runtimeProvider, + [], + { status: 1, stdout: "", stderr: "Error: sandbox alpha not found" }, + true, + ); + + expect(first.result).toMatchObject({ ok: false, deleteConfirmed: true }); + expect(retry.result).toMatchObject({ ok: true, alreadyGone: true }); + expect(retry.events.slice(-2)).toEqual(["sandbox delete alpha", "cleanup"]); + expect(runtimeProvider.destroy).toHaveBeenCalledTimes(2); + }); }); diff --git a/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts b/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts index 9dc200a96e1..27568ff1fe5 100644 --- a/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts +++ b/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts @@ -1,32 +1,64 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { serializedHostLocalInferenceReceipt } from "../../../../test/helpers/host-local-inference-receipt"; import { resolveTestAgentBaselinePolicy } from "../../../../test/support/snapshot-policy-test-fixture"; +import type { RuntimeProviderBundle } from "../../onboard/runtime-provider/contract"; import type { SnapshotStreamSandboxCreateMock } from "./snapshot-create-stream-test-types"; -const captureOpenshellMock = vi.fn(() => ({ status: 0, output: "alpha Ready\n" })); -const getSandboxMock = vi.fn((name?: string) => - name === "alpha" - ? { - name: "alpha", - agent: "openclaw", - gatewayName: "nemoclaw", - imageTag: "nemoclaw-alpha:test", - openshellDriver: "docker", - provider: "nvidia-nim", - model: "nvidia/model-a", - } - : null, -); -const registerSandboxMock = vi.fn(); +const harness = vi.hoisted(() => ({ + entries: new Map>(), + preserveForRebuild: vi.fn((value: unknown) => value), + prepareDestroy: vi.fn((value: unknown) => value), + destroy: vi.fn((value: unknown) => ({ status: "removed", receipt: value })), +})); +const captureOpenshellMock = vi.fn(() => ({ status: 0, output: "alpha Ready\nbeta Ready\n" })); +const getSandboxMock = vi.fn((name?: string) => harness.entries.get(name ?? "") ?? null); +const registerSandboxMock = vi.fn((entry: Record) => { + harness.entries.set(String(entry.name), entry); +}); const restoreSandboxStateMock = vi.fn(); +const captureSnapshotRestoreAuthorityMock = vi.fn(); const streamSandboxCreateMock = vi.fn(async () => ({ status: 7, output: "create failed before registry write", sawProgress: false, forcedReady: false, })); +const removeSandboxRegistryEntryOutcomeMock = vi.fn((name: string) => { + const removed = harness.entries.delete(name); + return { status: removed ? ("complete" as const) : ("not-found" as const), removed }; +}); + +const runtimeProvider = { + identity: { contractVersion: 1, id: "mxc", displayName: "MXC" }, + hostLocalInference: { + providerId: "mxc", + supported: true, + runtime: { + providerId: "mxc", + authorityId: "mxc:host-local", + services: ["ollama", "nim", "vllm"], + preserveForRebuild: harness.preserveForRebuild, + prepareDestroy: harness.prepareDestroy, + destroy: harness.destroy, + }, + }, +} as unknown as RuntimeProviderBundle; + +function sourceEntry(receipt?: string): Record { + return { + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw", + imageTag: "nemoclaw-alpha:test", + openshellDriver: receipt ? "mxc" : "docker", + provider: "nvidia-nim", + model: "nvidia/model-a", + ...(receipt ? { hostLocalInferenceReceipt: receipt } : {}), + }; +} vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => ""), @@ -110,13 +142,17 @@ vi.mock("../../state/mcp-lifecycle-lock", () => ({ })); vi.mock("../../state/registry", () => ({ getSandbox: getSandboxMock, - listSandboxes: vi.fn(() => ({ sandboxes: [getSandboxMock("alpha")], defaultSandbox: "alpha" })), + listSandboxes: vi.fn(() => ({ + sandboxes: [...harness.entries.values()], + defaultSandbox: "alpha", + })), registerSandbox: registerSandboxMock, - removeSandbox: vi.fn(), + removeSandbox: vi.fn((name: string) => harness.entries.delete(name)), updateSandbox: vi.fn(), })); vi.mock("../../state/sandbox", () => ({ backupSandboxState: vi.fn(), + captureSnapshotRestoreAuthority: captureSnapshotRestoreAuthorityMock, findBackup: vi.fn(() => ({ match: null })), getLatestBackup: vi.fn(() => ({ timestamp: "2026-06-15T00:00:00.000Z", @@ -127,17 +163,36 @@ vi.mock("../../state/sandbox", () => ({ })); vi.mock("./destroy", () => ({ cleanupShieldsDestroyArtifacts: vi.fn(), - removeSandboxRegistryEntry: vi.fn(), + removeSandboxRegistryEntryOutcome: removeSandboxRegistryEntryOutcomeMock, + requireSandboxDestructiveCleanupAuthority: vi.fn(() => ({ provider: runtimeProvider })), +})); +vi.mock("./restore-gateway-pairing", () => ({ + establishRestoredSandboxGatewayPairing: vi.fn(), + waitForRestoredSandboxGatewaySupervisor: vi.fn(() => true), })); vi.mock("./sandbox-gateway-routing", () => ({ probeGatewayRunning: vi.fn(() => true), selectSandboxGatewayIfRegistered: vi.fn(() => true), - usesGatewayMetadataProbe: vi.fn( - (driver?: string | null) => driver === "docker" || driver === "vm", - ), + usesGatewayMetadataProbe: vi.fn(() => true), +})); +vi.mock("./snapshot/dependencies", async (importOriginal) => ({ + ...(await importOriginal()), + requireCurrentSnapshotRuntimeProvider: vi.fn(() => runtimeProvider), })); describe("snapshot restore auto-create failures", () => { + beforeEach(() => { + vi.clearAllMocks(); + harness.entries.clear(); + harness.entries.set("alpha", sourceEntry()); + streamSandboxCreateMock.mockResolvedValue({ + status: 7, + output: "create failed before registry write", + sawProgress: false, + forcedReady: false, + }); + }); + it("does not register a ghost sandbox when auto-create fails", async () => { vi.spyOn(console, "error").mockImplementation(() => {}); vi.spyOn(console, "log").mockImplementation(() => {}); @@ -158,4 +213,49 @@ describe("snapshot restore auto-create failures", () => { expect(registerSandboxMock).not.toHaveBeenCalled(); expect(restoreSandboxStateMock).not.toHaveBeenCalled(); }); + + it("removes a registered clone when live inference re-proof fails", async () => { + const receipt = serializedHostLocalInferenceReceipt("mxc"); + harness.entries.set("alpha", sourceEntry(receipt)); + harness.preserveForRebuild + .mockImplementationOnce((value) => value) + .mockImplementationOnce(() => { + throw new Error("injected live route failure"); + }); + streamSandboxCreateMock.mockResolvedValue({ + status: 0, + output: "beta Ready", + sawProgress: true, + forcedReady: false, + }); + const { getLatestBackup } = await import("../../state/sandbox"); + vi.mocked(getLatestBackup).mockReturnValue({ + timestamp: "2026-08-02T00-00-00-000Z", + backupPath: "/tmp/backup-alpha", + hostLocalInferenceReceipt: receipt, + } as ReturnType); + captureSnapshotRestoreAuthorityMock.mockReturnValue({ + schemaVersion: 1, + backupPath: "/tmp/backup-alpha", + contentSha256: "e".repeat(64), + }); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(console, "log").mockImplementation(() => {}); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect( + runSandboxSnapshot("alpha", { kind: "restore", to: "beta" }), + ).rejects.toMatchObject({ exitCode: 1 }); + + expect(harness.preserveForRebuild).toHaveBeenCalledTimes(2); + expect(registerSandboxMock).toHaveBeenCalledWith( + expect.objectContaining({ name: "beta", hostLocalInferenceReceipt: receipt }), + ); + expect(harness.prepareDestroy).toHaveBeenCalledTimes(2); + expect(harness.destroy).not.toHaveBeenCalled(); + expect(removeSandboxRegistryEntryOutcomeMock).toHaveBeenCalledWith("beta"); + expect(getSandboxMock("beta")).toBeNull(); + expect(consoleError.mock.calls.flat().join("\n")).toContain("injected live route failure"); + expect(restoreSandboxStateMock).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 9ab371ad9ec..bb134fda05b 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -1364,10 +1364,14 @@ async function runSnapshotRestoreUnlocked( ); } catch (error) { console.error( - ` Cannot bind clone '${targetSandbox}' to snapshot inference authority: ${ + ` Cannot re-prove clone '${targetSandbox}' against snapshot inference authority: ${ error instanceof Error ? error.message : String(error) }.`, ); + console.error( + ` Removing incomplete clone '${targetSandbox}' while its exact provider ownership is still registered.`, + ); + deleteSandboxForRestore(targetSandbox); snapshotExit(1); } } diff --git a/src/lib/onboard/runtime-provider/host-local-inference.ts b/src/lib/onboard/runtime-provider/host-local-inference.ts index bab4a6fa073..e3ec455e4fc 100644 --- a/src/lib/onboard/runtime-provider/host-local-inference.ts +++ b/src/lib/onboard/runtime-provider/host-local-inference.ts @@ -124,11 +124,21 @@ export interface HostLocalInferenceRuntime { startManaged(input: HostLocalManagedInferenceInput): HostLocalInferenceReceipt; inspectManaged(receipt: HostLocalInferenceReceipt): HostLocalManagedInferenceInspection; stopManaged(receipt: HostLocalInferenceReceipt): HostLocalManagedInferenceInspection; - /** Re-prove the same out-of-sandbox service before carrying it into a rebuild. */ + /** + * Re-prove the same out-of-sandbox service before carrying it across a + * lifecycle boundary. Every invocation must perform a fresh provider-native + * identity inspection and network health probe; cached or receipt-only + * validation does not satisfy this contract. + */ preserveForRebuild(receipt: HostLocalInferenceReceipt): HostLocalInferenceReceipt; /** Prove exact ownership for teardown without requiring the service to be healthy. */ prepareDestroy(receipt: HostLocalInferenceReceipt): HostLocalInferenceReceipt; - /** Retire only the exact provider-owned runtime; host processes remain externally owned. */ + /** + * Retire only the exact provider-owned runtime; host processes remain + * externally owned. Managed cleanup must converge safely when repeated so a + * retained ownership journal can resume teardown after a process crash or + * provider failure. + */ destroy(receipt: HostLocalInferenceReceipt): HostLocalInferenceDestroyResult; }