From fdb1136efcd95af8558f944e192643099cbfe77d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 1 Aug 2026 08:26:11 -0700 Subject: [PATCH 01/10] feat(runtime): persist host-local inference authority Signed-off-by: Aaron Erickson --- .../sandbox/snapshot/backup-authority.test.ts | 127 +++++++++++++++++- .../sandbox/snapshot/backup-authority.ts | 57 +++++++- src/lib/onboard.ts | 2 + .../host-local-inference-lifecycle.ts | 38 ++++++ src/lib/onboard/sandbox-registration.ts | 21 ++- src/lib/onboard/setup-inference.ts | 8 +- src/lib/state/registry.ts | 10 ++ .../registry/host-local-inference.test.ts | 22 +++ .../state/registry/host-local-inference.ts | 38 ++++++ src/lib/state/registry/persistence.ts | 26 ++++ src/lib/state/registry/types.ts | 2 + src/lib/state/sandbox.ts | 24 ++++ ...board-host-local-inference-routing.test.ts | 10 ++ test/registry.test.ts | 37 +++++ test/runtime-provider-source-shape.test.ts | 1 + 15 files changed, 414 insertions(+), 9 deletions(-) create mode 100644 src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts create mode 100644 src/lib/state/registry/host-local-inference.test.ts create mode 100644 src/lib/state/registry/host-local-inference.ts diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts index 8221b1535bc..ce02d721f63 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts @@ -12,6 +12,11 @@ import { } from "../../../onboard/managed-image/contract"; import { encodeManagedStartupProfile } from "../../../onboard/managed-startup/profile"; import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; +import { + type HostLocalInferenceReceipt, + type HostLocalInferenceRuntime, + serializeHostLocalInferenceReceipt, +} from "../../../onboard/runtime-provider/host-local-inference"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../../state/registry/types"; import type { BackupOptions, BackupResult } from "../../../state/sandbox"; import { backupSandboxStateWithManagedAuthority } from "./backup-authority"; @@ -70,7 +75,10 @@ function runtime(handle = "session-1") { } as const; } -function provider(acceptsReceipt = true): RuntimeProviderBundle { +function provider( + acceptsReceipt = true, + hostLocalInferenceRuntime?: HostLocalInferenceRuntime, +): RuntimeProviderBundle { return { identity: { contractVersion: 1, id: "mxc", displayName: "MXC" }, workload: { @@ -84,9 +92,55 @@ function provider(acceptsReceipt = true): RuntimeProviderBundle { }, acceptsReceipt: () => acceptsReceipt, }, + hostLocalInference: hostLocalInferenceRuntime + ? { providerId: "mxc", supported: true, runtime: hostLocalInferenceRuntime } + : { providerId: "mxc", supported: false, reason: "not configured" }, } as unknown as RuntimeProviderBundle; } +function hostLocalReceipt(port = 8000): HostLocalInferenceReceipt { + return { + schemaVersion: 1, + providerId: "mxc", + service: "vllm", + engineAuthority: { + schemaVersion: 1, + providerId: "mxc", + operation: "host-local-inference", + engineId: "mxc", + authorityId: "mxc:host-local", + bindingSha256: "c".repeat(64), + }, + endpoint: { host: "mxc.internal", port, networkName: "mxc-network" }, + runtime: { + kind: "container", + runtimeId: "mxc-vllm-runtime", + name: "nemoclaw-vllm", + imageRef: `nvcr.io/nvidia/vllm@sha256:${"d".repeat(64)}`, + specSha256: "e".repeat(64), + gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, + }, + }; +} + +function hostLocalRuntime(): HostLocalInferenceRuntime { + return { + providerId: "mxc", + authorityId: "mxc:host-local", + services: ["ollama", "nim", "vllm"], + translateContainerArgs: (args) => args, + qualifyOllama: vi.fn(() => { + throw new Error("not used"); + }), + startManaged: vi.fn(() => { + throw new Error("not used"); + }), + inspectManaged: vi.fn((receipt) => ({ running: true, receipt })), + stopManaged: vi.fn((receipt) => ({ running: false, receipt })), + preserveForRebuild: vi.fn((receipt) => receipt), + }; +} + function successfulBackup(options: BackupOptions): BackupResult { try { options.validateBeforePublish?.(); @@ -182,6 +236,77 @@ describe("managed snapshot backup authority", () => { expect(captureRuntime).not.toHaveBeenCalled(); }); + it("re-proves and republishes an MXC-style host-local receipt without managed workload state", () => { + const receipt = serializeHostLocalInferenceReceipt(hostLocalReceipt()); + const entry = { + name: "alpha", + agent: "openclaw", + openshellDriver: "mxc", + hostLocalInferenceReceipt: receipt, + } satisfies SandboxEntry; + const inference = hostLocalRuntime(); + const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); + const captureRuntime = vi.fn(); + + const result = backupSandboxStateWithManagedAuthority( + "alpha", + { name: "host-local" }, + { + getSandbox: () => entry, + requireProvider: () => provider(true, inference), + captureRuntime: captureRuntime as never, + backup, + }, + ); + + expect(result.success).toBe(true); + expect(backup).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + name: "host-local", + hostLocalInferenceReceipt: receipt, + validateBeforePublish: expect.any(Function), + }), + ); + expect(inference.preserveForRebuild).toHaveBeenCalledTimes(2); + expect(captureRuntime).not.toHaveBeenCalled(); + }); + + it("rejects host-local route drift before manifest publication", () => { + const initialReceipt = serializeHostLocalInferenceReceipt(hostLocalReceipt()); + const changedReceipt = serializeHostLocalInferenceReceipt(hostLocalReceipt(8001)); + const entries = [initialReceipt, changedReceipt].map( + (hostLocalInferenceReceipt) => + ({ + name: "alpha", + agent: "hermes", + openshellDriver: "mxc", + hostLocalInferenceReceipt, + }) satisfies SandboxEntry, + ); + const getSandbox = vi + .fn<() => SandboxEntry | null>() + .mockReturnValueOnce(entries[0]) + .mockReturnValueOnce(entries[1]); + const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); + + const result = backupSandboxStateWithManagedAuthority( + "alpha", + {}, + { + getSandbox, + requireProvider: () => provider(true, hostLocalRuntime()), + captureRuntime: vi.fn() as never, + backup, + }, + ); + + expect(result).toMatchObject({ + success: false, + error: expect.stringContaining("host-local inference changed during backup"), + }); + }); + it("fails before filesystem capture when the provider rejects managed authority", () => { const entry = sandbox("openclaw"); const backup = vi.fn(); diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.ts b/src/lib/actions/sandbox/snapshot/backup-authority.ts index bc8ccb371b3..a6cbe6fea42 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority.ts @@ -5,6 +5,7 @@ import { isDeepStrictEqual } from "node:util"; import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provider/current"; +import { reproveHostLocalInferenceReceipt } from "../../../onboard/runtime-provider/host-local-inference-lifecycle"; import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime-provider/registry"; import type { SandboxEntry } from "../../../state/registry/types"; import * as sandboxState from "../../../state/sandbox"; @@ -13,13 +14,14 @@ import { captureSandboxRuntimeSnapshot } from "./provider-lifecycle"; type SnapshotBackupAuthority = Pick< sandboxState.BackupOptions, - "runtimeSnapshot" | "workload" | "validateBeforePublish" + "runtimeSnapshot" | "workload" | "hostLocalInferenceReceipt" | "validateBeforePublish" >; interface SnapshotBackupAuthorityDependencies { readonly getSandbox: (sandboxName: string) => SandboxEntry | null; readonly requireProvider: (sandbox: SandboxEntry) => RuntimeProviderBundle; readonly captureRuntime: typeof captureSandboxRuntimeSnapshot; + readonly reproveHostLocalInference: typeof reproveHostLocalInferenceReceipt; readonly backup: typeof sandboxState.backupSandboxState; } @@ -27,6 +29,7 @@ const defaultDependencies: Omit requireRuntimeProviderBundleForSandbox(sandbox, CURRENT_RUNTIME_PROVIDER_BUNDLES), captureRuntime: captureSandboxRuntimeSnapshot, + reproveHostLocalInference: reproveHostLocalInferenceReceipt, // Keep the call late-bound so tests and alternative state stores can replace // the module export without this adapter retaining an import-time reference. backup: (...args) => sandboxState.backupSandboxState(...args), @@ -106,6 +109,56 @@ function captureManagedAuthority( }; } +function captureHostLocalInferenceAuthority( + entry: SandboxEntry, + dependencies: SnapshotBackupAuthorityDependencies, +): Pick | null { + const receipt = entry.hostLocalInferenceReceipt; + if (typeof receipt !== "string") return null; + const provider = dependencies.requireProvider(entry); + const reproved = dependencies.reproveHostLocalInference(provider, receipt); + if (reproved !== receipt) { + throw new Error("host-local inference authority changed before backup"); + } + return { + hostLocalInferenceReceipt: receipt, + validateBeforePublish: () => { + const current = dependencies.getSandbox(entry.name); + if (!current) throw new Error(`sandbox '${entry.name}' is no longer registered`); + if (current.hostLocalInferenceReceipt !== receipt) { + throw new Error(`sandbox '${entry.name}' host-local inference changed during backup`); + } + const currentProvider = dependencies.requireProvider(current); + if (currentProvider.identity.id !== provider.identity.id) { + throw new Error(`sandbox '${entry.name}' runtime provider changed during backup`); + } + if (dependencies.reproveHostLocalInference(currentProvider, receipt) !== receipt) { + throw new Error(`sandbox '${entry.name}' host-local inference changed during backup`); + } + }, + }; +} + +function captureSnapshotAuthority( + entry: SandboxEntry, + dependencies: SnapshotBackupAuthorityDependencies, +): SnapshotBackupAuthority | null { + const managed = captureManagedAuthority(entry, dependencies); + const hostLocal = captureHostLocalInferenceAuthority(entry, dependencies); + if (!managed && !hostLocal) return null; + return { + ...(managed?.runtimeSnapshot === undefined ? {} : { runtimeSnapshot: managed.runtimeSnapshot }), + ...(managed?.workload === undefined ? {} : { workload: managed.workload }), + ...(hostLocal?.hostLocalInferenceReceipt === undefined + ? {} + : { hostLocalInferenceReceipt: hostLocal.hostLocalInferenceReceipt }), + validateBeforePublish: () => { + managed?.validateBeforePublish?.(); + hostLocal?.validateBeforePublish?.(); + }, + }; +} + /** * Capture one managed workload and runtime authority pair around the complete * filesystem copy. The state layer publishes the manifest only after the @@ -123,7 +176,7 @@ export function backupSandboxStateWithManagedAuthority( let authority: SnapshotBackupAuthority | null; try { - authority = captureManagedAuthority(entry, dependencies); + authority = captureSnapshotAuthority(entry, dependencies); } catch (error) { return failure(error); } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 7ea054cda6e..451c0d2c040 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2838,6 +2838,8 @@ async function createSandboxWithBaseImageResolution( agent, agentVersionKnown: !fromDockerfile, imageTag: resolvedImageTag, + hostLocalInferenceReceipt: + registry.getSandbox(sandboxName)?.hostLocalInferenceReceipt ?? null, openclawImagePluginInstalls, appliedPolicies: initialSandboxPolicy.appliedPresets, toolDisclosure: effectiveToolDisclosure, diff --git a/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts new file mode 100644 index 00000000000..952a2d42fc9 --- /dev/null +++ b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { RuntimeProviderBundle } from "./contract"; +import { + parseHostLocalInferenceReceipt, + serializeHostLocalInferenceReceipt, +} from "./host-local-inference"; + +/** + * Re-prove an exact durable host-local route through its owning provider. + * Central lifecycle code handles only the canonical receipt transport; the + * provider remains responsible for engine-specific inspection and authority. + */ +export function reproveHostLocalInferenceReceipt( + provider: RuntimeProviderBundle, + serialized: string, +): string { + const surface = provider.hostLocalInference; + if (!surface.supported) { + throw new Error( + `Runtime provider '${provider.identity.id}' does not support host-local inference.`, + ); + } + const receipt = parseHostLocalInferenceReceipt(serialized); + if ( + surface.providerId !== provider.identity.id || + surface.runtime.providerId !== provider.identity.id || + receipt.providerId !== provider.identity.id + ) { + throw new Error("Host-local inference receipt belongs to a different runtime provider."); + } + const reproved = serializeHostLocalInferenceReceipt(surface.runtime.preserveForRebuild(receipt)); + if (reproved !== serialized) { + throw new Error("Host-local inference authority changed while it was being preserved."); + } + return reproved; +} diff --git a/src/lib/onboard/sandbox-registration.ts b/src/lib/onboard/sandbox-registration.ts index a4d6b181e78..8875c1ec507 100644 --- a/src/lib/onboard/sandbox-registration.ts +++ b/src/lib/onboard/sandbox-registration.ts @@ -16,20 +16,21 @@ import type { SandboxMessagingState, } from "../state/registry"; import * as registry from "../state/registry"; +import { cloneSandboxHostLocalInferenceReceipt } from "../state/registry/host-local-inference"; import { cloneSandboxWorkloadReceipt } from "../state/registry/workload"; import { DEFAULT_TOOL_DISCLOSURE, type ToolDisclosure } from "../tool-disclosure"; import type { DcodeAutoApprovalMode } from "./dcode-auto-approval"; +import { + getHermesDashboardRegistryFields, + type HermesDashboardOnboardState, +} from "./hermes-dashboard"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, RuntimeProviderBundleRegistry, + RuntimeProviderSelectionError, requireRuntimeProviderBundleForSandbox, requireRuntimeProviderMutationAuthority, - RuntimeProviderSelectionError, } from "./runtime-provider/access"; -import { - getHermesDashboardRegistryFields, - type HermesDashboardOnboardState, -} from "./hermes-dashboard"; import { getSandboxAgentRegistryFields } from "./sandbox-agent"; export type CreatedSandboxRuntimeFields = Pick< @@ -52,6 +53,7 @@ export interface CreatedSandboxRegistryEntryInput { agentVersionKnown: boolean; imageTag: string | null; workload?: SandboxEntry["workload"]; + hostLocalInferenceReceipt?: SandboxEntry["hostLocalInferenceReceipt"]; openclawImagePluginInstalls?: readonly OpenClawImagePluginInstall[]; appliedPolicies: string[]; toolDisclosure?: ToolDisclosure; @@ -182,6 +184,14 @@ export function buildCreatedSandboxRegistryEntry( "Sandbox workload ownership receipt failed closed validation.", ); } + const hostLocalInferenceReceipt = cloneSandboxHostLocalInferenceReceipt( + input.hostLocalInferenceReceipt, + ); + if (input.hostLocalInferenceReceipt !== undefined && hostLocalInferenceReceipt === undefined) { + throw new RuntimeProviderSelectionError( + "Sandbox host-local inference receipt failed closed validation.", + ); + } return { name: input.sandboxName, @@ -190,6 +200,7 @@ export function buildCreatedSandboxRegistryEntry( ...getSandboxAgentRegistryFields(input.agent, input.agentVersionKnown), imageTag: input.imageTag, workload, + ...(hostLocalInferenceReceipt !== undefined ? { hostLocalInferenceReceipt } : {}), ...(input.openclawImagePluginInstalls !== undefined ? { openclawImagePluginInstalls: input.openclawImagePluginInstalls.map((install) => ({ diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index 152e190e898..eb9a300baa3 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -50,7 +50,10 @@ import type { import * as inferenceProviders from "./inference-providers"; import { createLocalInferenceRouteApplier } from "./local-inference-route"; import type { ProviderInferenceSetupOptions } from "./machine/handlers/provider-inference"; -import type { HostLocalInferenceRuntime } from "./runtime-provider/host-local-inference"; +import { + type HostLocalInferenceRuntime, + serializeHostLocalInferenceReceipt, +} from "./runtime-provider/host-local-inference"; import { type HostLocalInferenceStartupRoute, prepareHostLocalInferenceStartup, @@ -359,6 +362,7 @@ export function createSetupInference( gatewayName, ); let routeReserved = false; + let hostLocalInferenceReceipt: string | null = null; const reserveRoute = (name: string, selectedProvider: string, selectedModel: string) => { if (routeReserved) return true; const reserved = deps.updateSandbox(name, { @@ -370,6 +374,7 @@ export function createSetupInference( preferredInferenceApi: options.preferredInferenceApi ?? null, gatewayName, reservationSessionId: options.reservationSessionId, + hostLocalInferenceReceipt, }); routeReserved = reserved; return reserved; @@ -409,6 +414,7 @@ export function createSetupInference( provider, options.hostLocalInference, ); + hostLocalInferenceReceipt = serializeHostLocalInferenceReceipt(hostLocalRoute.receipt); } catch (error) { deps.error(` ${error instanceof Error ? error.message : String(error)}`); return deps.exitProcess(1); diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index a6b360998ef..8119d9e2f0e 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -14,6 +14,7 @@ import { isValidExtraProviderName, readExtraProviders, } from "./extra-providers"; +import { cloneSandboxHostLocalInferenceReceipt } from "./registry/host-local-inference"; import { withLock } from "./registry/lock"; import { load, save } from "./registry/persistence"; import { cloneSandboxWorkloadReceipt } from "./registry/workload"; @@ -111,6 +112,12 @@ export function registerSandbox(entry: SandboxEntry): void { if (retainedDefaultSandbox(data.defaultSandbox, data.sandboxes) === null) { data.defaultSandbox = null; } + const hostLocalInferenceReceipt = cloneSandboxHostLocalInferenceReceipt( + entry.hostLocalInferenceReceipt, + ); + if (entry.hostLocalInferenceReceipt !== undefined && hostLocalInferenceReceipt === undefined) { + throw new Error("Cannot register a sandbox with an invalid host-local inference receipt"); + } data.sandboxes[entry.name] = { name: entry.name, createdAt: entry.createdAt || new Date().toISOString(), @@ -165,6 +172,7 @@ export function registerSandbox(entry: SandboxEntry): void { : null, imageTag: entry.imageTag || null, workload: cloneSandboxWorkloadReceipt(entry.workload), + ...(hostLocalInferenceReceipt !== undefined ? { hostLocalInferenceReceipt } : {}), lifecycleGeneration: entry.lifecycleGeneration, lifecycleLiveIdentityFingerprint: entry.lifecycleLiveIdentityFingerprint, messaging: cloneSandboxMessagingState(entry.messaging), @@ -197,6 +205,7 @@ type SandboxInferenceRouteReservation = Pick< > & { gatewayName: string; reservationSessionId?: string; + hostLocalInferenceReceipt?: string | null; }; /** @@ -222,6 +231,7 @@ export function reserveSandboxInferenceRoute( endpointSource: normalized.endpointSource, credentialEnv: normalized.credentialEnv, preferredInferenceApi: normalized.preferredInferenceApi, + hostLocalInferenceReceipt: route.hostLocalInferenceReceipt ?? null, gatewayName: route.gatewayName, gatewayPort: undefined, }; diff --git a/src/lib/state/registry/host-local-inference.test.ts b/src/lib/state/registry/host-local-inference.test.ts new file mode 100644 index 00000000000..80ed424623c --- /dev/null +++ b/src/lib/state/registry/host-local-inference.test.ts @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { cloneSandboxHostLocalInferenceReceipt } from "./host-local-inference"; + +describe("sandbox host-local inference receipt transport", () => { + it("clones only canonical bounded object transports", () => { + const receipt = `${JSON.stringify({ schemaVersion: 1, providerId: "mxc" })}\n`; + + expect(cloneSandboxHostLocalInferenceReceipt(receipt)).toBe(receipt); + expect(cloneSandboxHostLocalInferenceReceipt(null)).toBeNull(); + expect(cloneSandboxHostLocalInferenceReceipt(undefined)).toBeUndefined(); + expect(cloneSandboxHostLocalInferenceReceipt(receipt.trimEnd())).toBeUndefined(); + expect(cloneSandboxHostLocalInferenceReceipt("[]\n")).toBeUndefined(); + expect(cloneSandboxHostLocalInferenceReceipt('{"providerId": "mxc"}\n')).toBeUndefined(); + expect( + cloneSandboxHostLocalInferenceReceipt(`{"value":"${"a".repeat(33 * 1024)}"}\n`), + ).toBeUndefined(); + }); +}); diff --git a/src/lib/state/registry/host-local-inference.ts b/src/lib/state/registry/host-local-inference.ts new file mode 100644 index 00000000000..20d92fde29d --- /dev/null +++ b/src/lib/state/registry/host-local-inference.ts @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const MAX_RECEIPT_BYTES = 32 * 1024; + +/** + * Clone the canonical, secret-free receipt transport without teaching the + * state layer about any concrete runtime provider. Provider consumers still + * parse the complete receipt schema and fail closed before lifecycle use. + */ +export function cloneSandboxHostLocalInferenceReceipt( + value: string | null | undefined, +): string | null | undefined { + if (value === undefined || value === null) return value; + if ( + typeof value !== "string" || + value.length === 0 || + value.includes("\0") || + Buffer.byteLength(value, "utf8") > MAX_RECEIPT_BYTES + ) { + return undefined; + } + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + return undefined; + } + if ( + typeof parsed !== "object" || + parsed === null || + Array.isArray(parsed) || + `${JSON.stringify(parsed)}\n` !== value + ) { + return undefined; + } + return value; +} diff --git a/src/lib/state/registry/persistence.ts b/src/lib/state/registry/persistence.ts index c0974507ea8..675a8530c4f 100644 --- a/src/lib/state/registry/persistence.ts +++ b/src/lib/state/registry/persistence.ts @@ -19,6 +19,7 @@ import { } from "../registry-normalization"; import * as reversibleRemoval from "../registry-reversible-removal"; import { nemoclawStateRoot } from "../state-root"; +import { cloneSandboxHostLocalInferenceReceipt } from "./host-local-inference"; import type { SandboxEntry, SandboxRegistry } from "./types"; import { cloneSandboxWorkloadReceipt } from "./workload"; @@ -33,6 +34,19 @@ function cloneSandboxWorkloadReceiptOrThrow( return workload; } +function cloneHostLocalInferenceReceiptOrThrow( + value: SandboxEntry["hostLocalInferenceReceipt"], + operation: "load" | "save", +): SandboxEntry["hostLocalInferenceReceipt"] { + const receipt = cloneSandboxHostLocalInferenceReceipt(value); + if (value !== undefined && receipt === undefined) { + throw new Error( + `Cannot ${operation} a sandbox entry with an invalid host-local inference receipt`, + ); + } + return receipt; +} + export const REGISTRY_FILE = path.join( nemoclawStateRoot(process.env.HOME || "/tmp", GATEWAY_PORT), "sandboxes.json", @@ -96,6 +110,10 @@ function serializeRegistryForDisk(data: SandboxRegistry): SandboxRegistry { function normalizeSandboxEntryForRuntime(entry: SandboxEntry): SandboxEntry { const messaging = cloneSandboxMessagingState(entry.messaging); const workload = cloneSandboxWorkloadReceiptOrThrow(entry.workload, "load"); + const hostLocalInferenceReceipt = cloneHostLocalInferenceReceiptOrThrow( + entry.hostLocalInferenceReceipt, + "load", + ); const mcp = normalizeSandboxMcpState(entry.mcp); const baselineExclusions = normalizeBaselineExclusions(entry.baselineExclusions); const baselineExclusionTransition = normalizeBaselineExclusionTransition( @@ -104,6 +122,7 @@ function normalizeSandboxEntryForRuntime(entry: SandboxEntry): SandboxEntry { const { messaging: _messaging, workload: _workload, + hostLocalInferenceReceipt: _hostLocalInferenceReceipt, mcp: _mcp, baselineExclusions: _baselineExclusions, baselineExclusionTransition: _baselineExclusionTransition, @@ -112,6 +131,7 @@ function normalizeSandboxEntryForRuntime(entry: SandboxEntry): SandboxEntry { return { ...rest, ...(workload ? { workload } : {}), + ...(hostLocalInferenceReceipt !== undefined ? { hostLocalInferenceReceipt } : {}), ...(messaging ? { messaging } : {}), ...(mcp ? { mcp } : {}), ...(baselineExclusions ? { baselineExclusions } : {}), @@ -141,6 +161,10 @@ function serializeSandboxEntryForDisk(entry: SandboxEntry): SandboxEntry { }; const messaging = serializeSandboxMessagingStateForDisk(durable.messaging); const workload = cloneSandboxWorkloadReceiptOrThrow(durable.workload, "save"); + const hostLocalInferenceReceipt = cloneHostLocalInferenceReceiptOrThrow( + durable.hostLocalInferenceReceipt, + "save", + ); const mcp = serializeSandboxMcpStateForDisk(durable.mcp); const baselineExclusions = normalizeBaselineExclusions(durable.baselineExclusions); const baselineExclusionTransition = normalizeBaselineExclusionTransition( @@ -149,6 +173,7 @@ function serializeSandboxEntryForDisk(entry: SandboxEntry): SandboxEntry { const { messaging: _messaging, workload: _workload, + hostLocalInferenceReceipt: _hostLocalInferenceReceipt, mcp: _mcp, baselineExclusions: _baselineExclusions, baselineExclusionTransition: _baselineExclusionTransition, @@ -158,6 +183,7 @@ function serializeSandboxEntryForDisk(entry: SandboxEntry): SandboxEntry { ...rest, ...(rest.dashboardPort === 0 ? { dashboardPort: null } : {}), ...(workload ? { workload } : {}), + ...(hostLocalInferenceReceipt !== undefined ? { hostLocalInferenceReceipt } : {}), ...(messaging ? { messaging } : {}), ...(mcp ? { mcp } : {}), ...(baselineExclusions ? { baselineExclusions } : {}), diff --git a/src/lib/state/registry/types.ts b/src/lib/state/registry/types.ts index 386bb16ff54..3c53a251bc8 100644 --- a/src/lib/state/registry/types.ts +++ b/src/lib/state/registry/types.ts @@ -125,6 +125,8 @@ export interface SandboxEntry extends Partial { * through per-sandbox image deletion. */ workload?: SandboxWorkloadReceipt; + /** Canonical provider-neutral receipt for an out-of-sandbox inference runtime. */ + hostLocalInferenceReceipt?: string | null; messaging?: SandboxMessagingState; mcp?: SandboxMcpState; hermesToolGateways?: string[]; diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index bcbb710d867..b65f16adf74 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -60,6 +60,7 @@ import { parseOpenClawImagePluginInstalls, planOpenClawPluginRestore, } from "./openclaw-plugin-restore.js"; +import { cloneSandboxHostLocalInferenceReceipt } from "./registry/host-local-inference.js"; import { cloneSandboxRuntimeSnapshot, type SandboxRuntimeSnapshot, @@ -131,6 +132,8 @@ export interface RebuildManifest { * snapshot. Older and explicit Dockerfile snapshots omit this field. */ workload?: SandboxWorkloadReceipt; + /** Exact provider-neutral authority for out-of-sandbox inference. */ + hostLocalInferenceReceipt?: string; instances?: InstanceBackup[]; // Optional user-provided label for `snapshot restore `. name?: string; @@ -145,6 +148,7 @@ export interface BackupOptions { name?: string | null; runtimeSnapshot?: SandboxRuntimeSnapshot; workload?: SandboxWorkloadReceipt; + hostLocalInferenceReceipt?: string; /** * Internal publication fence for provider-backed backups. The callback * runs after data capture and sanitization but before the manifest becomes @@ -320,6 +324,9 @@ function isRebuildManifest(value: unknown): value is RebuildManifest { : cloneSandboxRuntimeSnapshot(value.runtimeSnapshot); const workload = value.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(value.workload as never); + const hostLocalInferenceReceipt = cloneSandboxHostLocalInferenceReceipt( + value.hostLocalInferenceReceipt as string | null | undefined, + ); return ( typeof value.version === "number" && typeof value.sandboxName === "string" && @@ -347,6 +354,8 @@ function isRebuildManifest(value: unknown): value is RebuildManifest { (value.customPolicies === undefined || isCustomPolicyEntryArray(value.customPolicies)) && (value.runtimeSnapshot === undefined || runtimeSnapshot !== undefined) && (value.workload === undefined || workload !== undefined) && + (value.hostLocalInferenceReceipt === undefined || + (typeof hostLocalInferenceReceipt === "string" && hostLocalInferenceReceipt.length > 0)) && (workload?.kind !== "managed-image" || runtimeSnapshot !== undefined) && (value.instances === undefined || (Array.isArray(value.instances) && @@ -944,6 +953,7 @@ export { isSshTransportFailure }; function normalizeSnapshotBackupAuthority(options: BackupOptions): { readonly runtimeSnapshot?: SandboxRuntimeSnapshot; readonly workload?: SandboxWorkloadReceipt; + readonly hostLocalInferenceReceipt?: string; readonly error?: string; } { const runtimeSnapshot = @@ -952,18 +962,28 @@ function normalizeSnapshotBackupAuthority(options: BackupOptions): { : cloneSandboxRuntimeSnapshot(options.runtimeSnapshot); const workload = options.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(options.workload); + const hostLocalInferenceReceipt = cloneSandboxHostLocalInferenceReceipt( + options.hostLocalInferenceReceipt, + ); if (options.runtimeSnapshot !== undefined && runtimeSnapshot === undefined) { return { error: "snapshot runtime state is invalid or cannot be represented" }; } if (options.workload !== undefined && workload === undefined) { return { error: "snapshot workload authority is invalid" }; } + if ( + options.hostLocalInferenceReceipt !== undefined && + typeof hostLocalInferenceReceipt !== "string" + ) { + return { error: "snapshot host-local inference authority is invalid" }; + } if (workload?.kind === "managed-image" && runtimeSnapshot === undefined) { return { error: "managed snapshot is missing provider runtime state" }; } return { ...(runtimeSnapshot === undefined ? {} : { runtimeSnapshot }), ...(workload === undefined ? {} : { workload }), + ...(typeof hostLocalInferenceReceipt === "string" ? { hostLocalInferenceReceipt } : {}), }; } @@ -2203,6 +2223,9 @@ function readManifest(backupPath: string): RebuildManifest | null { : cloneSandboxRuntimeSnapshot(manifest.runtimeSnapshot); const workload = manifest.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(manifest.workload); + const hostLocalInferenceReceipt = cloneSandboxHostLocalInferenceReceipt( + manifest.hostLocalInferenceReceipt, + ); return { ...manifest, dir, @@ -2212,6 +2235,7 @@ function readManifest(backupPath: string): RebuildManifest | null { blueprintDigest: manifest.blueprintDigest ?? null, ...(runtimeSnapshot === undefined ? {} : { runtimeSnapshot }), ...(workload === undefined ? {} : { workload }), + ...(typeof hostLocalInferenceReceipt === "string" ? { hostLocalInferenceReceipt } : {}), }; } catch { return null; diff --git a/test/onboard-host-local-inference-routing.test.ts b/test/onboard-host-local-inference-routing.test.ts index d4268227b35..39c803a0def 100644 --- a/test/onboard-host-local-inference-routing.test.ts +++ b/test/onboard-host-local-inference-routing.test.ts @@ -6,6 +6,7 @@ import type { HostLocalInferenceReceipt, HostLocalInferenceRuntime, } from "../src/lib/onboard/runtime-provider/host-local-inference.js"; +import { parseHostLocalInferenceReceipt } from "../src/lib/onboard/runtime-provider/host-local-inference.js"; import { createSetupInference } from "../src/lib/onboard/setup-inference.js"; import { createDirectSetupInferenceHarnessFactory } from "./support/setup-inference-test-harness.js"; @@ -108,6 +109,15 @@ describe("setupInference host-local runtime integration", () => { "ollama-local", "qwen3.5:9b", ); + const reservation = harness.updateSandbox.mock.calls.at(-1)?.[1]; + expect(reservation?.hostLocalInferenceReceipt).toEqual(expect.any(String)); + expect( + parseHostLocalInferenceReceipt(reservation?.hostLocalInferenceReceipt ?? ""), + ).toMatchObject({ + providerId: "mxc", + service: "ollama", + endpoint: { port: 11434, networkName: "mxc-network" }, + }); }); it("fails before gateway mutation when the selected provider does not match the startup service", async () => { diff --git a/test/registry.test.ts b/test/registry.test.ts index e1e49702452..bb5429a130f 100644 --- a/test/registry.test.ts +++ b/test/registry.test.ts @@ -961,6 +961,43 @@ describe("registry", () => { expect(fs.existsSync(regFile)).toBe(false); }); + it("round-trips a canonical host-local inference receipt without rewriting it", () => { + const receipt = `${JSON.stringify({ schemaVersion: 1, providerId: "mxc" })}\n`; + registry.registerSandbox({ + name: "host-local", + hostLocalInferenceReceipt: receipt, + }); + + expect(registry.getSandbox("host-local").hostLocalInferenceReceipt).toBe(receipt); + const data = JSON.parse(fs.readFileSync(regFile, "utf-8")); + expect(data.sandboxes["host-local"].hostLocalInferenceReceipt).toBe(receipt); + }); + + it("rejects malformed host-local inference receipt transports on load and save", () => { + fs.mkdirSync(path.dirname(regFile), { recursive: true }); + fs.writeFileSync( + regFile, + JSON.stringify({ + defaultSandbox: "alpha", + sandboxes: { + alpha: { name: "alpha", hostLocalInferenceReceipt: '{"providerId": "mxc"}\n' }, + }, + }), + ); + expect(() => registry.getSandbox("alpha")).toThrow(/invalid host-local inference receipt/); + + fs.rmSync(regFile, { force: true }); + expect(() => + registry.save({ + defaultSandbox: "alpha", + sandboxes: { + alpha: { name: "alpha", hostLocalInferenceReceipt: "not-json\n" }, + }, + }), + ).toThrow(/invalid host-local inference receipt/); + expect(fs.existsSync(regFile)).toBe(false); + }); + it("skips malformed sandbox entries while loading the registry", () => { fs.mkdirSync(path.dirname(regFile), { recursive: true }); fs.writeFileSync( diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 92bc046f750..7717de389f8 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -165,6 +165,7 @@ describe("runtime provider central source boundary", () => { "src/lib/onboard/runtime-provider/contract.ts", "src/lib/onboard/runtime-provider/current.ts", "src/lib/onboard/runtime-provider/docker.ts", + "src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts", "src/lib/onboard/runtime-provider/host-local-inference-routing.ts", "src/lib/onboard/runtime-provider/host-local-inference.ts", "src/lib/onboard/runtime-provider/persisted-engine-authority.ts", From 9165574b518211474ec916bc53510689c66f4f6c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 1 Aug 2026 08:31:11 -0700 Subject: [PATCH 02/10] test(runtime): type host-local route reservation Signed-off-by: Aaron Erickson --- test/onboard-host-local-inference-routing.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/onboard-host-local-inference-routing.test.ts b/test/onboard-host-local-inference-routing.test.ts index 39c803a0def..02a85ed45a6 100644 --- a/test/onboard-host-local-inference-routing.test.ts +++ b/test/onboard-host-local-inference-routing.test.ts @@ -109,7 +109,11 @@ describe("setupInference host-local runtime integration", () => { "ollama-local", "qwen3.5:9b", ); - const reservation = harness.updateSandbox.mock.calls.at(-1)?.[1]; + const reservation = ( + harness.updateSandbox.mock.calls.at(-1) as unknown as + | [string, { hostLocalInferenceReceipt?: string }] + | undefined + )?.[1]; expect(reservation?.hostLocalInferenceReceipt).toEqual(expect.any(String)); expect( parseHostLocalInferenceReceipt(reservation?.hostLocalInferenceReceipt ?? ""), From 304d28d08e77f2c2dcb2907bdf3b27e558eb6378 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 06:58:22 -0700 Subject: [PATCH 03/10] refactor(runtime): narrow durable inference ownership Signed-off-by: Aaron Erickson --- .../sandbox/snapshot/backup-authority.test.ts | 127 +----------------- .../sandbox/snapshot/backup-authority.ts | 57 +------- src/lib/onboard.ts | 2 - .../host-local-inference-lifecycle.ts | 38 ------ src/lib/onboard/sandbox-registration.test.ts | 8 ++ src/lib/onboard/sandbox-registration.ts | 11 +- src/lib/onboard/setup-inference.ts | 2 +- src/lib/state/sandbox.ts | 24 ---- test/runtime-provider-source-shape.test.ts | 1 - 9 files changed, 22 insertions(+), 248 deletions(-) delete mode 100644 src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts index ce02d721f63..8221b1535bc 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts @@ -12,11 +12,6 @@ import { } from "../../../onboard/managed-image/contract"; import { encodeManagedStartupProfile } from "../../../onboard/managed-startup/profile"; import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; -import { - type HostLocalInferenceReceipt, - type HostLocalInferenceRuntime, - serializeHostLocalInferenceReceipt, -} from "../../../onboard/runtime-provider/host-local-inference"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../../state/registry/types"; import type { BackupOptions, BackupResult } from "../../../state/sandbox"; import { backupSandboxStateWithManagedAuthority } from "./backup-authority"; @@ -75,10 +70,7 @@ function runtime(handle = "session-1") { } as const; } -function provider( - acceptsReceipt = true, - hostLocalInferenceRuntime?: HostLocalInferenceRuntime, -): RuntimeProviderBundle { +function provider(acceptsReceipt = true): RuntimeProviderBundle { return { identity: { contractVersion: 1, id: "mxc", displayName: "MXC" }, workload: { @@ -92,55 +84,9 @@ function provider( }, acceptsReceipt: () => acceptsReceipt, }, - hostLocalInference: hostLocalInferenceRuntime - ? { providerId: "mxc", supported: true, runtime: hostLocalInferenceRuntime } - : { providerId: "mxc", supported: false, reason: "not configured" }, } as unknown as RuntimeProviderBundle; } -function hostLocalReceipt(port = 8000): HostLocalInferenceReceipt { - return { - schemaVersion: 1, - providerId: "mxc", - service: "vllm", - engineAuthority: { - schemaVersion: 1, - providerId: "mxc", - operation: "host-local-inference", - engineId: "mxc", - authorityId: "mxc:host-local", - bindingSha256: "c".repeat(64), - }, - endpoint: { host: "mxc.internal", port, networkName: "mxc-network" }, - runtime: { - kind: "container", - runtimeId: "mxc-vllm-runtime", - name: "nemoclaw-vllm", - imageRef: `nvcr.io/nvidia/vllm@sha256:${"d".repeat(64)}`, - specSha256: "e".repeat(64), - gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, - }, - }; -} - -function hostLocalRuntime(): HostLocalInferenceRuntime { - return { - providerId: "mxc", - authorityId: "mxc:host-local", - services: ["ollama", "nim", "vllm"], - translateContainerArgs: (args) => args, - qualifyOllama: vi.fn(() => { - throw new Error("not used"); - }), - startManaged: vi.fn(() => { - throw new Error("not used"); - }), - inspectManaged: vi.fn((receipt) => ({ running: true, receipt })), - stopManaged: vi.fn((receipt) => ({ running: false, receipt })), - preserveForRebuild: vi.fn((receipt) => receipt), - }; -} - function successfulBackup(options: BackupOptions): BackupResult { try { options.validateBeforePublish?.(); @@ -236,77 +182,6 @@ describe("managed snapshot backup authority", () => { expect(captureRuntime).not.toHaveBeenCalled(); }); - it("re-proves and republishes an MXC-style host-local receipt without managed workload state", () => { - const receipt = serializeHostLocalInferenceReceipt(hostLocalReceipt()); - const entry = { - name: "alpha", - agent: "openclaw", - openshellDriver: "mxc", - hostLocalInferenceReceipt: receipt, - } satisfies SandboxEntry; - const inference = hostLocalRuntime(); - const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); - const captureRuntime = vi.fn(); - - const result = backupSandboxStateWithManagedAuthority( - "alpha", - { name: "host-local" }, - { - getSandbox: () => entry, - requireProvider: () => provider(true, inference), - captureRuntime: captureRuntime as never, - backup, - }, - ); - - expect(result.success).toBe(true); - expect(backup).toHaveBeenCalledWith( - "alpha", - expect.objectContaining({ - name: "host-local", - hostLocalInferenceReceipt: receipt, - validateBeforePublish: expect.any(Function), - }), - ); - expect(inference.preserveForRebuild).toHaveBeenCalledTimes(2); - expect(captureRuntime).not.toHaveBeenCalled(); - }); - - it("rejects host-local route drift before manifest publication", () => { - const initialReceipt = serializeHostLocalInferenceReceipt(hostLocalReceipt()); - const changedReceipt = serializeHostLocalInferenceReceipt(hostLocalReceipt(8001)); - const entries = [initialReceipt, changedReceipt].map( - (hostLocalInferenceReceipt) => - ({ - name: "alpha", - agent: "hermes", - openshellDriver: "mxc", - hostLocalInferenceReceipt, - }) satisfies SandboxEntry, - ); - const getSandbox = vi - .fn<() => SandboxEntry | null>() - .mockReturnValueOnce(entries[0]) - .mockReturnValueOnce(entries[1]); - const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); - - const result = backupSandboxStateWithManagedAuthority( - "alpha", - {}, - { - getSandbox, - requireProvider: () => provider(true, hostLocalRuntime()), - captureRuntime: vi.fn() as never, - backup, - }, - ); - - expect(result).toMatchObject({ - success: false, - error: expect.stringContaining("host-local inference changed during backup"), - }); - }); - it("fails before filesystem capture when the provider rejects managed authority", () => { const entry = sandbox("openclaw"); const backup = vi.fn(); diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.ts b/src/lib/actions/sandbox/snapshot/backup-authority.ts index a6cbe6fea42..bc8ccb371b3 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority.ts @@ -5,7 +5,6 @@ import { isDeepStrictEqual } from "node:util"; import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provider/current"; -import { reproveHostLocalInferenceReceipt } from "../../../onboard/runtime-provider/host-local-inference-lifecycle"; import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime-provider/registry"; import type { SandboxEntry } from "../../../state/registry/types"; import * as sandboxState from "../../../state/sandbox"; @@ -14,14 +13,13 @@ import { captureSandboxRuntimeSnapshot } from "./provider-lifecycle"; type SnapshotBackupAuthority = Pick< sandboxState.BackupOptions, - "runtimeSnapshot" | "workload" | "hostLocalInferenceReceipt" | "validateBeforePublish" + "runtimeSnapshot" | "workload" | "validateBeforePublish" >; interface SnapshotBackupAuthorityDependencies { readonly getSandbox: (sandboxName: string) => SandboxEntry | null; readonly requireProvider: (sandbox: SandboxEntry) => RuntimeProviderBundle; readonly captureRuntime: typeof captureSandboxRuntimeSnapshot; - readonly reproveHostLocalInference: typeof reproveHostLocalInferenceReceipt; readonly backup: typeof sandboxState.backupSandboxState; } @@ -29,7 +27,6 @@ const defaultDependencies: Omit requireRuntimeProviderBundleForSandbox(sandbox, CURRENT_RUNTIME_PROVIDER_BUNDLES), captureRuntime: captureSandboxRuntimeSnapshot, - reproveHostLocalInference: reproveHostLocalInferenceReceipt, // Keep the call late-bound so tests and alternative state stores can replace // the module export without this adapter retaining an import-time reference. backup: (...args) => sandboxState.backupSandboxState(...args), @@ -109,56 +106,6 @@ function captureManagedAuthority( }; } -function captureHostLocalInferenceAuthority( - entry: SandboxEntry, - dependencies: SnapshotBackupAuthorityDependencies, -): Pick | null { - const receipt = entry.hostLocalInferenceReceipt; - if (typeof receipt !== "string") return null; - const provider = dependencies.requireProvider(entry); - const reproved = dependencies.reproveHostLocalInference(provider, receipt); - if (reproved !== receipt) { - throw new Error("host-local inference authority changed before backup"); - } - return { - hostLocalInferenceReceipt: receipt, - validateBeforePublish: () => { - const current = dependencies.getSandbox(entry.name); - if (!current) throw new Error(`sandbox '${entry.name}' is no longer registered`); - if (current.hostLocalInferenceReceipt !== receipt) { - throw new Error(`sandbox '${entry.name}' host-local inference changed during backup`); - } - const currentProvider = dependencies.requireProvider(current); - if (currentProvider.identity.id !== provider.identity.id) { - throw new Error(`sandbox '${entry.name}' runtime provider changed during backup`); - } - if (dependencies.reproveHostLocalInference(currentProvider, receipt) !== receipt) { - throw new Error(`sandbox '${entry.name}' host-local inference changed during backup`); - } - }, - }; -} - -function captureSnapshotAuthority( - entry: SandboxEntry, - dependencies: SnapshotBackupAuthorityDependencies, -): SnapshotBackupAuthority | null { - const managed = captureManagedAuthority(entry, dependencies); - const hostLocal = captureHostLocalInferenceAuthority(entry, dependencies); - if (!managed && !hostLocal) return null; - return { - ...(managed?.runtimeSnapshot === undefined ? {} : { runtimeSnapshot: managed.runtimeSnapshot }), - ...(managed?.workload === undefined ? {} : { workload: managed.workload }), - ...(hostLocal?.hostLocalInferenceReceipt === undefined - ? {} - : { hostLocalInferenceReceipt: hostLocal.hostLocalInferenceReceipt }), - validateBeforePublish: () => { - managed?.validateBeforePublish?.(); - hostLocal?.validateBeforePublish?.(); - }, - }; -} - /** * Capture one managed workload and runtime authority pair around the complete * filesystem copy. The state layer publishes the manifest only after the @@ -176,7 +123,7 @@ export function backupSandboxStateWithManagedAuthority( let authority: SnapshotBackupAuthority | null; try { - authority = captureSnapshotAuthority(entry, dependencies); + authority = captureManagedAuthority(entry, dependencies); } catch (error) { return failure(error); } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 451c0d2c040..7ea054cda6e 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2838,8 +2838,6 @@ async function createSandboxWithBaseImageResolution( agent, agentVersionKnown: !fromDockerfile, imageTag: resolvedImageTag, - hostLocalInferenceReceipt: - registry.getSandbox(sandboxName)?.hostLocalInferenceReceipt ?? null, openclawImagePluginInstalls, appliedPolicies: initialSandboxPolicy.appliedPresets, toolDisclosure: effectiveToolDisclosure, diff --git a/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts deleted file mode 100644 index 952a2d42fc9..00000000000 --- a/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts +++ /dev/null @@ -1,38 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { RuntimeProviderBundle } from "./contract"; -import { - parseHostLocalInferenceReceipt, - serializeHostLocalInferenceReceipt, -} from "./host-local-inference"; - -/** - * Re-prove an exact durable host-local route through its owning provider. - * Central lifecycle code handles only the canonical receipt transport; the - * provider remains responsible for engine-specific inspection and authority. - */ -export function reproveHostLocalInferenceReceipt( - provider: RuntimeProviderBundle, - serialized: string, -): string { - const surface = provider.hostLocalInference; - if (!surface.supported) { - throw new Error( - `Runtime provider '${provider.identity.id}' does not support host-local inference.`, - ); - } - const receipt = parseHostLocalInferenceReceipt(serialized); - if ( - surface.providerId !== provider.identity.id || - surface.runtime.providerId !== provider.identity.id || - receipt.providerId !== provider.identity.id - ) { - throw new Error("Host-local inference receipt belongs to a different runtime provider."); - } - const reproved = serializeHostLocalInferenceReceipt(surface.runtime.preserveForRebuild(receipt)); - if (reproved !== serialized) { - throw new Error("Host-local inference authority changed while it was being preserved."); - } - return reproved; -} diff --git a/src/lib/onboard/sandbox-registration.test.ts b/src/lib/onboard/sandbox-registration.test.ts index ef14dc58bb2..049f8529b0c 100644 --- a/src/lib/onboard/sandbox-registration.test.ts +++ b/src/lib/onboard/sandbox-registration.test.ts @@ -448,6 +448,12 @@ describe("selection", () => { describe("registerCreatedSandbox", () => { it("passes the built entry to the supplied registry writer", () => { const registerSandbox = vi.fn(); + const hostLocalInferenceReceipt = `${JSON.stringify({ schemaVersion: 1, providerId: "docker" })}\n`; + const registry = requireDist("../state/registry.js"); + const getSandbox = vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "demo", + hostLocalInferenceReceipt, + }); const input = { sandboxName: "demo", @@ -487,6 +493,7 @@ describe("registerCreatedSandbox", () => { expect(entry.name).toBe("demo"); expect(entry.openclawImagePluginInstalls).toEqual([]); expect(entry.workload).toEqual(input.workload); + expect(entry.hostLocalInferenceReceipt).toBe(hostLocalInferenceReceipt); expect(() => registerCreatedSandbox({ ...input, @@ -494,6 +501,7 @@ describe("registerCreatedSandbox", () => { }), ).toThrow(/workload ownership receipt failed closed validation/u); expect(registerSandbox).toHaveBeenCalledTimes(1); + getSandbox.mockRestore(); }); it("fails before registry mutation for an unknown durable provider identity", () => { diff --git a/src/lib/onboard/sandbox-registration.ts b/src/lib/onboard/sandbox-registration.ts index 8875c1ec507..1b4121fbb57 100644 --- a/src/lib/onboard/sandbox-registration.ts +++ b/src/lib/onboard/sandbox-registration.ts @@ -237,7 +237,16 @@ export function buildCreatedSandboxRegistryEntry( } export function registerCreatedSandbox(input: CreatedSandboxRegistrationInput): SandboxEntry { - const entry = buildCreatedSandboxRegistryEntry(input); + const pendingHostLocalInferenceReceipt = + input.hostLocalInferenceReceipt !== undefined + ? input.hostLocalInferenceReceipt + : registry.getSandbox(input.sandboxName)?.hostLocalInferenceReceipt; + const entry = buildCreatedSandboxRegistryEntry({ + ...input, + ...(pendingHostLocalInferenceReceipt === undefined + ? {} + : { hostLocalInferenceReceipt: pendingHostLocalInferenceReceipt }), + }); const provider = requireRuntimeProviderBundleForSandbox( entry, input.runtimeProviders ?? CURRENT_RUNTIME_PROVIDER_BUNDLES, diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index eb9a300baa3..dd073b07110 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -374,7 +374,7 @@ export function createSetupInference( preferredInferenceApi: options.preferredInferenceApi ?? null, gatewayName, reservationSessionId: options.reservationSessionId, - hostLocalInferenceReceipt, + ...(hostLocalInferenceReceipt === null ? {} : { hostLocalInferenceReceipt }), }); routeReserved = reserved; return reserved; diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index b65f16adf74..bcbb710d867 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -60,7 +60,6 @@ import { parseOpenClawImagePluginInstalls, planOpenClawPluginRestore, } from "./openclaw-plugin-restore.js"; -import { cloneSandboxHostLocalInferenceReceipt } from "./registry/host-local-inference.js"; import { cloneSandboxRuntimeSnapshot, type SandboxRuntimeSnapshot, @@ -132,8 +131,6 @@ export interface RebuildManifest { * snapshot. Older and explicit Dockerfile snapshots omit this field. */ workload?: SandboxWorkloadReceipt; - /** Exact provider-neutral authority for out-of-sandbox inference. */ - hostLocalInferenceReceipt?: string; instances?: InstanceBackup[]; // Optional user-provided label for `snapshot restore `. name?: string; @@ -148,7 +145,6 @@ export interface BackupOptions { name?: string | null; runtimeSnapshot?: SandboxRuntimeSnapshot; workload?: SandboxWorkloadReceipt; - hostLocalInferenceReceipt?: string; /** * Internal publication fence for provider-backed backups. The callback * runs after data capture and sanitization but before the manifest becomes @@ -324,9 +320,6 @@ function isRebuildManifest(value: unknown): value is RebuildManifest { : cloneSandboxRuntimeSnapshot(value.runtimeSnapshot); const workload = value.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(value.workload as never); - const hostLocalInferenceReceipt = cloneSandboxHostLocalInferenceReceipt( - value.hostLocalInferenceReceipt as string | null | undefined, - ); return ( typeof value.version === "number" && typeof value.sandboxName === "string" && @@ -354,8 +347,6 @@ function isRebuildManifest(value: unknown): value is RebuildManifest { (value.customPolicies === undefined || isCustomPolicyEntryArray(value.customPolicies)) && (value.runtimeSnapshot === undefined || runtimeSnapshot !== undefined) && (value.workload === undefined || workload !== undefined) && - (value.hostLocalInferenceReceipt === undefined || - (typeof hostLocalInferenceReceipt === "string" && hostLocalInferenceReceipt.length > 0)) && (workload?.kind !== "managed-image" || runtimeSnapshot !== undefined) && (value.instances === undefined || (Array.isArray(value.instances) && @@ -953,7 +944,6 @@ export { isSshTransportFailure }; function normalizeSnapshotBackupAuthority(options: BackupOptions): { readonly runtimeSnapshot?: SandboxRuntimeSnapshot; readonly workload?: SandboxWorkloadReceipt; - readonly hostLocalInferenceReceipt?: string; readonly error?: string; } { const runtimeSnapshot = @@ -962,28 +952,18 @@ function normalizeSnapshotBackupAuthority(options: BackupOptions): { : cloneSandboxRuntimeSnapshot(options.runtimeSnapshot); const workload = options.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(options.workload); - const hostLocalInferenceReceipt = cloneSandboxHostLocalInferenceReceipt( - options.hostLocalInferenceReceipt, - ); if (options.runtimeSnapshot !== undefined && runtimeSnapshot === undefined) { return { error: "snapshot runtime state is invalid or cannot be represented" }; } if (options.workload !== undefined && workload === undefined) { return { error: "snapshot workload authority is invalid" }; } - if ( - options.hostLocalInferenceReceipt !== undefined && - typeof hostLocalInferenceReceipt !== "string" - ) { - return { error: "snapshot host-local inference authority is invalid" }; - } if (workload?.kind === "managed-image" && runtimeSnapshot === undefined) { return { error: "managed snapshot is missing provider runtime state" }; } return { ...(runtimeSnapshot === undefined ? {} : { runtimeSnapshot }), ...(workload === undefined ? {} : { workload }), - ...(typeof hostLocalInferenceReceipt === "string" ? { hostLocalInferenceReceipt } : {}), }; } @@ -2223,9 +2203,6 @@ function readManifest(backupPath: string): RebuildManifest | null { : cloneSandboxRuntimeSnapshot(manifest.runtimeSnapshot); const workload = manifest.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(manifest.workload); - const hostLocalInferenceReceipt = cloneSandboxHostLocalInferenceReceipt( - manifest.hostLocalInferenceReceipt, - ); return { ...manifest, dir, @@ -2235,7 +2212,6 @@ function readManifest(backupPath: string): RebuildManifest | null { blueprintDigest: manifest.blueprintDigest ?? null, ...(runtimeSnapshot === undefined ? {} : { runtimeSnapshot }), ...(workload === undefined ? {} : { workload }), - ...(typeof hostLocalInferenceReceipt === "string" ? { hostLocalInferenceReceipt } : {}), }; } catch { return null; diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 7717de389f8..92bc046f750 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -165,7 +165,6 @@ describe("runtime provider central source boundary", () => { "src/lib/onboard/runtime-provider/contract.ts", "src/lib/onboard/runtime-provider/current.ts", "src/lib/onboard/runtime-provider/docker.ts", - "src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts", "src/lib/onboard/runtime-provider/host-local-inference-routing.ts", "src/lib/onboard/runtime-provider/host-local-inference.ts", "src/lib/onboard/runtime-provider/persisted-engine-authority.ts", From 5dd4ca8b34ce337cf6907e616f5dfe0bbc6c6703 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 08:15:14 -0700 Subject: [PATCH 04/10] fix(state): validate inference ownership receipts Signed-off-by: Aaron Erickson --- src/lib/onboard/sandbox-registration.test.ts | 3 +- .../registry/host-local-inference.test.ts | 12 ++++-- .../state/registry/host-local-inference.ts | 30 ++++----------- test/helpers/host-local-inference-receipt.ts | 37 +++++++++++++++++++ test/registry.test.ts | 3 +- 5 files changed, 56 insertions(+), 29 deletions(-) create mode 100644 test/helpers/host-local-inference-receipt.ts diff --git a/src/lib/onboard/sandbox-registration.test.ts b/src/lib/onboard/sandbox-registration.test.ts index 049f8529b0c..7c096175a14 100644 --- a/src/lib/onboard/sandbox-registration.test.ts +++ b/src/lib/onboard/sandbox-registration.test.ts @@ -3,6 +3,7 @@ import { createRequire } from "node:module"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { serializedHostLocalInferenceReceipt } from "../../../test/helpers/host-local-inference-receipt"; const requireDist = createRequire(import.meta.url); const onboardSession = requireDist("../state/onboard-session.js"); @@ -448,7 +449,7 @@ describe("selection", () => { describe("registerCreatedSandbox", () => { it("passes the built entry to the supplied registry writer", () => { const registerSandbox = vi.fn(); - const hostLocalInferenceReceipt = `${JSON.stringify({ schemaVersion: 1, providerId: "docker" })}\n`; + const hostLocalInferenceReceipt = serializedHostLocalInferenceReceipt("docker"); const registry = requireDist("../state/registry.js"); const getSandbox = vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "demo", diff --git a/src/lib/state/registry/host-local-inference.test.ts b/src/lib/state/registry/host-local-inference.test.ts index 80ed424623c..92d29dd0801 100644 --- a/src/lib/state/registry/host-local-inference.test.ts +++ b/src/lib/state/registry/host-local-inference.test.ts @@ -3,18 +3,22 @@ import { describe, expect, it } from "vitest"; +import { serializedHostLocalInferenceReceipt } from "../../../../test/helpers/host-local-inference-receipt"; import { cloneSandboxHostLocalInferenceReceipt } from "./host-local-inference"; describe("sandbox host-local inference receipt transport", () => { - it("clones only canonical bounded object transports", () => { - const receipt = `${JSON.stringify({ schemaVersion: 1, providerId: "mxc" })}\n`; + it("clones only complete canonical receipt transports", () => { + const valid = serializedHostLocalInferenceReceipt(); - expect(cloneSandboxHostLocalInferenceReceipt(receipt)).toBe(receipt); + expect(cloneSandboxHostLocalInferenceReceipt(valid)).toBe(valid); expect(cloneSandboxHostLocalInferenceReceipt(null)).toBeNull(); expect(cloneSandboxHostLocalInferenceReceipt(undefined)).toBeUndefined(); - expect(cloneSandboxHostLocalInferenceReceipt(receipt.trimEnd())).toBeUndefined(); + expect(cloneSandboxHostLocalInferenceReceipt(valid.trimEnd())).toBeUndefined(); expect(cloneSandboxHostLocalInferenceReceipt("[]\n")).toBeUndefined(); expect(cloneSandboxHostLocalInferenceReceipt('{"providerId": "mxc"}\n')).toBeUndefined(); + expect( + cloneSandboxHostLocalInferenceReceipt('{"apiKey":"must-not-persist"}\n'), + ).toBeUndefined(); expect( cloneSandboxHostLocalInferenceReceipt(`{"value":"${"a".repeat(33 * 1024)}"}\n`), ).toBeUndefined(); diff --git a/src/lib/state/registry/host-local-inference.ts b/src/lib/state/registry/host-local-inference.ts index 20d92fde29d..aa898a5851e 100644 --- a/src/lib/state/registry/host-local-inference.ts +++ b/src/lib/state/registry/host-local-inference.ts @@ -1,38 +1,22 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -const MAX_RECEIPT_BYTES = 32 * 1024; +import { + parseHostLocalInferenceReceipt, + serializeHostLocalInferenceReceipt, +} from "../../onboard/runtime-provider/host-local-inference"; /** - * Clone the canonical, secret-free receipt transport without teaching the - * state layer about any concrete runtime provider. Provider consumers still - * parse the complete receipt schema and fail closed before lifecycle use. + * Clone a canonical, secret-free provider-neutral receipt. The state boundary + * validates the complete schema before accepting durable authority. */ export function cloneSandboxHostLocalInferenceReceipt( value: string | null | undefined, ): string | null | undefined { if (value === undefined || value === null) return value; - if ( - typeof value !== "string" || - value.length === 0 || - value.includes("\0") || - Buffer.byteLength(value, "utf8") > MAX_RECEIPT_BYTES - ) { - return undefined; - } - let parsed: unknown; try { - parsed = JSON.parse(value); + return serializeHostLocalInferenceReceipt(parseHostLocalInferenceReceipt(value)); } catch { return undefined; } - if ( - typeof parsed !== "object" || - parsed === null || - Array.isArray(parsed) || - `${JSON.stringify(parsed)}\n` !== value - ) { - return undefined; - } - return value; } diff --git a/test/helpers/host-local-inference-receipt.ts b/test/helpers/host-local-inference-receipt.ts new file mode 100644 index 00000000000..6874bb2b3c9 --- /dev/null +++ b/test/helpers/host-local-inference-receipt.ts @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + type HostLocalInferenceReceipt, + serializeHostLocalInferenceReceipt, +} from "../../src/lib/onboard/runtime-provider/host-local-inference"; + +export function hostLocalInferenceReceipt(providerId = "mxc"): HostLocalInferenceReceipt { + return { + schemaVersion: 1, + providerId, + service: "vllm", + engineAuthority: { + schemaVersion: 1, + providerId, + operation: "host-local-inference", + engineId: providerId, + authorityId: `${providerId}:host-local`, + bindingSha256: "a".repeat(64), + }, + endpoint: { host: `${providerId}.internal`, port: 8000, networkName: `${providerId}-network` }, + runtime: { + kind: "container", + runtimeId: `${providerId}-vllm`, + name: "nemoclaw-vllm", + imageRef: `nvcr.io/nvidia/vllm@sha256:${"b".repeat(64)}`, + probeImageRef: `quay.io/curl/curl@sha256:${"c".repeat(64)}`, + specSha256: "d".repeat(64), + gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, + }, + }; +} + +export function serializedHostLocalInferenceReceipt(providerId = "mxc"): string { + return serializeHostLocalInferenceReceipt(hostLocalInferenceReceipt(providerId)); +} diff --git a/test/registry.test.ts b/test/registry.test.ts index bb5429a130f..213a505cf5d 100644 --- a/test/registry.test.ts +++ b/test/registry.test.ts @@ -6,6 +6,7 @@ import { createRequire } from "node:module"; import os from "node:os"; import path from "node:path"; import { beforeEach, describe, expect, it } from "vitest"; +import { serializedHostLocalInferenceReceipt } from "./helpers/host-local-inference-receipt"; // Use a temp dir so tests don't touch real ~/.nemoclaw. // HOME must be set before loading registry (it reads HOME at require time), @@ -962,7 +963,7 @@ describe("registry", () => { }); it("round-trips a canonical host-local inference receipt without rewriting it", () => { - const receipt = `${JSON.stringify({ schemaVersion: 1, providerId: "mxc" })}\n`; + const receipt = serializedHostLocalInferenceReceipt(); registry.registerSandbox({ name: "host-local", hostLocalInferenceReceipt: receipt, From ddadc80cb22cf03d63cb4021bc3bb89622606976 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 08:33:10 -0700 Subject: [PATCH 05/10] test(state): cover explicit inference ownership clear Signed-off-by: Aaron Erickson --- src/lib/onboard/sandbox-registration.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib/onboard/sandbox-registration.test.ts b/src/lib/onboard/sandbox-registration.test.ts index 7c096175a14..caca41d49c3 100644 --- a/src/lib/onboard/sandbox-registration.test.ts +++ b/src/lib/onboard/sandbox-registration.test.ts @@ -495,13 +495,19 @@ describe("registerCreatedSandbox", () => { expect(entry.openclawImagePluginInstalls).toEqual([]); expect(entry.workload).toEqual(input.workload); expect(entry.hostLocalInferenceReceipt).toBe(hostLocalInferenceReceipt); + const clearedEntry = registerCreatedSandbox({ + ...input, + hostLocalInferenceReceipt: null, + }); + expect(clearedEntry.hostLocalInferenceReceipt).toBeNull(); + expect(registerSandbox).toHaveBeenLastCalledWith(clearedEntry); expect(() => registerCreatedSandbox({ ...input, workload: { ...input.workload, reference: "" }, }), ).toThrow(/workload ownership receipt failed closed validation/u); - expect(registerSandbox).toHaveBeenCalledTimes(1); + expect(registerSandbox).toHaveBeenCalledTimes(2); getSandbox.mockRestore(); }); From ddb5feb0453fe3b62918a672db2fe3cbbf20b0b1 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 09:52:38 -0700 Subject: [PATCH 06/10] fix(state): exclude route ownership from recreate fingerprint Signed-off-by: Aaron Erickson --- src/lib/onboard/sandbox-recreate-transaction.test.ts | 9 ++++++--- src/lib/onboard/sandbox-recreate-transaction.ts | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/sandbox-recreate-transaction.test.ts b/src/lib/onboard/sandbox-recreate-transaction.test.ts index ecc221286c9..2a27d3c95e2 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.test.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.test.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { serializedHostLocalInferenceReceipt } from "../../../test/helpers/host-local-inference-receipt"; import { decisionSelected } from "../state/onboard-checkpoint-decision"; import { deriveCheckpointFromSession } from "../state/onboard-checkpoint-migrate"; import type { @@ -673,6 +674,7 @@ describe("source registry fingerprint", () => { const journaled = fingerprintSandboxRegistryEntry( registry.getSandbox("alpha") as SandboxEntry, ); + const hostLocalInferenceReceipt = serializedHostLocalInferenceReceipt("podman"); expect( registry.reserveSandboxInferenceRoute("alpha", { @@ -684,11 +686,12 @@ describe("source registry fingerprint", () => { preferredInferenceApi: "openai-responses", gatewayName: "nemoclaw", reservationSessionId: "session-9", + hostLocalInferenceReceipt, }), ).toBe(true); - expect(fingerprintSandboxRegistryEntry(registry.getSandbox("alpha") as SandboxEntry)).toBe( - journaled, - ); + const reserved = registry.getSandbox("alpha") as SandboxEntry; + expect(reserved.hostLocalInferenceReceipt).toBe(hostLocalInferenceReceipt); + expect(fingerprintSandboxRegistryEntry(reserved)).toBe(journaled); } finally { await fs.rm(home, { recursive: true, force: true }); } diff --git a/src/lib/onboard/sandbox-recreate-transaction.ts b/src/lib/onboard/sandbox-recreate-transaction.ts index 387a92f67b6..c5cb7b8da97 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.ts @@ -176,6 +176,7 @@ const ROUTE_RESERVATION_FIELDS: readonly (keyof SandboxEntry)[] = [ "endpointSource", "credentialEnv", "preferredInferenceApi", + "hostLocalInferenceReceipt", "gatewayName", "gatewayPort", ]; From 990a5f5634d3df19c7d60b96e0e4b8c438936f20 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 11:47:15 -0700 Subject: [PATCH 07/10] fix(state): preserve reserved inference receipt Signed-off-by: Aaron Erickson --- .../state/registry-route-reservation.test.ts | 72 +++++++++++++++++++ src/lib/state/registry.ts | 4 +- 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/src/lib/state/registry-route-reservation.test.ts b/src/lib/state/registry-route-reservation.test.ts index 07abaa5e931..1c60a7a9c31 100644 --- a/src/lib/state/registry-route-reservation.test.ts +++ b/src/lib/state/registry-route-reservation.test.ts @@ -5,6 +5,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { serializedHostLocalInferenceReceipt } from "../../../test/helpers/host-local-inference-receipt"; describe("sandbox inference route reservation", () => { afterEach(() => { @@ -96,6 +97,77 @@ describe("sandbox inference route reservation", () => { } }); + it("preserves an omitted host-local receipt through creation registration", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-route-reservation-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const registry = await import("./registry"); + const { registerCreatedSandbox } = await import("../onboard/sandbox-registration"); + const receipt = serializedHostLocalInferenceReceipt("docker"); + const route = { + provider: "compatible-endpoint", + model: "model-a", + endpointUrl: "https://api.example.test/v1", + credentialEnv: "CUSTOM_API_KEY", + preferredInferenceApi: "openai-responses", + gatewayName: "nemoclaw-9090", + } as const; + + registry.reserveSandboxInferenceRoute("alpha", { + ...route, + hostLocalInferenceReceipt: receipt, + }); + registry.reserveSandboxInferenceRoute("alpha", { ...route, model: "model-b" }); + + expect(registry.getSandbox("alpha")?.hostLocalInferenceReceipt).toBe(receipt); + const entry = registerCreatedSandbox({ + sandboxName: "alpha", + inferenceSelection: { + provider: route.provider, + model: "model-b", + endpointUrl: route.endpointUrl, + endpointSource: null, + credentialEnv: route.credentialEnv, + preferredInferenceApi: route.preferredInferenceApi, + compatibleEndpointReasoning: null, + compatibleEndpointReasoningEffort: null, + nimContainer: null, + }, + runtimeFields: { + gpuEnabled: false, + hostGpuDetected: false, + sandboxGpuEnabled: false, + sandboxGpuMode: "auto", + sandboxGpuDevice: null, + openshellDriver: "docker", + openshellVersion: "0.1.2", + }, + agent: null, + agentVersionKnown: true, + imageTag: null, + workload: { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: null, + shared: false, + }, + appliedPolicies: [], + plannedMessagingState: undefined, + hermesToolGateways: [], + hermesDashboardState: { enabled: false, config: null }, + dashboardPort: 18789, + gatewayName: route.gatewayName, + gatewayPort: 9090, + }); + + expect(entry.hostLocalInferenceReceipt).toBe(receipt); + expect(registry.getSandbox("alpha")?.hostLocalInferenceReceipt).toBe(receipt); + } finally { + await fs.rm(home, { recursive: true, force: true }); + } + }); + it("stamps the owning onboard session on the reservation (#6562)", async () => { const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-route-reservation-")); vi.stubEnv("HOME", home); diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index 8119d9e2f0e..599340f3c64 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -231,7 +231,9 @@ export function reserveSandboxInferenceRoute( endpointSource: normalized.endpointSource, credentialEnv: normalized.credentialEnv, preferredInferenceApi: normalized.preferredInferenceApi, - hostLocalInferenceReceipt: route.hostLocalInferenceReceipt ?? null, + ...(route.hostLocalInferenceReceipt !== undefined + ? { hostLocalInferenceReceipt: route.hostLocalInferenceReceipt } + : {}), gatewayName: route.gatewayName, gatewayPort: undefined, }; From c96cf8d5fc7ef01840222932ddc2070dd95d4bdc Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 04:15:37 -0700 Subject: [PATCH 08/10] feat(runtime): complete host-local inference lifecycle Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/destroy-execution.ts | 45 ++++ .../destroy-host-local-inference.test.ts | 175 ++++++++++++++ src/lib/actions/sandbox/destroy.ts | 10 + .../snapshot-auto-create-failure.test.ts | 144 +++++++++-- ...pshot-command-host-local-authority.test.ts | 221 +++++++++++++++++ src/lib/actions/sandbox/snapshot.test.ts | 6 +- src/lib/actions/sandbox/snapshot.ts | 193 ++++++++++++--- .../sandbox/snapshot/backup-authority.test.ts | 101 ++++++++ .../sandbox/snapshot/backup-authority.ts | 58 ++++- .../actions/sandbox/snapshot/dependencies.ts | 8 + .../sandbox/snapshot/restore-authority.ts | 119 +++++---- .../restore-host-local-authority.test.ts | 226 ++++++++++++++++++ .../host-local-inference-lifecycle.test.ts | 139 +++++++++++ .../host-local-inference-lifecycle.ts | 170 +++++++++++++ .../host-local-inference-routing.test.ts | 2 + .../runtime-provider/host-local-inference.ts | 27 ++- .../podman-host-local-inference.test.ts | 38 +++ .../podman-host-local-inference.ts | 42 ++++ .../onboard/runtime-provider/podman.test.ts | 2 + src/lib/onboard/runtime-provider/registry.ts | 2 + .../runtime-provider-contract.test.ts | 4 + src/lib/state/registry.ts | 1 + src/lib/state/sandbox.ts | 37 ++- ...odman-host-local-inference-test-harness.ts | 13 +- ...board-host-local-inference-routing.test.ts | 2 + test/runtime-provider-source-shape.test.ts | 1 + 26 files changed, 1668 insertions(+), 118 deletions(-) create mode 100644 src/lib/actions/sandbox/destroy-host-local-inference.test.ts create mode 100644 src/lib/actions/sandbox/snapshot-command-host-local-authority.test.ts create mode 100644 src/lib/actions/sandbox/snapshot/restore-host-local-authority.test.ts create mode 100644 src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts create mode 100644 src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts diff --git a/src/lib/actions/sandbox/destroy-execution.ts b/src/lib/actions/sandbox/destroy-execution.ts index 91bf4cd38a5..223c6583f81 100644 --- a/src/lib/actions/sandbox/destroy-execution.ts +++ b/src/lib/actions/sandbox/destroy-execution.ts @@ -9,6 +9,11 @@ import { type RuntimeProviderBundleRegistry, requireRuntimeProviderDestructiveCleanupAuthority, } from "../../onboard/runtime-provider/access"; +import { + type PreparedHostLocalInferenceAuthority, + prepareSandboxHostLocalInferenceDestroyAuthority, + retirePreparedHostLocalInferenceAuthority, +} from "../../onboard/runtime-provider/host-local-inference-lifecycle"; import { type DetachSandboxProvidersResult, runSandboxProviderPreDeleteCleanup, @@ -34,6 +39,8 @@ export function redactDestroyError(error: unknown): string { type SandboxDestroyExecutionInput = { cleanupShieldsArtifacts: (sandboxName: string) => void; force: boolean; + getSandbox: (sandboxName: string) => SandboxEntry | null; + listSandboxes: () => { sandboxes: SandboxEntry[] }; runOpenshell: DestroyRunOpenshell; sandbox: SandboxEntry | null; sandboxConfirmedAbsent: boolean; @@ -62,6 +69,8 @@ export type SandboxDestroyExecutionResult = mcpOwnershipRequiresGateway: boolean; mcpRecoveryFailure?: string; shieldsRelockRequiresGateway: boolean; + hostLocalInferenceCleanupFailure?: string; + deleteConfirmed?: boolean; }; type HardenedDeleteState = { @@ -242,6 +251,8 @@ async function finalizeMcpDestroy( export async function executeSandboxDestroy({ cleanupShieldsArtifacts, force, + getSandbox, + listSandboxes, runOpenshell, sandbox, sandboxConfirmedAbsent, @@ -251,6 +262,7 @@ export async function executeSandboxDestroy({ }: SandboxDestroyExecutionInput): Promise { return withTimerBoundShieldsMutationLockAsync(sandboxName, "destroy sandbox", async () => { let runtimeProvider: RuntimeProviderBundle | null = null; + let hostLocalInferenceAuthority: PreparedHostLocalInferenceAuthority | null = null; if (sandbox) { try { runtimeProvider = requireRuntimeProviderDestructiveCleanupAuthority( @@ -258,6 +270,10 @@ export async function executeSandboxDestroy({ sandbox, runtimeProviders, ).provider; + hostLocalInferenceAuthority = prepareSandboxHostLocalInferenceDestroyAuthority( + runtimeProvider, + sandbox, + ); } catch (error) { return { ok: false as const, @@ -331,6 +347,35 @@ export async function executeSandboxDestroy({ if (!forcedLocalCleanup) { await finalizeMcpDestroy(sandboxName, mcpPreparation, force); } + if (!forcedLocalCleanup && runtimeProvider && sandbox && hostLocalInferenceAuthority) { + // Keep retirement after confirmed sandbox deletion: retiring first could + // leave a still-live sandbox without inference when its delete fails. + // The registry row is the durable cleanup journal. A retirement failure + // returns before that row is removed, and a retry takes the already-gone + // path to converge the provider's idempotent exact-runtime teardown. + try { + const current = getSandbox(sandboxName); + if (!current) { + throw new Error(`sandbox '${sandboxName}' is no longer registered`); + } + retirePreparedHostLocalInferenceAuthority( + runtimeProvider, + current, + hostLocalInferenceAuthority, + listSandboxes().sandboxes, + ); + } catch (error) { + return { + ok: false as const, + deleteOutput, + exitCode: 1, + gatewayUnreachable: false, + mcpOwnershipRequiresGateway: false, + hostLocalInferenceCleanupFailure: redactDestroyError(error), + deleteConfirmed: true, + }; + } + } return { ok: true as const, detachOutcome, diff --git a/src/lib/actions/sandbox/destroy-host-local-inference.test.ts b/src/lib/actions/sandbox/destroy-host-local-inference.test.ts new file mode 100644 index 00000000000..2b6041515d7 --- /dev/null +++ b/src/lib/actions/sandbox/destroy-host-local-inference.test.ts @@ -0,0 +1,175 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { createInMemoryRuntimeProviderBundle } from "../../../../test/helpers/runtime-provider-bundle"; +import { + type HostLocalInferenceReceipt, + serializeHostLocalInferenceReceipt, +} from "../../onboard/runtime-provider/host-local-inference"; +import type { SandboxEntry } from "../../state/registry"; +import { executeSandboxDestroy } from "./destroy-execution"; + +function receipt(): HostLocalInferenceReceipt { + return { + schemaVersion: 1, + providerId: "mxc", + service: "vllm", + engineAuthority: { + schemaVersion: 1, + providerId: "mxc", + operation: "host-local-inference", + engineId: "mxc", + authorityId: "mxc:host-local", + bindingSha256: "a".repeat(64), + }, + endpoint: { host: "mxc.internal", port: 8000, networkName: "mxc-network" }, + runtime: { + kind: "container", + runtimeId: "mxc-vllm", + name: "nemoclaw-vllm", + imageRef: `nvcr.io/nvidia/vllm@sha256:${"b".repeat(64)}`, + probeImageRef: `quay.io/curl/curl@sha256:${"d".repeat(64)}`, + specSha256: "c".repeat(64), + gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, + }, + }; +} + +function sandbox(name = "alpha"): SandboxEntry { + return { + name, + agent: "openclaw", + openshellDriver: "mxc", + hostLocalInferenceReceipt: serializeHostLocalInferenceReceipt(receipt()), + }; +} + +function destroySuccessfully(value: HostLocalInferenceReceipt) { + return { status: "removed" as const, receipt: value }; +} + +function failDestroy(message: string) { + return (_value: HostLocalInferenceReceipt): never => { + throw new Error(message); + }; +} + +function provider( + destroyRuntime: (value: HostLocalInferenceReceipt) => { + status: "removed"; + receipt: HostLocalInferenceReceipt; + } = destroySuccessfully, +) { + const preserveForRebuild = vi.fn((value: HostLocalInferenceReceipt) => value); + const prepareDestroy = vi.fn((value: HostLocalInferenceReceipt) => value); + const destroy = vi.fn(destroyRuntime); + const bundle = createInMemoryRuntimeProviderBundle({ + providerId: "mxc", + workloadProfile: { + support: null, + hostArchitectures: ["amd64"], + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: false, + }, + hostLocalInferenceRuntime: { + providerId: "mxc", + authorityId: "mxc:host-local", + services: ["ollama", "nim", "vllm"], + translateContainerArgs: (args: readonly string[]) => args, + qualifyOllama: vi.fn(), + startManaged: vi.fn(), + inspectManaged: vi.fn(), + stopManaged: vi.fn(), + preserveForRebuild, + prepareDestroy, + destroy, + }, + }); + return { bundle, destroy, prepareDestroy, preserveForRebuild }; +} + +async function runDestroy( + runtimeProvider: ReturnType, + peers: SandboxEntry[], + deleteResult: { status: number; stdout: string; stderr: string } = { + status: 0, + stdout: "", + stderr: "", + }, + sandboxConfirmedAbsent = false, +) { + const entry = sandbox(); + const events: string[] = []; + const result = await executeSandboxDestroy({ + cleanupShieldsArtifacts: () => events.push("cleanup"), + force: false, + getSandbox: () => entry, + listSandboxes: () => ({ sandboxes: [entry, ...peers] }), + runOpenshell: (args) => { + events.push(args.join(" ")); + return deleteResult; + }, + sandbox: entry, + sandboxConfirmedAbsent, + sandboxName: "alpha", + runtimeProviders: { mxc: runtimeProvider.bundle }, + deps: { + readTimerMarker: () => null, + wipeSandboxState: () => undefined, + }, + }); + return { events, result }; +} + +describe("sandbox destroy host-local inference transaction", () => { + it("deletes the sandbox before retiring the exact unshared runtime", async () => { + const runtimeProvider = provider(); + const { events, result } = await runDestroy(runtimeProvider, []); + + expect(result).toMatchObject({ ok: true }); + expect(events.slice(-2)).toEqual(["sandbox delete alpha", "cleanup"]); + expect(runtimeProvider.prepareDestroy).toHaveBeenCalledTimes(2); + expect(runtimeProvider.destroy).toHaveBeenCalledOnce(); + }); + + it("keeps a runtime referenced by another sandbox", async () => { + const runtimeProvider = provider(); + const { result } = await runDestroy(runtimeProvider, [sandbox("beta")]); + + expect(result).toMatchObject({ ok: true }); + expect(runtimeProvider.destroy).not.toHaveBeenCalled(); + }); + + it("preserves local ownership when exact runtime retirement fails", async () => { + const runtimeProvider = provider(failDestroy("injected runtime removal failure")); + const { events, result } = await runDestroy(runtimeProvider, []); + + expect(result).toMatchObject({ + ok: false, + deleteConfirmed: true, + hostLocalInferenceCleanupFailure: "injected runtime removal failure", + }); + expect(events.slice(-2)).toEqual(["sandbox delete alpha", "cleanup"]); + }); + + it("reconciles retained ownership when destroy is retried after confirmed deletion", async () => { + const destroyRuntime = vi + .fn(destroySuccessfully) + .mockImplementationOnce(failDestroy("injected runtime removal failure")); + const runtimeProvider = provider(destroyRuntime); + + const first = await runDestroy(runtimeProvider, []); + const retry = await runDestroy( + runtimeProvider, + [], + { status: 1, stdout: "", stderr: "Error: sandbox alpha not found" }, + true, + ); + + expect(first.result).toMatchObject({ ok: false, deleteConfirmed: true }); + expect(retry.result).toMatchObject({ ok: true, alreadyGone: true }); + expect(retry.events.slice(-2)).toEqual(["sandbox delete alpha", "cleanup"]); + expect(runtimeProvider.destroy).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 4a9ac783596..ecf5ff71d96 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -463,12 +463,22 @@ async function destroySandboxUnlocked( const destructiveResult = await executeSandboxDestroy({ cleanupShieldsArtifacts: cleanupShieldsDestroyArtifacts, force: normalized.force === true, + getSandbox: registry.getSandbox, + listSandboxes: registry.listSandboxes, runOpenshell, sandbox, sandboxConfirmedAbsent, sandboxName, }); if (!destructiveResult.ok) { + if (destructiveResult.hostLocalInferenceCleanupFailure) { + console.error( + ` Sandbox '${sandboxName}' is gone, but its exact host-local inference cleanup failed: ${destructiveResult.hostLocalInferenceCleanupFailure}`, + ); + console.error( + ` Local ownership state was preserved. Re-run '${CLI_NAME} ${sandboxName} destroy --yes' to reconcile only the recorded provider runtime.`, + ); + } if (destructiveResult.deleteOutput) { console.error(` ${destructiveResult.deleteOutput}`); } diff --git a/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts b/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts index 9dc200a96e1..27568ff1fe5 100644 --- a/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts +++ b/src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts @@ -1,32 +1,64 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { serializedHostLocalInferenceReceipt } from "../../../../test/helpers/host-local-inference-receipt"; import { resolveTestAgentBaselinePolicy } from "../../../../test/support/snapshot-policy-test-fixture"; +import type { RuntimeProviderBundle } from "../../onboard/runtime-provider/contract"; import type { SnapshotStreamSandboxCreateMock } from "./snapshot-create-stream-test-types"; -const captureOpenshellMock = vi.fn(() => ({ status: 0, output: "alpha Ready\n" })); -const getSandboxMock = vi.fn((name?: string) => - name === "alpha" - ? { - name: "alpha", - agent: "openclaw", - gatewayName: "nemoclaw", - imageTag: "nemoclaw-alpha:test", - openshellDriver: "docker", - provider: "nvidia-nim", - model: "nvidia/model-a", - } - : null, -); -const registerSandboxMock = vi.fn(); +const harness = vi.hoisted(() => ({ + entries: new Map>(), + preserveForRebuild: vi.fn((value: unknown) => value), + prepareDestroy: vi.fn((value: unknown) => value), + destroy: vi.fn((value: unknown) => ({ status: "removed", receipt: value })), +})); +const captureOpenshellMock = vi.fn(() => ({ status: 0, output: "alpha Ready\nbeta Ready\n" })); +const getSandboxMock = vi.fn((name?: string) => harness.entries.get(name ?? "") ?? null); +const registerSandboxMock = vi.fn((entry: Record) => { + harness.entries.set(String(entry.name), entry); +}); const restoreSandboxStateMock = vi.fn(); +const captureSnapshotRestoreAuthorityMock = vi.fn(); const streamSandboxCreateMock = vi.fn(async () => ({ status: 7, output: "create failed before registry write", sawProgress: false, forcedReady: false, })); +const removeSandboxRegistryEntryOutcomeMock = vi.fn((name: string) => { + const removed = harness.entries.delete(name); + return { status: removed ? ("complete" as const) : ("not-found" as const), removed }; +}); + +const runtimeProvider = { + identity: { contractVersion: 1, id: "mxc", displayName: "MXC" }, + hostLocalInference: { + providerId: "mxc", + supported: true, + runtime: { + providerId: "mxc", + authorityId: "mxc:host-local", + services: ["ollama", "nim", "vllm"], + preserveForRebuild: harness.preserveForRebuild, + prepareDestroy: harness.prepareDestroy, + destroy: harness.destroy, + }, + }, +} as unknown as RuntimeProviderBundle; + +function sourceEntry(receipt?: string): Record { + return { + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw", + imageTag: "nemoclaw-alpha:test", + openshellDriver: receipt ? "mxc" : "docker", + provider: "nvidia-nim", + model: "nvidia/model-a", + ...(receipt ? { hostLocalInferenceReceipt: receipt } : {}), + }; +} vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => ""), @@ -110,13 +142,17 @@ vi.mock("../../state/mcp-lifecycle-lock", () => ({ })); vi.mock("../../state/registry", () => ({ getSandbox: getSandboxMock, - listSandboxes: vi.fn(() => ({ sandboxes: [getSandboxMock("alpha")], defaultSandbox: "alpha" })), + listSandboxes: vi.fn(() => ({ + sandboxes: [...harness.entries.values()], + defaultSandbox: "alpha", + })), registerSandbox: registerSandboxMock, - removeSandbox: vi.fn(), + removeSandbox: vi.fn((name: string) => harness.entries.delete(name)), updateSandbox: vi.fn(), })); vi.mock("../../state/sandbox", () => ({ backupSandboxState: vi.fn(), + captureSnapshotRestoreAuthority: captureSnapshotRestoreAuthorityMock, findBackup: vi.fn(() => ({ match: null })), getLatestBackup: vi.fn(() => ({ timestamp: "2026-06-15T00:00:00.000Z", @@ -127,17 +163,36 @@ vi.mock("../../state/sandbox", () => ({ })); vi.mock("./destroy", () => ({ cleanupShieldsDestroyArtifacts: vi.fn(), - removeSandboxRegistryEntry: vi.fn(), + removeSandboxRegistryEntryOutcome: removeSandboxRegistryEntryOutcomeMock, + requireSandboxDestructiveCleanupAuthority: vi.fn(() => ({ provider: runtimeProvider })), +})); +vi.mock("./restore-gateway-pairing", () => ({ + establishRestoredSandboxGatewayPairing: vi.fn(), + waitForRestoredSandboxGatewaySupervisor: vi.fn(() => true), })); vi.mock("./sandbox-gateway-routing", () => ({ probeGatewayRunning: vi.fn(() => true), selectSandboxGatewayIfRegistered: vi.fn(() => true), - usesGatewayMetadataProbe: vi.fn( - (driver?: string | null) => driver === "docker" || driver === "vm", - ), + usesGatewayMetadataProbe: vi.fn(() => true), +})); +vi.mock("./snapshot/dependencies", async (importOriginal) => ({ + ...(await importOriginal()), + requireCurrentSnapshotRuntimeProvider: vi.fn(() => runtimeProvider), })); describe("snapshot restore auto-create failures", () => { + beforeEach(() => { + vi.clearAllMocks(); + harness.entries.clear(); + harness.entries.set("alpha", sourceEntry()); + streamSandboxCreateMock.mockResolvedValue({ + status: 7, + output: "create failed before registry write", + sawProgress: false, + forcedReady: false, + }); + }); + it("does not register a ghost sandbox when auto-create fails", async () => { vi.spyOn(console, "error").mockImplementation(() => {}); vi.spyOn(console, "log").mockImplementation(() => {}); @@ -158,4 +213,49 @@ describe("snapshot restore auto-create failures", () => { expect(registerSandboxMock).not.toHaveBeenCalled(); expect(restoreSandboxStateMock).not.toHaveBeenCalled(); }); + + it("removes a registered clone when live inference re-proof fails", async () => { + const receipt = serializedHostLocalInferenceReceipt("mxc"); + harness.entries.set("alpha", sourceEntry(receipt)); + harness.preserveForRebuild + .mockImplementationOnce((value) => value) + .mockImplementationOnce(() => { + throw new Error("injected live route failure"); + }); + streamSandboxCreateMock.mockResolvedValue({ + status: 0, + output: "beta Ready", + sawProgress: true, + forcedReady: false, + }); + const { getLatestBackup } = await import("../../state/sandbox"); + vi.mocked(getLatestBackup).mockReturnValue({ + timestamp: "2026-08-02T00-00-00-000Z", + backupPath: "/tmp/backup-alpha", + hostLocalInferenceReceipt: receipt, + } as ReturnType); + captureSnapshotRestoreAuthorityMock.mockReturnValue({ + schemaVersion: 1, + backupPath: "/tmp/backup-alpha", + contentSha256: "e".repeat(64), + }); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(console, "log").mockImplementation(() => {}); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect( + runSandboxSnapshot("alpha", { kind: "restore", to: "beta" }), + ).rejects.toMatchObject({ exitCode: 1 }); + + expect(harness.preserveForRebuild).toHaveBeenCalledTimes(2); + expect(registerSandboxMock).toHaveBeenCalledWith( + expect.objectContaining({ name: "beta", hostLocalInferenceReceipt: receipt }), + ); + expect(harness.prepareDestroy).toHaveBeenCalledTimes(2); + expect(harness.destroy).not.toHaveBeenCalled(); + expect(removeSandboxRegistryEntryOutcomeMock).toHaveBeenCalledWith("beta"); + expect(getSandboxMock("beta")).toBeNull(); + expect(consoleError.mock.calls.flat().join("\n")).toContain("injected live route failure"); + expect(restoreSandboxStateMock).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/actions/sandbox/snapshot-command-host-local-authority.test.ts b/src/lib/actions/sandbox/snapshot-command-host-local-authority.test.ts new file mode 100644 index 00000000000..92a59c683d8 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot-command-host-local-authority.test.ts @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + hostLocalInferenceReceipt, + serializedHostLocalInferenceReceipt, +} from "../../../../test/helpers/host-local-inference-receipt"; +import type { RuntimeProviderBundle } from "../../onboard/runtime-provider/contract"; +import { serializeHostLocalInferenceReceipt } from "../../onboard/runtime-provider/host-local-inference"; +import type { SandboxEntry } from "../../state/registry/types"; +import type { RebuildManifest, RestoreResult, SnapshotRestoreOptions } from "../../state/sandbox"; +import { runSandboxSnapshot } from "./snapshot"; + +const harness = vi.hoisted(() => { + let registryEntry: unknown = null; + const events: string[] = []; + return { + events, + getSandbox: vi.fn(() => registryEntry), + setRegistryEntry: (entry: unknown) => { + registryEntry = entry; + }, + getLatestBackup: vi.fn(), + captureSnapshotRestoreAuthority: vi.fn(), + restoreSandboxState: vi.fn(), + preserveForRebuild: vi.fn((receipt: unknown) => { + events.push("reprove"); + return receipt; + }), + }; +}); + +const provider = { + identity: { contractVersion: 1, id: "docker", displayName: "Docker" }, + hostLocalInference: { + providerId: "docker", + supported: true, + runtime: { + providerId: "docker", + authorityId: "docker:host-local", + services: ["vllm"], + preserveForRebuild: harness.preserveForRebuild, + }, + }, +} as unknown as RuntimeProviderBundle; + +vi.mock("../../adapters/openshell/runtime", () => ({ + captureOpenshell: vi.fn(() => ({ status: 0, output: "alpha Ready\n" })), + getOpenshellBinary: vi.fn(() => "openshell"), + runOpenshell: vi.fn(() => ({ status: 0, output: "" })), +})); + +vi.mock("../../policy", () => ({ + applyPreset: vi.fn(() => true), + applyPresetContent: vi.fn(() => true), + getAppliedPresets: vi.fn(() => []), + getPresetContentGatewayState: vi.fn(() => "absent"), + loadPresetForSandbox: vi.fn(() => null), + removePreset: vi.fn(() => true), +})); + +vi.mock("../../runtime-recovery", () => ({ + parseLiveSandboxNames: vi.fn(() => new Set(["alpha"])), +})); + +vi.mock("../../shields", () => ({ + isShieldsDown: vi.fn(() => true), + repairMutableConfigPerms: vi.fn(() => ({ applied: true, verified: true, errors: [] })), +})); + +vi.mock("../../shields/timer-bound-lock", () => ({ + withTimerBoundShieldsMutationLock: vi.fn( + (_name: string, _operation: string, callback: () => unknown) => callback(), + ), +})); + +vi.mock("../../state/mcp-lifecycle-lock", () => ({ + withSandboxMutationLock: vi.fn((_name: string, callback: () => Promise) => callback()), +})); + +vi.mock("../../state/registry", () => ({ + getBaselineExclusions: vi.fn(() => []), + getCustomPolicies: vi.fn(() => []), + getSandbox: harness.getSandbox, + listSandboxes: vi.fn(() => ({ + sandboxes: [harness.getSandbox()].filter(Boolean), + defaultSandbox: "alpha", + })), + updateSandbox: vi.fn(), +})); + +vi.mock("../../state/sandbox", () => ({ + captureSnapshotRestoreAuthority: harness.captureSnapshotRestoreAuthority, + findBackup: vi.fn(() => ({ match: null })), + getLatestBackup: harness.getLatestBackup, + listBackups: vi.fn(() => []), + restoreSandboxState: harness.restoreSandboxState, +})); + +vi.mock("./sandbox-gateway-routing", () => ({ + probeGatewayRunning: vi.fn(() => true), + selectSandboxGatewayIfRegistered: vi.fn(() => true), + usesGatewayMetadataProbe: vi.fn(() => false), +})); + +vi.mock("./snapshot/dependencies", async (importOriginal) => ({ + ...(await importOriginal()), + requireCurrentSnapshotRuntimeProvider: vi.fn(() => provider), +})); + +function receiptAtPort(port: number): string { + const receipt = hostLocalInferenceReceipt("docker"); + return serializeHostLocalInferenceReceipt({ + ...receipt, + endpoint: { ...receipt.endpoint, port }, + }); +} + +function manifest(receipt: string): RebuildManifest { + return { + version: 1, + sandboxName: "alpha", + timestamp: "2026-08-02T00-00-00-000Z", + agentType: "openclaw", + agentVersion: null, + expectedVersion: null, + stateDirs: [], + dir: "/sandbox", + backupPath: "/tmp/backup-alpha", + blueprintDigest: null, + hostLocalInferenceReceipt: receipt, + }; +} + +function sandbox(receipt: string): SandboxEntry { + return { + name: "alpha", + agent: "openclaw", + openshellDriver: "docker", + hostLocalInferenceReceipt: receipt, + }; +} + +function successfulRestore( + _name: string, + _path: string, + options: SnapshotRestoreOptions = {}, +): RestoreResult { + harness.events.push("restore-start"); + options.validateBeforeMutation?.(); + harness.events.push("restore-complete"); + return { + success: true, + restoredDirs: [], + failedDirs: [], + restoredFiles: [], + failedFiles: [], + }; +} + +describe("snapshot command host-local inference authority", () => { + beforeEach(() => { + vi.clearAllMocks(); + harness.events.length = 0; + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("re-proves authority before restore, at the mutation fence, and after success", async () => { + const receipt = serializedHostLocalInferenceReceipt("docker"); + harness.setRegistryEntry(sandbox(receipt)); + harness.getLatestBackup.mockReturnValue(manifest(receipt)); + harness.captureSnapshotRestoreAuthority.mockReturnValue({ + schemaVersion: 1, + backupPath: "/tmp/backup-alpha", + contentSha256: "e".repeat(64), + }); + harness.restoreSandboxState.mockImplementation(successfulRestore); + vi.spyOn(console, "log").mockImplementation(() => {}); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(harness.preserveForRebuild).toHaveBeenCalledTimes(4); + expect(harness.events).toEqual([ + "reprove", + "reprove", + "restore-start", + "reprove", + "restore-complete", + "reprove", + ]); + expect(harness.restoreSandboxState).toHaveBeenCalledWith( + "alpha", + "/tmp/backup-alpha", + expect.objectContaining({ + authority: expect.objectContaining({ contentSha256: "e".repeat(64) }), + validateBeforeMutation: expect.any(Function), + }), + ); + }); + + it("rejects mismatched target authority before filesystem restore", async () => { + const snapshotReceipt = receiptAtPort(8000); + harness.setRegistryEntry(sandbox(receiptAtPort(8001))); + harness.getLatestBackup.mockReturnValue(manifest(snapshotReceipt)); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + + await expect(runSandboxSnapshot("alpha", { kind: "restore" })).rejects.toMatchObject({ + exitCode: 1, + }); + + expect(harness.restoreSandboxState).not.toHaveBeenCalled(); + expect(consoleError.mock.calls.flat().join("\n")).toContain( + "different host-local inference authority", + ); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 19cf6428de0..9575f19b995 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -91,7 +91,6 @@ const lifecycleMock = vi.hoisted(() => { ), }; }); - const backupSandboxStateMock = vi.fn(); const captureOpenshellMock = vi.fn< (args: string[], opts?: Record) => OpenshellCaptureResult @@ -239,12 +238,13 @@ vi.mock("./restore-gateway-pairing", () => ({ establishRestoredSandboxGatewayPairing: vi.fn(), waitForRestoredSandboxGatewaySupervisor: vi.fn(() => true), })); - vi.mock("./destroy", () => ({ cleanupShieldsDestroyArtifacts: lifecycleMock.cleanupShieldsDestroyArtifactsMock, removeSandboxRegistryEntry: vi.fn(), removeSandboxRegistryEntryOutcome: vi.fn(() => ({ status: "complete", removed: true })), - requireSandboxDestructiveCleanupAuthority: vi.fn(), + requireSandboxDestructiveCleanupAuthority: vi.fn(() => ({ + provider: { identity: { id: "docker" } }, + })), })); describe("runSandboxSnapshot", () => { beforeEach(() => { diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 77595a0ab18..2223fa4a9de 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -81,13 +81,19 @@ import { } from "./sandbox-gateway-routing"; import { backupSandboxStateWithManagedAuthority, + confirmHostLocalInferenceAuthority, confirmSandboxRuntimeRestore, + type PreparedHostLocalInferenceAuthority, type PreparedSandboxRuntimeRestore, + prepareHostLocalInferenceAuthority, prepareManagedSnapshotProfileRestore, + prepareSandboxHostLocalInferenceDestroyAuthority, prepareSandboxRuntimeRestore, + type RuntimeProviderBundle, readManagedSnapshotProfileAuthority, rejectManagedSnapshotCloneUntilRebind, requireCurrentSnapshotRuntimeProvider, + retirePreparedHostLocalInferenceAuthority, } from "./snapshot/dependencies"; import { formatSnapshotBaselineExclusionSummary } from "./snapshot-baseline-exclusion-summary"; import { printHermesGatewayRestoreHint } from "./snapshot-hermes-gateway-hint"; @@ -489,8 +495,14 @@ function deleteSandboxForRestore(name: string): void { ); snapshotExit(1); } + let runtimeProvider: RuntimeProviderBundle; + let hostLocalInferenceAuthority: PreparedHostLocalInferenceAuthority | null; try { - requireSandboxDestructiveCleanupAuthority(name, sbMeta); + runtimeProvider = requireSandboxDestructiveCleanupAuthority(name, sbMeta).provider; + hostLocalInferenceAuthority = prepareSandboxHostLocalInferenceDestroyAuthority( + runtimeProvider, + sbMeta, + ); } catch (error) { const detail = error instanceof Error ? error.message : String(error); console.error( @@ -526,6 +538,25 @@ function deleteSandboxForRestore(name: string): void { ); snapshotExit(1); } + if (hostLocalInferenceAuthority) { + try { + const current = registry.getSandbox(name); + if (!current) throw new Error(`sandbox '${name}' is no longer registered`); + retirePreparedHostLocalInferenceAuthority( + runtimeProvider, + current, + hostLocalInferenceAuthority, + registry.listSandboxes().sandboxes, + ); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + console.error( + ` Destination '${name}' is gone, but its host-local inference cleanup failed: ${detail}`, + ); + console.error(" Local ownership state was preserved; retry the restore to reconcile it."); + snapshotExit(1); + } + } // Destination-only cleanup so the recreated sandbox does not inherit stale // host-side state or hit provider-name conflicts (Codex #3796 P2): // - /tmp/nemoclaw-services-: PID dir for this sandbox's services @@ -1059,6 +1090,7 @@ async function runSnapshotRestoreUnlocked( }; const currentSourceEntry = registry.getSandbox(sandboxName); let hasManagedProfileAuthority = false; + const hostLocalInferenceReceipt = resolvedSnapshot.hostLocalInferenceReceipt; let snapshotRestoreAuthority: sandboxState.SnapshotRestoreAuthority | null = null; try { const snapshotAuthority = readManagedSnapshotProfileAuthority(snapshotProfileSource); @@ -1079,7 +1111,18 @@ async function runSnapshotRestoreUnlocked( if (isCrossSandboxRestore && hasManagedProfileAuthority) { rejectManagedSnapshotCloneUntilRebind(snapshotProfileSource, targetSandbox); } - if (hasManagedProfileAuthority) { + if (typeof hostLocalInferenceReceipt === "string") { + if (!currentSourceEntry) { + throw new Error("host-local inference snapshot source is no longer registered"); + } + const sourceProvider = requireCurrentSnapshotRuntimeProvider(currentSourceEntry); + prepareHostLocalInferenceAuthority( + sourceProvider, + currentSourceEntry, + hostLocalInferenceReceipt, + ); + } + if (hasManagedProfileAuthority || typeof hostLocalInferenceReceipt === "string") { snapshotRestoreAuthority = sandboxState.captureSnapshotRestoreAuthority( backupPath, resolvedSnapshot, @@ -1090,7 +1133,7 @@ async function runSnapshotRestoreUnlocked( } } catch (error) { console.error( - ` Cannot restore managed snapshot authority: ${ + ` Cannot restore provider snapshot authority: ${ error instanceof Error ? error.message : String(error) }.`, ); @@ -1099,6 +1142,7 @@ async function runSnapshotRestoreUnlocked( } let preparedRuntimeRestore: PreparedSandboxRuntimeRestore | null = null; + let preparedHostLocalInferenceRestore: PreparedHostLocalInferenceAuthority | null = null; if (!isCrossSandboxRestore) { // Self-restore: target is `sandboxName`. Cannot auto-create; the // source pod is the target, so it must already be live. @@ -1139,6 +1183,30 @@ async function runSnapshotRestoreUnlocked( snapshotExit(1); } } + if (typeof hostLocalInferenceReceipt === "string") { + const currentTarget = registry.getSandbox(targetSandbox); + if (!currentTarget) { + console.error( + ` Cannot restore host-local inference snapshot '${sandboxName}': target authority is missing.`, + ); + snapshotExit(1); + } + try { + const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); + preparedHostLocalInferenceRestore = prepareHostLocalInferenceAuthority( + provider, + currentTarget, + hostLocalInferenceReceipt, + ); + } catch (error) { + console.error( + ` Cannot preflight host-local inference snapshot restore: ${ + error instanceof Error ? error.message : String(error) + }.`, + ); + snapshotExit(1); + } + } } else { // #3756: cross-sandbox restore into a destination that already exists // used to overlay onto the live filesystem silently. Refuse by default @@ -1279,73 +1347,122 @@ async function runSnapshotRestoreUnlocked( await withDashboardPortReservationLock(() => withGatewayRouteMutationLock(sourceGatewayName, createAndRegisterClone), ); + if (typeof hostLocalInferenceReceipt === "string") { + const currentTarget = registry.getSandbox(targetSandbox); + if (!currentTarget) { + console.error( + ` Clone '${targetSandbox}' was created without durable host-local inference authority.`, + ); + snapshotExit(1); + } + try { + const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); + preparedHostLocalInferenceRestore = prepareHostLocalInferenceAuthority( + provider, + currentTarget, + hostLocalInferenceReceipt, + ); + } catch (error) { + console.error( + ` Cannot re-prove clone '${targetSandbox}' against snapshot inference authority: ${ + error instanceof Error ? error.message : String(error) + }.`, + ); + console.error( + ` Removing incomplete clone '${targetSandbox}' while its exact provider ownership is still registered.`, + ); + deleteSandboxForRestore(targetSandbox); + snapshotExit(1); + } + } } withTimerBoundShieldsMutationLock(targetSandbox, "restore sandbox snapshot", () => { // Serialize filesystem restore, mutable-permission repair, and policy // reconciliation under the active timer generation. At the absolute // deadline, auto-restore keeps the outer lifecycle gate closed and waits // for this exact owner to finish before restoring lockdown. - const validateManagedRestoreBeforeMutation = preparedRuntimeRestore - ? () => { - const currentTarget = registry.getSandbox(targetSandbox); - if (!currentTarget) { - throw new Error(`target '${targetSandbox}' is no longer registered`); - } - const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); - const profileRestore = prepareManagedSnapshotProfileRestore( - snapshotProfileSource, - currentTarget, - provider, - ); - if (!profileRestore) { - throw new Error("managed profile restore authority is missing"); + const validateProviderRestoreBeforeMutation = + preparedRuntimeRestore || preparedHostLocalInferenceRestore + ? () => { + const currentTarget = registry.getSandbox(targetSandbox); + if (!currentTarget) { + throw new Error(`target '${targetSandbox}' is no longer registered`); + } + const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); + if (preparedRuntimeRestore) { + const profileRestore = prepareManagedSnapshotProfileRestore( + snapshotProfileSource, + currentTarget, + provider, + ); + if (!profileRestore) { + throw new Error("managed profile restore authority is missing"); + } + const prepared = preparedRuntimeRestore; + if (!prepared) throw new Error("managed runtime restore authority is missing"); + // The state layer invokes this after local tar staging and + // immediately before its first remote filesystem mutation. + preparedRuntimeRestore = prepareSandboxRuntimeRestore( + provider, + currentTarget, + prepared.source, + profileRestore.providerRestoreAuthority, + ); + } + if (typeof hostLocalInferenceReceipt === "string") { + preparedHostLocalInferenceRestore = prepareHostLocalInferenceAuthority( + provider, + currentTarget, + hostLocalInferenceReceipt, + ); + } } - const prepared = preparedRuntimeRestore; - if (!prepared) throw new Error("managed runtime restore authority is missing"); - // The state layer invokes this after local tar staging and - // immediately before its first remote filesystem mutation. - preparedRuntimeRestore = prepareSandboxRuntimeRestore( - provider, - currentTarget, - prepared.source, - profileRestore.providerRestoreAuthority, - ); - } - : null; + : null; if (targetSandbox !== sandboxName) { console.log(` Restoring snapshot from '${sandboxName}' into '${targetSandbox}'...`); } else { console.log(` Restoring snapshot into '${sandboxName}'...`); } - if (Boolean(snapshotRestoreAuthority) !== Boolean(validateManagedRestoreBeforeMutation)) { + if (Boolean(snapshotRestoreAuthority) !== Boolean(validateProviderRestoreBeforeMutation)) { console.error( - ` Cannot restore managed snapshot '${sandboxName}': content authority and the runtime mutation fence must both be present.`, + ` Cannot restore provider snapshot '${sandboxName}': content authority and the runtime mutation fence must both be present.`, ); console.error(` Destination '${targetSandbox}' was not changed.`); snapshotExit(1); } const result = - snapshotRestoreAuthority && validateManagedRestoreBeforeMutation + snapshotRestoreAuthority && validateProviderRestoreBeforeMutation ? sandboxState.restoreSandboxState(targetSandbox, backupPath, { authority: snapshotRestoreAuthority, - validateBeforeMutation: validateManagedRestoreBeforeMutation, + validateBeforeMutation: validateProviderRestoreBeforeMutation, }) : sandboxState.restoreSandboxState(targetSandbox, backupPath); if (result.success) { - if (preparedRuntimeRestore) { + if (preparedRuntimeRestore || preparedHostLocalInferenceRestore) { const currentTarget = registry.getSandbox(targetSandbox); if (!currentTarget) { console.error( - ` Managed snapshot state was restored, but target '${targetSandbox}' is no longer registered.`, + ` Provider snapshot state was restored, but target '${targetSandbox}' is no longer registered.`, ); snapshotExit(1); } try { - const provider = requireCurrentSnapshotRuntimeProvider(currentTarget); - confirmSandboxRuntimeRestore(provider, currentTarget, preparedRuntimeRestore); + const provider = preparedRuntimeRestore + ? requireCurrentSnapshotRuntimeProvider(currentTarget) + : requireCurrentSnapshotRuntimeProvider(currentTarget); + if (preparedRuntimeRestore) { + confirmSandboxRuntimeRestore(provider, currentTarget, preparedRuntimeRestore); + } + if (preparedHostLocalInferenceRestore) { + confirmHostLocalInferenceAuthority( + provider, + currentTarget, + preparedHostLocalInferenceRestore, + ); + } } catch (error) { console.error( - ` Managed snapshot state was restored, but provider restore proof failed: ${ + ` Provider snapshot state was restored, but provider restore proof failed: ${ error instanceof Error ? error.message : String(error) }.`, ); diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts index 8221b1535bc..fda55c091a1 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts @@ -12,6 +12,7 @@ import { } from "../../../onboard/managed-image/contract"; import { encodeManagedStartupProfile } from "../../../onboard/managed-startup/profile"; import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; +import { serializeHostLocalInferenceReceipt } from "../../../onboard/runtime-provider/host-local-inference"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../../state/registry/types"; import type { BackupOptions, BackupResult } from "../../../state/sandbox"; import { backupSandboxStateWithManagedAuthority } from "./backup-authority"; @@ -121,6 +122,32 @@ function successfulBackup(options: BackupOptions): BackupResult { }; } +function hostLocalReceipt(port = 8000): string { + return serializeHostLocalInferenceReceipt({ + schemaVersion: 1, + providerId: "mxc", + service: "vllm", + engineAuthority: { + schemaVersion: 1, + providerId: "mxc", + operation: "host-local-inference", + engineId: "mxc", + authorityId: "mxc:host-local", + bindingSha256: "c".repeat(64), + }, + endpoint: { host: "mxc.internal", port, networkName: "mxc-network" }, + runtime: { + kind: "container", + runtimeId: "mxc-vllm-runtime", + name: "nemoclaw-vllm", + imageRef: `nvcr.io/nvidia/vllm@sha256:${"d".repeat(64)}`, + probeImageRef: `quay.io/curl/curl@sha256:${"f".repeat(64)}`, + specSha256: "e".repeat(64), + gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, + }, + }); +} + describe("managed snapshot backup authority", () => { it.each([ "openclaw", @@ -182,6 +209,80 @@ describe("managed snapshot backup authority", () => { expect(captureRuntime).not.toHaveBeenCalled(); }); + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("captures and republishes exact %s host-local inference authority", (agent) => { + const receipt = hostLocalReceipt(); + const entry = { + name: "alpha", + agent, + openshellDriver: "mxc", + hostLocalInferenceReceipt: receipt, + } satisfies SandboxEntry; + const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); + const reproveHostLocalInference = vi.fn((_provider, serialized: string) => serialized); + + const result = backupSandboxStateWithManagedAuthority( + "alpha", + { name: "host-local" }, + { + getSandbox: () => entry, + requireProvider: () => provider(), + captureRuntime: vi.fn() as never, + reproveHostLocalInference, + backup, + }, + ); + + expect(result.success).toBe(true); + expect(backup).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + name: "host-local", + hostLocalInferenceReceipt: receipt, + validateBeforePublish: expect.any(Function), + }), + ); + expect(reproveHostLocalInference).toHaveBeenCalledTimes(2); + }); + + it("rejects host-local route drift before manifest publication", () => { + const receipts = [hostLocalReceipt(), hostLocalReceipt(8001)]; + const entries = receipts.map( + (hostLocalInferenceReceipt) => + ({ + name: "alpha", + agent: "hermes", + openshellDriver: "mxc", + hostLocalInferenceReceipt, + }) satisfies SandboxEntry, + ); + const getSandbox = vi + .fn<() => SandboxEntry | null>() + .mockReturnValueOnce(entries[0]) + .mockReturnValueOnce(entries[1]); + const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); + + const result = backupSandboxStateWithManagedAuthority( + "alpha", + {}, + { + getSandbox, + requireProvider: () => provider(), + captureRuntime: vi.fn() as never, + reproveHostLocalInference: (_provider, serialized) => serialized, + backup, + }, + ); + + expect(result).toMatchObject({ + success: false, + error: expect.stringContaining("host-local inference changed during backup"), + }); + }); + it("fails before filesystem capture when the provider rejects managed authority", () => { const entry = sandbox("openclaw"); const backup = vi.fn(); diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.ts b/src/lib/actions/sandbox/snapshot/backup-authority.ts index bc8ccb371b3..3b374e4b514 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority.ts @@ -5,6 +5,7 @@ import { isDeepStrictEqual } from "node:util"; import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provider/current"; +import { reproveHostLocalInferenceReceipt } from "../../../onboard/runtime-provider/host-local-inference-lifecycle"; import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime-provider/registry"; import type { SandboxEntry } from "../../../state/registry/types"; import * as sandboxState from "../../../state/sandbox"; @@ -13,13 +14,14 @@ import { captureSandboxRuntimeSnapshot } from "./provider-lifecycle"; type SnapshotBackupAuthority = Pick< sandboxState.BackupOptions, - "runtimeSnapshot" | "workload" | "validateBeforePublish" + "runtimeSnapshot" | "workload" | "hostLocalInferenceReceipt" | "validateBeforePublish" >; interface SnapshotBackupAuthorityDependencies { readonly getSandbox: (sandboxName: string) => SandboxEntry | null; readonly requireProvider: (sandbox: SandboxEntry) => RuntimeProviderBundle; readonly captureRuntime: typeof captureSandboxRuntimeSnapshot; + readonly reproveHostLocalInference: typeof reproveHostLocalInferenceReceipt; readonly backup: typeof sandboxState.backupSandboxState; } @@ -27,6 +29,7 @@ const defaultDependencies: Omit requireRuntimeProviderBundleForSandbox(sandbox, CURRENT_RUNTIME_PROVIDER_BUNDLES), captureRuntime: captureSandboxRuntimeSnapshot, + reproveHostLocalInference: reproveHostLocalInferenceReceipt, // Keep the call late-bound so tests and alternative state stores can replace // the module export without this adapter retaining an import-time reference. backup: (...args) => sandboxState.backupSandboxState(...args), @@ -40,7 +43,7 @@ function failure(error: unknown): sandboxState.BackupResult { failedDirs: [], backedUpFiles: [], failedFiles: [], - error: `Cannot capture managed snapshot authority: ${detail}.`, + error: `Cannot capture provider snapshot authority: ${detail}.`, }; } @@ -106,6 +109,55 @@ function captureManagedAuthority( }; } +function captureHostLocalInferenceAuthority( + entry: SandboxEntry, + dependencies: SnapshotBackupAuthorityDependencies, +): Pick | null { + const receipt = entry.hostLocalInferenceReceipt; + if (typeof receipt !== "string") return null; + const provider = dependencies.requireProvider(entry); + if (dependencies.reproveHostLocalInference(provider, receipt) !== receipt) { + throw new Error("host-local inference authority changed before backup"); + } + return { + hostLocalInferenceReceipt: receipt, + validateBeforePublish: () => { + const current = dependencies.getSandbox(entry.name); + if (!current) throw new Error(`sandbox '${entry.name}' is no longer registered`); + if (current.hostLocalInferenceReceipt !== receipt) { + throw new Error(`sandbox '${entry.name}' host-local inference changed during backup`); + } + const currentProvider = dependencies.requireProvider(current); + if (currentProvider.identity.id !== provider.identity.id) { + throw new Error(`sandbox '${entry.name}' runtime provider changed during backup`); + } + if (dependencies.reproveHostLocalInference(currentProvider, receipt) !== receipt) { + throw new Error(`sandbox '${entry.name}' host-local inference changed during backup`); + } + }, + }; +} + +function captureSnapshotAuthority( + entry: SandboxEntry, + dependencies: SnapshotBackupAuthorityDependencies, +): SnapshotBackupAuthority | null { + const managed = captureManagedAuthority(entry, dependencies); + const hostLocal = captureHostLocalInferenceAuthority(entry, dependencies); + if (!managed && !hostLocal) return null; + return { + ...(managed?.runtimeSnapshot === undefined ? {} : { runtimeSnapshot: managed.runtimeSnapshot }), + ...(managed?.workload === undefined ? {} : { workload: managed.workload }), + ...(hostLocal?.hostLocalInferenceReceipt === undefined + ? {} + : { hostLocalInferenceReceipt: hostLocal.hostLocalInferenceReceipt }), + validateBeforePublish: () => { + managed?.validateBeforePublish?.(); + hostLocal?.validateBeforePublish?.(); + }, + }; +} + /** * Capture one managed workload and runtime authority pair around the complete * filesystem copy. The state layer publishes the manifest only after the @@ -123,7 +175,7 @@ export function backupSandboxStateWithManagedAuthority( let authority: SnapshotBackupAuthority | null; try { - authority = captureManagedAuthority(entry, dependencies); + authority = captureSnapshotAuthority(entry, dependencies); } catch (error) { return failure(error); } diff --git a/src/lib/actions/sandbox/snapshot/dependencies.ts b/src/lib/actions/sandbox/snapshot/dependencies.ts index 957d60dd625..079ed824eaf 100644 --- a/src/lib/actions/sandbox/snapshot/dependencies.ts +++ b/src/lib/actions/sandbox/snapshot/dependencies.ts @@ -6,6 +6,13 @@ import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provi import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime-provider/registry"; import type { SandboxEntry } from "../../../state/registry/types"; +export { + confirmHostLocalInferenceAuthority, + type PreparedHostLocalInferenceAuthority, + prepareHostLocalInferenceAuthority, + prepareSandboxHostLocalInferenceDestroyAuthority, + retirePreparedHostLocalInferenceAuthority, +} from "../../../onboard/runtime-provider/host-local-inference-lifecycle"; export type { ManagedWorkloadCloneSnapshot, PreparedManagedWorkloadCloneHandoff, @@ -38,6 +45,7 @@ export { prepareSandboxRuntimeRestore, SandboxSnapshotProviderError, } from "./provider-lifecycle"; +export type { RuntimeProviderBundle }; /** * Resolve the one already-registered provider bundle for a durable sandbox. diff --git a/src/lib/actions/sandbox/snapshot/restore-authority.ts b/src/lib/actions/sandbox/snapshot/restore-authority.ts index 8d5b2198e58..3a81212500b 100644 --- a/src/lib/actions/sandbox/snapshot/restore-authority.ts +++ b/src/lib/actions/sandbox/snapshot/restore-authority.ts @@ -3,6 +3,11 @@ import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provider/current"; +import { + confirmHostLocalInferenceAuthority, + type PreparedHostLocalInferenceAuthority, + prepareHostLocalInferenceAuthority, +} from "../../../onboard/runtime-provider/host-local-inference-lifecycle"; import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime-provider/registry"; import type { SandboxEntry } from "../../../state/registry/types"; import * as sandboxState from "../../../state/sandbox"; @@ -16,14 +21,14 @@ import { prepareSandboxRuntimeRestore, } from "./provider-lifecycle"; -interface ManagedRestoreAuthorityDependencies { +interface ProviderRestoreAuthorityDependencies { readonly getSandbox: (sandboxName: string) => SandboxEntry | null; readonly requireProvider: (sandbox: SandboxEntry) => RuntimeProviderBundle; readonly captureContentAuthority: typeof sandboxState.captureSnapshotRestoreAuthority; readonly restore: typeof sandboxState.restoreRecreatedSandboxState; } -const defaultDependencies: Omit = { +const defaultDependencies: Omit = { requireProvider: (sandbox) => requireRuntimeProviderBundleForSandbox(sandbox, CURRENT_RUNTIME_PROVIDER_BUNDLES), captureContentAuthority: (...args) => sandboxState.captureSnapshotRestoreAuthority(...args), @@ -38,21 +43,21 @@ function failure(error: unknown): sandboxState.RestoreResult { failedDirs: ["manifest"], restoredFiles: [], failedFiles: [], - error: `Cannot restore managed snapshot authority: ${detail}.`, + error: `Cannot restore provider snapshot authority: ${detail}.`, }; } /** - * Restore a rebuild backup through the same provider and content authority - * boundary as an explicit snapshot restore. Legacy/custom-image manifests - * retain their existing state-only path. + * Restore a rebuild backup through its provider runtime and content authority. + * Legacy/custom-image manifests without provider-backed state retain the + * existing state-only path. */ export function restoreRecreatedSandboxStateWithManagedAuthority( sandboxName: string, manifest: sandboxState.RebuildManifest, options: sandboxState.RecreatedSandboxRestoreOptions, - overrides: Pick & - Partial>, + overrides: Pick & + Partial>, ): sandboxState.RestoreResult { const dependencies = { ...defaultDependencies, ...overrides }; let snapshotProfile; @@ -65,14 +70,16 @@ export function restoreRecreatedSandboxStateWithManagedAuthority( } catch (error) { return failure(error); } - if (!snapshotProfile) { + const hostLocalInferenceReceipt = manifest.hostLocalInferenceReceipt; + if (!snapshotProfile && typeof hostLocalInferenceReceipt !== "string") { return dependencies.restore(sandboxName, manifest.backupPath, options); } - if (!manifest.runtimeSnapshot) { + if (snapshotProfile && !manifest.runtimeSnapshot) { return failure("managed snapshot is missing provider runtime authority"); } - let prepared: PreparedSandboxRuntimeRestore; + let preparedRuntime: PreparedSandboxRuntimeRestore | null = null; + let preparedHostLocal: PreparedHostLocalInferenceAuthority | null = null; let providerId: string; let contentAuthority: sandboxState.SnapshotRestoreAuthority; try { @@ -80,25 +87,34 @@ export function restoreRecreatedSandboxStateWithManagedAuthority( if (!target) throw new Error(`target '${sandboxName}' is not registered`); const provider = dependencies.requireProvider(target); providerId = provider.identity.id; - const profileRestore = prepareManagedSnapshotProfileRestore( - { - sandboxName: manifest.sandboxName, - agentType: manifest.agentType, - workload: manifest.workload, - }, - target, - provider, - ); - if (!profileRestore) throw new Error("managed profile restore authority is missing"); + if (snapshotProfile) { + const profileRestore = prepareManagedSnapshotProfileRestore( + { + sandboxName: manifest.sandboxName, + agentType: manifest.agentType, + workload: manifest.workload, + }, + target, + provider, + ); + if (!profileRestore) throw new Error("managed profile restore authority is missing"); + preparedRuntime = prepareSandboxRuntimeRestore( + provider, + target, + manifest.runtimeSnapshot!, + profileRestore.providerRestoreAuthority, + ); + } + if (typeof hostLocalInferenceReceipt === "string") { + preparedHostLocal = prepareHostLocalInferenceAuthority( + provider, + target, + hostLocalInferenceReceipt, + ); + } const captured = dependencies.captureContentAuthority(manifest.backupPath, manifest); if (!captured) throw new Error("selected snapshot content changed during restore preflight"); contentAuthority = captured; - prepared = prepareSandboxRuntimeRestore( - provider, - target, - manifest.runtimeSnapshot, - profileRestore.providerRestoreAuthority, - ); } catch (error) { return failure(error); } @@ -113,22 +129,32 @@ export function restoreRecreatedSandboxStateWithManagedAuthority( if (provider.identity.id !== providerId) { throw new Error(`target '${sandboxName}' runtime provider changed before restore`); } - const profileRestore = prepareManagedSnapshotProfileRestore( - { - sandboxName: manifest.sandboxName, - agentType: manifest.agentType, - workload: manifest.workload, - }, - current, - provider, - ); - if (!profileRestore) throw new Error("managed profile restore authority is missing"); - prepared = prepareSandboxRuntimeRestore( - provider, - current, - prepared.source, - profileRestore.providerRestoreAuthority, - ); + if (snapshotProfile) { + const profileRestore = prepareManagedSnapshotProfileRestore( + { + sandboxName: manifest.sandboxName, + agentType: manifest.agentType, + workload: manifest.workload, + }, + current, + provider, + ); + if (!profileRestore) throw new Error("managed profile restore authority is missing"); + if (!preparedRuntime) throw new Error("managed runtime restore authority is missing"); + preparedRuntime = prepareSandboxRuntimeRestore( + provider, + current, + preparedRuntime.source, + profileRestore.providerRestoreAuthority, + ); + } + if (typeof hostLocalInferenceReceipt === "string") { + preparedHostLocal = prepareHostLocalInferenceAuthority( + provider, + current, + hostLocalInferenceReceipt, + ); + } }, }); if (!restore.success) return restore; @@ -140,7 +166,10 @@ export function restoreRecreatedSandboxStateWithManagedAuthority( if (provider.identity.id !== providerId) { throw new Error(`target '${sandboxName}' runtime provider changed during restore`); } - confirmSandboxRuntimeRestore(provider, current, prepared); + if (preparedRuntime) confirmSandboxRuntimeRestore(provider, current, preparedRuntime); + if (preparedHostLocal) { + confirmHostLocalInferenceAuthority(provider, current, preparedHostLocal); + } return restore; } catch (error) { const detail = error instanceof Error ? error.message : String(error); @@ -148,7 +177,7 @@ export function restoreRecreatedSandboxStateWithManagedAuthority( ...restore, success: false, error: - `State was restored, but managed runtime proof failed: ${detail}. ` + + `State was restored, but provider runtime proof failed: ${detail}. ` + `Retry this exact snapshot after the runtime stabilizes.`, }; } diff --git a/src/lib/actions/sandbox/snapshot/restore-host-local-authority.test.ts b/src/lib/actions/sandbox/snapshot/restore-host-local-authority.test.ts new file mode 100644 index 00000000000..f2c2af57329 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/restore-host-local-authority.test.ts @@ -0,0 +1,226 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; +import { + type HostLocalInferenceReceipt, + serializeHostLocalInferenceReceipt, +} from "../../../onboard/runtime-provider/host-local-inference"; +import type { SandboxEntry } from "../../../state/registry/types"; +import type { + RebuildManifest, + RecreatedSandboxRestoreOptions, + RestoreResult, +} from "../../../state/sandbox"; +import { restoreRecreatedSandboxStateWithManagedAuthority } from "./restore-authority"; + +type Agent = "openclaw" | "hermes" | "langchain-deepagents-code"; +type Service = "ollama" | "nim" | "vllm"; + +function receipt(service: Service, port = 8000): HostLocalInferenceReceipt { + return { + schemaVersion: 1, + providerId: "mxc", + service, + engineAuthority: { + schemaVersion: 1, + providerId: "mxc", + operation: "host-local-inference", + engineId: "mxc", + authorityId: "mxc:host-local", + bindingSha256: "a".repeat(64), + }, + endpoint: { host: "mxc.internal", port, networkName: "mxc-network" }, + runtime: + service === "ollama" + ? { + kind: "host", + probeImageRef: `quay.io/curl/curl@sha256:${"b".repeat(64)}`, + } + : { + kind: "container", + runtimeId: `mxc-${service}-runtime`, + name: `nemoclaw-${service}`, + imageRef: `nvcr.io/nvidia/${service}@sha256:${"c".repeat(64)}`, + probeImageRef: `quay.io/curl/curl@sha256:${"b".repeat(64)}`, + specSha256: "d".repeat(64), + gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, + }, + }; +} + +function manifest(agent: Agent, service: Service, port = 8000): RebuildManifest { + return { + version: 1, + sandboxName: "alpha", + timestamp: "2026-08-02T00-00-00-000Z", + agentType: agent, + agentVersion: null, + expectedVersion: null, + stateDirs: [], + dir: "/sandbox", + backupPath: "/tmp/alpha", + blueprintDigest: null, + hostLocalInferenceReceipt: serializeHostLocalInferenceReceipt(receipt(service, port)), + }; +} + +function sandbox(agent: Agent, service: Service, port = 8000): SandboxEntry { + return { + name: "alpha", + agent, + openshellDriver: "mxc", + hostLocalInferenceReceipt: serializeHostLocalInferenceReceipt(receipt(service, port)), + }; +} + +function provider() { + const preserveForRebuild = vi.fn((value: HostLocalInferenceReceipt) => value); + const bundle = { + identity: { contractVersion: 1, id: "mxc", displayName: "MXC" }, + hostLocalInference: { + providerId: "mxc", + supported: true, + runtime: { + providerId: "mxc", + authorityId: "mxc:host-local", + services: ["ollama", "nim", "vllm"], + translateContainerArgs: (args: readonly string[]) => args, + qualifyOllama: vi.fn(), + startManaged: vi.fn(), + inspectManaged: vi.fn(), + stopManaged: vi.fn(), + preserveForRebuild, + prepareDestroy: vi.fn((value: HostLocalInferenceReceipt) => value), + destroy: vi.fn((value: HostLocalInferenceReceipt) => ({ + status: "removed" as const, + receipt: value, + })), + }, + }, + } as unknown as RuntimeProviderBundle; + return { bundle, preserveForRebuild }; +} + +function successfulRestore(options: RecreatedSandboxRestoreOptions): RestoreResult { + try { + options.validateBeforeMutation?.(); + return { + success: true, + restoredDirs: ["workspace"], + failedDirs: [], + restoredFiles: [], + failedFiles: [], + }; + } catch (error) { + return { + success: false, + restoredDirs: [], + failedDirs: ["workspace"], + restoredFiles: [], + failedFiles: [], + error: error instanceof Error ? error.message : String(error), + }; + } +} + +describe("host-local inference snapshot restore authority", () => { + it.each([ + ["openclaw", "ollama"], + ["openclaw", "nim"], + ["openclaw", "vllm"], + ["hermes", "ollama"], + ["hermes", "nim"], + ["hermes", "vllm"], + ["langchain-deepagents-code", "ollama"], + ["langchain-deepagents-code", "nim"], + ["langchain-deepagents-code", "vllm"], + ] as const)("re-proves exact %s %s authority before, at, and after restore", (agent, service) => { + const target = sandbox(agent, service); + const runtimeProvider = provider(); + const restore = vi.fn((_name, _path, options: RecreatedSandboxRestoreOptions) => + successfulRestore(options), + ); + + const result = restoreRecreatedSandboxStateWithManagedAuthority( + "alpha", + manifest(agent, service), + { targetAgentType: agent }, + { + getSandbox: () => target, + requireProvider: () => runtimeProvider.bundle, + captureContentAuthority: () => ({ + schemaVersion: 1, + backupPath: "/tmp/alpha", + contentSha256: "e".repeat(64), + }), + restore, + }, + ); + + expect(result.success).toBe(true); + expect(runtimeProvider.preserveForRebuild).toHaveBeenCalledTimes(3); + expect(restore).toHaveBeenCalledWith( + "alpha", + "/tmp/alpha", + expect.objectContaining({ + authority: expect.objectContaining({ contentSha256: "e".repeat(64) }), + validateBeforeMutation: expect.any(Function), + }), + ); + }); + + it("fails before mutation when the target route differs from the manifest", () => { + const runtimeProvider = provider(); + const restore = vi.fn(); + const result = restoreRecreatedSandboxStateWithManagedAuthority( + "alpha", + manifest("hermes", "vllm"), + { targetAgentType: "hermes" }, + { + getSandbox: () => sandbox("hermes", "vllm", 8001), + requireProvider: () => runtimeProvider.bundle, + captureContentAuthority: vi.fn(), + restore, + }, + ); + + expect(result).toMatchObject({ + success: false, + error: expect.stringContaining("different host-local inference authority"), + }); + expect(restore).not.toHaveBeenCalled(); + }); + + it("fails closed when the route changes at the filesystem mutation fence", () => { + const runtimeProvider = provider(); + const entries = [sandbox("openclaw", "ollama"), sandbox("openclaw", "ollama", 11435)]; + const getSandbox = vi + .fn<() => SandboxEntry | null>() + .mockReturnValueOnce(entries[0]) + .mockReturnValueOnce(entries[1]); + + const result = restoreRecreatedSandboxStateWithManagedAuthority( + "alpha", + manifest("openclaw", "ollama"), + { targetAgentType: "openclaw" }, + { + getSandbox, + requireProvider: () => runtimeProvider.bundle, + captureContentAuthority: () => ({ + schemaVersion: 1, + backupPath: "/tmp/alpha", + contentSha256: "f".repeat(64), + }), + restore: (_name, _path, options) => successfulRestore(options), + }, + ); + + expect(result).toMatchObject({ + success: false, + error: expect.stringContaining("different host-local inference authority"), + }); + }); +}); diff --git a/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts new file mode 100644 index 00000000000..8d29fe3b76a --- /dev/null +++ b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.test.ts @@ -0,0 +1,139 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { SandboxEntry } from "../../state/registry/types"; +import type { RuntimeProviderBundle } from "./contract"; +import { + type HostLocalInferenceReceipt, + serializeHostLocalInferenceReceipt, +} from "./host-local-inference"; +import { + prepareSandboxHostLocalInferenceDestroyAuthority, + retirePreparedHostLocalInferenceAuthority, +} from "./host-local-inference-lifecycle"; + +function serializedReceipt(service: "ollama" | "nim" | "vllm" = "vllm"): string { + return serializeHostLocalInferenceReceipt({ + schemaVersion: 1, + providerId: "mxc", + service, + engineAuthority: { + schemaVersion: 1, + providerId: "mxc", + operation: "host-local-inference", + engineId: "mxc", + authorityId: "mxc:host-local", + bindingSha256: "a".repeat(64), + }, + endpoint: { host: "mxc.internal", port: 8000, networkName: "mxc-network" }, + runtime: + service === "ollama" + ? { kind: "host", probeImageRef: `quay.io/curl/curl@sha256:${"b".repeat(64)}` } + : { + kind: "container", + runtimeId: `mxc-${service}`, + name: `nemoclaw-${service}`, + imageRef: `nvcr.io/nvidia/${service}@sha256:${"c".repeat(64)}`, + probeImageRef: `quay.io/curl/curl@sha256:${"b".repeat(64)}`, + specSha256: "d".repeat(64), + gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] }, + }, + }); +} + +function sandbox(name: string, receipt = serializedReceipt()): SandboxEntry { + return { name, agent: "openclaw", openshellDriver: "mxc", hostLocalInferenceReceipt: receipt }; +} + +function provider() { + const preserveForRebuild = vi.fn((receipt: HostLocalInferenceReceipt) => receipt); + const prepareDestroy = vi.fn((receipt: HostLocalInferenceReceipt) => receipt); + const destroy = vi.fn((receipt: HostLocalInferenceReceipt) => ({ + status: receipt.runtime.kind === "host" ? ("retained" as const) : ("removed" as const), + ...(receipt.runtime.kind === "host" ? { reason: "host-process" as const } : {}), + receipt, + })); + const bundle = { + identity: { contractVersion: 1, id: "mxc", displayName: "MXC" }, + hostLocalInference: { + providerId: "mxc", + supported: true, + runtime: { + providerId: "mxc", + authorityId: "mxc:host-local", + services: ["ollama", "nim", "vllm"], + translateContainerArgs: (args: readonly string[]) => args, + qualifyOllama: vi.fn(), + startManaged: vi.fn(), + inspectManaged: vi.fn(), + stopManaged: vi.fn(), + preserveForRebuild, + prepareDestroy, + destroy, + }, + }, + } as unknown as RuntimeProviderBundle; + return { bundle, destroy, prepareDestroy, preserveForRebuild }; +} + +describe("host-local inference lifecycle authority", () => { + it.each([ + "ollama", + "nim", + "vllm", + ] as const)("retires exact unshared %s authority through an MXC-style provider", (service) => { + const runtimeProvider = provider(); + const entry = sandbox("alpha", serializedReceipt(service)); + const prepared = prepareSandboxHostLocalInferenceDestroyAuthority( + runtimeProvider.bundle, + entry, + ); + + expect(prepared).not.toBeNull(); + expect( + retirePreparedHostLocalInferenceAuthority(runtimeProvider.bundle, entry, prepared!, [entry]) + .status, + ).toBe(service === "ollama" ? "retained" : "removed"); + expect(runtimeProvider.destroy).toHaveBeenCalledOnce(); + expect(runtimeProvider.prepareDestroy).toHaveBeenCalledTimes(2); + }); + + it("keeps a managed runtime while another sandbox owns the exact receipt", () => { + const runtimeProvider = provider(); + const alpha = sandbox("alpha"); + const beta = sandbox("beta"); + const prepared = prepareSandboxHostLocalInferenceDestroyAuthority( + runtimeProvider.bundle, + alpha, + ); + + expect( + retirePreparedHostLocalInferenceAuthority(runtimeProvider.bundle, alpha, prepared!, [ + alpha, + beta, + ]).status, + ).toBe("shared"); + expect(runtimeProvider.destroy).not.toHaveBeenCalled(); + }); + + it("rejects registry drift before provider mutation", () => { + const runtimeProvider = provider(); + const alpha = sandbox("alpha"); + const prepared = prepareSandboxHostLocalInferenceDestroyAuthority( + runtimeProvider.bundle, + alpha, + ); + + expect(() => + retirePreparedHostLocalInferenceAuthority( + runtimeProvider.bundle, + sandbox("alpha", serializedReceipt("nim")), + prepared!, + [], + ), + ).toThrow("destroy target changed runtime identity"); + expect(runtimeProvider.destroy).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts new file mode 100644 index 00000000000..06af9fe63a8 --- /dev/null +++ b/src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxEntry } from "../../state/registry/types"; +import type { RuntimeProviderBundle } from "./contract"; +import { + type HostLocalInferenceDestroyResult, + type HostLocalInferenceReceipt, + type HostLocalInferenceRuntime, + parseHostLocalInferenceReceipt, + serializeHostLocalInferenceReceipt, +} from "./host-local-inference"; + +export interface PreparedHostLocalInferenceAuthority { + readonly providerId: string; + readonly sandboxName: string; + readonly serializedReceipt: string; + readonly receipt: HostLocalInferenceReceipt; +} + +export type HostLocalInferenceRetirementResult = + | HostLocalInferenceDestroyResult + | { + readonly status: "shared"; + readonly receipt: HostLocalInferenceReceipt; + }; + +function requireRuntime( + provider: RuntimeProviderBundle, + receipt: HostLocalInferenceReceipt, +): HostLocalInferenceRuntime { + const surface = provider.hostLocalInference; + if (!surface.supported) { + throw new Error( + `Runtime provider '${provider.identity.id}' does not support host-local inference.`, + ); + } + if ( + surface.providerId !== provider.identity.id || + surface.runtime.providerId !== provider.identity.id || + receipt.providerId !== provider.identity.id + ) { + throw new Error("Host-local inference receipt belongs to a different runtime provider."); + } + if (!surface.runtime.services.includes(receipt.service)) { + throw new Error( + `Runtime provider '${provider.identity.id}' does not support host-local ${receipt.service}.`, + ); + } + return surface.runtime; +} + +/** Re-prove an exact durable route through its owning provider. */ +export function reproveHostLocalInferenceReceipt( + provider: RuntimeProviderBundle, + serialized: string, +): string { + const receipt = parseHostLocalInferenceReceipt(serialized); + const runtime = requireRuntime(provider, receipt); + const reproved = serializeHostLocalInferenceReceipt(runtime.preserveForRebuild(receipt)); + if (reproved !== serialized) { + throw new Error("Host-local inference authority changed while it was being preserved."); + } + return reproved; +} + +export function prepareHostLocalInferenceAuthority( + provider: RuntimeProviderBundle, + sandbox: Pick, + serialized: string, +): PreparedHostLocalInferenceAuthority { + if (sandbox.hostLocalInferenceReceipt !== serialized) { + throw new Error(`target '${sandbox.name}' has different host-local inference authority`); + } + reproveHostLocalInferenceReceipt(provider, serialized); + return Object.freeze({ + providerId: provider.identity.id, + sandboxName: sandbox.name, + serializedReceipt: serialized, + receipt: parseHostLocalInferenceReceipt(serialized), + }); +} + +export function prepareSandboxHostLocalInferenceAuthority( + provider: RuntimeProviderBundle, + sandbox: Pick, +): PreparedHostLocalInferenceAuthority | null { + return typeof sandbox.hostLocalInferenceReceipt === "string" + ? prepareHostLocalInferenceAuthority(provider, sandbox, sandbox.hostLocalInferenceReceipt) + : null; +} + +export function prepareSandboxHostLocalInferenceDestroyAuthority( + provider: RuntimeProviderBundle, + sandbox: Pick, +): PreparedHostLocalInferenceAuthority | null { + const serialized = sandbox.hostLocalInferenceReceipt; + if (typeof serialized !== "string") return null; + const receipt = parseHostLocalInferenceReceipt(serialized); + const runtime = requireRuntime(provider, receipt); + if (serializeHostLocalInferenceReceipt(runtime.prepareDestroy(receipt)) !== serialized) { + throw new Error("Host-local inference authority changed during destroy preflight."); + } + return Object.freeze({ + providerId: provider.identity.id, + sandboxName: sandbox.name, + serializedReceipt: serialized, + receipt, + }); +} + +export function confirmHostLocalInferenceAuthority( + provider: RuntimeProviderBundle, + sandbox: Pick, + prepared: PreparedHostLocalInferenceAuthority, +): void { + if (provider.identity.id !== prepared.providerId || sandbox.name !== prepared.sandboxName) { + throw new Error("Host-local inference restore target changed runtime identity."); + } + prepareHostLocalInferenceAuthority(provider, sandbox, prepared.serializedReceipt); +} + +function confirmHostLocalInferenceDestroyAuthority( + provider: RuntimeProviderBundle, + sandbox: Pick, + prepared: PreparedHostLocalInferenceAuthority, +): void { + if ( + provider.identity.id !== prepared.providerId || + sandbox.name !== prepared.sandboxName || + sandbox.hostLocalInferenceReceipt !== prepared.serializedReceipt + ) { + throw new Error("Host-local inference destroy target changed runtime identity."); + } + const runtime = requireRuntime(provider, prepared.receipt); + if ( + serializeHostLocalInferenceReceipt(runtime.prepareDestroy(prepared.receipt)) !== + prepared.serializedReceipt + ) { + throw new Error("Host-local inference authority changed before destroy mutation."); + } +} + +/** + * Retire an exact managed runtime only after the caller has confirmed sandbox + * deletion. Shared receipts remain live for their peer sandboxes, and host + * Ollama is always retained because NemoClaw does not own that process. + */ +export function retirePreparedHostLocalInferenceAuthority( + provider: RuntimeProviderBundle, + sandbox: Pick, + prepared: PreparedHostLocalInferenceAuthority, + peers: readonly Pick[], +): HostLocalInferenceRetirementResult { + confirmHostLocalInferenceDestroyAuthority(provider, sandbox, prepared); + if ( + peers.some( + (peer) => + peer.name !== sandbox.name && peer.hostLocalInferenceReceipt === prepared.serializedReceipt, + ) + ) { + return Object.freeze({ status: "shared" as const, receipt: prepared.receipt }); + } + const runtime = requireRuntime(provider, prepared.receipt); + const result = runtime.destroy(prepared.receipt); + if (serializeHostLocalInferenceReceipt(result.receipt) !== prepared.serializedReceipt) { + throw new Error("Host-local inference destroy returned different runtime authority."); + } + return result; +} diff --git a/src/lib/onboard/runtime-provider/host-local-inference-routing.test.ts b/src/lib/onboard/runtime-provider/host-local-inference-routing.test.ts index c9345c697d0..5eef0f0653a 100644 --- a/src/lib/onboard/runtime-provider/host-local-inference-routing.test.ts +++ b/src/lib/onboard/runtime-provider/host-local-inference-routing.test.ts @@ -58,6 +58,8 @@ function runtime(): HostLocalInferenceRuntime { inspectManaged: vi.fn((value) => ({ running: true, receipt: value })), stopManaged: vi.fn((value) => ({ running: false, receipt: value })), preserveForRebuild: vi.fn((value) => value), + prepareDestroy: vi.fn((value) => value), + destroy: vi.fn((value) => ({ status: "removed" as const, receipt: value })), }; } diff --git a/src/lib/onboard/runtime-provider/host-local-inference.ts b/src/lib/onboard/runtime-provider/host-local-inference.ts index e890dd294ca..e3ec455e4fc 100644 --- a/src/lib/onboard/runtime-provider/host-local-inference.ts +++ b/src/lib/onboard/runtime-provider/host-local-inference.ts @@ -82,6 +82,17 @@ export interface HostLocalManagedInferenceInspection { readonly receipt: HostLocalInferenceReceipt; } +export type HostLocalInferenceDestroyResult = + | { + readonly status: "retained"; + readonly reason: "host-process"; + readonly receipt: HostLocalInferenceReceipt; + } + | { + readonly status: "removed" | "already-absent"; + readonly receipt: HostLocalInferenceReceipt; + }; + export interface HostLocalInferenceRouteAuthority { readonly schemaVersion: 1; readonly providerId: string; @@ -113,8 +124,22 @@ export interface HostLocalInferenceRuntime { startManaged(input: HostLocalManagedInferenceInput): HostLocalInferenceReceipt; inspectManaged(receipt: HostLocalInferenceReceipt): HostLocalManagedInferenceInspection; stopManaged(receipt: HostLocalInferenceReceipt): HostLocalManagedInferenceInspection; - /** Re-prove the same out-of-sandbox service before carrying it into a rebuild. */ + /** + * Re-prove the same out-of-sandbox service before carrying it across a + * lifecycle boundary. Every invocation must perform a fresh provider-native + * identity inspection and network health probe; cached or receipt-only + * validation does not satisfy this contract. + */ preserveForRebuild(receipt: HostLocalInferenceReceipt): HostLocalInferenceReceipt; + /** Prove exact ownership for teardown without requiring the service to be healthy. */ + prepareDestroy(receipt: HostLocalInferenceReceipt): HostLocalInferenceReceipt; + /** + * Retire only the exact provider-owned runtime; host processes remain + * externally owned. Managed cleanup must converge safely when repeated so a + * retained ownership journal can resume teardown after a process crash or + * provider failure. + */ + destroy(receipt: HostLocalInferenceReceipt): HostLocalInferenceDestroyResult; } const PROVIDER_ID = /^[a-z][a-z0-9-]{0,62}$/u; diff --git a/src/lib/onboard/runtime-provider/podman-host-local-inference.test.ts b/src/lib/onboard/runtime-provider/podman-host-local-inference.test.ts index aaeec215ca8..f1d25676189 100644 --- a/src/lib/onboard/runtime-provider/podman-host-local-inference.test.ts +++ b/src/lib/onboard/runtime-provider/podman-host-local-inference.test.ts @@ -68,6 +68,25 @@ describe("Podman host-local inference runtime", () => { ]); }); + it("retains externally owned Ollama while proving the exact route during destroy", () => { + const host = runtimeHarness(); + const receipt = host.runtime.qualifyOllama({ + networkName: "openshell", + hostPort: 11434, + probeImageRef: PROBE_IMAGE_REF, + }); + host.setProbeFailure("host process is temporarily unavailable"); + + expect(host.runtime.destroy(receipt)).toEqual({ + status: "retained", + reason: "host-process", + receipt, + }); + expect( + host.capture.mock.calls.map(([args]) => args).some(([operation]) => operation === "rm"), + ).toBe(false); + }); + it.each([ "nim", "vllm", @@ -115,6 +134,25 @@ describe("Podman host-local inference runtime", () => { ]); }); + it.each([ + "nim", + "vllm", + ] as const)("removes exact managed %s authority and makes destroy idempotent", (service) => { + const host = runtimeHarness(); + const receipt = host.runtime.startManaged(managedInput(service)); + const runtimeId = + receipt.runtime.kind === "container" ? receipt.runtime.runtimeId : "unreachable"; + + expect(host.runtime.destroy(receipt)).toEqual({ status: "removed", receipt }); + expect(host.containers.has(runtimeId)).toBe(false); + expect(host.runtime.destroy(receipt)).toEqual({ status: "already-absent", receipt }); + expect(host.capture.mock.calls.map(([args]) => args)).toContainEqual([ + "rm", + "--force", + runtimeId, + ]); + }); + it("removes only a newly created container when exact post-start inspection fails", () => { const host = runtimeHarness(); host.setInspectImageOverride(`nvcr.io/nvidia/other@sha256:${"d".repeat(64)}`); diff --git a/src/lib/onboard/runtime-provider/podman-host-local-inference.ts b/src/lib/onboard/runtime-provider/podman-host-local-inference.ts index 078f2a355ac..b7db608e549 100644 --- a/src/lib/onboard/runtime-provider/podman-host-local-inference.ts +++ b/src/lib/onboard/runtime-provider/podman-host-local-inference.ts @@ -390,6 +390,16 @@ function inspectContainer(engine: ContainerEngine, runtimeId: string): ManagedCo }); } +function exactContainerExists(engine: ContainerEngine, runtimeId: string): boolean { + const result = engine.capture(["container", "exists", runtimeId], PROBE_TIMEOUT_MS); + if (result.error) { + throw new Error(`Podman inference container existence check failed: ${commandDetail(result)}`); + } + if (result.status === 0) return true; + if (result.status === 1) return false; + throw new Error(`Podman inference container existence check failed: ${commandDetail(result)}`); +} + function requireManagedIdentity( container: ManagedContainer, expected: { @@ -776,5 +786,37 @@ export function createPodmanHostLocalInferenceRuntime( } return normalized; }, + prepareDestroy(receipt: HostLocalInferenceReceipt) { + const normalized = authorizeReceipt(receipt); + if ( + normalized.runtime.kind === "container" && + exactContainerExists(engine, normalized.runtime.runtimeId) + ) { + inspectReceipt(normalized); + } + return normalized; + }, + destroy(receipt: HostLocalInferenceReceipt) { + const normalized = authorizeReceipt(receipt); + if (normalized.runtime.kind === "host") { + return Object.freeze({ + status: "retained" as const, + reason: "host-process" as const, + receipt: normalized, + }); + } + if (!exactContainerExists(engine, normalized.runtime.runtimeId)) { + return Object.freeze({ status: "already-absent" as const, receipt: normalized }); + } + const inspected = inspectReceipt(normalized); + requireSuccess( + "container removal", + engine.capture(["rm", "--force", inspected.container.runtimeId], MUTATION_TIMEOUT_MS), + ); + if (exactContainerExists(engine, inspected.container.runtimeId)) { + throw new Error("Podman inference removal left the exact managed container present."); + } + return Object.freeze({ status: "removed" as const, receipt: inspected.receipt }); + }, }); } diff --git a/src/lib/onboard/runtime-provider/podman.test.ts b/src/lib/onboard/runtime-provider/podman.test.ts index 50f34b286a1..33b93e09462 100644 --- a/src/lib/onboard/runtime-provider/podman.test.ts +++ b/src/lib/onboard/runtime-provider/podman.test.ts @@ -143,6 +143,8 @@ function inferenceRuntime(authorityId = AUTHORITY_ID): HostLocalInferenceRuntime inspectManaged: unavailable, stopManaged: unavailable, preserveForRebuild: unavailable, + prepareDestroy: unavailable, + destroy: unavailable, }; } diff --git a/src/lib/onboard/runtime-provider/registry.ts b/src/lib/onboard/runtime-provider/registry.ts index bf8bd2cc086..a651720a787 100644 --- a/src/lib/onboard/runtime-provider/registry.ts +++ b/src/lib/onboard/runtime-provider/registry.ts @@ -377,6 +377,8 @@ function validateHostLocalInferenceSurface( "inspectManaged", "stopManaged", "preserveForRebuild", + "prepareDestroy", + "destroy", ] as const) { requireFunction(runtime, operation, "hostLocalInference runtime"); } diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index 01d81d18d88..47f5b7e8f6f 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -109,6 +109,8 @@ function mxcInferenceRuntime(): HostLocalInferenceRuntime { inspectManaged: unavailable, stopManaged: unavailable, preserveForRebuild: unavailable, + prepareDestroy: unavailable, + destroy: unavailable, }; } @@ -947,6 +949,8 @@ describe("socket-free MXC action contract", () => { executeSandboxDestroy({ cleanupShieldsArtifacts, force: false, + getSandbox, + listSandboxes: () => ({ sandboxes: [entry] }), runOpenshell, sandbox: entry, sandboxConfirmedAbsent: false, diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index 599340f3c64..fd397adb1e0 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -63,6 +63,7 @@ export { withLock, } from "./registry/lock"; export { load, REGISTRY_FILE, save } from "./registry/persistence"; +export { cloneSandboxHostLocalInferenceReceipt } from "./registry/host-local-inference"; export type { BaselineExclusionEntry, BaselineExclusionTransition, diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 05fed8025a5..1ced8e3427e 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -88,6 +88,8 @@ export const OPENCLAW_IMAGE_PLUGIN_PROVENANCE_RESTORE_ERROR = "custom-image OpenClaw plugin provenance is missing or invalid"; export const MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR = "managed snapshot restore requires exact content and runtime authority"; +export const HOST_LOCAL_INFERENCE_SNAPSHOT_RESTORE_AUTHORITY_ERROR = + "host-local inference snapshot restore requires exact content and runtime authority"; function parseJson(text: string): T { return JSON.parse(text); @@ -140,6 +142,8 @@ export interface RebuildManifest { * snapshot. Older and explicit Dockerfile snapshots omit this field. */ workload?: SandboxWorkloadReceipt; + /** Exact provider-neutral authority for out-of-sandbox inference. */ + hostLocalInferenceReceipt?: string; instances?: InstanceBackup[]; // Optional user-provided label for `snapshot restore `. name?: string; @@ -154,6 +158,7 @@ export interface BackupOptions { name?: string | null; runtimeSnapshot?: SandboxRuntimeSnapshot; workload?: SandboxWorkloadReceipt; + hostLocalInferenceReceipt?: string; /** * Internal publication fence for provider-backed backups. The callback * runs after data capture and sanitization but before the manifest becomes @@ -329,6 +334,9 @@ function isRebuildManifest(value: unknown): value is RebuildManifest { : cloneSandboxRuntimeSnapshot(value.runtimeSnapshot); const workload = value.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(value.workload as never); + const hostLocalInferenceReceipt = registry.cloneSandboxHostLocalInferenceReceipt( + value.hostLocalInferenceReceipt as string | null | undefined, + ); return ( typeof value.version === "number" && typeof value.sandboxName === "string" && @@ -359,6 +367,8 @@ function isRebuildManifest(value: unknown): value is RebuildManifest { validatePreservedEnvFiles(value.preservedEnv, HERMES_PRESERVED_ENV_INVENTORY))) && (value.runtimeSnapshot === undefined || runtimeSnapshot !== undefined) && (value.workload === undefined || workload !== undefined) && + (value.hostLocalInferenceReceipt === undefined || + (typeof hostLocalInferenceReceipt === "string" && hostLocalInferenceReceipt.length > 0)) && (workload?.kind !== "managed-image" || runtimeSnapshot !== undefined) && (value.instances === undefined || (Array.isArray(value.instances) && @@ -1032,6 +1042,7 @@ export { isSshTransportFailure }; function normalizeSnapshotBackupAuthority(options: BackupOptions): { readonly runtimeSnapshot?: SandboxRuntimeSnapshot; readonly workload?: SandboxWorkloadReceipt; + readonly hostLocalInferenceReceipt?: string; readonly error?: string; } { const runtimeSnapshot = @@ -1040,18 +1051,28 @@ function normalizeSnapshotBackupAuthority(options: BackupOptions): { : cloneSandboxRuntimeSnapshot(options.runtimeSnapshot); const workload = options.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(options.workload); + const hostLocalInferenceReceipt = registry.cloneSandboxHostLocalInferenceReceipt( + options.hostLocalInferenceReceipt, + ); if (options.runtimeSnapshot !== undefined && runtimeSnapshot === undefined) { return { error: "snapshot runtime state is invalid or cannot be represented" }; } if (options.workload !== undefined && workload === undefined) { return { error: "snapshot workload authority is invalid" }; } + if ( + options.hostLocalInferenceReceipt !== undefined && + typeof hostLocalInferenceReceipt !== "string" + ) { + return { error: "snapshot host-local inference authority is invalid" }; + } if (workload?.kind === "managed-image" && runtimeSnapshot === undefined) { return { error: "managed snapshot is missing provider runtime state" }; } return { ...(runtimeSnapshot === undefined ? {} : { runtimeSnapshot }), ...(workload === undefined ? {} : { workload }), + ...(typeof hostLocalInferenceReceipt === "string" ? { hostLocalInferenceReceipt } : {}), }; } @@ -1879,11 +1900,13 @@ function restoreSandboxStateInternal( error, }; }; - if ( - manifest.workload?.kind === "managed-image" && - (!options.authority || !options.validateBeforeMutation) - ) { - return failRestoreContract(MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR); + if (!options.authority || !options.validateBeforeMutation) { + if (manifest.workload?.kind === "managed-image") { + return failRestoreContract(MANAGED_SNAPSHOT_RESTORE_AUTHORITY_ERROR); + } + if (typeof manifest.hostLocalInferenceReceipt === "string") { + return failRestoreContract(HOST_LOCAL_INFERENCE_SNAPSHOT_RESTORE_AUTHORITY_ERROR); + } } if (options.targetAgentType !== manifest.agentType) { return failRestoreContract( @@ -2302,6 +2325,9 @@ function readManifest(backupPath: string): RebuildManifest | null { : cloneSandboxRuntimeSnapshot(manifest.runtimeSnapshot); const workload = manifest.workload === undefined ? undefined : cloneSandboxWorkloadReceipt(manifest.workload); + const hostLocalInferenceReceipt = registry.cloneSandboxHostLocalInferenceReceipt( + manifest.hostLocalInferenceReceipt, + ); return { ...manifest, dir, @@ -2311,6 +2337,7 @@ function readManifest(backupPath: string): RebuildManifest | null { blueprintDigest: manifest.blueprintDigest ?? null, ...(runtimeSnapshot === undefined ? {} : { runtimeSnapshot }), ...(workload === undefined ? {} : { workload }), + ...(typeof hostLocalInferenceReceipt === "string" ? { hostLocalInferenceReceipt } : {}), }; } catch { return null; diff --git a/test/helpers/podman-host-local-inference-test-harness.ts b/test/helpers/podman-host-local-inference-test-harness.ts index 7798df86f89..7158991126f 100644 --- a/test/helpers/podman-host-local-inference-test-harness.ts +++ b/test/helpers/podman-host-local-inference-test-harness.ts @@ -152,7 +152,18 @@ function engineHarness(authorityId = AUTHORITY_ID) { }; } case "container": { - assert.equal(args[1], "inspect", "unexpected container operation"); + switch (args[1]) { + case "exists": + return { + status: containers.has(String(args[2])) ? 0 : 1, + stdout: "", + stderr: "", + }; + case "inspect": + break; + default: + assert.fail("unexpected container operation"); + } const container = requireContainer(containers, String(args[2])); return { status: 0, diff --git a/test/onboard-host-local-inference-routing.test.ts b/test/onboard-host-local-inference-routing.test.ts index ddf50eaca41..7d1316a7f1b 100644 --- a/test/onboard-host-local-inference-routing.test.ts +++ b/test/onboard-host-local-inference-routing.test.ts @@ -61,6 +61,8 @@ function runtime(): HostLocalInferenceRuntime { inspectManaged: vi.fn((receipt) => ({ running: true, receipt })), stopManaged: vi.fn((receipt) => ({ running: false, receipt })), preserveForRebuild: vi.fn((receipt) => receipt), + prepareDestroy: vi.fn((receipt) => receipt), + destroy: vi.fn((receipt) => ({ status: "removed" as const, receipt })), }; } diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 19b093bbf42..f03e3af3406 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -170,6 +170,7 @@ describe("runtime provider central source boundary", () => { "src/lib/onboard/runtime-provider/contract.ts", "src/lib/onboard/runtime-provider/current.ts", "src/lib/onboard/runtime-provider/docker.ts", + "src/lib/onboard/runtime-provider/host-local-inference-lifecycle.ts", "src/lib/onboard/runtime-provider/host-local-inference-routing.ts", "src/lib/onboard/runtime-provider/host-local-inference.ts", "src/lib/onboard/runtime-provider/mxc.ts", From e09c1f22fd44e66e0d901eb9cb38af4b460ad847 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 04:31:21 -0700 Subject: [PATCH 09/10] docs(runtime): state cleanup authority contract Signed-off-by: Aaron Erickson --- src/lib/onboard/runtime-provider/host-local-inference.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/runtime-provider/host-local-inference.ts b/src/lib/onboard/runtime-provider/host-local-inference.ts index e3ec455e4fc..94d94dd9e2e 100644 --- a/src/lib/onboard/runtime-provider/host-local-inference.ts +++ b/src/lib/onboard/runtime-provider/host-local-inference.ts @@ -135,9 +135,10 @@ export interface HostLocalInferenceRuntime { prepareDestroy(receipt: HostLocalInferenceReceipt): HostLocalInferenceReceipt; /** * Retire only the exact provider-owned runtime; host processes remain - * externally owned. Managed cleanup must converge safely when repeated so a - * retained ownership journal can resume teardown after a process crash or - * provider failure. + * externally owned. Managed cleanup must remain idempotent across retries and + * revalidate exact runtime authority before each deletion so a retained + * ownership journal can resume teardown after a process crash or provider + * failure. */ destroy(receipt: HostLocalInferenceReceipt): HostLocalInferenceDestroyResult; } From 4fa744cffdbab8b3691df3a770dff5d09fd07333 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 04:47:06 -0700 Subject: [PATCH 10/10] fix(runtime): complete destroy failure contract Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/destroy-execution.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/actions/sandbox/destroy-execution.ts b/src/lib/actions/sandbox/destroy-execution.ts index 223c6583f81..dfe716c0ba2 100644 --- a/src/lib/actions/sandbox/destroy-execution.ts +++ b/src/lib/actions/sandbox/destroy-execution.ts @@ -371,6 +371,7 @@ export async function executeSandboxDestroy({ exitCode: 1, gatewayUnreachable: false, mcpOwnershipRequiresGateway: false, + shieldsRelockRequiresGateway: false, hostLocalInferenceCleanupFailure: redactDestroyError(error), deleteConfirmed: true, };