feat(runtime): complete host-local inference lifecycle

src/lib/actions/sandbox/destroy-execution.ts

@@ -9,6 +9,11 @@ import {
   type RuntimeProviderBundleRegistry,
   requireRuntimeProviderDestructiveCleanupAuthority,
 } from "../../onboard/runtime-provider/access";
+import {
+  type PreparedHostLocalInferenceAuthority,
+  prepareSandboxHostLocalInferenceDestroyAuthority,
+  retirePreparedHostLocalInferenceAuthority,
+} from "../../onboard/runtime-provider/host-local-inference-lifecycle";
 import {
   type DetachSandboxProvidersResult,
   runSandboxProviderPreDeleteCleanup,
@@ -34,6 +39,8 @@ export function redactDestroyError(error: unknown): string {
 type SandboxDestroyExecutionInput = {
   cleanupShieldsArtifacts: (sandboxName: string) => void;
   force: boolean;
+  getSandbox: (sandboxName: string) => SandboxEntry | null;
+  listSandboxes: () => { sandboxes: SandboxEntry[] };
   runOpenshell: DestroyRunOpenshell;
   sandbox: SandboxEntry | null;
   sandboxConfirmedAbsent: boolean;
@@ -61,6 +68,8 @@ export type SandboxDestroyExecutionResult =
       gatewayUnreachable: boolean;
       mcpOwnershipRequiresGateway: boolean;
       mcpRecoveryFailure?: string;
+      hostLocalInferenceCleanupFailure?: string;
+      deleteConfirmed?: boolean;
     };
 
 type HardenedDeleteState = {
@@ -191,6 +200,8 @@ async function finalizeMcpDestroy(
 export async function executeSandboxDestroy({
   cleanupShieldsArtifacts,
   force,
+  getSandbox,
+  listSandboxes,
   runOpenshell,
   sandbox,
   sandboxConfirmedAbsent,
@@ -200,13 +211,18 @@ export async function executeSandboxDestroy({
 }: SandboxDestroyExecutionInput): Promise<SandboxDestroyExecutionResult> {
   return withTimerBoundShieldsMutationLockAsync(sandboxName, "destroy sandbox", async () => {
     let runtimeProvider: RuntimeProviderBundle | null = null;
+    let hostLocalInferenceAuthority: PreparedHostLocalInferenceAuthority | null = null;
     if (sandbox) {
       try {
         runtimeProvider = requireRuntimeProviderDestructiveCleanupAuthority(
           sandboxName,
           sandbox,
           runtimeProviders,
         ).provider;
+        hostLocalInferenceAuthority = prepareSandboxHostLocalInferenceDestroyAuthority(
+          runtimeProvider,
+          sandbox,
+        );
       } catch (error) {
         return {
           ok: false as const,
@@ -268,6 +284,35 @@ export async function executeSandboxDestroy({
     if (!forcedLocalCleanup) {
       await finalizeMcpDestroy(sandboxName, mcpPreparation, force);
     }
+    if (!forcedLocalCleanup && runtimeProvider && sandbox && hostLocalInferenceAuthority) {
+      // Keep retirement after confirmed sandbox deletion: retiring first could
+      // leave a still-live sandbox without inference when its delete fails.
+      // The registry row is the durable cleanup journal. A retirement failure
+      // returns before that row is removed, and a retry takes the already-gone
+      // path to converge the provider's idempotent exact-runtime teardown.
+      try {
+        const current = getSandbox(sandboxName);
+        if (!current) {
+          throw new Error(`sandbox '${sandboxName}' is no longer registered`);
+        }
+        retirePreparedHostLocalInferenceAuthority(
+          runtimeProvider,
+          current,
+          hostLocalInferenceAuthority,
+          listSandboxes().sandboxes,
+        );
+      } catch (error) {
+        return {
+          ok: false as const,
+          deleteOutput,
+          exitCode: 1,
+          gatewayUnreachable: false,
+          mcpOwnershipRequiresGateway: false,
+          hostLocalInferenceCleanupFailure: redactDestroyError(error),
+          deleteConfirmed: true,
+        };
+      }
+    }
     return {
       ok: true as const,
       detachOutcome,

src/lib/actions/sandbox/destroy-host-local-inference.test.ts

@@ -0,0 +1,175 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it, vi } from "vitest";
+import { createInMemoryRuntimeProviderBundle } from "../../../../test/helpers/runtime-provider-bundle";
+import {
+  type HostLocalInferenceReceipt,
+  serializeHostLocalInferenceReceipt,
+} from "../../onboard/runtime-provider/host-local-inference";
+import type { SandboxEntry } from "../../state/registry";
+import { executeSandboxDestroy } from "./destroy-execution";
+
+function receipt(): HostLocalInferenceReceipt {
+  return {
+    schemaVersion: 1,
+    providerId: "mxc",
+    service: "vllm",
+    engineAuthority: {
+      schemaVersion: 1,
+      providerId: "mxc",
+      operation: "host-local-inference",
+      engineId: "mxc",
+      authorityId: "mxc:host-local",
+      bindingSha256: "a".repeat(64),
+    },
+    endpoint: { host: "mxc.internal", port: 8000, networkName: "mxc-network" },
+    runtime: {
+      kind: "container",
+      runtimeId: "mxc-vllm",
+      name: "nemoclaw-vllm",
+      imageRef: `nvcr.io/nvidia/vllm@sha256:${"b".repeat(64)}`,
+      probeImageRef: `quay.io/curl/curl@sha256:${"d".repeat(64)}`,
+      specSha256: "c".repeat(64),
+      gpu: { vendor: "nvidia", devices: ["nvidia.com/gpu=all"] },
+    },
+  };
+}
+
+function sandbox(name = "alpha"): SandboxEntry {
+  return {
+    name,
+    agent: "openclaw",
+    openshellDriver: "mxc",
+    hostLocalInferenceReceipt: serializeHostLocalInferenceReceipt(receipt()),
+  };
+}
+
+function destroySuccessfully(value: HostLocalInferenceReceipt) {
+  return { status: "removed" as const, receipt: value };
+}
+
+function failDestroy(message: string) {
+  return (_value: HostLocalInferenceReceipt): never => {
+    throw new Error(message);
+  };
+}
+
+function provider(
+  destroyRuntime: (value: HostLocalInferenceReceipt) => {
+    status: "removed";
+    receipt: HostLocalInferenceReceipt;
+  } = destroySuccessfully,
+) {
+  const preserveForRebuild = vi.fn((value: HostLocalInferenceReceipt) => value);
+  const prepareDestroy = vi.fn((value: HostLocalInferenceReceipt) => value);
+  const destroy = vi.fn(destroyRuntime);
+  const bundle = createInMemoryRuntimeProviderBundle({
+    providerId: "mxc",
+    workloadProfile: {
+      support: null,
+      hostArchitectures: ["amd64"],
+      managedImageSelectionPolicy: "prefer-managed",
+      legacyDockerfileBuilds: false,
+    },
+    hostLocalInferenceRuntime: {
+      providerId: "mxc",
+      authorityId: "mxc:host-local",
+      services: ["ollama", "nim", "vllm"],
+      translateContainerArgs: (args: readonly string[]) => args,
+      qualifyOllama: vi.fn(),
+      startManaged: vi.fn(),
+      inspectManaged: vi.fn(),
+      stopManaged: vi.fn(),
+      preserveForRebuild,
+      prepareDestroy,
+      destroy,
+    },
+  });
+  return { bundle, destroy, prepareDestroy, preserveForRebuild };
+}
+
+async function runDestroy(
+  runtimeProvider: ReturnType<typeof provider>,
+  peers: SandboxEntry[],
+  deleteResult: { status: number; stdout: string; stderr: string } = {
+    status: 0,
+    stdout: "",
+    stderr: "",
+  },
+  sandboxConfirmedAbsent = false,
+) {
+  const entry = sandbox();
+  const events: string[] = [];
+  const result = await executeSandboxDestroy({
+    cleanupShieldsArtifacts: () => events.push("cleanup"),
+    force: false,
+    getSandbox: () => entry,
+    listSandboxes: () => ({ sandboxes: [entry, ...peers] }),
+    runOpenshell: (args) => {
+      events.push(args.join(" "));
+      return deleteResult;
+    },
+    sandbox: entry,
+    sandboxConfirmedAbsent,
+    sandboxName: "alpha",
+    runtimeProviders: { mxc: runtimeProvider.bundle },
+    deps: {
+      readTimerMarker: () => null,
+      wipeSandboxState: () => undefined,
+    },
+  });
+  return { events, result };
+}
+
+describe("sandbox destroy host-local inference transaction", () => {
+  it("deletes the sandbox before retiring the exact unshared runtime", async () => {
+    const runtimeProvider = provider();
+    const { events, result } = await runDestroy(runtimeProvider, []);
+
+    expect(result).toMatchObject({ ok: true });
+    expect(events.slice(-2)).toEqual(["sandbox delete alpha", "cleanup"]);
+    expect(runtimeProvider.prepareDestroy).toHaveBeenCalledTimes(2);
+    expect(runtimeProvider.destroy).toHaveBeenCalledOnce();
+  });
+
+  it("keeps a runtime referenced by another sandbox", async () => {
+    const runtimeProvider = provider();
+    const { result } = await runDestroy(runtimeProvider, [sandbox("beta")]);
+
+    expect(result).toMatchObject({ ok: true });
+    expect(runtimeProvider.destroy).not.toHaveBeenCalled();
+  });
+
+  it("preserves local ownership when exact runtime retirement fails", async () => {
+    const runtimeProvider = provider(failDestroy("injected runtime removal failure"));
+    const { events, result } = await runDestroy(runtimeProvider, []);
+
+    expect(result).toMatchObject({
+      ok: false,
+      deleteConfirmed: true,
+      hostLocalInferenceCleanupFailure: "injected runtime removal failure",
+    });
+    expect(events.slice(-2)).toEqual(["sandbox delete alpha", "cleanup"]);
+  });
+
+  it("reconciles retained ownership when destroy is retried after confirmed deletion", async () => {
+    const destroyRuntime = vi
+      .fn(destroySuccessfully)
+      .mockImplementationOnce(failDestroy("injected runtime removal failure"));
+    const runtimeProvider = provider(destroyRuntime);
+
+    const first = await runDestroy(runtimeProvider, []);
+    const retry = await runDestroy(
+      runtimeProvider,
+      [],
+      { status: 1, stdout: "", stderr: "Error: sandbox alpha not found" },
+      true,
+    );
+
+    expect(first.result).toMatchObject({ ok: false, deleteConfirmed: true });
+    expect(retry.result).toMatchObject({ ok: true, alreadyGone: true });
+    expect(retry.events.slice(-2)).toEqual(["sandbox delete alpha", "cleanup"]);
+    expect(runtimeProvider.destroy).toHaveBeenCalledTimes(2);
+  });
+});

src/lib/actions/sandbox/destroy.ts

@@ -416,12 +416,22 @@ async function destroySandboxUnlocked(
   const destructiveResult = await executeSandboxDestroy({
     cleanupShieldsArtifacts: cleanupShieldsDestroyArtifacts,
     force: normalized.force === true,
+    getSandbox: registry.getSandbox,
+    listSandboxes: registry.listSandboxes,
     runOpenshell,
     sandbox,
     sandboxConfirmedAbsent,
     sandboxName,
   });
   if (!destructiveResult.ok) {
+    if (destructiveResult.hostLocalInferenceCleanupFailure) {
+      console.error(
+        `  Sandbox '${sandboxName}' is gone, but its exact host-local inference cleanup failed: ${destructiveResult.hostLocalInferenceCleanupFailure}`,
+      );
+      console.error(
+        `  Local ownership state was preserved. Re-run '${CLI_NAME} ${sandboxName} destroy --yes' to reconcile only the recorded provider runtime.`,
+      );
+    }
     if (destructiveResult.deleteOutput) {
       console.error(`  ${destructiveResult.deleteOutput}`);
     }