From 06cdc5bd65c6022aca20f9acd97a442e419a446c Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 20 Aug 2026 16:13:56 +0200 Subject: [PATCH 1/4] feat(cli): spawn agents in Cloud sandboxes --- CHANGELOG.md | 1 + packages/cli/README.md | 21 +- packages/cli/src/cli/commands/fleet.test.ts | 182 +++++++++++++++ packages/cli/src/cli/commands/fleet.ts | 206 ++++++++++++++--- packages/cloud/src/fleet-sandbox.test.ts | 160 +++++++++++++ packages/cloud/src/fleet-sandbox.ts | 241 ++++++++++++++++++++ packages/cloud/src/index.ts | 10 + 7 files changed, 790 insertions(+), 31 deletions(-) create mode 100644 packages/cloud/src/fleet-sandbox.test.ts create mode 100644 packages/cloud/src/fleet-sandbox.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fd962366..bfdeceac7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `agent-relay fleet spawn --sandbox` now provisions a fresh Cloud Daytona node, requires the current Relayfile workspace to mount, waits for node readiness, and spawns the agent in `/workspace` in one command. Workspace-only shells automatically use and remove a short-lived launcher identity, so `--token` is no longer required for this path. - `agent-relay observer` mints a scoped, read-only observer token and prints the observer URL built from it, so sharing a live follow-along view no longer requires hand-rolling a `POST /v1/observer-tokens` call. Defaults to a 24-hour token with agent DMs excluded; `--channels`, `--include-dms`, and `--expires` widen it, and `observer list` / `observer revoke ` manage existing tokens. - `get_observer_url` MCP tool does the same for an orchestrating agent, so a lead can hand the user a follow-along link without shelling out. - `@agent-relay/sdk` exports `createObserverToken`, `listObserverTokens`, and `revokeObserverToken`. diff --git a/packages/cli/README.md b/packages/cli/README.md index 98b463b69..e69c1ec18 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -147,6 +147,13 @@ agent-relay fleet spawn codex \ # Omit --node for automatic eligible-node placement. agent-relay fleet spawn codex --name api-worker --task "Review the current diff." +# Provision a fresh Daytona node, require the current Relayfile workspace to +# mount at /workspace, wait for readiness, then spawn Codex there. +agent-relay fleet spawn codex \ + --sandbox \ + --name daytona-worker \ + --task "Review the current workspace and wait for follow-up." + agent-relay message dm send api-worker "Detailed task instructions" # wait is the default: it queues for the recipient's next safe idle boundary and # can remain unread while that recipient is busy. steer requests immediate @@ -157,11 +164,19 @@ agent-relay message inbox check --limit 20 agent-relay fleet release api-worker --reason "Work accepted" ``` -Commands use the workspace session pinned to the current project. Targeted +Commands use the workspace session pinned to the current project. Exact-node spawn and messaging operations also need an agent identity: pass `--token` or set `RELAY_AGENT_TOKEN` to the token returned by -`agent-relay agent register `. Automatic placement and release need -only the workspace key. +`agent-relay agent register `. `fleet spawn --sandbox` needs a Cloud +login (`agent-relay cloud login`) but does not need an agent token: when one is +absent, it creates and removes a short-lived launcher identity automatically. +Automatic placement and release need only the workspace key. + +The sandbox path provisions a fresh Daytona instance and makes the Relayfile +mount mandatory by default, so the spawned worker starts in `/workspace` and +sees the same synced Relayfile workspace. Pass `--no-sandbox-relayfile` only +when a deliberately bare sandbox is desired. If provisioning times out or the +spawn fails, Relay asks Cloud to delete the newly created sandbox. `--session-ref` is a real CLI resume, not a logical collaboration label. Pass the actual Claude session ID or Codex thread ID and target its origin node. diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index ae9f08a31..3dbbeb5c5 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -560,6 +560,188 @@ describe('fleet command support', () => { }); }); + it('fleet spawn --sandbox provisions Daytona, mounts Relayfile, and uses a temporary launcher', async () => { + const previousToken = process.env.RELAY_AGENT_TOKEN; + delete process.env.RELAY_AGENT_TOKEN; + const placement = { + spawn: vi.fn(async () => ({ + invocationId: 'inv_sandbox', + node: { name: 'daytona-codex' }, + })), + }; + const register = vi.fn(async () => ({ token: 'at_live_launcher' })); + const release = vi.fn(async () => ({ released: true, deleted: true })); + const createWorkspaceRelay = vi.fn(() => ({ + workspace: { + info: vi.fn(async () => ({ id: 'rw_abc' })), + register, + release, + }, + })); + const createAgentRelay = vi.fn(() => ({ messaging: { placement } })); + const ensureCloudFleetSandbox = vi.fn(async () => ({ + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-1', + nodeName: 'daytona-codex', + sandboxId: 'sandbox-1', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + relayfileMountPath: '/workspace', + })); + const deleteCloudFleetSandbox = vi.fn(async () => undefined); + const logs: string[] = []; + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: createAgentRelay as never, + createWorkspaceRelay: createWorkspaceRelay as never, + createWorkspace: vi.fn() as never, + log: (message: unknown) => logs.push(String(message)), + error: vi.fn(), + exit: vi.fn() as never, + }, + ensureCloudFleetSandbox, + deleteCloudFleetSandbox, + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + try { + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-name', + 'daytona-codex', + '--name', + 'sandbox-worker', + '--task', + 'Wait for VERIFY', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); + } finally { + if (previousToken === undefined) delete process.env.RELAY_AGENT_TOKEN; + else process.env.RELAY_AGENT_TOKEN = previousToken; + } + + expect(ensureCloudFleetSandbox).toHaveBeenCalledWith({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + maxAgents: 1, + mountRelayfile: true, + forceProvision: true, + waitTimeoutMs: 90_000, + name: 'daytona-codex', + }); + expect(register).toHaveBeenCalledWith( + expect.objectContaining({ + name: expect.stringMatching(/^fleet-sandbox-launcher-[a-f0-9]{8}$/), + metadata: { purpose: 'fleet-sandbox-launcher' }, + }), + { strict: true } + ); + expect(createAgentRelay).toHaveBeenCalledWith({ + workspaceKey: 'rk_live_test', + token: 'at_live_launcher', + baseUrl: undefined, + }); + expect(placement.spawn).toHaveBeenCalledWith( + expect.objectContaining({ + capability: 'spawn:codex', + node: 'daytona-codex', + confirm: true, + input: expect.objectContaining({ + name: 'sandbox-worker', + worker_cwd: '/workspace', + }), + }) + ); + expect(release).toHaveBeenCalledWith( + expect.objectContaining({ + name: expect.stringMatching(/^fleet-sandbox-launcher-/), + deleteAgent: true, + }) + ); + expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); + expect(JSON.parse(logs[0]!)).toMatchObject({ + sandbox: { nodeName: 'daytona-codex', relayfileMountPath: '/workspace' }, + invocation: { invocationId: 'inv_sandbox' }, + attachCommand: "agent-relay node agent attach 'sandbox-worker' --node 'daytona-codex' --mode drive", + }); + }); + + it('fleet spawn --sandbox deletes a freshly provisioned sandbox when dispatch fails', async () => { + const placement = { spawn: vi.fn(async () => Promise.reject(new Error('dispatch failed'))) }; + const deleteCloudFleetSandbox = vi.fn(async () => undefined); + const errors: string[] = []; + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn(() => ({ messaging: { placement } })) as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: (...args: unknown[]) => errors.push(args.join(' ')), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox: vi.fn(async () => ({ + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-1', + nodeName: 'daytona-codex', + sandboxId: 'sandbox-1', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + relayfileMountPath: '/workspace', + })), + deleteCloudFleetSandbox, + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + '--token', + 'at_live_lead', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + + expect(errors.join('\n')).toContain('dispatch failed'); + expect(deleteCloudFleetSandbox).toHaveBeenCalledWith({ + cloudWorkspaceId: 'cloud-workspace', + sandboxId: 'sandbox-1', + }); + }); + it('fleet spawn --no-confirm accepts an unconfirmed targeted dispatch', async () => { const placement = { spawn: vi.fn(async () => ({ diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 6c1f5c305..fa1a4b0d8 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -1,4 +1,11 @@ +import { randomUUID } from 'node:crypto'; + import { InvalidArgumentError, type Command } from 'commander'; +import { + deleteCloudFleetSandbox, + ensureCloudFleetSandbox, + type EnsureCloudFleetSandboxResult, +} from '@agent-relay/cloud'; import { HarnessDriverClient } from '@agent-relay/harness-driver'; import { createWorkspaceClient, type RelayWorkspaceThinClient, type RelayNode } from '@agent-relay/sdk'; @@ -41,6 +48,8 @@ export interface FleetCommandDependencies { core: CoreDependencies; sdk: SdkCommandDeps; createFleetWorkspaceClient: (options: SdkClientOptions) => RelayWorkspaceThinClient; + ensureCloudFleetSandbox: typeof ensureCloudFleetSandbox; + deleteCloudFleetSandbox: typeof deleteCloudFleetSandbox; log: (...args: unknown[]) => void; warn: (...args: unknown[]) => void; error: (...args: unknown[]) => void; @@ -58,6 +67,8 @@ function withFleetDefaults(overrides: Partial = {}): F workspaceKey: resolveWorkspaceKey(options), baseUrl: resolveBaseUrl(options), }), + ensureCloudFleetSandbox, + deleteCloudFleetSandbox, log: (...args: unknown[]) => console.log(...args), warn: (...args: unknown[]) => console.warn(...args), error: (...args: unknown[]) => console.error(...args), @@ -142,6 +153,12 @@ export function registerFleetCommands( .requiredOption('--task ', 'Initial task instructions') .option('--node ', 'Target a specific fleet node') .option('--target-node ', 'Alias for --node') + .option( + '--sandbox', + 'Provision a fresh Cloud Daytona node, mount this Relayfile workspace, and spawn there' + ) + .option('--sandbox-name ', 'Name for the provisioned Daytona fleet node') + .option('--no-sandbox-relayfile', 'Provision the sandbox without mounting Relayfile') .option('--channel ', 'Channel for the worker to join') .option('--persona ', 'Worker persona (automatic placement)') .option('--model ', 'Model powering the worker') @@ -167,11 +184,22 @@ export function registerFleetCommands( const clientOptions = sdkOptionsFromOpts(options); const name = requiredText(options.name, 'Worker name'); const task = requiredText(options.task, 'Task'); - const targetNode = - optionalText(options.targetNode, 'Target node') ?? optionalText(options.node, 'Node'); + let targetNode = optionalText(options.targetNode, 'Target node') ?? optionalText(options.node, 'Node'); + const useSandbox = options.sandbox === true; + const sandboxName = optionalText(options.sandboxName, 'Sandbox name'); + const mountSandboxRelayfile = options.sandboxRelayfile !== false; + if (useSandbox && targetNode) { + throw new Error('--sandbox cannot be combined with --node or --target-node.'); + } + if (!useSandbox && sandboxName) { + throw new Error('--sandbox-name requires --sandbox.'); + } + if (!useSandbox && options.sandboxRelayfile === false) { + throw new Error('--no-sandbox-relayfile requires --sandbox.'); + } const channel = optionalText(options.channel, 'Channel'); const model = optionalText(options.model, 'Model'); - const workerCwd = optionalText(options.cwd, 'Worker cwd'); + let workerCwd = optionalText(options.cwd, 'Worker cwd'); const organization = optionalText(options.organization, 'Organization'); const project = optionalText(options.project, 'Project'); const workstream = optionalText(options.workstream, 'Workstream'); @@ -188,36 +216,153 @@ export function registerFleetCommands( throw new Error('--confirm-timeout must be a positive number of milliseconds.'); } + let sandbox: EnsureCloudFleetSandboxResult | undefined; + let workspaceRelay: ReturnType | undefined; + if (useSandbox) { + workspaceRelay = deps.sdk.createWorkspaceRelay(clientOptions); + const workspaceInfo = await workspaceRelay.workspace.info(); + const relayWorkspaceId = workspaceInfo.id?.trim(); + if (!relayWorkspaceId) { + throw new Error('The current Relay workspace did not report an ID for Cloud provisioning.'); + } + sandbox = await deps.ensureCloudFleetSandbox({ + workspaceId: relayWorkspaceId, + requiredCapability: `spawn:${cli}`, + maxAgents: 1, + mountRelayfile: mountSandboxRelayfile, + forceProvision: true, + waitTimeoutMs: 90_000, + ...(sandboxName ? { name: sandboxName } : {}), + }); + if (sandbox.outcome === 'provisioning_timeout') { + await deps + .deleteCloudFleetSandbox({ + cloudWorkspaceId: sandbox.cloudWorkspaceId, + sandboxId: sandbox.sandboxId, + }) + .catch((error) => { + deps.warn( + `The timed-out sandbox could not be cleaned up automatically: ${ + error instanceof Error ? error.message : String(error) + }` + ); + }); + throw new Error( + `Daytona node '${sandbox.nodeName}' did not become ready within ${sandbox.waitedMs}ms.` + ); + } + if ( + mountSandboxRelayfile && + (sandbox.outcome !== 'provisioned' || sandbox.relayfileMounted !== true) + ) { + if (sandbox.outcome === 'provisioned') { + await deps + .deleteCloudFleetSandbox({ + cloudWorkspaceId: sandbox.cloudWorkspaceId, + sandboxId: sandbox.sandboxId, + }) + .catch(() => undefined); + } + throw new Error('Cloud returned a Daytona node without the required Relayfile mount.'); + } + targetNode = sandbox.nodeName; + if (!workerCwd && sandbox.outcome === 'provisioned' && sandbox.relayfileMounted) { + workerCwd = sandbox.relayfileMountPath ?? '/workspace'; + } + } + if (targetNode) { - if (!resolveAgentToken(clientOptions)) { + if (!useSandbox && !resolveAgentToken(clientOptions)) { throw new Error( 'Targeted Fleet spawn requires an agent token. Pass --token or set RELAY_AGENT_TOKEN.' ); } - const relay = deps.sdk.createAgentRelay(clientOptions); - // Placement alone only proves the node accepted the dispatch. A node - // running an obsolete broker advertises `spawn:` capacity, acks - // the invocation and launches nothing, which is indistinguishable from - // success here — so wait for the node to confirm unless asked not to. - const confirm = options.confirm !== false; - const invocation = await relay.messaging.placement.spawn({ - capability: `spawn:${cli}`, - node: targetNode, - failFast: true, - confirm, - ...(confirm ? { confirmTimeoutMs: confirmTimeoutMs } : {}), - input: { - name, - cli, - task, - ...(channel ? { channels: [channel] } : {}), - ...(model ? { model } : {}), - ...(workerCwd ? { worker_cwd: workerCwd } : {}), - ...registrationMetadata, - ...(sessionRef ? { session_ref: sessionRef } : {}), - }, - }); - printJson(deps.sdk, { invocation }); + let launcherName: string | undefined; + try { + let agentToken = resolveAgentToken(clientOptions); + if (!agentToken) { + workspaceRelay ??= deps.sdk.createWorkspaceRelay(clientOptions); + launcherName = `fleet-sandbox-launcher-${randomUUID().slice(0, 8)}`; + const launcher = await workspaceRelay.workspace.register( + { + name: launcherName, + metadata: { purpose: 'fleet-sandbox-launcher' }, + }, + { strict: true } + ); + agentToken = launcher.token; + if (!agentToken) { + throw new Error('The temporary fleet sandbox launcher did not receive an agent token.'); + } + } + + const relay = deps.sdk.createAgentRelay({ ...clientOptions, token: agentToken }); + // Placement alone only proves the node accepted the dispatch. A node + // running an obsolete broker advertises `spawn:` capacity, acks + // the invocation and launches nothing, which is indistinguishable from + // success here — so wait for the node to confirm unless asked not to. + const confirm = options.confirm !== false; + const invocation = await relay.messaging.placement.spawn({ + capability: `spawn:${cli}`, + node: targetNode, + failFast: true, + confirm, + ...(confirm ? { confirmTimeoutMs: confirmTimeoutMs } : {}), + input: { + name, + cli, + task, + ...(channel ? { channels: [channel] } : {}), + ...(model ? { model } : {}), + ...(workerCwd ? { worker_cwd: workerCwd } : {}), + ...registrationMetadata, + ...(sessionRef ? { session_ref: sessionRef } : {}), + }, + }); + printJson(deps.sdk, { + ...(sandbox + ? { + sandbox, + attachCommand: + `agent-relay node agent attach ${shellQuote(name)} ` + + `--node ${shellQuote(targetNode)} --mode drive`, + } + : {}), + invocation, + }); + } catch (error) { + if (sandbox?.outcome === 'provisioned') { + await deps + .deleteCloudFleetSandbox({ + cloudWorkspaceId: sandbox.cloudWorkspaceId, + sandboxId: sandbox.sandboxId, + }) + .catch((cleanupError) => { + deps.warn( + `Spawn failed and the sandbox could not be cleaned up automatically: ${ + cleanupError instanceof Error ? cleanupError.message : String(cleanupError) + }` + ); + }); + } + throw error; + } finally { + if (launcherName && workspaceRelay) { + await workspaceRelay.workspace + .release({ + name: launcherName, + reason: 'Temporary fleet sandbox launcher completed', + deleteAgent: true, + }) + .catch((error) => { + deps.warn( + `Temporary launcher cleanup failed: ${ + error instanceof Error ? error.message : String(error) + }` + ); + }); + } + } return; } @@ -355,6 +500,11 @@ function optionalText(value: unknown, label: string): string | undefined { return requiredText(value, label); } +/** Quote untrusted names in the copy-pasteable attach command printed on success. */ +function shellQuote(value: string): string { + return `'${value.replace(/'/g, `'\\''`)}'`; +} + /** * Warn (on stderr, so it never pollutes the JSON on stdout) when the workspace * key was inferred from the project's persisted session rather than named diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts new file mode 100644 index 000000000..e544ff09d --- /dev/null +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -0,0 +1,160 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + ensureCloudSession: vi.fn(), + authorizedApiFetch: vi.fn(), +})); + +vi.mock('./auth.js', () => ({ + ensureCloudSession: mocks.ensureCloudSession, + authorizedApiFetch: mocks.authorizedApiFetch, +})); + +import { deleteCloudFleetSandbox, ensureCloudFleetSandbox } from './fleet-sandbox.js'; + +const auth = { + accessToken: 'access', + refreshToken: 'refresh', + accessTokenExpiresAt: '2099-01-01T00:00:00Z', + apiUrl: 'https://agentrelay.test/cloud', +}; +const refreshedAuth = { ...auth, accessToken: 'refreshed' }; + +describe('Cloud fleet sandbox client', () => { + beforeEach(() => { + vi.resetAllMocks(); + mocks.ensureCloudSession.mockResolvedValue({ auth, client: {} }); + }); + + it('resolves the unified workspace and provisions a ready mounted sandbox', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99' }), + auth: refreshedAuth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + nodeId: 'node-1', + nodeName: 'daytona-codex', + sandboxId: 'sandbox-1', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + relayfileMountPath: '/workspace', + }, + { status: 201 } + ), + auth: refreshedAuth, + }); + + const result = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + name: 'daytona-codex', + requiredCapability: 'spawn:codex', + maxAgents: 1, + mountRelayfile: true, + forceProvision: true, + waitTimeoutMs: 90_000, + }); + + expect(mocks.authorizedApiFetch).toHaveBeenNthCalledWith( + 1, + auth, + '/api/v1/workspaces/rw_abc/resolve', + { method: 'GET' }, + { interactive: false } + ); + const ensureCall = mocks.authorizedApiFetch.mock.calls[1]; + expect(ensureCall?.[0]).toEqual(refreshedAuth); + expect(ensureCall?.[1]).toBe('/api/v1/fleet/nodes/sandbox/ensure'); + expect(JSON.parse(String(ensureCall?.[2]?.body))).toEqual({ + workspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', + name: 'daytona-codex', + requiredCapability: 'spawn:codex', + maxAgents: 1, + mountRelayfile: true, + forceProvision: true, + waitTimeoutMs: 90_000, + }); + expect(result).toEqual({ + outcome: 'provisioned', + cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', + nodeId: 'node-1', + nodeName: 'daytona-codex', + sandboxId: 'sandbox-1', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + relayfileMountPath: '/workspace', + }); + }); + + it('preserves a bounded provisioning timeout so the CLI can report it', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: 'cloud-workspace' }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioning_timeout', + sandboxId: 'sandbox-1', + relayWorkspaceId: 'rw_abc', + nodeName: 'daytona-codex', + waitedMs: 90_000, + }, + { status: 202 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + }) + ).resolves.toMatchObject({ + outcome: 'provisioning_timeout', + sandboxId: 'sandbox-1', + nodeName: 'daytona-codex', + waitedMs: 90_000, + }); + }); + + it('turns Cloud authorization failures into actionable errors', async () => { + mocks.authorizedApiFetch.mockResolvedValueOnce({ + response: Response.json({ error: 'Forbidden' }, { status: 403 }), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + }) + ).rejects.toThrow('owner or admin'); + }); + + it('deletes only the named sandbox in the resolved Cloud workspace', async () => { + mocks.authorizedApiFetch.mockResolvedValueOnce({ + response: Response.json({ sandboxId: 'sandbox-1', deleted: true }), + auth, + }); + + await deleteCloudFleetSandbox({ + cloudWorkspaceId: 'cloud-workspace', + sandboxId: 'sandbox-1', + }); + + expect(mocks.authorizedApiFetch).toHaveBeenCalledWith( + auth, + '/api/v1/fleet/nodes/sandbox/sandbox-1', + { + method: 'DELETE', + body: JSON.stringify({ workspaceId: 'cloud-workspace' }), + }, + { interactive: false } + ); + }); +}); diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts new file mode 100644 index 000000000..86d559af3 --- /dev/null +++ b/packages/cloud/src/fleet-sandbox.ts @@ -0,0 +1,241 @@ +import { authorizedApiFetch, ensureCloudSession } from './auth.js'; +import { redactCredentialValues } from './redact.js'; +import { defaultApiUrl } from './types.js'; + +type JsonRecord = Record; + +export type EnsureCloudFleetSandboxInput = { + /** Cloud UUID or unified rw_* workspace id. */ + workspaceId: string; + name?: string; + requiredCapability: string; + maxAgents?: number; + mountRelayfile?: boolean; + forceProvision?: boolean; + waitTimeoutMs?: number; +}; + +export type CloudFleetSandboxReady = { + outcome: 'provisioned'; + cloudWorkspaceId: string; + nodeId: string; + nodeName: string; + sandboxId: string; + relayWorkspaceId: string; + relayfileMounted: boolean; + relayfileMountPath?: string; +}; + +export type CloudFleetSandboxReused = { + outcome: 'reused'; + cloudWorkspaceId: string; + nodeId: string; + nodeName: string; + status: string; + activeAgents: number | null; + maxAgents: number | null; +}; + +export type CloudFleetSandboxProvisioningTimeout = { + outcome: 'provisioning_timeout'; + cloudWorkspaceId: string; + sandboxId: string; + relayWorkspaceId: string; + nodeName: string; + waitedMs: number; +}; + +export type EnsureCloudFleetSandboxResult = + | CloudFleetSandboxReady + | CloudFleetSandboxReused + | CloudFleetSandboxProvisioningTimeout; + +export type DeleteCloudFleetSandboxInput = { + cloudWorkspaceId: string; + sandboxId: string; +}; + +function isObject(value: unknown): value is JsonRecord { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function readString(payload: JsonRecord, key: string): string | undefined { + const value = payload[key]; + return typeof value === 'string' && value.trim() ? value.trim() : undefined; +} + +function readNumber(payload: JsonRecord, key: string): number | undefined { + const value = payload[key]; + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + +async function readJson(response: Response): Promise { + try { + return await response.json(); + } catch { + return null; + } +} + +function endpointError(action: string, response: Response, payload: unknown): Error { + if (response.status === 401) { + return new Error(`Cloud login required. Run \`agent-relay cloud login\` and retry ${action}.`); + } + if (response.status === 403) { + return new Error(`Cloud workspace owner or admin access is required to ${action}.`); + } + if (response.status === 429) { + const retryAfter = response.headers.get('retry-after')?.trim(); + return new Error( + `Cloud rate limit exceeded while trying to ${action}.${ + retryAfter ? ` Retry after ${retryAfter} seconds.` : '' + }` + ); + } + const detail = isObject(payload) + ? (readString(payload, 'error') ?? readString(payload, 'message') ?? response.statusText) + : response.statusText; + return new Error( + redactCredentialValues(`Failed to ${action}: ${response.status}${detail ? ` ${detail}` : ''}`) + ); +} + +function requiredString(payload: JsonRecord, key: string, context: string): string { + const value = readString(payload, key); + if (!value) throw new Error(`${context} response is missing ${key}.`); + return value; +} + +async function resolveCloudWorkspaceId( + workspaceId: string, + auth: Awaited>['auth'] +): Promise<{ + cloudWorkspaceId: string; + auth: Awaited>['auth']; +}> { + const { response, auth: activeAuth } = await authorizedApiFetch( + auth, + `/api/v1/workspaces/${encodeURIComponent(workspaceId)}/resolve`, + { method: 'GET' }, + { interactive: false } + ); + const payload = await readJson(response); + if (!response.ok) throw endpointError('resolve the Cloud workspace', response, payload); + if (!isObject(payload)) throw new Error('Cloud workspace resolver returned an invalid response.'); + return { + cloudWorkspaceId: requiredString(payload, 'cloudWorkspaceId', 'Cloud workspace resolver'), + auth: activeAuth, + }; +} + +function normalizeEnsureResult(payload: unknown, cloudWorkspaceId: string): EnsureCloudFleetSandboxResult { + if (!isObject(payload)) throw new Error('Cloud fleet sandbox response was not valid JSON.'); + const outcome = readString(payload, 'outcome'); + const nodeName = requiredString(payload, 'nodeName', 'Cloud fleet sandbox'); + + if (outcome === 'provisioned') { + if (typeof payload.relayfileMounted !== 'boolean') { + throw new Error('Cloud fleet sandbox response is missing relayfileMounted.'); + } + return { + outcome, + cloudWorkspaceId, + nodeId: requiredString(payload, 'nodeId', 'Cloud fleet sandbox'), + nodeName, + sandboxId: requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'), + relayWorkspaceId: requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'), + relayfileMounted: payload.relayfileMounted, + ...(readString(payload, 'relayfileMountPath') + ? { relayfileMountPath: readString(payload, 'relayfileMountPath') } + : {}), + }; + } + + if (outcome === 'reused') { + return { + outcome, + cloudWorkspaceId, + nodeId: requiredString(payload, 'nodeId', 'Cloud fleet sandbox'), + nodeName, + status: requiredString(payload, 'status', 'Cloud fleet sandbox'), + activeAgents: readNumber(payload, 'activeAgents') ?? null, + maxAgents: readNumber(payload, 'maxAgents') ?? null, + }; + } + + if (outcome === 'provisioning_timeout') { + return { + outcome, + cloudWorkspaceId, + sandboxId: requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'), + relayWorkspaceId: requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'), + nodeName, + waitedMs: readNumber(payload, 'waitedMs') ?? 0, + }; + } + + throw new Error('Cloud fleet sandbox response has an unknown outcome.'); +} + +/** Resolve a Relay workspace in Cloud, provision/reuse a node, and wait for readiness. */ +export async function ensureCloudFleetSandbox( + input: EnsureCloudFleetSandboxInput, + options: { apiUrl?: string } = {} +): Promise { + const workspaceId = input.workspaceId.trim(); + const requiredCapability = input.requiredCapability.trim(); + if (!workspaceId) throw new Error('A workspace ID is required to provision a fleet sandbox.'); + if (!requiredCapability) throw new Error('A spawn capability is required to provision a fleet sandbox.'); + + const session = await ensureCloudSession({ + apiUrl: options.apiUrl || defaultApiUrl(), + interactive: false, + }); + const resolved = await resolveCloudWorkspaceId(workspaceId, session.auth); + const { response } = await authorizedApiFetch( + resolved.auth, + '/api/v1/fleet/nodes/sandbox/ensure', + { + method: 'POST', + body: JSON.stringify({ + workspaceId: resolved.cloudWorkspaceId, + requiredCapability, + ...(input.name ? { name: input.name } : {}), + ...(input.maxAgents !== undefined ? { maxAgents: input.maxAgents } : {}), + ...(input.mountRelayfile !== undefined ? { mountRelayfile: input.mountRelayfile } : {}), + ...(input.forceProvision !== undefined ? { forceProvision: input.forceProvision } : {}), + ...(input.waitTimeoutMs !== undefined ? { waitTimeoutMs: input.waitTimeoutMs } : {}), + }), + }, + { interactive: false } + ); + const payload = await readJson(response); + if (!response.ok) throw endpointError('provision the fleet sandbox', response, payload); + return normalizeEnsureResult(payload, resolved.cloudWorkspaceId); +} + +/** Best-effort-safe deletion for a Cloud-owned Daytona fleet sandbox. */ +export async function deleteCloudFleetSandbox( + input: DeleteCloudFleetSandboxInput, + options: { apiUrl?: string } = {} +): Promise { + const cloudWorkspaceId = input.cloudWorkspaceId.trim(); + const sandboxId = input.sandboxId.trim(); + if (!cloudWorkspaceId || !sandboxId) throw new Error('Cloud workspace and sandbox IDs are required.'); + + const session = await ensureCloudSession({ + apiUrl: options.apiUrl || defaultApiUrl(), + interactive: false, + }); + const { response } = await authorizedApiFetch( + session.auth, + `/api/v1/fleet/nodes/sandbox/${encodeURIComponent(sandboxId)}`, + { + method: 'DELETE', + body: JSON.stringify({ workspaceId: cloudWorkspaceId }), + }, + { interactive: false } + ); + const payload = await readJson(response); + if (!response.ok) throw endpointError('delete the fleet sandbox', response, payload); +} diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index 36d75bf6b..c4a5fccce 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -77,6 +77,16 @@ export { export { createWorkspace, issueWorkspaceToken, resolveActiveWorkspace } from './workspaces.js'; export { redactCredentialValues } from './redact.js'; +export { + ensureCloudFleetSandbox, + deleteCloudFleetSandbox, + type EnsureCloudFleetSandboxInput, + type EnsureCloudFleetSandboxResult, + type CloudFleetSandboxReady, + type CloudFleetSandboxReused, + type CloudFleetSandboxProvisioningTimeout, + type DeleteCloudFleetSandboxInput, +} from './fleet-sandbox.js'; export { acknowledgeCloudWorkerAssignment, From 7f394535703c1890b5bc95a6ace0ecaeceef74a8 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 20 Aug 2026 16:27:30 +0200 Subject: [PATCH 2/4] fix(cli): surface sandbox cleanup failures --- packages/cli/src/cli/commands/fleet.test.ts | 59 +++++++++++++++++++++ packages/cli/src/cli/commands/fleet.ts | 8 ++- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 3dbbeb5c5..93824da97 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -742,6 +742,65 @@ describe('fleet command support', () => { }); }); + it('warns when an unmounted sandbox cannot be cleaned up automatically', async () => { + const warnings: string[] = []; + const errors: string[] = []; + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn() as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: (...args: unknown[]) => errors.push(args.join(' ')), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox: vi.fn(async () => ({ + outcome: 'provisioned' as const, + cloudWorkspaceId: 'cloud-workspace', + nodeId: 'node-1', + nodeName: 'daytona-codex', + sandboxId: 'sandbox-1', + relayWorkspaceId: 'rw_abc', + relayfileMounted: false, + })), + deleteCloudFleetSandbox: vi.fn(async () => Promise.reject(new Error('delete failed'))), + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: (...args: unknown[]) => warnings.push(args.join(' ')), + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + '--token', + 'at_live_lead', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + + expect(errors.join('\n')).toContain('without the required Relayfile mount'); + expect(warnings.join('\n')).toContain('may still be running'); + expect(warnings.join('\n')).toContain('delete failed'); + }); + it('fleet spawn --no-confirm accepts an unconfirmed targeted dispatch', async () => { const placement = { spawn: vi.fn(async () => ({ diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index fa1a4b0d8..96bb5c6d3 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -261,7 +261,13 @@ export function registerFleetCommands( cloudWorkspaceId: sandbox.cloudWorkspaceId, sandboxId: sandbox.sandboxId, }) - .catch(() => undefined); + .catch((error) => { + deps.warn( + `The unmounted sandbox could not be cleaned up automatically and may still be running: ${ + error instanceof Error ? error.message : String(error) + }` + ); + }); } throw new Error('Cloud returned a Daytona node without the required Relayfile mount.'); } From b574e81bc253f8a1ae9a277e64fe3af05524d78f Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 20 Aug 2026 17:18:58 +0200 Subject: [PATCH 3/4] fix(cli): harden sandbox provisioning failures --- CHANGELOG.md | 2 +- packages/cli/src/cli/commands/fleet.test.ts | 166 +++++++++++++++++--- packages/cli/src/cli/commands/fleet.ts | 44 ++++-- packages/cloud/src/fleet-sandbox.test.ts | 98 +++++++++++- packages/cloud/src/fleet-sandbox.ts | 160 ++++++++++++++++--- packages/cloud/src/index.ts | 2 + 6 files changed, 402 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfdeceac7..9589bb364 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `agent-relay fleet spawn --sandbox` now provisions a fresh Cloud Daytona node, requires the current Relayfile workspace to mount, waits for node readiness, and spawns the agent in `/workspace` in one command. Workspace-only shells automatically use and remove a short-lived launcher identity, so `--token` is no longer required for this path. +- `agent-relay fleet spawn --sandbox` now spawns an agent in a fresh Cloud Daytona node in one command, with no `--token` required in workspace-only shells. It mounts the current Relayfile workspace at `/workspace` by default; pass `--no-sandbox-relayfile` for a deliberately bare sandbox. - `agent-relay observer` mints a scoped, read-only observer token and prints the observer URL built from it, so sharing a live follow-along view no longer requires hand-rolling a `POST /v1/observer-tokens` call. Defaults to a 24-hour token with agent DMs excluded; `--channels`, `--include-dms`, and `--expires` widen it, and `observer list` / `observer revoke ` manage existing tokens. - `get_observer_url` MCP tool does the same for an orchestrating agent, so a lead can hand the user a follow-along link without shelling out. - `@agent-relay/sdk` exports `createObserverToken`, `listObserverTokens`, and `revokeObserverToken`. diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 93824da97..0f905de65 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -3,8 +3,11 @@ import os from 'node:os'; import path from 'node:path'; import { Command } from 'commander'; +import { CloudFleetSandboxProvisionError } from '@agent-relay/cloud'; import { defineNode, invokeNodeHandler, spawn as fleetSpawn } from '@agent-relay/fleet'; -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +afterEach(() => vi.unstubAllEnvs()); // `fleet status` fetches the broker session (which carries the node token and // workspace key) and queries the engine nodes API; stub both so the redaction @@ -561,8 +564,7 @@ describe('fleet command support', () => { }); it('fleet spawn --sandbox provisions Daytona, mounts Relayfile, and uses a temporary launcher', async () => { - const previousToken = process.env.RELAY_AGENT_TOKEN; - delete process.env.RELAY_AGENT_TOKEN; + vi.stubEnv('RELAY_AGENT_TOKEN', undefined); const placement = { spawn: vi.fn(async () => ({ invocationId: 'inv_sandbox', @@ -610,29 +612,23 @@ describe('fleet command support', () => { error: () => undefined, }); - try { - await program.parseAsync( - [ - 'fleet', - 'spawn', - 'codex', - '--sandbox', - '--sandbox-name', - 'daytona-codex', - '--name', - 'sandbox-worker', - '--task', - 'Wait for VERIFY', - '--workspace-key', - 'rk_live_test', - ], - { from: 'user' } - ); - } finally { - if (previousToken === undefined) delete process.env.RELAY_AGENT_TOKEN; - else process.env.RELAY_AGENT_TOKEN = previousToken; - } - + await program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-name', + 'daytona-codex', + '--name', + 'sandbox-worker', + '--task', + 'Wait for VERIFY', + '--workspace-key', + 'rk_live_test', + ], + { from: 'user' } + ); expect(ensureCloudFleetSandbox).toHaveBeenCalledWith({ workspaceId: 'rw_abc', requiredCapability: 'spawn:codex', @@ -742,6 +738,124 @@ describe('fleet command support', () => { }); }); + it('cleans up when Cloud reports a post-provision response failure with a sandbox ID', async () => { + const deleteCloudFleetSandbox = vi.fn(async () => undefined); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn() as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox: vi.fn(async () => { + throw new CloudFleetSandboxProvisionError('malformed response', { + cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', + sandboxId: 'sandbox-1', + nodeName: 'daytona-codex', + outcomeUnknown: true, + }); + }), + deleteCloudFleetSandbox, + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: () => undefined, + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-name', + 'daytona-codex', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + '--token', + 'at_live_lead', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + + expect(deleteCloudFleetSandbox).toHaveBeenCalledWith({ + cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', + sandboxId: 'sandbox-1', + }); + }); + + it('identifies the requested node when the provisioning response is interrupted', async () => { + const warnings: string[] = []; + const deleteCloudFleetSandbox = vi.fn(); + const program = new Command(); + program.exitOverride(); + registerFleetCommands(program, { + sdk: { + createAgentRelay: vi.fn() as never, + createWorkspaceRelay: vi.fn(() => ({ + workspace: { info: vi.fn(async () => ({ id: 'rw_abc' })) }, + })) as never, + createWorkspace: vi.fn() as never, + log: vi.fn(), + error: vi.fn(), + exit: (() => { + throw new Error('__exit__'); + }) as never, + }, + ensureCloudFleetSandbox: vi.fn(async () => { + throw new CloudFleetSandboxProvisionError('request interrupted', { + cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', + nodeName: 'daytona-codex', + outcomeUnknown: true, + }); + }), + deleteCloudFleetSandbox, + createFleetWorkspaceClient: vi.fn() as never, + log: () => undefined, + warn: (...args: unknown[]) => warnings.push(args.join(' ')), + error: () => undefined, + }); + + await expect( + program.parseAsync( + [ + 'fleet', + 'spawn', + 'codex', + '--sandbox', + '--sandbox-name', + 'daytona-codex', + '--name', + 'sandbox-worker', + '--task', + 'Work', + '--workspace-key', + 'rk_live_test', + '--token', + 'at_live_lead', + ], + { from: 'user' } + ) + ).rejects.toThrow('__exit__'); + + expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); + expect(warnings.join('\n')).toContain("check Cloud Fleet for node 'daytona-codex'"); + }); + it('warns when an unmounted sandbox cannot be cleaned up automatically', async () => { const warnings: string[] = []; const errors: string[] = []; diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 96bb5c6d3..ef506a385 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto'; import { InvalidArgumentError, type Command } from 'commander'; import { + CloudFleetSandboxProvisionError, deleteCloudFleetSandbox, ensureCloudFleetSandbox, type EnsureCloudFleetSandboxResult, @@ -225,15 +226,40 @@ export function registerFleetCommands( if (!relayWorkspaceId) { throw new Error('The current Relay workspace did not report an ID for Cloud provisioning.'); } - sandbox = await deps.ensureCloudFleetSandbox({ - workspaceId: relayWorkspaceId, - requiredCapability: `spawn:${cli}`, - maxAgents: 1, - mountRelayfile: mountSandboxRelayfile, - forceProvision: true, - waitTimeoutMs: 90_000, - ...(sandboxName ? { name: sandboxName } : {}), - }); + const requestedSandboxName = sandboxName ?? `fleet-sandbox-${randomUUID().slice(0, 8)}`; + try { + sandbox = await deps.ensureCloudFleetSandbox({ + workspaceId: relayWorkspaceId, + requiredCapability: `spawn:${cli}`, + maxAgents: 1, + mountRelayfile: mountSandboxRelayfile, + forceProvision: true, + waitTimeoutMs: 90_000, + name: requestedSandboxName, + }); + } catch (error) { + if (error instanceof CloudFleetSandboxProvisionError && error.cloudWorkspaceId && error.sandboxId) { + await deps + .deleteCloudFleetSandbox({ + cloudWorkspaceId: error.cloudWorkspaceId, + sandboxId: error.sandboxId, + }) + .catch((cleanupError) => { + deps.warn( + `Provisioning failed after Daytona created sandbox '${error.sandboxId}', and automatic cleanup failed: ${ + cleanupError instanceof Error ? cleanupError.message : String(cleanupError) + }` + ); + }); + } else if (error instanceof CloudFleetSandboxProvisionError && error.outcomeUnknown) { + deps.warn( + `Cloud did not return a complete provisioning response. The outcome is unknown; check Cloud Fleet for node '${ + error.nodeName ?? requestedSandboxName + }' before retrying so a Daytona sandbox is not left running.` + ); + } + throw error; + } if (sandbox.outcome === 'provisioning_timeout') { await deps .deleteCloudFleetSandbox({ diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index e544ff09d..8abb8d65f 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -10,7 +10,11 @@ vi.mock('./auth.js', () => ({ authorizedApiFetch: mocks.authorizedApiFetch, })); -import { deleteCloudFleetSandbox, ensureCloudFleetSandbox } from './fleet-sandbox.js'; +import { + CloudFleetSandboxProvisionError, + deleteCloudFleetSandbox, + ensureCloudFleetSandbox, +} from './fleet-sandbox.js'; const auth = { accessToken: 'access', @@ -19,6 +23,7 @@ const auth = { apiUrl: 'https://agentrelay.test/cloud', }; const refreshedAuth = { ...auth, accessToken: 'refreshed' }; +const CLOUD_WORKSPACE_ID = '50587328-441d-4acb-b8f3-dbe1b3c5de99'; describe('Cloud fleet sandbox client', () => { beforeEach(() => { @@ -29,7 +34,7 @@ describe('Cloud fleet sandbox client', () => { it('resolves the unified workspace and provisions a ready mounted sandbox', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ - response: Response.json({ cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99' }), + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), auth: refreshedAuth, }) .mockResolvedValueOnce({ @@ -62,14 +67,14 @@ describe('Cloud fleet sandbox client', () => { 1, auth, '/api/v1/workspaces/rw_abc/resolve', - { method: 'GET' }, + { method: 'GET', signal: expect.any(AbortSignal) }, { interactive: false } ); const ensureCall = mocks.authorizedApiFetch.mock.calls[1]; expect(ensureCall?.[0]).toEqual(refreshedAuth); expect(ensureCall?.[1]).toBe('/api/v1/fleet/nodes/sandbox/ensure'); expect(JSON.parse(String(ensureCall?.[2]?.body))).toEqual({ - workspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', + workspaceId: CLOUD_WORKSPACE_ID, name: 'daytona-codex', requiredCapability: 'spawn:codex', maxAgents: 1, @@ -79,7 +84,7 @@ describe('Cloud fleet sandbox client', () => { }); expect(result).toEqual({ outcome: 'provisioned', - cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99', + cloudWorkspaceId: CLOUD_WORKSPACE_ID, nodeId: 'node-1', nodeName: 'daytona-codex', sandboxId: 'sandbox-1', @@ -92,7 +97,7 @@ describe('Cloud fleet sandbox client', () => { it('preserves a bounded provisioning timeout so the CLI can report it', async () => { mocks.authorizedApiFetch .mockResolvedValueOnce({ - response: Response.json({ cloudWorkspaceId: 'cloud-workspace' }), + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), auth, }) .mockResolvedValueOnce({ @@ -136,6 +141,82 @@ describe('Cloud fleet sandbox client', () => { ).rejects.toThrow('owner or admin'); }); + it('rejects a malformed Cloud workspace identity before provisioning', async () => { + mocks.authorizedApiFetch.mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: 'not-a-uuid' }), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + }) + ).rejects.toThrow('invalid cloudWorkspaceId'); + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(1); + }); + + it('preserves the sandbox identity from a malformed successful response', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioned', + nodeName: 'daytona-codex', + sandboxId: 'sandbox-1', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + }, + { status: 201 } + ), + auth, + }); + + const error = await ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(CloudFleetSandboxProvisionError); + expect(error).toMatchObject({ + cloudWorkspaceId: CLOUD_WORKSPACE_ID, + sandboxId: 'sandbox-1', + nodeName: 'daytona-codex', + outcomeUnknown: true, + }); + }); + + it('rejects a timeout response that omits waitedMs', async () => { + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json( + { + outcome: 'provisioning_timeout', + sandboxId: 'sandbox-1', + relayWorkspaceId: 'rw_abc', + nodeName: 'daytona-codex', + }, + { status: 202 } + ), + auth, + }); + + await expect( + ensureCloudFleetSandbox({ + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + }) + ).rejects.toThrow('missing waitedMs'); + }); + it('deletes only the named sandbox in the resolved Cloud workspace', async () => { mocks.authorizedApiFetch.mockResolvedValueOnce({ response: Response.json({ sandboxId: 'sandbox-1', deleted: true }), @@ -143,7 +224,7 @@ describe('Cloud fleet sandbox client', () => { }); await deleteCloudFleetSandbox({ - cloudWorkspaceId: 'cloud-workspace', + cloudWorkspaceId: CLOUD_WORKSPACE_ID, sandboxId: 'sandbox-1', }); @@ -152,7 +233,8 @@ describe('Cloud fleet sandbox client', () => { '/api/v1/fleet/nodes/sandbox/sandbox-1', { method: 'DELETE', - body: JSON.stringify({ workspaceId: 'cloud-workspace' }), + signal: expect.any(AbortSignal), + body: JSON.stringify({ workspaceId: CLOUD_WORKSPACE_ID }), }, { interactive: false } ); diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 86d559af3..85af10512 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -4,6 +4,45 @@ import { defaultApiUrl } from './types.js'; type JsonRecord = Record; +const CLOUD_WORKSPACE_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const DEFAULT_ENSURE_TIMEOUT_MS = 120_000; +const DEFAULT_DELETE_TIMEOUT_MS = 30_000; + +export type CloudFleetSandboxRequestOptions = { + apiUrl?: string; + signal?: AbortSignal; + timeoutMs?: number; +}; + +/** + * Carries every safe identifier Cloud returned when provisioning failed after + * the request may have created a billable sandbox. + */ +export class CloudFleetSandboxProvisionError extends Error { + readonly cloudWorkspaceId?: string; + readonly sandboxId?: string; + readonly nodeName?: string; + readonly outcomeUnknown: boolean; + + constructor( + message: string, + identity: { + cloudWorkspaceId?: string; + sandboxId?: string; + nodeName?: string; + outcomeUnknown?: boolean; + cause?: unknown; + } = {} + ) { + super(message, identity.cause === undefined ? undefined : { cause: identity.cause }); + this.name = 'CloudFleetSandboxProvisionError'; + this.cloudWorkspaceId = identity.cloudWorkspaceId; + this.sandboxId = identity.sandboxId; + this.nodeName = identity.nodeName; + this.outcomeUnknown = identity.outcomeUnknown === true; + } +} + export type EnsureCloudFleetSandboxInput = { /** Cloud UUID or unified rw_* workspace id. */ workspaceId: string; @@ -69,6 +108,21 @@ function readNumber(payload: JsonRecord, key: string): number | undefined { return typeof value === 'number' && Number.isFinite(value) ? value : undefined; } +function requiredNumber(payload: JsonRecord, key: string, context: string): number { + const value = readNumber(payload, key); + if (value === undefined) throw new Error(`${context} response is missing ${key}.`); + return value; +} + +function boundedSignal(options: CloudFleetSandboxRequestOptions, defaultTimeoutMs: number): AbortSignal { + const timeoutMs = options.timeoutMs ?? defaultTimeoutMs; + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new Error('Cloud fleet sandbox request timeout must be a positive number of milliseconds.'); + } + const timeoutSignal = AbortSignal.timeout(timeoutMs); + return options.signal ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal; +} + async function readJson(response: Response): Promise { try { return await response.json(); @@ -108,7 +162,8 @@ function requiredString(payload: JsonRecord, key: string, context: string): stri async function resolveCloudWorkspaceId( workspaceId: string, - auth: Awaited>['auth'] + auth: Awaited>['auth'], + signal: AbortSignal ): Promise<{ cloudWorkspaceId: string; auth: Awaited>['auth']; @@ -116,14 +171,18 @@ async function resolveCloudWorkspaceId( const { response, auth: activeAuth } = await authorizedApiFetch( auth, `/api/v1/workspaces/${encodeURIComponent(workspaceId)}/resolve`, - { method: 'GET' }, + { method: 'GET', signal }, { interactive: false } ); const payload = await readJson(response); if (!response.ok) throw endpointError('resolve the Cloud workspace', response, payload); if (!isObject(payload)) throw new Error('Cloud workspace resolver returned an invalid response.'); + const cloudWorkspaceId = requiredString(payload, 'cloudWorkspaceId', 'Cloud workspace resolver'); + if (!CLOUD_WORKSPACE_ID_PATTERN.test(cloudWorkspaceId)) { + throw new Error('Cloud workspace resolver returned an invalid cloudWorkspaceId.'); + } return { - cloudWorkspaceId: requiredString(payload, 'cloudWorkspaceId', 'Cloud workspace resolver'), + cloudWorkspaceId, auth: activeAuth, }; } @@ -170,7 +229,7 @@ function normalizeEnsureResult(payload: unknown, cloudWorkspaceId: string): Ensu sandboxId: requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'), relayWorkspaceId: requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'), nodeName, - waitedMs: readNumber(payload, 'waitedMs') ?? 0, + waitedMs: requiredNumber(payload, 'waitedMs', 'Cloud fleet sandbox'), }; } @@ -180,7 +239,7 @@ function normalizeEnsureResult(payload: unknown, cloudWorkspaceId: string): Ensu /** Resolve a Relay workspace in Cloud, provision/reuse a node, and wait for readiness. */ export async function ensureCloudFleetSandbox( input: EnsureCloudFleetSandboxInput, - options: { apiUrl?: string } = {} + options: CloudFleetSandboxRequestOptions = {} ): Promise { const workspaceId = input.workspaceId.trim(); const requiredCapability = input.requiredCapability.trim(); @@ -191,33 +250,80 @@ export async function ensureCloudFleetSandbox( apiUrl: options.apiUrl || defaultApiUrl(), interactive: false, }); - const resolved = await resolveCloudWorkspaceId(workspaceId, session.auth); - const { response } = await authorizedApiFetch( - resolved.auth, - '/api/v1/fleet/nodes/sandbox/ensure', - { - method: 'POST', - body: JSON.stringify({ - workspaceId: resolved.cloudWorkspaceId, - requiredCapability, - ...(input.name ? { name: input.name } : {}), - ...(input.maxAgents !== undefined ? { maxAgents: input.maxAgents } : {}), - ...(input.mountRelayfile !== undefined ? { mountRelayfile: input.mountRelayfile } : {}), - ...(input.forceProvision !== undefined ? { forceProvision: input.forceProvision } : {}), - ...(input.waitTimeoutMs !== undefined ? { waitTimeoutMs: input.waitTimeoutMs } : {}), - }), - }, - { interactive: false } - ); + const signal = boundedSignal(options, DEFAULT_ENSURE_TIMEOUT_MS); + const resolved = await resolveCloudWorkspaceId(workspaceId, session.auth, signal); + let response: Response; + try { + ({ response } = await authorizedApiFetch( + resolved.auth, + '/api/v1/fleet/nodes/sandbox/ensure', + { + method: 'POST', + signal, + body: JSON.stringify({ + workspaceId: resolved.cloudWorkspaceId, + requiredCapability, + ...(input.name ? { name: input.name } : {}), + ...(input.maxAgents !== undefined ? { maxAgents: input.maxAgents } : {}), + ...(input.mountRelayfile !== undefined ? { mountRelayfile: input.mountRelayfile } : {}), + ...(input.forceProvision !== undefined ? { forceProvision: input.forceProvision } : {}), + ...(input.waitTimeoutMs !== undefined ? { waitTimeoutMs: input.waitTimeoutMs } : {}), + }), + }, + { interactive: false } + )); + } catch (error) { + throw new CloudFleetSandboxProvisionError( + redactCredentialValues( + `Cloud fleet sandbox request ended without a complete response: ${ + error instanceof Error ? error.message : String(error) + }` + ), + { + cloudWorkspaceId: resolved.cloudWorkspaceId, + ...(input.name ? { nodeName: input.name } : {}), + outcomeUnknown: true, + cause: error, + } + ); + } const payload = await readJson(response); - if (!response.ok) throw endpointError('provision the fleet sandbox', response, payload); - return normalizeEnsureResult(payload, resolved.cloudWorkspaceId); + if (!response.ok) { + const error = endpointError('provision the fleet sandbox', response, payload); + if (isObject(payload) && readString(payload, 'sandboxId')) { + throw new CloudFleetSandboxProvisionError(error.message, { + cloudWorkspaceId: resolved.cloudWorkspaceId, + sandboxId: readString(payload, 'sandboxId'), + nodeName: readString(payload, 'nodeName') ?? input.name, + cause: error, + }); + } + throw error; + } + try { + return normalizeEnsureResult(payload, resolved.cloudWorkspaceId); + } catch (error) { + throw new CloudFleetSandboxProvisionError( + error instanceof Error ? error.message : 'Cloud fleet sandbox response was invalid.', + { + cloudWorkspaceId: resolved.cloudWorkspaceId, + ...(isObject(payload) && readString(payload, 'sandboxId') + ? { sandboxId: readString(payload, 'sandboxId') } + : {}), + ...(isObject(payload) && (readString(payload, 'nodeName') ?? input.name) + ? { nodeName: readString(payload, 'nodeName') ?? input.name } + : {}), + outcomeUnknown: true, + cause: error, + } + ); + } } /** Best-effort-safe deletion for a Cloud-owned Daytona fleet sandbox. */ export async function deleteCloudFleetSandbox( input: DeleteCloudFleetSandboxInput, - options: { apiUrl?: string } = {} + options: CloudFleetSandboxRequestOptions = {} ): Promise { const cloudWorkspaceId = input.cloudWorkspaceId.trim(); const sandboxId = input.sandboxId.trim(); @@ -227,11 +333,13 @@ export async function deleteCloudFleetSandbox( apiUrl: options.apiUrl || defaultApiUrl(), interactive: false, }); + const signal = boundedSignal(options, DEFAULT_DELETE_TIMEOUT_MS); const { response } = await authorizedApiFetch( session.auth, `/api/v1/fleet/nodes/sandbox/${encodeURIComponent(sandboxId)}`, { method: 'DELETE', + signal, body: JSON.stringify({ workspaceId: cloudWorkspaceId }), }, { interactive: false } diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index c4a5fccce..4f7f5f107 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -80,12 +80,14 @@ export { redactCredentialValues } from './redact.js'; export { ensureCloudFleetSandbox, deleteCloudFleetSandbox, + CloudFleetSandboxProvisionError, type EnsureCloudFleetSandboxInput, type EnsureCloudFleetSandboxResult, type CloudFleetSandboxReady, type CloudFleetSandboxReused, type CloudFleetSandboxProvisioningTimeout, type DeleteCloudFleetSandboxInput, + type CloudFleetSandboxRequestOptions, } from './fleet-sandbox.js'; export { From 96c2cd656d2b43f28f3db67b14414bccaa51eba7 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 20 Aug 2026 17:28:33 +0200 Subject: [PATCH 4/4] fix(cloud): isolate provisioning request budget --- packages/cloud/src/fleet-sandbox.test.ts | 45 ++++++++++++++++++++++++ packages/cloud/src/fleet-sandbox.ts | 3 +- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index 8abb8d65f..5db4237d2 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -127,6 +127,51 @@ describe('Cloud fleet sandbox client', () => { }); }); + it('starts a fresh request budget after a delayed workspace resolution', async () => { + const signals: AbortSignal[] = []; + mocks.authorizedApiFetch + .mockImplementationOnce(async (_auth, _path, init) => { + signals.push(init.signal as AbortSignal); + await new Promise((resolve) => setTimeout(resolve, 60)); + return { + response: Response.json({ cloudWorkspaceId: CLOUD_WORKSPACE_ID }), + auth, + }; + }) + .mockImplementationOnce(async (_auth, _path, init) => { + signals.push(init.signal as AbortSignal); + await new Promise((resolve) => setTimeout(resolve, 60)); + return { + response: Response.json( + { + outcome: 'provisioned', + nodeId: 'node-1', + nodeName: 'daytona-codex', + sandboxId: 'sandbox-1', + relayWorkspaceId: 'rw_abc', + relayfileMounted: true, + }, + { status: 201 } + ), + auth, + }; + }); + + await expect( + ensureCloudFleetSandbox( + { + workspaceId: 'rw_abc', + requiredCapability: 'spawn:codex', + }, + { timeoutMs: 100 } + ) + ).resolves.toMatchObject({ outcome: 'provisioned', sandboxId: 'sandbox-1' }); + + expect(signals).toHaveLength(2); + expect(signals[0]).not.toBe(signals[1]); + expect(signals[1]?.aborted).toBe(false); + }); + it('turns Cloud authorization failures into actionable errors', async () => { mocks.authorizedApiFetch.mockResolvedValueOnce({ response: Response.json({ error: 'Forbidden' }, { status: 403 }), diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 85af10512..f59321c7f 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -250,8 +250,9 @@ export async function ensureCloudFleetSandbox( apiUrl: options.apiUrl || defaultApiUrl(), interactive: false, }); + const resolutionSignal = boundedSignal(options, DEFAULT_ENSURE_TIMEOUT_MS); + const resolved = await resolveCloudWorkspaceId(workspaceId, session.auth, resolutionSignal); const signal = boundedSignal(options, DEFAULT_ENSURE_TIMEOUT_MS); - const resolved = await resolveCloudWorkspaceId(workspaceId, session.auth, signal); let response: Response; try { ({ response } = await authorizedApiFetch(