From 036bbe8483ed60aa5cd6430559ac08301110ec0e Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 20:19:20 +0000 Subject: [PATCH 01/16] feat(client): project sandbox generation rollout and optional E2B key updates --- .../agents-client/src/sandbox-client.test.ts | 35 ++++++++++-- packages/agents-client/src/sandbox-client.ts | 57 +++++++++++++++---- 2 files changed, 78 insertions(+), 14 deletions(-) diff --git a/packages/agents-client/src/sandbox-client.test.ts b/packages/agents-client/src/sandbox-client.test.ts index b8c9edbbe..d881e167c 100644 --- a/packages/agents-client/src/sandbox-client.test.ts +++ b/packages/agents-client/src/sandbox-client.test.ts @@ -8,6 +8,7 @@ function response(value: unknown, status = 200) { return new Response(JSON.strin const created = "2026-09-25T08:00:00.123456789Z"; /** A ready node: Core omits `diagnostic`. */ const node = { + rollout: { state: "ready", ready_generation: 1 }, id: "3b0c1f4e-8a2d-4c6b-9e7f-1a2b3c4d5e6f", name: "core-01", provider: "docker", online: true, provider_ready: true, cpu_count: 16, available_memory_bytes: 8589934592, available_disk_bytes: 107374182400, running: 1, snapshots: 0, last_seen_at: "2026-09-25T09:00:00Z", max_active: 2, max_retained: 2, active: 1, reserved: 0, retained: 1, cleanup_pending: 0, @@ -15,7 +16,7 @@ const node = { }; /** Never heard from, enrolled before Core recorded enrollment IDs, with a fixed readiness code. */ const unready = { - ...node, id: "7f6e5d4c-3b2a-4190-8f7e-6d5c4b3a2918", online: false, provider_ready: false, diagnostic: "kvm_unavailable", + ...node, rollout: { state: "unknown", ready_generation: 1 }, id: "7f6e5d4c-3b2a-4190-8f7e-6d5c4b3a2918", online: false, provider_ready: false, diagnostic: "kvm_unavailable", cpu_count: null, available_memory_bytes: null, available_disk_bytes: null, running: 0, last_seen_at: null, active: 0, retained: 0, enrollment_id: null, }; const detail = { @@ -32,14 +33,15 @@ const unobserved = { history: { resolution_seconds: 900, points: [] }, }; const allocation = { + deployment_generation: 1, id: "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d", node_id: node.id, tenant_id: "1b2c3d4e-5f6a-4b7c-8d9e-0f1a2b3c4d5e", session_id: "2c3d4e5f-6a7b-4c8d-9e0f-1a2b3c4d5e6f", environment_id: "3d4e5f6a-7b8c-4d9e-8f1a-2b3c4d5e6f7a", state: "running", compute_phase: "running", compute_phase_changed_at: null, diagnostic: "", initialization: "ready", created_at: created, }; const runtime = { source_commit: "a".repeat(40), image_id: "sha256:" + "b".repeat(64), image_manifest_digest: "sha256:" + "c".repeat(64), microsandbox_ref: "oac-runtime@sha256:" + "d".repeat(64), runtime_sha256: "e".repeat(64), firmware_sha256: "f".repeat(64) }; -const unconfigured = { installation_id: "", provider: "", core_url: "https://core.example", reset: null, owner_epoch: 0, generation: 0, mode: "", resources: { allocations: 0, pending: 0 }, suspension: null }; +const unconfigured = { rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: null }, installation_id: "", provider: "", core_url: "https://core.example", reset: null, owner_epoch: 0, generation: 0, mode: "", resources: { allocations: 0, pending: 0 }, suspension: null }; const docker = { - ...unconfigured, installation_id: "94be54a1-138c-4f30-bc87-b13686272dbe", provider: "docker", owner_epoch: 1, generation: 1, mode: "nodes", + ...unconfigured, installation_id: "94be54a1-138c-4f30-bc87-b13686272dbe", provider: "docker", rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 2, pending: 1 }, specification: { resources: { cpus: 2, memory_mib: 2048 }, runtime }, specification_digest: "0".repeat(64), }; const microsandbox = { @@ -48,7 +50,7 @@ const microsandbox = { }; /** An E2B selection saved before Core recorded its template build. */ const e2bDeployment = { - ...docker, provider: "e2b", mode: "direct", specification: { resources: { cpus: 2, memory_mib: 2048 } }, + ...docker, provider: "e2b", rollout: unconfigured.rollout, mode: "direct", specification: { resources: { cpus: 2, memory_mib: 2048 } }, e2b: { template: "runtime:00000000-0000-0000-0000-000000000001", credential_configured: true, template_build: { status: null, resources: { cpus: null, memory_mib: null, root_disk_mib: null } } }, }; const reads: Record Promise> = { @@ -311,3 +313,28 @@ describe("durable reset projection", () => { expect(fetch).toHaveBeenCalledTimes(1); }); }); + +it("uses the authoritative rollout signal instead of retained old Sessions for polling", async () => { + const settled = { ...e2bDeployment, rollout: { state: "settled", previous_generation_sandboxes: 2, nodes: null } }; + const client = new SandboxAdminClient({ fetch: async () => response(settled) }); + expect((await client.retrieveDeployment()).rollout).toEqual(settled.rollout); + for (const rollout of [ + { ...settled.rollout, state: "preparing" }, + { ...settled.rollout, previous_generation_sandboxes: 4 }, + { ...docker.rollout, state: "preparing" }, + { ...docker.rollout, nodes: { ...docker.rollout.nodes, unknown: -1 } }, + ]) { + const invalid = new SandboxAdminClient({ fetch: async () => response({ ...docker, rollout }) }); + await expect(invalid.retrieveDeployment()).rejects.toMatchObject({ code: "invalid_admin_response" }); + } +}); + +it("omits a preserved key and submits an explicit same key once without retry", async () => { + const fetch = vi.fn().mockImplementation(async () => response(e2bDeployment)); + const client = new SandboxAdminClient({ fetch }); + await client.updateDeployment({ provider: "e2b", expected_generation: 1, e2b: { template: e2bDeployment.e2b.template } }); + await client.updateDeployment({ provider: "e2b", expected_generation: 1, e2b: { template: e2bDeployment.e2b.template, api_key: "same-key" } }); + expect(fetch).toHaveBeenCalledTimes(2); + expect(JSON.parse(String(fetch.mock.calls[0]![1]!.body)).e2b).not.toHaveProperty("api_key"); + expect(JSON.parse(String(fetch.mock.calls[1]![1]!.body)).e2b.api_key).toBe("same-key"); +}); diff --git a/packages/agents-client/src/sandbox-client.ts b/packages/agents-client/src/sandbox-client.ts index 560e9d622..ab8cab540 100644 --- a/packages/agents-client/src/sandbox-client.ts +++ b/packages/agents-client/src/sandbox-client.ts @@ -31,7 +31,22 @@ export interface InitializeSandboxDeployment { runtime?: SandboxRuntimeRelease; e2b?: { api_key: string; template: string }; } -export interface UpdateSandboxDeployment extends InitializeSandboxDeployment { expected_generation: number } +export interface UpdateSandboxDeployment extends Omit { + /** Omit api_key to preserve the current key. Supplying it, even unchanged, verifies and replaces it once. */ + e2b?: { api_key?: string; template: string }; +} +export interface SandboxRollout { + /** Poll at high frequency only while preparing, independently of old Session retention. */ + state: "settled" | "preparing"; + previous_generation_sandboxes: number; + nodes: { ready: number; preparing: number; failed: number; update_required: number; unknown: number } | null; +} +export interface SandboxNodeRollout { + state: "ready" | "preparing" | "failed" | "update_required" | "unknown"; + /** Durable serving pin; this alone does not imply current connection readiness. */ + ready_generation: number | null; + diagnostic?: SandboxNodeDiagnostic; +} export interface StartSandboxReset { expected_generation: number; clear: "auto" | "force"; deadline_seconds?: number } export interface SandboxReset { clear: "auto" | "force"; @@ -45,6 +60,7 @@ export interface SandboxReset { } export interface SandboxDeployment { + rollout: SandboxRollout; specification?: SandboxSpecification; specification_digest?: string; installation_id: string; @@ -66,6 +82,7 @@ export interface SandboxE2BTemplateBuild { resources: { cpus: number | null; memory_mib: number | null; root_disk_mib: number | null }; } export interface SandboxNode { + rollout: SandboxNodeRollout; id: string; name: string; provider: string; @@ -128,6 +145,7 @@ export interface SandboxNodeUpdate { max_retained: number; } export interface SandboxAllocation { + deployment_generation: number; id: string; node_id: string; tenant_id: string; @@ -204,10 +222,24 @@ function projectReset(value: unknown, held: number): SandboxReset | null { new Set(nodes.map(node => node.node_id)).size === nodes.length); return { ...reset, remaining: { ...remaining, offline_nodes: nodes } } as unknown as SandboxReset; } +function projectRollout(value: unknown, mode: unknown, held: number): SandboxRollout { + const rollout = members(value, ["state", "previous_generation_sandboxes", "nodes"]); + valid((rollout.state === "settled" || rollout.state === "preparing") && isNonnegativeInteger(rollout.previous_generation_sandboxes) && Number(rollout.previous_generation_sandboxes) <= held); + const nodes = rollout.nodes === null ? null : members(rollout.nodes, ["ready", "preparing", "failed", "update_required", "unknown"]); + valid((nodes !== null) === (mode === "nodes") && (nodes === null || Object.values(nodes).every(isNonnegativeInteger)) && (rollout.state === "preparing") === (nodes !== null && Number(nodes.preparing) > 0)); + return { ...rollout, nodes: nodes && { ...nodes } } as unknown as SandboxRollout; +} +function projectNodeRollout(value: unknown, online: unknown): SandboxNodeRollout { + const rollout = members(value, ["state", "ready_generation"], ["diagnostic"]); + valid(["ready", "preparing", "failed", "update_required", "unknown"].includes(rollout.state as string) && nullable(isNonnegativeInteger)(rollout.ready_generation) && + (online !== false || rollout.state === "unknown") && (rollout.state !== "ready" || rollout.ready_generation !== null) && + (rollout.diagnostic === undefined || (typeof rollout.diagnostic === "string" && rollout.diagnostic !== "" && rollout.state === "failed"))); + return { ...rollout, ...(rollout.diagnostic !== undefined ? { diagnostic: nodeDiagnostics.has(rollout.diagnostic as string) ? rollout.diagnostic : "provider_unavailable" } : {}) } as unknown as SandboxNodeRollout; +} /** `specification` and `specification_digest` appear together once configured; `e2b` appears exactly for E2B. */ function projectDeployment(value: unknown): SandboxDeployment { const e2b = isRecord(value) && value.provider === "e2b"; - const fields = ["installation_id", "provider", "core_url", "reset", "owner_epoch", "generation", "mode", "resources", "suspension"]; + const fields = ["installation_id", "provider", "core_url", "reset", "rollout", "owner_epoch", "generation", "mode", "resources", "suspension"]; const deployment = members(value, e2b ? [...fields, "e2b"] : fields, ["specification", "specification_digest"]); const resources = members(deployment.resources, ["allocations", "pending"]); const suspension = deployment.suspension === null ? null : members(deployment.suspension, ["idle_seconds", "retention_seconds"]); @@ -217,18 +249,19 @@ function projectDeployment(value: unknown): SandboxDeployment { (suspension === null || [suspension.idle_seconds, suspension.retention_seconds].every(isNonnegativeInteger)) && configured === hasOwn(deployment, "specification_digest") && (!configured || (typeof deployment.specification_digest === "string" && deployment.specification_digest !== ""))); return { - ...deployment, reset: projectReset(deployment.reset, Number(resources.allocations) + Number(resources.pending)), resources: { ...resources } as SandboxDeployment["resources"], suspension: suspension && { ...suspension } as SandboxDeployment["suspension"], + ...deployment, rollout: projectRollout(deployment.rollout, deployment.mode, Number(resources.allocations) + Number(resources.pending)), reset: projectReset(deployment.reset, Number(resources.allocations) + Number(resources.pending)), resources: { ...resources } as SandboxDeployment["resources"], suspension: suspension && { ...suspension } as SandboxDeployment["suspension"], ...(configured ? { specification: projectSpecification(deployment.specification) } : {}), ...(e2b ? { e2b: projectE2B(deployment.e2b) } : {}), } as unknown as SandboxDeployment; } -const nodeFields = ["id", "name", "provider", "online", "provider_ready", "cpu_count", "available_memory_bytes", "available_disk_bytes", "running", "snapshots", +const nodeFields = ["rollout", "id", "name", "provider", "online", "provider_ready", "cpu_count", "available_memory_bytes", "available_disk_bytes", "running", "snapshots", "last_seen_at", "max_active", "max_retained", "active", "reserved", "retained", "cleanup_pending", "created_at", "core_url", "enrollment_id"]; const nodeDiagnostics = new Set(["provider_unavailable", "docker_unavailable", "docker_limits_unsupported", "runtime_image_unavailable", "kvm_unavailable", "microsandbox_artifacts_unavailable", "capacity_insufficient"]); /** Core omits an empty `diagnostic`, so a present one is a code; an unknown code reads as provider_unavailable. */ function projectNode(node: Record): SandboxNode { - const { diagnostic, ...fields } = node; + const { diagnostic, rollout, ...rest } = node; + const fields = { ...rest, rollout: projectNodeRollout(rollout, node.online) }; valid(strings(node, ["id", "name", "provider", "core_url"]) && typeof node.online === "boolean" && typeof node.provider_ready === "boolean" && [node.cpu_count, node.available_memory_bytes, node.available_disk_bytes].every(nullable(isNonnegativeInteger)) && [node.running, node.snapshots, node.max_active, node.max_retained, node.active, node.reserved, node.retained, node.cleanup_pending].every(isNonnegativeInteger) && @@ -265,8 +298,8 @@ function projectAllocations(value: unknown, nodeId: string): { data: SandboxAllo const list = members(value, ["data"]); valid(Array.isArray(list.data)); return { data: (list.data as unknown[]).map((entry) => { - const allocation = members(entry, [...allocationFields, "compute_phase_changed_at", "created_at"]); - valid(strings(allocation, allocationFields) && sameResourceId(allocation.node_id as string, nodeId) && allocationDiagnostics.has(allocation.diagnostic as string) && + const allocation = members(entry, [...allocationFields, "deployment_generation", "compute_phase_changed_at", "created_at"]); + valid(isNonnegativeInteger(allocation.deployment_generation) && strings(allocation, allocationFields) && sameResourceId(allocation.node_id as string, nodeId) && allocationDiagnostics.has(allocation.diagnostic as string) && nullable(timestamp)(allocation.compute_phase_changed_at) && timestamp(allocation.created_at)); return { ...allocation } as unknown as SandboxAllocation; }) }; @@ -304,24 +337,28 @@ export class SandboxAdminClient { return projectDeployment(await this.#core.json("/deployment", options, method, input)); } catch (error) { if (!input.e2b) throw error; - if (error instanceof AgentCoreError && error.status === 409) { + if (error instanceof AgentCoreError && [400, 409, 503].includes(error.status)) { const messages: Record = { sandbox_generation_stale: "The sandbox configuration changed. Refresh before submitting again.", sandbox_reset_required: "Reset the sandbox deployment before changing its backend.", sandbox_reset_in_progress: "A sandbox reset is in progress.", sandbox_not_configured: "The sandbox deployment is not configured.", sandbox_in_use: "Hosted sandbox resources still belong to this deployment.", + e2b_team_mismatch: "This E2B key cannot manage the retained deployment. Reset before changing teams.", + e2b_api_key_invalid: "The E2B API key was rejected.", + e2b_template_build_invalid: "Select a ready immutable E2B template build with matching resources.", + e2b_request_unconfirmed: "E2B verification could not be confirmed. Refresh before submitting again.", }; if (error.code && messages[error.code]) { // Credential-bearing errors expose fixed local copy and allowlisted // numeric facts only; never echo arbitrary message, param or details. const fields = error.code === "sandbox_generation_stale" ? ["current_generation"] : error.code === "sandbox_in_use" ? ["allocations", "pending"] : []; const details = Object.fromEntries(fields.filter(field => isNonnegativeInteger(error.details?.[field])).map(field => [field, Number(error.details![field])])); - throw new AgentCoreError(messages[error.code]!, 409, error.code, null, undefined, Object.keys(details).length ? details : undefined); + throw new AgentCoreError(messages[error.code]!, error.status, error.code, null, undefined, Object.keys(details).length ? details : undefined); } } // The public-URL rejection is safe to show unless it somehow reflects the key. - const key = input.e2b.api_key; + const key = input.e2b.api_key ?? ""; if (error instanceof AgentCoreError && error.status === 409 && error.code === "sandbox_configuration_error" && !error.message.includes(key) && !(error.param ?? "").includes(key)) { // Only Core's message and param pass through; nothing else from the response does. From 2b0874432e0d5a9e513a31f777ef85502dbc3160 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 20:24:15 +0000 Subject: [PATCH 02/16] fix(client): keep omitted-key URL rejection safe and actionable --- .../agents-client/src/sandbox-client.test.ts | 16 ++++++++++++++-- packages/agents-client/src/sandbox-client.ts | 10 ++++------ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/packages/agents-client/src/sandbox-client.test.ts b/packages/agents-client/src/sandbox-client.test.ts index d881e167c..4c7fc0fcc 100644 --- a/packages/agents-client/src/sandbox-client.test.ts +++ b/packages/agents-client/src/sandbox-client.test.ts @@ -244,7 +244,7 @@ describe("hosted provider configuration", () => { }); expect(fetch).toHaveBeenCalledTimes(2); }); - it("shows Core's public-URL rejection unless it reflects the E2B key", async () => { + it("projects public-URL rejection to fixed copy even when the upstream message reflects a key", async () => { const rejection = { message: "E2B sandboxes reach Core over the internet. Set an HTTPS public URL that is not loopback.", code: "sandbox_configuration_error", param: null, type: "reflected" }; const fetch = vi.fn() .mockResolvedValueOnce(response({ error: rejection }, 409)) @@ -253,7 +253,7 @@ describe("hosted provider configuration", () => { const shown = await client.initializeDeployment({ expected_generation: 0, provider: "e2b", e2b }).catch((error: unknown) => error); expect(shown).toMatchObject({ status: 409, code: "sandbox_configuration_error", message: rejection.message, param: null }); expect((shown as AgentCoreError).errorType).toBeUndefined(); - await expect(client.initializeDeployment({ expected_generation: 0, provider: "e2b", e2b })).rejects.toMatchObject({ status: 409, code: "sandbox_configuration_unconfirmed" }); + await expect(client.initializeDeployment({ expected_generation: 0, provider: "e2b", e2b })).rejects.toMatchObject({ status: 409, code: "sandbox_configuration_error", message: rejection.message, param: null }); }); it("never retries uncertain switch or reset writes", async () => { const fetch = vi.fn().mockRejectedValue(new TypeError("Connection lost")); @@ -338,3 +338,15 @@ it("omits a preserved key and submits an explicit same key once without retry", expect(JSON.parse(String(fetch.mock.calls[0]![1]!.body)).e2b).not.toHaveProperty("api_key"); expect(JSON.parse(String(fetch.mock.calls[1]![1]!.body)).e2b.api_key).toBe("same-key"); }); + +it("keeps omitted-key public-URL errors actionable without reflecting a stored key", async () => { + const secret = "previously-stored-secret"; + const fetch = vi.fn().mockResolvedValue(response({ error: { code: "sandbox_configuration_error", message: `arbitrary upstream ${secret}`, param: secret, details: { credential: secret } } }, 409)); + const client = new SandboxAdminClient({ fetch }); + const error = await client.updateDeployment({ provider: "e2b", expected_generation: 1, e2b: { template: e2bDeployment.e2b.template } }).catch(error => error); + expect(error).toMatchObject({ status: 409, code: "sandbox_configuration_error", param: null, message: "E2B sandboxes reach Core over the internet. Set an HTTPS public URL that is not loopback." }); + expect(JSON.stringify(error)).not.toContain(secret); + expect(error.message).not.toContain(secret); + expect(error.details).toBeUndefined(); + expect(fetch).toHaveBeenCalledTimes(1); +}); diff --git a/packages/agents-client/src/sandbox-client.ts b/packages/agents-client/src/sandbox-client.ts index ab8cab540..f8c00f198 100644 --- a/packages/agents-client/src/sandbox-client.ts +++ b/packages/agents-client/src/sandbox-client.ts @@ -357,12 +357,10 @@ export class SandboxAdminClient { throw new AgentCoreError(messages[error.code]!, error.status, error.code, null, undefined, Object.keys(details).length ? details : undefined); } } - // The public-URL rejection is safe to show unless it somehow reflects the key. - const key = input.e2b.api_key ?? ""; - if (error instanceof AgentCoreError && error.status === 409 && error.code === "sandbox_configuration_error" - && !error.message.includes(key) && !(error.param ?? "").includes(key)) { - // Only Core's message and param pass through; nothing else from the response does. - throw new AgentCoreError(error.message, 409, "sandbox_configuration_error", error.param ?? null); + // This code has one fixed Core meaning. Never forward its raw message or + // param: an omitted key cannot be used to detect a reflected stored key. + if (error instanceof AgentCoreError && error.status === 409 && error.code === "sandbox_configuration_error") { + throw new AgentCoreError("E2B sandboxes reach Core over the internet. Set an HTTPS public URL that is not loopback.", 409, "sandbox_configuration_error", null); } // Any other credential-bearing rejection may reflect the key in any error field. throw new AgentCoreError("Sandbox configuration could not be confirmed. Refresh before submitting again.", error instanceof AgentCoreError ? error.status : 0, "sandbox_configuration_unconfirmed"); From b9b13b186089b317db271ab83b10f86b48168f66 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 20:35:47 +0000 Subject: [PATCH 03/16] feat(core): retain sandbox generations and apply verified E2B changes online --- CONTRIBUTING.md | 40 ++- contracts/agents-api/admin-api.md | 6 + contracts/agents-api/core-errors.md | 17 + contracts/agents-api/core.openapi.yaml | 57 ++- contracts/agents-api/harness-selection.md | 4 +- contracts/agents-api/sandbox-deployment.md | 102 +++++- .../agents-client/src/sandbox-client.test.ts | 6 + packages/agents-client/src/sandbox-client.ts | 2 + services/agents-api/HOSTED-SANDBOX-MANAGER.md | 31 +- .../cmd/server/managed_generations.go | 201 +++++++++++ .../cmd/server/managed_generations_test.go | 105 ++++++ .../agents-api/cmd/server/managed_setup.go | 17 +- .../server/managed_setup_preflight_test.go | 12 +- .../cmd/server/managed_setup_test.go | 11 + services/agents-api/internal/api/errors.go | 9 + .../api/sandbox_deployment_changes_test.go | 1 + .../internal/api/sandbox_deployment_setup.go | 32 +- .../db/queries/runtime_allocations.sql | 4 +- .../internal/db/queries/runtime_nodes.sql | 24 +- .../db/queries/sandbox_deployment_setup.sql | 2 +- .../db/queries/sandbox_generations.sql | 22 ++ .../internal/db/queries/sandbox_reset.sql | 24 +- .../db/sqlc/initial_environment_files.sql.go | 6 +- .../agents-api/internal/db/sqlc/models.go | 23 +- .../db/sqlc/runtime_allocations.sql.go | 38 +- .../db/sqlc/runtime_lifecycle_nodes.sql.go | 3 +- .../internal/db/sqlc/runtime_nodes.sql.go | 64 ++-- .../db/sqlc/runtime_suspension.sql.go | 3 +- .../db/sqlc/sandbox_deployment_setup.sql.go | 2 +- .../db/sqlc/sandbox_generations.sql.go | 100 ++++++ .../internal/db/sqlc/sandbox_reset.sql.go | 28 +- .../internal/execution/runtime_manager.go | 10 + .../execution/sandbox_deployment_setup.go | 2 + .../execution/sandbox_deployment_switch.go | 33 +- .../execution/sandbox_generations_test.go | 113 ++++++ .../internal/sandbox/e2b/credential.go | 44 +++ .../agents-api/internal/sandbox/e2b/fence.go | 64 ++++ .../internal/sandbox/e2b/process.go | 10 +- .../internal/sandbox/e2b/process_test.go | 65 ++++ .../internal/sandbox/e2b/provider.go | 2 + .../internal/store/runtime_allocations.go | 7 +- .../internal/store/runtime_node_types.go | 38 +- .../internal/store/runtime_nodes.go | 33 +- .../store/runtime_worker_recovery_test.go | 4 +- .../store/sandbox_deployment_mutations.go | 41 ++- .../store/sandbox_deployment_setup.go | 4 + .../internal/store/sandbox_generations.go | 151 ++++++++ .../store/sandbox_generations_test.go | 334 ++++++++++++++++++ .../internal/store/sandbox_reset.go | 3 + .../migrations/000080_sandbox_generations.sql | 67 ++++ .../agents-api/tools/e2b-provider/README.md | 17 +- .../tools/e2b-provider/credential_test.py | 68 ++++ .../agents-api/tools/e2b-provider/provider.py | 59 +++- services/agents-api/tools/e2b-provider/sdk.py | 29 +- 54 files changed, 2031 insertions(+), 163 deletions(-) create mode 100644 services/agents-api/cmd/server/managed_generations.go create mode 100644 services/agents-api/cmd/server/managed_generations_test.go create mode 100644 services/agents-api/internal/db/queries/sandbox_generations.sql create mode 100644 services/agents-api/internal/db/sqlc/sandbox_generations.sql.go create mode 100644 services/agents-api/internal/execution/sandbox_generations_test.go create mode 100644 services/agents-api/internal/sandbox/e2b/credential.go create mode 100644 services/agents-api/internal/sandbox/e2b/fence.go create mode 100644 services/agents-api/internal/store/sandbox_generations.go create mode 100644 services/agents-api/internal/store/sandbox_generations_test.go create mode 100644 services/agents-api/migrations/000080_sandbox_generations.sql create mode 100644 services/agents-api/tools/e2b-provider/credential_test.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d9fc0c533..8d7bd92ba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -758,7 +758,7 @@ inspection enforce the declared CPU/memory and exact image. E2B setup verifies t exact ready template build and matching CPU/memory through the pinned SDK before saving its encrypted account key, and records the build as read for the safe view; an omitted E2B `resources` adopts that build's CPU and memory. Loading a committed selection reconstructs its -provider from the original credentials and receipts without repeating candidate +provider from its owned generation, current committed credential and receipts without repeating candidate template validation; a template endpoint outage must not block cleanup of existing sandboxes. Creation and instance inspection still enforce the saved resources. E2B uses direct placement without a node. Node @@ -767,9 +767,9 @@ OCI manifest digest, microsandbox image reference, Runtime and firmware hashes. These identities are distinct and cannot substitute for each other. Startup claims the stable installation identity and a new owner epoch before -provider selection. The existing runtime manager loads one immutable generation. +provider selection. The runtime manager retains generation-aware provider facades. Initial setup and replacement prepare and validate candidates before database -writes. Rejected candidates preserve the active configuration and workers. A clean +writes. Rejected candidates preserve the active configuration and workers. A node replacement uses the existing mutation gate, pauses manager admission, drains old calls and loops, then repeats the resource/generation guards in the commit transaction. A changed selection, its generation and retirement of old nodes and @@ -791,10 +791,40 @@ read. All sandbox writes require the observed generation, including initial POST at zero; reject stale state before provider preparation, reset or no-op checks, and repeat it under the committing row lock. Same-provider PUT currently requires no -reset and zero held resources. A different backend or unspecified old selection -requires reset. No online rollout is implemented at this stage; omit that projection. +reset. Node PUT keeps the zero-held-resource guard until the complete multi-generation +node protocol ships. E2B changes apply online without retiring nodes, tokens or the +owner epoch. A different backend, E2B team or unspecified old selection requires reset. Preserve historical allocation ownership and placement, never migrate a Session. +Persist immutable allocation and placement deployment generations, distinct from +compute generations. Node allocations copy their placement; E2B binds at reservation. +Keep superseded specification/build rows without credentials while current, owned by +an unreleased allocation/placement, or pinned by any nonremoved node. A durable node +serving pin survives offline state and zero resources. GC uses the deployment lock +and bounded pages; reset clears pins only after release. Never downgrade facts needed +for generation routing. Node readiness and no-gap placement remain PR-N work; do not +relax the node PUT guard or introduce half a protocol. + +Use one E2B classifier. Omitted key preserves the current key; identical selection +with omitted key is a no-op. Explicit key submission, including identical bytes, +verifies and increments generation. Check the candidate and retained builds, paginated +team-owned template membership and every settled live receipt under the candidate key. +Public template readability is not team ownership. Missing/unsettled receipts and +unconfirmed reads reject. Fence credential commits against all provider calls and +actual helper subprocess completion after cancellation, then reverify. Helper exit +never settles unknown remote Create. Bounded failure keeps the original key and +lifecycles. Resolve each allocation's immutable specification and current key in one +snapshot; no stale-key cache, fallback to current spec, or provider-map unloading. +Audit the committed change without secret fields. + +Project rollout and reset from the same database snapshot. Old retained resources +alone never imply preparation or high-frequency polling. Rollout state is settled +unless actual target preparation is active. Offline/unconfirmed nodes are unknown; +only exact current connection/epoch/generation readiness is ready. A durable pin is +not connectivity. Fixed diagnostics alone may explain failed preparation. PR-G reports +no synthetic preparing state; old online v1 nodes can be update_required while their +valid pin remains available to the eventual PR-N placement contract. + Reset is durable execution state, advanced by the existing manager outside its counted work. Serialize start, escalation, cancel, setup, update and finalization through the mutation gate. Auto waits only for in-progress/waiting root/subagent diff --git a/contracts/agents-api/admin-api.md b/contracts/agents-api/admin-api.md index aafe9d933..d81275c99 100644 --- a/contracts/agents-api/admin-api.md +++ b/contracts/agents-api/admin-api.md @@ -198,3 +198,9 @@ Sandbox deployment reset records `reset_start`, explicit `reset_force`, automati Background archives retain the reset requester's credential/actor/request/trace provenance and recover each Session's real Project scope. Audit failure rolls back the corresponding state transition. Cancel does not undo an archive already committed. + +Online E2B deployment updates audit `change`, or `replace_credential` when an API +key is explicitly supplied (including the existing key), under resource type +`sandbox_deployment` and the installation ID. The audit and generation/credential +write commit together; verification failures and omitted-key no-ops produce no +mutation audit. No credential, request body or provider response text is recorded. diff --git a/contracts/agents-api/core-errors.md b/contracts/agents-api/core-errors.md index 38210a623..2172e3b95 100644 --- a/contracts/agents-api/core-errors.md +++ b/contracts/agents-api/core-errors.md @@ -58,3 +58,20 @@ Existing operation-specific codes remain documented in the [administrator contract](admin-api.md), [sandbox deployment contract](sandbox-deployment.md), [executor credential contract](environment-executor-credentials.md) and related resource contracts. This change does not reclassify their validation failures. + +## E2B online changes + +The same-provider deployment PUT uses fixed safe errors. No provider response +message, template name, key or unlisted-resource count is returned in details. + +| HTTP | Code | Meaning | Param | +| --- | --- | --- | --- | +| 400 | `e2b_api_key_invalid` | Provider explicitly rejected authentication | `e2b.api_key` | +| 400 | `e2b_template_build_invalid` | Candidate immutable build is invalid or does not match resources | `e2b.template` | +| 409 | `e2b_team_mismatch` | Candidate key does not prove ownership/manageability of the retained deployment | `e2b.api_key` | +| 503 | `e2b_request_unconfirmed` | Verification, receipt settlement or bounded credential fencing could not be confirmed | null | + +Missing or unsettled Create receipts are uncertainty, never evidence of a different +team or released compute. The typed client projects these codes to fixed local +messages. It also projects `409 sandbox_configuration_error` to fixed public-URL +guidance, even when a PUT omitted its key; arbitrary upstream text is never echoed. diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml index 4ca1b429d..00dc1bbbe 100644 --- a/contracts/agents-api/core.openapi.yaml +++ b/contracts/agents-api/core.openapi.yaml @@ -939,6 +939,8 @@ definitions: x-nullable: true resources: $ref: '#/definitions/store.SandboxDeploymentResources' + rollout: + $ref: '#/definitions/store.SandboxRollout' specification: $ref: '#/definitions/sandbox.DeploymentSpec' specification_digest: @@ -1003,6 +1005,8 @@ definitions: type: integer retained: type: integer + rollout: + $ref: '#/definitions/store.SandboxNodeRollout' running: type: integer snapshots: @@ -1018,6 +1022,8 @@ definitions: x-nullable: true created_at: type: string + deployment_generation: + type: integer diagnostic: type: string environment_id: @@ -1093,6 +1099,8 @@ definitions: type: integer retained: type: integer + rollout: + $ref: '#/definitions/store.SandboxNodeRollout' running: type: integer snapshots: @@ -1156,6 +1164,24 @@ definitions: - $ref: '#/definitions/store.SandboxE2BTemplateBuildView' description: The fixed template build as Core read it when this selection was saved. type: object + store.SandboxNodeRollout: + properties: + diagnostic: + type: string + ready_generation: + description: Durable serving-generation pin; online and provider_ready still gate placement. + type: integer + x-nullable: true + state: + description: Target preparation, independent of an old pin's serving readiness. + enum: + - ready + - preparing + - failed + - update_required + - unknown + type: string + type: object store.SandboxResetOfflineNode: properties: name: @@ -1213,6 +1239,33 @@ definitions: requested_at: type: string type: object + store.SandboxRollout: + properties: + nodes: + allOf: + - $ref: '#/definitions/store.SandboxRolloutNodes' + x-nullable: true + previous_generation_sandboxes: + type: integer + state: + enum: + - settled + - preparing + type: string + type: object + store.SandboxRolloutNodes: + properties: + failed: + type: integer + preparing: + type: integer + ready: + type: integer + unknown: + type: integer + update_required: + type: integer + type: object store.SandboxSuspensionView: properties: idle_seconds: @@ -5799,7 +5852,7 @@ paths: put: consumes: - application/json - description: Requires the current generation, the same backend type, no active reset and verified cleanup of all old resources. A backend type change requires reset. Credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory. A core_url member is rejected with 400; the address comes from the installation public URL. Historical records are retained; old node credentials and enrollments are retired. Never automatically retry an uncertain write. + description: 'Requires the observed generation, the same backend type and no active reset. E2B same-team changes apply online: allocations retain immutable generation and current credentials; omitted api_key preserves it, explicit submission including the same key verifies and advances generation. Other teams require explicit reset. Node providers retain the zero-resource guard and retire old nodes/tokens on change. Core rejects core_url input. Never automatically replay an uncertain write; rollout.state is the authoritative preparation polling signal.' parameters: - description: Replacement deployment selection in: body @@ -5836,7 +5889,7 @@ paths: $ref: '#/definitions/api.CoreErrorResponse' security: - DeploymentAdminAuth: [] - summary: Change a fully drained deployment's sandbox configuration + summary: Change the sandbox deployment configuration tags: - Sandbox Manager /core/v1/sandbox/deployment/reset: diff --git a/contracts/agents-api/harness-selection.md b/contracts/agents-api/harness-selection.md index 4b7b5870f..341f0ecd3 100644 --- a/contracts/agents-api/harness-selection.md +++ b/contracts/agents-api/harness-selection.md @@ -55,8 +55,8 @@ another harness does not choose another image or backend; the selected Runtime image must contain and qualify each enabled harness. Capability checks still apply. -Same-provider resource/Runtime edits currently require no reset and verified zero -held resources. Changing provider requires explicit durable reset, confirmed +Node resource/Runtime edits still require no reset and verified zero held resources. +E2B same-team changes apply online with immutable per-allocation generations. Changing provider requires explicit durable reset, confirmed cleanup and a new setup at the resulting generation. Existing Sessions never move providers. See the [deployment procedure](../../services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-reset). diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index 733134705..4bf263884 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -21,7 +21,7 @@ for the operator workflow. Generated schemas cover the | --- | --- | --- | | `GET /core/v1/sandbox/deployment` | Core key | Read the safe active configuration and retained-resource counts | | `POST /core/v1/sandbox/deployment` | Core key | Select the initial provider, resources and Runtime | -| `PUT /core/v1/sandbox/deployment` | Core key | Update the same provider at zero retained and pending resources | +| `PUT /core/v1/sandbox/deployment` | Core key | Update E2B online; node providers still require zero retained and pending resources | | `POST /core/v1/sandbox/deployment/reset` | Core key | Start or escalate a durable hosted clear | | `DELETE /core/v1/sandbox/deployment/reset?expected_generation=N` | Core key | Cancel the remaining clear without restoring archived work | | `GET /api/v1/sandbox-node/configuration` | Enrollment token or retained node credential | Read the active node installation configuration without consuming enrollment | @@ -74,7 +74,7 @@ resource and same-selection conditions, including an identical old request body. | `provider` | Exactly one of `docker`, `microsandbox`, `e2b` | | `resources` | Per-sandbox resource limits described below; required for Docker/microsandbox, optional for E2B | | `runtime` | Required immutable distribution identity for Docker/microsandbox; absent for E2B | -| `e2b` | Required only for E2B: write-only `api_key` and immutable `template` build selector | +| `e2b` | Required only for E2B: immutable `template` build selector; write-only `api_key` required on POST, optional on same-provider PUT | The request has no Core address. Core derives the deployment's `core_url` from the installation public URL (`public_url` in `config.json`, `OAC_PUBLIC_URL` for @@ -148,14 +148,14 @@ before registration and retains the exact local image identity it imports. E2B instead uses `e2b.template` in `template-id:build-uuid` form. The build UUID must be canonical and nonzero; a mutable template alias alone is insufficient. Omit `runtime`. The API key is encrypted in PostgreSQL and never returned in a safe -view, bootstrap configuration, command argument or log. Replacing the key or -build uses the same zero-resource update as changing resources. +view, bootstrap configuration, command argument or log. Same-team key or build +changes apply online while old sandboxes retain their original specification. ## Safe response GET and successful mutations return `installation_id`, `provider`, `core_url` (read-only: the installation public URL, present before configuration), `mode`, -`generation`, `owner_epoch`, `reset`, `suspension` and resource +`generation`, `owner_epoch`, `reset`, `rollout`, `suspension` and resource accounting. A configured deployment also returns `specification` and `specification_digest`. E2B returns only `e2b.template`, `e2b.credential_configured` and `e2b.template_build`; the `e2b` object is absent @@ -168,7 +168,8 @@ the values describe the immutable build at selection time. Validation admits onl a `ready` build whose CPU count and memory equal the selected `cpus` and `memory_mib`. `root_disk_mib` is the build's native disk size, which Core does not enforce separately. Unknown values are null, including every value of a selection -saved before Core recorded them; re-saving the selection records them. +saved before Core recorded them; a verified write records them. An omitted-key +identical PUT is a no-op and does not refresh provider metadata. `suspension` is `{idle_seconds, retention_seconds}` for microsandbox, the only provider Core suspends (currently 300 and 86400). Docker, E2B and unconfigured @@ -203,15 +204,86 @@ Missing prerequisites or failed preparation leave the committed provider intact. A different backend, or an old node selection without a specification, requires reset first. POST also initializes after a completed reset, using its new generation. -PUT currently accepts only the same provider, no active reset and zero unreleased -allocations plus pending hosted Environments. Send the complete selection and the -observed generation once. Core prepares the candidate, drains manager calls outside -any database transaction, repeats guards under the deployment lock, and commits a -changed selection, generation and node/token retirement together. A no-op keeps its -generation. Failed final writes restore the committed provider using the owner's -bounded recovery context before releasing mutation ownership. If recovery fails, -the owner stops with admission fenced. There is no online rollout in this contract; -responses omit `rollout` until that feature exists. +PUT accepts the same provider and no active reset. Send the observed generation +once; never replay an uncertain mutation automatically. E2B changes apply online: +new allocations use the newly committed specification and existing allocations +retain their immutable deployment generation. No node, token or owner epoch is +retired by an E2B update. Docker/microsandbox still require zero held resources, +prepare and drain before commit, and retire old nodes/tokens. Their online update +protocol is not implemented yet. + +For E2B PUT, omit `e2b.api_key` to preserve the current key. Omitted-key identical +selection is a no-op. Explicit nonempty key submission, including the same key, +always verifies and advances generation. Null or empty keys are invalid. A key-only +change uses the same full DTO: provider, existing template, optional resources and +expected_generation, plus the new api_key. It has no separate route or implicit reset. + +Verification reads the candidate and every retained build with the candidate key, +proves ownership using the SDK's paginated team-template listing, and confirms each +settled live receipt in the installation-labelled sandbox listing. Public template +readability alone is insufficient. Missing or unsettled receipts and unconfirmed +reads fail closed. Different-team ownership gives `409 e2b_team_mismatch`; explicitly +reset before initializing another team. Provider 401/403 gives `400 e2b_api_key_invalid`; +an invalid candidate build gives `400 e2b_template_build_invalid`; an unconfirmed +verification gives `503 e2b_request_unconfirmed`. No provider text or credential is +returned. The write and `change` or `replace_credential` audit share one transaction. + +A credential replacement briefly fences provider calls, waits for actual helper +process completion even after caller cancellation, and repeats verification before +commit. Helper exit is not evidence that remote Create settled. The fence and reads +are bounded; failure preserves the old key and lifecycles. After the successful +response, all retained-generation management uses the committed key. Only then +revoke the previous key in E2B. Template/resource changes do not drain lifecycles. + +### Generation ownership and rollout + +`runtime_deployment` owns the current specification. Superseded rows contain only +immutable specification/build metadata, never another E2B credential. An E2B +allocation binds its generation at reservation. Node placements bind at Session +admission and allocations copy that generation, even after repeated updates. +Inspection, renewal, command execution and cleanup route the allocation's original +specification with the current credential; a missing generation never falls back +to the current specification. Released historical generation identifiers remain. + +Retain a generation while it is current, referenced by an unreleased allocation or +placement, or pinned by a nonremoved node. The durable node serving pin survives +offline state and zero resources; it is separate from current connection readiness. +Collection shares the deployment lock with updates and admission and deletes at +most 32 eligible generation rows per pass. Reset retires pins and clears superseded +rows only after confirmed resource release. Downgrade refuses ownership or serving +pins that still need generation routing. + +Every deployment response includes: + +```json +"rollout": {"state":"settled", "previous_generation_sandboxes":3, + "nodes":null} +``` + +The previous count uses the same snapshot as resource totals: old-generation +unreleased allocations plus old-generation pending placements without allocations, +never counting one resource twice. E2B and unconfigured deployments return null +nodes. A node deployment returns counts `{ready,preparing,failed,update_required,unknown}`. +Every nonremoved node belongs to exactly one bucket. Offline or unconfirmed current +connections are `unknown` first; an online v1 node enrolled at an older target is +`update_required`; online exact-generation provider readiness is `ready`; an online +reported diagnostic is `failed`; otherwise it is `unknown`. The existing 45-second +heartbeat and owner-epoch rules define online. Pins alone never imply readiness. Target rollout is distinct from serving readiness: +unknown/preparing/failed/update_required must not erase independently confirmed old +serving readiness. provider_ready is the last provider report and requires online; +current v1 reports its enrolled provider. Future node protocol must bind serving +readiness to the exact pin and current connection/epoch. Valid PR-G v1 online ready +nodes at the current target project ready; the client also permits an old qualified +pin with unknown target preparation for the subsequent protocol. + +Each node adds `rollout: {state, ready_generation, diagnostic?}`. ready_generation is +the nullable durable serving pin. Diagnostic is a fixed node reason; unknown values +project as `provider_unavailable`. Allocation items add `deployment_generation`. +`rollout.state` is the authoritative high-frequency polling signal: poll every five +seconds only while it is `preparing` or reset is nonnull. Old Sessions, failed, +update-required and offline nodes alone do not keep polling active. PR-G reports +`settled`: actual node multi-generation preparation and no-gap placement remain +PR-N work, with the zero-resource node PUT guard intact. To change backend, explicitly start reset: diff --git a/packages/agents-client/src/sandbox-client.test.ts b/packages/agents-client/src/sandbox-client.test.ts index 4c7fc0fcc..1ec2a3f87 100644 --- a/packages/agents-client/src/sandbox-client.test.ts +++ b/packages/agents-client/src/sandbox-client.test.ts @@ -350,3 +350,9 @@ it("keeps omitted-key public-URL errors actionable without reflecting a stored k expect(error.details).toBeUndefined(); expect(fetch).toHaveBeenCalledTimes(1); }); + +it.each(["unknown", "preparing", "failed", "update_required"])("does not erase old serving readiness for target %s", async (state) => { + const value = { ...node, rollout: { state, ready_generation: 1, ...(state === "failed" ? { diagnostic: "runtime_image_unavailable" } : {}) } }; + const client = new SandboxAdminClient({ fetch: async () => response({ data: [value] }) }); + expect((await client.listNodes()).data[0]).toEqual(value); +}); diff --git a/packages/agents-client/src/sandbox-client.ts b/packages/agents-client/src/sandbox-client.ts index f8c00f198..8a6dbb4fd 100644 --- a/packages/agents-client/src/sandbox-client.ts +++ b/packages/agents-client/src/sandbox-client.ts @@ -42,6 +42,7 @@ export interface SandboxRollout { nodes: { ready: number; preparing: number; failed: number; update_required: number; unknown: number } | null; } export interface SandboxNodeRollout { + /** Target preparation; unknown/failed/preparing does not invalidate a qualified old serving pin. */ state: "ready" | "preparing" | "failed" | "update_required" | "unknown"; /** Durable serving pin; this alone does not imply current connection readiness. */ ready_generation: number | null; @@ -87,6 +88,7 @@ export interface SandboxNode { name: string; provider: string; online: boolean; + /** Last provider report; combine with online. Target rollout state is independent of serving readiness. */ provider_ready: boolean; /** Absent while the provider is ready. */ diagnostic?: SandboxNodeDiagnostic; diff --git a/services/agents-api/HOSTED-SANDBOX-MANAGER.md b/services/agents-api/HOSTED-SANDBOX-MANAGER.md index c6136a186..76ec82a44 100644 --- a/services/agents-api/HOSTED-SANDBOX-MANAGER.md +++ b/services/agents-api/HOSTED-SANDBOX-MANAGER.md @@ -64,14 +64,13 @@ mode `0700`. The standard distribution prepares both. Back up this private state with the database and credential-encryption key; losing it can leave an uncertain allocation that cannot safely be reclaimed. Do not mount it into Web or Runtime. -Drain and confirm cleanup before revoking the configured E2B account key. Replacing -that key currently uses the same zero-resource configuration guard; in-place key -rotation with retained resources is not supported. If the key is revoked early, -Core keeps unverifiable allocations and blocks switching, even if compute was -removed through the provider console. Do not clear database allocations or private -receipts to bypass this check. A future credential-repair operation must verify -account/resource ownership before accepting a replacement key; an inaccessible -sandbox or empty listing from another account is not proof of cleanup. +Replace an E2B key through the full deployment PUT, using the current template and +observed generation. Core verifies team ownership and all retained resources before +committing it. Omit api_key to preserve it; explicitly supplying the same key still +verifies and advances generation. Revoke the old key only after a successful response. +Missing or unsettled receipts, another team, or unconfirmed provider reads reject +without changing the active key. Do not clear allocations or receipts to bypass this. +See [generation and credential guarantees](../../contracts/agents-api/sandbox-deployment.md#generation-ownership-and-rollout). For own machines, **Add node** generates a one-time command that uses the installation public URL. Before installing, the node installer reads the active @@ -276,11 +275,17 @@ an unreachable Core or a 403 from a proxy in front of it, restart the service ev 5 seconds without a start limit. Ordinary disconnects and host restarts reuse the original identity. -Same-provider resource/Runtime or E2B key/build edits currently require zero -unreleased allocations and pending hosted Environments, no reset and the current -generation. PUT the complete selection once. Core validates before draining, then -rechecks and commits the generation and node/token retirement together. A failed -commit recovers the previous provider. There is no online rollout in this release. +E2B same-team key/build/resource edits apply online; existing sandboxes retain their +original generation and new allocations use the new selection. Docker/microsandbox +resource/Runtime edits still require zero unreleased allocations and pending hosted +Environments, no reset and the observed generation. Their multi-generation preparation +protocol remains future work. Submit once and read back after uncertain responses. + +Deployment rollout reports old-generation resource counts separately from preparation. +Poll frequently only while reset is active or rollout.state is preparing. E2B updates +settle immediately; old Sessions can remain indefinitely without implying preparation. +Offline node rollout is unknown, and ready_generation is a durable pin rather than +proof that the node is online. Node update_required and failed states are settled. Changing backend requires an explicit reset. POST `/core/v1/sandbox/deployment/reset` with `expected_generation` and `clear: auto` diff --git a/services/agents-api/cmd/server/managed_generations.go b/services/agents-api/cmd/server/managed_generations.go new file mode 100644 index 000000000..484c3126b --- /dev/null +++ b/services/agents-api/cmd/server/managed_generations.go @@ -0,0 +1,201 @@ +package main + +import ( + "context" + "errors" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/runtimeobs" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/e2b" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +type generationStore interface { + GetSandboxAllocationSetup(context.Context, sandbox.Reference) (store.SandboxSetup, error) + GetSandboxSetup(context.Context) (store.SandboxSetup, error) + SandboxGenerationPage(context.Context, int64) ([]store.SandboxSetup, error) + SandboxCredentialAllocationPage(context.Context, string) ([]store.RuntimeAllocation, error) +} + +// Every facade, including already running lifecycles, resolves the allocation's +// immutable specification and current credential. There is no mutable provider +// map to unload and no current-generation fallback for a missing historical row. +type e2bGenerationRouter struct { + setup *managedSetup + store generationStore +} + +func (p *e2bGenerationRouter) route(ctx context.Context, r sandbox.Reference) (sandbox.Provider, func(), error) { + release, err := p.setup.e2bCalls.Enter(ctx) + if err != nil { + return nil, nil, err + } + setup, err := p.store.GetSandboxAllocationSetup(ctx, r) + if err != nil { + release() + return nil, nil, err + } + provider, err := p.setup.provider(setup) + if err != nil { + release() + return nil, nil, err + } + return provider, release, nil +} +func (p *e2bGenerationRouter) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { + v, done, err := p.route(ctx, b.Reference) + if err != nil { + return sandbox.Info{}, err + } + defer done() + return v.Create(ctx, b) +} +func (p *e2bGenerationRouter) GetInfo(ctx context.Context, r sandbox.Reference) (sandbox.Info, error) { + v, done, err := p.route(ctx, r) + if err != nil { + return sandbox.Info{}, err + } + defer done() + return v.GetInfo(ctx, r) +} +func (p *e2bGenerationRouter) Renew(ctx context.Context, r sandbox.Reference) (sandbox.Info, error) { + v, done, err := p.route(ctx, r) + if err != nil { + return sandbox.Info{}, err + } + defer done() + return v.Renew(ctx, r) +} +func (p *e2bGenerationRouter) Kill(ctx context.Context, r sandbox.Reference) error { + v, done, err := p.route(ctx, r) + if err != nil { + return err + } + defer done() + return v.Kill(ctx, r) +} +func (p *e2bGenerationRouter) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { + v, done, err := p.route(ctx, r) + if err != nil { + return sandbox.CommandResult{}, err + } + defer done() + return v.RunCommand(ctx, r, c) +} +func (p *e2bGenerationRouter) Observe(ctx context.Context, t runtimeobs.Target) (runtimeobs.Sample, error) { + v, done, err := p.route(ctx, sandbox.Reference{TenantID: t.TenantID, EnvironmentID: t.EnvironmentID, AllocationID: t.Instance.AllocationID}) + if err != nil { + return runtimeobs.Sample{}, err + } + defer done() + return v.(*e2b.Provider).Observe(ctx, t) +} + +func (s *managedSetup) routeGenerations(candidate execution.PreparedRuntimeDeployment, setup store.SandboxSetup) (execution.PreparedRuntimeDeployment, error) { + if setup.Provider != "e2b" { + return candidate, nil + } + db, ok := s.store.(generationStore) + if !ok { + return execution.PreparedRuntimeDeployment{}, errors.New("sandbox generation store is unavailable") + } + candidate.Config.Provider = &e2bGenerationRouter{setup: s, store: db} + candidate.FenceCredential = func(ctx context.Context) (func(), error) { + release, err := s.e2bCalls.Fence(ctx) + if err != nil { + return nil, e2b.ErrRequestUnconfirmed + } + return release, nil + } + candidate.VerifyCredential = func(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + // The candidate's template must itself belong to the same accessible team. + verify := func(value store.SandboxSetup, refs []sandbox.Reference) error { + value.InstallationID = setup.InstallationID + value.E2B = &store.SandboxE2BConfiguration{APIKey: setup.E2B.APIKey, Template: value.E2B.Template} + provider, err := s.provider(value) + if err != nil { + return err + } + return provider.(*e2b.Provider).VerifyCredential(ctx, refs) + } + if err := verify(setup, nil); err != nil { + return err + } + current, err := db.GetSandboxSetup(ctx) + if err != nil { + return err + } + if err := verify(current, nil); err != nil { + return err + } + for after := int64(-1); ; { + page, err := db.SandboxGenerationPage(ctx, after) + if err != nil { + return err + } + for _, g := range page { + if err := verify(g, nil); err != nil { + return err + } + after = int64(g.Generation) + } + if len(page) < 32 { + break + } + } + for after := ""; ; { + page, err := db.SandboxCredentialAllocationPage(ctx, after) + if err != nil { + return err + } + refs := make([]sandbox.Reference, 0, len(page)) + for _, a := range page { + refs = append(refs, sandbox.Reference{TenantID: a.TenantID, EnvironmentID: a.EnvironmentID, AllocationID: a.ID}) + after = a.ID + } + if len(refs) > 0 { + if err := verify(setup, refs); err != nil { + return err + } + } + if len(page) < 32 { + break + } + } + return nil + } + return candidate, nil +} + +// Metrics do not qualify a sandbox against the current build. The helper reads +// only labelled receipts and provider metrics, so one batch can span generations. +func (p *e2bGenerationRouter) ObserveBatch(ctx context.Context, targets []runtimeobs.Target) ([]runtimeobs.BatchResult, bool) { + result := make([]runtimeobs.BatchResult, len(targets)) + failed := func(err error) ([]runtimeobs.BatchResult, bool) { + for i := range result { + result[i].Err = err + } + return result, true + } + if len(targets) > runtimeobs.MaxBatchTargets { + return failed(sandbox.ErrInvalid) + } + release, err := p.setup.e2bCalls.Enter(ctx) + if err != nil { + return failed(err) + } + defer release() + setup, err := p.store.GetSandboxSetup(ctx) + if err != nil { + return failed(err) + } + provider, err := p.setup.provider(setup) + if err != nil { + return failed(err) + } + return provider.(*e2b.Provider).ObserveBatch(ctx, targets) +} diff --git a/services/agents-api/cmd/server/managed_generations_test.go b/services/agents-api/cmd/server/managed_generations_test.go new file mode 100644 index 000000000..d5d3643e4 --- /dev/null +++ b/services/agents-api/cmd/server/managed_generations_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" +) + +type routingSetupStore struct { + setupStore + old store.SandboxSetup + oldID string +} + +func (s *routingSetupStore) GetSandboxAllocationSetup(_ context.Context, ref sandbox.Reference) (store.SandboxSetup, error) { + value := s.value + if ref.AllocationID == s.oldID { + value = s.old + key := *value.E2B + key.APIKey = s.value.E2B.APIKey + value.E2B = &key + } + return value, nil +} +func TestE2BRouterKeepsOldSpecificationWithCommittedCredential(t *testing.T) { + state := t.TempDir() + if err := os.Chmod(state, 0700); err != nil { + t.Fatal(err) + } + helper := filepath.Join(t.TempDir(), "helper") + script := `#!/usr/bin/env python3 +import json,sys,pathlib +q=json.load(sys.stdin) +with (pathlib.Path(q['Config']['StateDir'])/'requests').open('a') as f: f.write(json.dumps(q)+'\n') +info=dict(q['Reference'],State='running',ProviderID='owned',CreateSettled=True) +if q['Operation']=='kill': info['State']='absent' +print(json.dumps({'Version':1,'Info':info})) +` + if err := os.WriteFile(helper, []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("OAC_E2B_PROVIDER_BIN", helper) + t.Setenv("OAC_E2B_STATE_DIR", state) + id := uuid.NewString() + old := store.SandboxSetup{InstallationID: id, Provider: "e2b", Mode: "direct", Generation: 1, Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}}, E2B: &store.SandboxE2BConfiguration{APIKey: "old-key", Template: "old:" + uuid.NewString()}} + current := old + current.Generation = 2 + current.Specification.Resources.CPUs = 4 + current.E2B = &store.SandboxE2BConfiguration{APIKey: "new-key", Template: "new:" + uuid.NewString()} + ref := sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()} + db := &routingSetupStore{setupStore: setupStore{value: current}, old: old, oldID: ref.AllocationID} + setup := &managedSetup{store: db, installationID: id} + // A facade retained by a generation-one lifecycle still reads current credentials. + router := &e2bGenerationRouter{setup: setup, store: db} + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + if _, err := router.GetInfo(ctx, ref); err != nil { + t.Fatal(err) + } + if _, err := router.Renew(ctx, ref); err != nil { + t.Fatal(err) + } + if err := router.Kill(ctx, ref); err != nil { + t.Fatal(err) + } + next := ref + next.AllocationID = uuid.NewString() + if _, err := router.GetInfo(ctx, next); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(filepath.Join(state, "requests")) + if err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(string(raw)), "\n") + if len(lines) != 4 { + t.Fatal(len(lines)) + } + for i, line := range lines { + var q struct { + Config struct { + APIKey, Template string + Resources sandbox.Resources + } + } + if err := json.Unmarshal([]byte(line), &q); err != nil { + t.Fatal(err) + } + expected := old + if i == 3 { + expected = current + } + if q.Config.APIKey != "new-key" || q.Config.Template != expected.E2B.Template || q.Config.Resources != expected.Specification.Resources { + t.Fatal("generation or credential mismatch", i) + } + } +} diff --git a/services/agents-api/cmd/server/managed_setup.go b/services/agents-api/cmd/server/managed_setup.go index e3d0494fb..11e71a2c4 100644 --- a/services/agents-api/cmd/server/managed_setup.go +++ b/services/agents-api/cmd/server/managed_setup.go @@ -30,6 +30,7 @@ type managedSetup struct { // publicURL is OAC_PUBLIC_URL; every sandbox reaches Core through it. publicURL string selected atomic.Pointer[managedSelection] + e2bCalls e2b.CallFence } // Empty selections retain their generation so a delayed provider load cannot @@ -71,6 +72,9 @@ func (s *managedSetup) load(ctx context.Context) (*execution.RuntimeProvider, er return selected.Config, nil } candidate, err := s.configuration(setup) + if err == nil { + candidate, err = s.routeGenerations(candidate, setup) + } if err != nil { log.Warn(ctx, "Hosted provider is unavailable; administrator recovery remains available", "provider", setup.Provider, "error", err) return nil, fmt.Errorf("%w: %v", execution.ErrExecutionUnavailable, err) @@ -90,10 +94,13 @@ func (s *managedSetup) prepare(ctx context.Context, setup store.SandboxSetup) (e if provider, ok := candidate.Config.Provider.(*e2b.Provider); ok { build, err := provider.ValidateDeployment(ctx) if err != nil { + if errors.Is(err, e2b.ErrCredentialInvalid) { + return execution.PreparedRuntimeDeployment{}, err + } if errors.Is(err, sandbox.ErrInvalid) { - return execution.PreparedRuntimeDeployment{}, &store.SandboxConfigurationError{Message: "E2B configuration was rejected; select a ready fixed template build whose CPU and memory match the deployment specification"} + return execution.PreparedRuntimeDeployment{}, e2b.ErrTemplateInvalid } - return execution.PreparedRuntimeDeployment{}, fmt.Errorf("%w: E2B validation could not be confirmed; verify the helper, credential, network and fixed template build before retrying", execution.ErrExecutionUnavailable) + return execution.PreparedRuntimeDeployment{}, e2b.ErrRequestUnconfirmed } if setup.Specification.Resources == (sandbox.Resources{}) { // Omitted E2B resources take the validated build's CPU and memory. @@ -111,7 +118,7 @@ func (s *managedSetup) prepare(ctx context.Context, setup store.SandboxSetup) (e candidate.E2BTemplateBuild.RootDiskMiB = &disk } } - return candidate, nil + return s.routeGenerations(candidate, setup) } // Loading an already committed selection must retain provider access to its @@ -192,8 +199,8 @@ func (s *managedSetup) provider(setup store.SandboxSetup) (sandbox.Provider, err if setup.Specification.Resources != (sandbox.Resources{}) { resources = &setup.Specification.Resources } - provider, err := e2b.New(e2b.Config{Binary: binary, StateDir: os.Getenv("OAC_E2B_STATE_DIR"), - Resources: resources, InstallationID: setup.InstallationID, APIKey: setup.E2B.APIKey, Template: setup.E2B.Template, TimeoutSeconds: 3600}) + provider, err := e2b.NewWithCaller(e2b.Config{Binary: binary, StateDir: os.Getenv("OAC_E2B_STATE_DIR"), + Resources: resources, InstallationID: setup.InstallationID, APIKey: setup.E2B.APIKey, Template: setup.E2B.Template, TimeoutSeconds: 3600}, &e2b.ProcessCaller{Fence: &s.e2bCalls}) if err != nil { return nil, errors.New("E2B provider cannot load; check the installed helper and private state directory") } diff --git a/services/agents-api/cmd/server/managed_setup_preflight_test.go b/services/agents-api/cmd/server/managed_setup_preflight_test.go index f7a48a07f..d75a2922a 100644 --- a/services/agents-api/cmd/server/managed_setup_preflight_test.go +++ b/services/agents-api/cmd/server/managed_setup_preflight_test.go @@ -30,8 +30,7 @@ func TestE2BRejectedSpecificationHasSafeActionableDiagnostic(t *testing.T) { s := &managedSetup{installationID: id} selection := store.SandboxSetup{InstallationID: id, Provider: "e2b", Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 3, MemoryMiB: 3072}}, E2B: &store.SandboxE2BConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} _, err := s.prepare(t.Context(), selection) - var invalid *store.SandboxConfigurationError - if !errors.As(err, &invalid) || !strings.Contains(invalid.Message, "CPU and memory") || strings.Contains(invalid.Message, "synthetic-private-key") || s.selected.Load() != nil { + if !errors.Is(err, e2b.ErrTemplateInvalid) || strings.Contains(err.Error(), "synthetic-private-key") || s.selected.Load() != nil { t.Fatal("rejected candidate lost its safe diagnostic or was published", err) } s.store = &setupStore{value: selection} @@ -107,7 +106,7 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { t.Setenv("OAC_E2B_PROVIDER_BIN", helper) t.Setenv("OAC_E2B_STATE_DIR", state) id := uuid.NewString() - s := &managedSetup{installationID: id} + s := &managedSetup{installationID: id, store: &setupStore{}} selection := store.SandboxSetup{InstallationID: id, Provider: "e2b", E2B: &store.SandboxE2BConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} candidate, err := s.prepare(t.Context(), selection) disk := int32(24063) @@ -116,7 +115,12 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { t.Fatalf("validated build was not recorded: %+v %v", candidate.E2BTemplateBuild, err) } // The published candidate enforces the adopted resources. - if _, err := candidate.Config.Provider.(*e2b.Provider).ValidateDeployment(t.Context()); err != nil { + selection.Specification.Resources = sandbox.Resources{CPUs: 4, MemoryMiB: 4096} + provider, err := s.provider(selection) + if err != nil { + t.Fatal(err) + } + if _, err := provider.(*e2b.Provider).ValidateDeployment(t.Context()); err != nil { t.Fatal(err) } logged, err := os.ReadFile(requests) diff --git a/services/agents-api/cmd/server/managed_setup_test.go b/services/agents-api/cmd/server/managed_setup_test.go index 127d76b1e..81dac915f 100644 --- a/services/agents-api/cmd/server/managed_setup_test.go +++ b/services/agents-api/cmd/server/managed_setup_test.go @@ -11,6 +11,7 @@ import ( "errors" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/node" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" "github.com/google/uuid" @@ -194,3 +195,13 @@ func TestManagedSetupResetTombstoneRejectsDelayedProviderLoad(t *testing.T) { t.Fatal("reset blocked subsequent configuration") } } + +func (s *setupStore) GetSandboxAllocationSetup(_ context.Context, _ sandbox.Reference) (store.SandboxSetup, error) { + return s.value, nil +} +func (s *setupStore) SandboxGenerationPage(context.Context, int64) ([]store.SandboxSetup, error) { + return nil, nil +} +func (s *setupStore) SandboxCredentialAllocationPage(context.Context, string) ([]store.RuntimeAllocation, error) { + return nil, nil +} diff --git a/services/agents-api/internal/api/errors.go b/services/agents-api/internal/api/errors.go index a0e6df808..d73197dbc 100644 --- a/services/agents-api/internal/api/errors.go +++ b/services/agents-api/internal/api/errors.go @@ -9,6 +9,7 @@ import ( v1 "github.com/MiniMax-AI-Dev/parsar/contracts/agents-api/v1" "github.com/MiniMax-AI-Dev/parsar/internal/obs/log" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/e2b" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" ) @@ -99,6 +100,14 @@ func writeStoreError(w http.ResponseWriter, r *http.Request, err error, notFound var resetRequired *store.SandboxResetRequiredError var inUse *store.SandboxInUseError switch { + case errors.Is(err, e2b.ErrRequestUnconfirmed): + writeError(w, http.StatusServiceUnavailable, "e2b_request_unconfirmed", "E2B verification could not be confirmed.") + case errors.Is(err, e2b.ErrTemplateInvalid): + writeError(w, http.StatusBadRequest, "e2b_template_build_invalid", "Select a ready immutable E2B template build with matching resources.", "e2b.template") + case errors.Is(err, e2b.ErrCredentialInvalid): + writeError(w, http.StatusBadRequest, "e2b_api_key_invalid", "The E2B API key was rejected.", "e2b.api_key") + case errors.Is(err, e2b.ErrTeamMismatch): + writeError(w, http.StatusConflict, "e2b_team_mismatch", "The E2B key cannot manage the retained deployment. Reset before changing teams.", "e2b.api_key") case errors.As(err, &stale): writeCoreError(w, http.StatusConflict, "sandbox_generation_stale", "The sandbox deployment generation changed. Refresh before submitting again.", CoreErrorDetails{"current_generation": CoreErrorNumber(float64(stale.CurrentGeneration))}) case errors.As(err, &resetRequired): diff --git a/services/agents-api/internal/api/sandbox_deployment_changes_test.go b/services/agents-api/internal/api/sandbox_deployment_changes_test.go index 3bdc00238..74a5c4d6a 100644 --- a/services/agents-api/internal/api/sandbox_deployment_changes_test.go +++ b/services/agents-api/internal/api/sandbox_deployment_changes_test.go @@ -44,6 +44,7 @@ func TestSandboxDeploymentChangesAuthenticateAndDecode(t *testing.T) { {"PATCH", "/deployment/maintenance", "caller", `{"maintenance":true,"expected_generation":2}`, 401}, {"PUT", "/deployment", "administrator", strings.Replace(selection, `"api_key"`, `"API_KEY"`, 1), 400}, {"PUT", "/deployment", "administrator", strings.Replace(selection, `"expected_generation":2,`, "", 1), 400}, + {"PUT", "/deployment", "administrator", strings.Replace(selection, `"synthetic-private-key"`, `null`, 1), 400}, {"PUT", "/deployment", "administrator", selection, 200}, {"POST", "/deployment/reset", "administrator", `{"expected_generation":2}`, 400}, {"POST", "/deployment/reset", "caller", `{"clear":"force","expected_generation":2}`, 401}, diff --git a/services/agents-api/internal/api/sandbox_deployment_setup.go b/services/agents-api/internal/api/sandbox_deployment_setup.go index 3b31b22b4..f547d431e 100644 --- a/services/agents-api/internal/api/sandbox_deployment_setup.go +++ b/services/agents-api/internal/api/sandbox_deployment_setup.go @@ -14,8 +14,8 @@ import ( // SandboxE2BInput is write-only provider configuration. Safe responses use the // store's separate deployment view and never serialize this request. type SandboxE2BInput struct { - APIKey string `json:"api_key"` - Template string `json:"template"` + APIKey *string `json:"api_key,omitempty"` + Template string `json:"template"` } type SandboxDeploymentInput struct { @@ -35,7 +35,11 @@ type SandboxDeploymentChangeInput struct { func (v SandboxDeploymentInput) request() store.SandboxDeploymentSetupRequest { input := store.SandboxDeploymentSetupRequest{ExpectedGeneration: *v.ExpectedGeneration, Provider: v.Provider, DeploymentSpec: sandbox.DeploymentSpec{Resources: v.Resources, Runtime: v.Runtime}} if v.E2B != nil { - input.E2B = &store.SandboxE2BConfiguration{APIKey: v.E2B.APIKey, Template: v.E2B.Template} + input.E2B = &store.SandboxE2BConfiguration{Template: v.E2B.Template} + if v.E2B.APIKey != nil { + input.E2B.APIKey = *v.E2B.APIKey + input.E2B.ReplaceCredential = true + } } return input } @@ -85,7 +89,7 @@ func (h *Handler) initializeSandboxDeployment(w http.ResponseWriter, r *http.Req return } var input SandboxDeploymentInput - if decodeInputObject(raw, &input, "provider", "e2b", "resources", "runtime", "expected_generation") != nil || input.ExpectedGeneration == nil { + if decodeInputObject(raw, &input, "provider", "e2b", "resources", "runtime", "expected_generation") != nil || input.ExpectedGeneration == nil || nullSandboxKey(raw) { writeStoreError(w, r, store.ErrInvalidInput) return } @@ -101,8 +105,8 @@ func (h *Handler) initializeSandboxDeployment(w http.ResponseWriter, r *http.Req writeJSON(w, http.StatusOK, result) } -// @Summary Change a fully drained deployment's sandbox configuration -// @Description Requires the current generation, the same backend type, no active reset and verified cleanup of all old resources. A backend type change requires reset. Credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory. A core_url member is rejected with 400; the address comes from the installation public URL. Historical records are retained; old node credentials and enrollments are retired. Never automatically retry an uncertain write. +// @Summary Change the sandbox deployment configuration +// @Description Requires the observed generation, the same backend type and no active reset. E2B same-team changes apply online: allocations retain immutable generation and current credentials; omitted api_key preserves it, explicit submission including the same key verifies and advances generation. Other teams require explicit reset. Node providers retain the zero-resource guard and retire old nodes/tokens on change. Core rejects core_url input. Never automatically replay an uncertain write; rollout.state is the authoritative preparation polling signal. // @Tags Sandbox Manager // @Produce json // @Security DeploymentAdminAuth @@ -120,7 +124,7 @@ func (h *Handler) updateSandboxDeployment(w http.ResponseWriter, r *http.Request return } var input SandboxDeploymentChangeInput - if decodeInputObject(raw, &input, "provider", "e2b", "resources", "runtime", "expected_generation") != nil || input.ExpectedGeneration == nil { + if decodeInputObject(raw, &input, "provider", "e2b", "resources", "runtime", "expected_generation") != nil || input.ExpectedGeneration == nil || nullSandboxKey(raw) { writeStoreError(w, r, store.ErrInvalidInput) return } @@ -156,7 +160,7 @@ func (h *Handler) startSandboxReset(w http.ResponseWriter, r *http.Request) { Clear string `json:"clear"` DeadlineSeconds *int32 `json:"deadline_seconds"` } - if decodeInputObject(raw, &input, "expected_generation", "clear", "deadline_seconds") != nil || input.ExpectedGeneration == nil { + if decodeInputObject(raw, &input, "expected_generation", "clear", "deadline_seconds") != nil || input.ExpectedGeneration == nil || nullSandboxKey(raw) { writeError(w, http.StatusBadRequest, "invalid_request_error", "A current expected_generation is required.", "expected_generation") return } @@ -221,3 +225,15 @@ func parseResetGeneration(r *http.Request) (uint64, error) { } return strconv.ParseUint(query.Get("expected_generation"), 10, 64) } + +// Null is an invalid explicit credential, not the omitted-key preservation path. +func nullSandboxKey(raw json.RawMessage) bool { + var body struct { + E2B map[string]json.RawMessage `json:"e2b"` + } + if json.Unmarshal(raw, &body) != nil { + return false + } + key, present := body.E2B["api_key"] + return present && string(key) == "null" +} diff --git a/services/agents-api/internal/db/queries/runtime_allocations.sql b/services/agents-api/internal/db/queries/runtime_allocations.sql index 4e1060ff3..529adb727 100644 --- a/services/agents-api/internal/db/queries/runtime_allocations.sql +++ b/services/agents-api/internal/db/queries/runtime_allocations.sql @@ -1,6 +1,6 @@ -- name: CreateRuntimeAllocation :one -INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, initialization) -VALUES ($1, $2, $3, $4, $5, CASE WHEN EXISTS (SELECT 1 FROM initial_environment_files f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) OR EXISTS (SELECT 1 FROM environment_setups f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) THEN 'pending' ELSE 'complete' END) RETURNING *; +INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation, initialization) +VALUES ($1, $2, $3, $4, $5, $6, CASE WHEN EXISTS (SELECT 1 FROM initial_environment_files f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) OR EXISTS (SELECT 1 FROM environment_setups f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) THEN 'pending' ELSE 'complete' END) RETURNING *; -- name: GetRuntimeAllocation :one SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired diff --git a/services/agents-api/internal/db/queries/runtime_nodes.sql b/services/agents-api/internal/db/queries/runtime_nodes.sql index 6e9d6ff34..efe1503c8 100644 --- a/services/agents-api/internal/db/queries/runtime_nodes.sql +++ b/services/agents-api/internal/db/queries/runtime_nodes.sql @@ -5,15 +5,15 @@ UPDATE runtime_deployment SET provider_kind=$1, local_node_id=$2, mode='nodes', SELECT * FROM runtime_deployment WHERE singleton=true; -- name: InsertRuntimeNode :one -INSERT INTO runtime_nodes(id,installation_id,name,backend_fingerprint,credential_sha256,max_active,max_retained,specification_digest,deployment_generation,core_url,enrollment_id) -VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) RETURNING *; +INSERT INTO runtime_nodes(id,installation_id,name,backend_fingerprint,credential_sha256,max_active,max_retained,specification_digest,deployment_generation,core_url,enrollment_id,ready_generation) +VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$9) RETURNING *; -- name: GetRuntimeNode :one SELECT * FROM runtime_nodes WHERE id=$1 AND removed_at IS NULL; -- name: ListRuntimeNodes :many SELECT n.*, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at > clock_timestamp()-interval '45 seconds')::boolean AS online, - d.provider_kind, + d.provider_kind, d.generation AS target_generation, (SELECT count(*) FROM runtime_placements p LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id WHERE p.node_id=n.id AND p.released_at IS NULL AND (a.id IS NULL OR a.compute_phase <> 'suspended'))::bigint AS active, (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL)::bigint AS retained, (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=p.environment_id))::bigint AS reserved, @@ -28,7 +28,7 @@ AND (sqlc.narg(node_id)::uuid IS NULL OR n.id=sqlc.narg(node_id)::uuid) ORDER BY UPDATE runtime_nodes SET name=$2,max_active=$3,max_retained=$4 WHERE id=$1 AND removed_at IS NULL RETURNING *; -- name: RemoveRuntimeNode :exec -UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL WHERE id=$1 AND removed_at IS NULL; +UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL,ready_generation=NULL WHERE id=$1 AND removed_at IS NULL; -- name: ConnectRuntimeNode :execrows UPDATE runtime_nodes SET connection_id=$2,provider_ready=false,connected_epoch=d.owner_epoch,last_seen_at=clock_timestamp() @@ -57,7 +57,7 @@ AND installation_id=(SELECT installation_id FROM runtime_deployment WHERE single SELECT * FROM runtime_node_enrollments WHERE token_sha256=$1; -- name: CreateRuntimePlacement :exec -INSERT INTO runtime_placements(environment_id,node_id) VALUES($1,$2); +INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) VALUES($1,$2,(SELECT generation FROM runtime_deployment)); -- name: GetRuntimePlacement :one SELECT p.*, n.name, (n.provider_ready AND n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at>clock_timestamp()-interval '45 seconds' AND n.removed_at IS NULL)::boolean AS available, @@ -74,13 +74,13 @@ WHERE environment_id IN(SELECT id FROM environments WHERE session_id=$1) AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=runtime_placements.environment_id); -- name: AdoptRuntimePlacements :exec -INSERT INTO runtime_placements(environment_id,node_id,released_at) -SELECT a.environment_id,$1,a.released_at FROM runtime_allocations a WHERE a.provider_key=$2 AND a.state<>'released' AND a.node_id IS NULL +INSERT INTO runtime_placements(environment_id,node_id,released_at,deployment_generation) +SELECT a.environment_id,$1,a.released_at,a.deployment_generation FROM runtime_allocations a WHERE a.provider_key=$2 AND a.state<>'released' AND a.node_id IS NULL ON CONFLICT(environment_id) DO NOTHING; -- name: AdoptPendingRuntimePlacements :exec -INSERT INTO runtime_placements(environment_id,node_id) -SELECT e.id,$1 FROM environments e JOIN sessions s ON s.id=e.session_id +INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) +SELECT e.id,$1,(SELECT generation FROM runtime_deployment) FROM environments e JOIN sessions s ON s.id=e.session_id WHERE s.deleted_at IS NULL AND e.status='pending' AND s.configuration->'environment'->>'type'='openai_hosted' AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=e.id) ON CONFLICT(environment_id) DO NOTHING; @@ -90,13 +90,13 @@ UPDATE runtime_allocations SET node_id=$1,compute_activity_at=clock_timestamp() WHERE provider_key=$2 AND node_id IS NULL AND state<>'released'; -- name: ListNodeRuntimeAllocations :many -SELECT a.id,a.node_id,a.observation_error,a.state,a.compute_phase,a.compute_phase_changed_at,a.initialization,a.created_at,a.environment_id,e.session_id,s.tenant_id +SELECT a.id,a.node_id,a.deployment_generation,a.observation_error,a.state,a.compute_phase,a.compute_phase_changed_at,a.initialization,a.created_at,a.environment_id,e.session_id,s.tenant_id FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id WHERE a.node_id=$1 AND a.state<>'released' ORDER BY a.created_at,a.id LIMIT 1000; -- name: CreateSessionRuntimePlacement :exec -INSERT INTO runtime_placements(environment_id,node_id) -SELECT id,$2 FROM environments WHERE session_id=$1; +INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) +SELECT id,$2,(SELECT generation FROM runtime_deployment) FROM environments WHERE session_id=$1; -- name: SetRuntimeObservation :exec UPDATE runtime_allocations SET observation_error=$4 WHERE id=$1 AND compute_revision=$2 AND state=$3 AND state<>'released'; diff --git a/services/agents-api/internal/db/queries/sandbox_deployment_setup.sql b/services/agents-api/internal/db/queries/sandbox_deployment_setup.sql index 871e74eba..56ff85a27 100644 --- a/services/agents-api/internal/db/queries/sandbox_deployment_setup.sql +++ b/services/agents-api/internal/db/queries/sandbox_deployment_setup.sql @@ -15,7 +15,7 @@ e2b_template_memory_mib=sqlc.narg(e2b_template_memory_mib), e2b_template_root_di updated_at=clock_timestamp() WHERE singleton=true AND provider_kind='e2b'; -- name: RetireSandboxNodes :exec -UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL,provider_ready=false WHERE removed_at IS NULL; +UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL,provider_ready=false,ready_generation=NULL WHERE removed_at IS NULL; -- name: RetireSandboxEnrollments :exec UPDATE runtime_node_enrollments SET expires_at=clock_timestamp() WHERE consumed_at IS NULL; diff --git a/services/agents-api/internal/db/queries/sandbox_generations.sql b/services/agents-api/internal/db/queries/sandbox_generations.sql new file mode 100644 index 000000000..a4dc344ef --- /dev/null +++ b/services/agents-api/internal/db/queries/sandbox_generations.sql @@ -0,0 +1,22 @@ +-- name: RetainSandboxGeneration :exec +INSERT INTO runtime_deployment_generations(generation,provider_kind,specification,e2b_template,e2b_template_build_status,e2b_template_cpus,e2b_template_memory_mib,e2b_template_root_disk_mib) +SELECT generation,provider_kind,specification,e2b_template,e2b_template_build_status,e2b_template_cpus,e2b_template_memory_mib,e2b_template_root_disk_mib +FROM runtime_deployment d WHERE provider_kind <> '' AND ( + EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.state <> 'released' AND a.deployment_generation=d.generation) + OR EXISTS(SELECT 1 FROM runtime_placements p WHERE p.released_at IS NULL AND p.deployment_generation=d.generation) + OR EXISTS(SELECT 1 FROM runtime_nodes n WHERE n.removed_at IS NULL AND n.ready_generation=d.generation)); + +-- name: CollectSandboxGenerations :exec +DELETE FROM runtime_deployment_generations WHERE generation IN (SELECT g.generation FROM runtime_deployment_generations g WHERE g.generation <> (SELECT generation FROM runtime_deployment) +AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.state <> 'released' AND a.deployment_generation=g.generation) +AND NOT EXISTS(SELECT 1 FROM runtime_placements p WHERE p.released_at IS NULL AND p.deployment_generation=g.generation) +AND NOT EXISTS(SELECT 1 FROM runtime_nodes n WHERE n.removed_at IS NULL AND n.ready_generation=g.generation) ORDER BY g.generation LIMIT 32); + +-- name: GetSandboxGeneration :one +SELECT * FROM runtime_deployment_generations WHERE generation=$1; + +-- name: ListSandboxGenerations :many +SELECT * FROM runtime_deployment_generations WHERE generation > $1 ORDER BY generation LIMIT 32; + +-- name: ClearSandboxGenerations :exec +DELETE FROM runtime_deployment_generations; diff --git a/services/agents-api/internal/db/queries/sandbox_reset.sql b/services/agents-api/internal/db/queries/sandbox_reset.sql index e65dbd830..77f30e4a2 100644 --- a/services/agents-api/internal/db/queries/sandbox_reset.sql +++ b/services/agents-api/internal/db/queries/sandbox_reset.sql @@ -57,7 +57,7 @@ SELECT id FROM projects WHERE tenant_id = $1; -- name: GetSandboxDeploymentSnapshot :one WITH observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), held AS ( - SELECT a.node_id, s.id AS session_id, e.id AS environment_id, false AS pending, + SELECT a.deployment_generation, a.node_id, s.id AS session_id, e.id AS environment_id, false AS pending, (a.state = 'cleanup_pending' OR s.deleted_at IS NOT NULL OR e.status IN ('failed', 'expired') OR CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= observed.as_of @@ -66,7 +66,7 @@ held AS ( JOIN sessions s ON s.id = e.session_id CROSS JOIN runtime_deployment d CROSS JOIN observed WHERE a.state <> 'released' UNION ALL - SELECT p.node_id, s.id, e.id, true, false + SELECT p.deployment_generation, p.node_id, s.id, e.id, true, false FROM environments e JOIN sessions s ON s.id = e.session_id LEFT JOIN runtime_placements p ON p.environment_id = e.id AND p.released_at IS NULL WHERE s.deleted_at IS NULL AND e.status = 'pending' @@ -81,6 +81,15 @@ held AS ( n.name, (h.node_id IS NOT NULL AND NOT COALESCE(n.removed_at IS NULL AND n.connection_id IS NOT NULL AND n.connected_epoch = d.owner_epoch AND n.last_seen_at > observed.as_of - interval '45 seconds', false)) AS offline FROM held h LEFT JOIN runtime_nodes n ON n.id = h.node_id CROSS JOIN runtime_deployment d CROSS JOIN observed +), rollout_nodes AS ( + SELECT CASE + WHEN n.connection_id IS NULL OR n.connected_epoch <> d.owner_epoch OR n.last_seen_at IS NULL OR n.last_seen_at <= observed.as_of - interval '45 seconds' THEN 'unknown' + WHEN n.deployment_generation <> d.generation THEN 'update_required' + WHEN n.provider_ready AND n.ready_generation = d.generation THEN 'ready' + WHEN COALESCE(n.health->>'diagnostic','') <> '' THEN 'failed' + ELSE 'unknown' END AS state + FROM runtime_nodes n CROSS JOIN runtime_deployment d CROSS JOIN observed + WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id ), offline AS ( SELECT node_id, name, count(*)::bigint AS resources FROM classified WHERE offline GROUP BY node_id, name ) @@ -93,5 +102,14 @@ SELECT sqlc.embed(d), 'cleanup', (SELECT count(*) FROM classified WHERE category = 'cleanup'), 'on_offline_nodes', (SELECT count(*) FROM classified WHERE offline), 'offline_nodes', COALESCE((SELECT jsonb_agg(jsonb_build_object('node_id', node_id, 'name', name, 'resources', resources) ORDER BY node_id) FROM offline), '[]'::jsonb) - )::jsonb AS remaining + )::jsonb AS remaining, + jsonb_build_object('state','settled', + 'previous_generation_sandboxes',(SELECT count(*) FROM held h WHERE h.deployment_generation <> d.generation), + 'nodes',CASE WHEN d.mode='nodes' THEN jsonb_build_object( + 'ready',(SELECT count(*) FROM rollout_nodes WHERE state='ready'), + 'preparing',0, + 'failed',(SELECT count(*) FROM rollout_nodes WHERE state='failed'), + 'update_required',(SELECT count(*) FROM rollout_nodes WHERE state='update_required'), + 'unknown',(SELECT count(*) FROM rollout_nodes WHERE state='unknown')) ELSE NULL END + )::jsonb AS rollout FROM runtime_deployment d; diff --git a/services/agents-api/internal/db/sqlc/initial_environment_files.sql.go b/services/agents-api/internal/db/sqlc/initial_environment_files.sql.go index 52a9ddea9..f90ae5bf5 100644 --- a/services/agents-api/internal/db/sqlc/initial_environment_files.sql.go +++ b/services/agents-api/internal/db/sqlc/initial_environment_files.sql.go @@ -15,7 +15,7 @@ const claimRuntimeInitialization = `-- name: ClaimRuntimeInitialization :one UPDATE runtime_allocations SET initialization = 'running' WHERE id = $1 AND initialization = 'pending' AND state = 'running' AND create_settled AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') -RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at +RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` func (q *Queries) ClaimRuntimeInitialization(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -41,6 +41,7 @@ func (q *Queries) ClaimRuntimeInitialization(ctx context.Context, id pgtype.UUID &i.NodeID, &i.ObservationError, &i.ComputePhaseChangedAt, + &i.DeploymentGeneration, ) return i, err } @@ -49,7 +50,7 @@ const completeRuntimeInitialization = `-- name: CompleteRuntimeInitialization :o UPDATE runtime_allocations SET initialization = 'complete' WHERE id = $1 AND initialization = 'running' AND state = 'running' AND create_settled AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') -RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at +RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` func (q *Queries) CompleteRuntimeInitialization(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -75,6 +76,7 @@ func (q *Queries) CompleteRuntimeInitialization(ctx context.Context, id pgtype.U &i.NodeID, &i.ObservationError, &i.ComputePhaseChangedAt, + &i.DeploymentGeneration, ) return i, err } diff --git a/services/agents-api/internal/db/sqlc/models.go b/services/agents-api/internal/db/sqlc/models.go index d4107f1ab..c1ff3d716 100644 --- a/services/agents-api/internal/db/sqlc/models.go +++ b/services/agents-api/internal/db/sqlc/models.go @@ -240,6 +240,7 @@ type RuntimeAllocation struct { NodeID pgtype.UUID `json:"node_id"` ObservationError string `json:"observation_error"` ComputePhaseChangedAt pgtype.Timestamptz `json:"compute_phase_changed_at"` + DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` } type RuntimeDeployment struct { @@ -270,6 +271,18 @@ type RuntimeDeployment struct { ResetAudit []byte `json:"reset_audit"` } +type RuntimeDeploymentGeneration struct { + Generation int64 `json:"generation"` + ProviderKind string `json:"provider_kind"` + Specification []byte `json:"specification"` + E2bTemplate string `json:"e2b_template"` + E2bTemplateBuildStatus pgtype.Text `json:"e2b_template_build_status"` + E2bTemplateCpus pgtype.Int4 `json:"e2b_template_cpus"` + E2bTemplateMemoryMib pgtype.Int4 `json:"e2b_template_memory_mib"` + E2bTemplateRootDiskMib pgtype.Int4 `json:"e2b_template_root_disk_mib"` + CreatedAt pgtype.Timestamptz `json:"created_at"` +} + type RuntimeDeviceAuthority struct { ID pgtype.UUID `json:"id"` TenantID pgtype.UUID `json:"tenant_id"` @@ -316,6 +329,7 @@ type RuntimeNode struct { DeploymentGeneration int64 `json:"deployment_generation"` CoreUrl string `json:"core_url"` EnrollmentID pgtype.UUID `json:"enrollment_id"` + ReadyGeneration pgtype.Int8 `json:"ready_generation"` } type RuntimeNodeEnrollment struct { @@ -330,10 +344,11 @@ type RuntimeNodeEnrollment struct { } type RuntimePlacement struct { - EnvironmentID pgtype.UUID `json:"environment_id"` - NodeID pgtype.UUID `json:"node_id"` - ReservedAt pgtype.Timestamptz `json:"reserved_at"` - ReleasedAt pgtype.Timestamptz `json:"released_at"` + EnvironmentID pgtype.UUID `json:"environment_id"` + NodeID pgtype.UUID `json:"node_id"` + ReservedAt pgtype.Timestamptz `json:"reserved_at"` + ReleasedAt pgtype.Timestamptz `json:"released_at"` + DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` } type Session struct { diff --git a/services/agents-api/internal/db/sqlc/runtime_allocations.sql.go b/services/agents-api/internal/db/sqlc/runtime_allocations.sql.go index 2781ccf9d..6aa63cef7 100644 --- a/services/agents-api/internal/db/sqlc/runtime_allocations.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_allocations.sql.go @@ -12,16 +12,17 @@ import ( ) const createRuntimeAllocation = `-- name: CreateRuntimeAllocation :one -INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, initialization) -VALUES ($1, $2, $3, $4, $5, CASE WHEN EXISTS (SELECT 1 FROM initial_environment_files f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) OR EXISTS (SELECT 1 FROM environment_setups f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) THEN 'pending' ELSE 'complete' END) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at +INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation, initialization) +VALUES ($1, $2, $3, $4, $5, $6, CASE WHEN EXISTS (SELECT 1 FROM initial_environment_files f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) OR EXISTS (SELECT 1 FROM environment_setups f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) THEN 'pending' ELSE 'complete' END) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` type CreateRuntimeAllocationParams struct { - ID pgtype.UUID `json:"id"` - EnvironmentID pgtype.UUID `json:"environment_id"` - DeviceID pgtype.UUID `json:"device_id"` - ProviderKey pgtype.UUID `json:"provider_key"` - NodeID pgtype.UUID `json:"node_id"` + ID pgtype.UUID `json:"id"` + EnvironmentID pgtype.UUID `json:"environment_id"` + DeviceID pgtype.UUID `json:"device_id"` + ProviderKey pgtype.UUID `json:"provider_key"` + NodeID pgtype.UUID `json:"node_id"` + DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` } func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntimeAllocationParams) (RuntimeAllocation, error) { @@ -31,6 +32,7 @@ func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntime arg.DeviceID, arg.ProviderKey, arg.NodeID, + arg.DeploymentGeneration, ) var i RuntimeAllocation err := row.Scan( @@ -53,12 +55,13 @@ func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntime &i.NodeID, &i.ObservationError, &i.ComputePhaseChangedAt, + &i.DeploymentGeneration, ) return i, err } const getRuntimeAllocation = `-- name: GetRuntimeAllocation :one -SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired +SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, a.deployment_generation, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id @@ -101,6 +104,7 @@ func (q *Queries) GetRuntimeAllocation(ctx context.Context, arg GetRuntimeAlloca &i.RuntimeAllocation.NodeID, &i.RuntimeAllocation.ObservationError, &i.RuntimeAllocation.ComputePhaseChangedAt, + &i.RuntimeAllocation.DeploymentGeneration, &i.SessionID, &i.TenantID, &i.DeletedAt, @@ -113,7 +117,7 @@ const keepRuntimeAllocation = `-- name: KeepRuntimeAllocation :one UPDATE runtime_allocations SET kept_at = clock_timestamp() WHERE id = $1 AND state = 'running' AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') -RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at +RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` func (q *Queries) KeepRuntimeAllocation(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -139,12 +143,13 @@ func (q *Queries) KeepRuntimeAllocation(ctx context.Context, id pgtype.UUID) (Ru &i.NodeID, &i.ObservationError, &i.ComputePhaseChangedAt, + &i.DeploymentGeneration, ) return i, err } const listRuntimeAllocations = `-- name: ListRuntimeAllocations :many -SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired +SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, a.deployment_generation, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id @@ -189,6 +194,7 @@ func (q *Queries) ListRuntimeAllocations(ctx context.Context, id pgtype.UUID) ([ &i.RuntimeAllocation.NodeID, &i.RuntimeAllocation.ObservationError, &i.RuntimeAllocation.ComputePhaseChangedAt, + &i.RuntimeAllocation.DeploymentGeneration, &i.SessionID, &i.TenantID, &i.DeletedAt, @@ -285,7 +291,7 @@ const observeRuntimeRunning = `-- name: ObserveRuntimeRunning :one UPDATE runtime_allocations SET state = 'running', create_settled = true WHERE id = $1 AND state IN ('creating', 'running') AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') -RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at +RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` func (q *Queries) ObserveRuntimeRunning(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -311,13 +317,14 @@ func (q *Queries) ObserveRuntimeRunning(ctx context.Context, id pgtype.UUID) (Ru &i.NodeID, &i.ObservationError, &i.ComputePhaseChangedAt, + &i.DeploymentGeneration, ) return i, err } const releaseRuntimeAllocation = `-- name: ReleaseRuntimeAllocation :one UPDATE runtime_allocations SET state = 'released', released_at = clock_timestamp() -WHERE id = $1 AND state = 'cleanup_pending' AND create_settled RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at +WHERE id = $1 AND state = 'cleanup_pending' AND create_settled RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` func (q *Queries) ReleaseRuntimeAllocation(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -343,13 +350,14 @@ func (q *Queries) ReleaseRuntimeAllocation(ctx context.Context, id pgtype.UUID) &i.NodeID, &i.ObservationError, &i.ComputePhaseChangedAt, + &i.DeploymentGeneration, ) return i, err } const requestRuntimeCleanup = `-- name: RequestRuntimeCleanup :one UPDATE runtime_allocations SET state = 'cleanup_pending' -WHERE id = $1 AND state <> 'released' RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at +WHERE id = $1 AND state <> 'released' RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` func (q *Queries) RequestRuntimeCleanup(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -375,13 +383,14 @@ func (q *Queries) RequestRuntimeCleanup(ctx context.Context, id pgtype.UUID) (Ru &i.NodeID, &i.ObservationError, &i.ComputePhaseChangedAt, + &i.DeploymentGeneration, ) return i, err } const settleRuntimeCreation = `-- name: SettleRuntimeCreation :one UPDATE runtime_allocations SET create_settled = true -WHERE id = $1 AND state <> 'released' RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at +WHERE id = $1 AND state <> 'released' RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` func (q *Queries) SettleRuntimeCreation(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -407,6 +416,7 @@ func (q *Queries) SettleRuntimeCreation(ctx context.Context, id pgtype.UUID) (Ru &i.NodeID, &i.ObservationError, &i.ComputePhaseChangedAt, + &i.DeploymentGeneration, ) return i, err } diff --git a/services/agents-api/internal/db/sqlc/runtime_lifecycle_nodes.sql.go b/services/agents-api/internal/db/sqlc/runtime_lifecycle_nodes.sql.go index f2ccd5149..be0d53eae 100644 --- a/services/agents-api/internal/db/sqlc/runtime_lifecycle_nodes.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_lifecycle_nodes.sql.go @@ -50,7 +50,7 @@ func (q *Queries) GetRuntimeLifecyclePlacement(ctx context.Context, arg GetRunti } const listRuntimeAllocationsForNode = `-- name: ListRuntimeAllocationsForNode :many -SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired +SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, a.deployment_generation, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id @@ -101,6 +101,7 @@ func (q *Queries) ListRuntimeAllocationsForNode(ctx context.Context, arg ListRun &i.RuntimeAllocation.NodeID, &i.RuntimeAllocation.ObservationError, &i.RuntimeAllocation.ComputePhaseChangedAt, + &i.RuntimeAllocation.DeploymentGeneration, &i.SessionID, &i.TenantID, &i.DeletedAt, diff --git a/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go b/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go index 71e03a966..4a577dd6e 100644 --- a/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go @@ -12,8 +12,8 @@ import ( ) const adoptPendingRuntimePlacements = `-- name: AdoptPendingRuntimePlacements :exec -INSERT INTO runtime_placements(environment_id,node_id) -SELECT e.id,$1 FROM environments e JOIN sessions s ON s.id=e.session_id +INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) +SELECT e.id,$1,(SELECT generation FROM runtime_deployment) FROM environments e JOIN sessions s ON s.id=e.session_id WHERE s.deleted_at IS NULL AND e.status='pending' AND s.configuration->'environment'->>'type'='openai_hosted' AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=e.id) ON CONFLICT(environment_id) DO NOTHING @@ -25,8 +25,8 @@ func (q *Queries) AdoptPendingRuntimePlacements(ctx context.Context, nodeID pgty } const adoptRuntimePlacements = `-- name: AdoptRuntimePlacements :exec -INSERT INTO runtime_placements(environment_id,node_id,released_at) -SELECT a.environment_id,$1,a.released_at FROM runtime_allocations a WHERE a.provider_key=$2 AND a.state<>'released' AND a.node_id IS NULL +INSERT INTO runtime_placements(environment_id,node_id,released_at,deployment_generation) +SELECT a.environment_id,$1,a.released_at,a.deployment_generation FROM runtime_allocations a WHERE a.provider_key=$2 AND a.state<>'released' AND a.node_id IS NULL ON CONFLICT(environment_id) DO NOTHING ` @@ -118,7 +118,7 @@ func (q *Queries) CreateRuntimeEnrollment(ctx context.Context, arg CreateRuntime } const createRuntimePlacement = `-- name: CreateRuntimePlacement :exec -INSERT INTO runtime_placements(environment_id,node_id) VALUES($1,$2) +INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) VALUES($1,$2,(SELECT generation FROM runtime_deployment)) ` type CreateRuntimePlacementParams struct { @@ -132,8 +132,8 @@ func (q *Queries) CreateRuntimePlacement(ctx context.Context, arg CreateRuntimeP } const createSessionRuntimePlacement = `-- name: CreateSessionRuntimePlacement :exec -INSERT INTO runtime_placements(environment_id,node_id) -SELECT id,$2 FROM environments WHERE session_id=$1 +INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) +SELECT id,$2,(SELECT generation FROM runtime_deployment) FROM environments WHERE session_id=$1 ` type CreateSessionRuntimePlacementParams struct { @@ -219,7 +219,7 @@ func (q *Queries) GetRuntimeEnrollment(ctx context.Context, tokenSha256 string) } const getRuntimeNode = `-- name: GetRuntimeNode :one -SELECT id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id FROM runtime_nodes WHERE id=$1 AND removed_at IS NULL +SELECT id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id, ready_generation FROM runtime_nodes WHERE id=$1 AND removed_at IS NULL ` func (q *Queries) GetRuntimeNode(ctx context.Context, id pgtype.UUID) (RuntimeNode, error) { @@ -244,27 +244,29 @@ func (q *Queries) GetRuntimeNode(ctx context.Context, id pgtype.UUID) (RuntimeNo &i.DeploymentGeneration, &i.CoreUrl, &i.EnrollmentID, + &i.ReadyGeneration, ) return i, err } const getRuntimePlacement = `-- name: GetRuntimePlacement :one -SELECT p.environment_id, p.node_id, p.reserved_at, p.released_at, n.name, (n.provider_ready AND n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at>clock_timestamp()-interval '45 seconds' AND n.removed_at IS NULL)::boolean AS available, +SELECT p.environment_id, p.node_id, p.reserved_at, p.released_at, p.deployment_generation, n.name, (n.provider_ready AND n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at>clock_timestamp()-interval '45 seconds' AND n.removed_at IS NULL)::boolean AS available, COALESCE(a.observation_error,'')::text AS observation_error, COALESCE(a.state,'reserved')::text AS state, COALESCE(a.compute_phase,'disabled')::text AS compute_phase FROM runtime_placements p JOIN runtime_nodes n ON n.id=p.node_id CROSS JOIN runtime_deployment d LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id WHERE p.environment_id=$1 ` type GetRuntimePlacementRow struct { - EnvironmentID pgtype.UUID `json:"environment_id"` - NodeID pgtype.UUID `json:"node_id"` - ReservedAt pgtype.Timestamptz `json:"reserved_at"` - ReleasedAt pgtype.Timestamptz `json:"released_at"` - Name string `json:"name"` - Available bool `json:"available"` - ObservationError string `json:"observation_error"` - State string `json:"state"` - ComputePhase string `json:"compute_phase"` + EnvironmentID pgtype.UUID `json:"environment_id"` + NodeID pgtype.UUID `json:"node_id"` + ReservedAt pgtype.Timestamptz `json:"reserved_at"` + ReleasedAt pgtype.Timestamptz `json:"released_at"` + DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` + Name string `json:"name"` + Available bool `json:"available"` + ObservationError string `json:"observation_error"` + State string `json:"state"` + ComputePhase string `json:"compute_phase"` } func (q *Queries) GetRuntimePlacement(ctx context.Context, environmentID pgtype.UUID) (GetRuntimePlacementRow, error) { @@ -275,6 +277,7 @@ func (q *Queries) GetRuntimePlacement(ctx context.Context, environmentID pgtype. &i.NodeID, &i.ReservedAt, &i.ReleasedAt, + &i.DeploymentGeneration, &i.Name, &i.Available, &i.ObservationError, @@ -312,8 +315,8 @@ func (q *Queries) HeartbeatRuntimeNode(ctx context.Context, arg HeartbeatRuntime } const insertRuntimeNode = `-- name: InsertRuntimeNode :one -INSERT INTO runtime_nodes(id,installation_id,name,backend_fingerprint,credential_sha256,max_active,max_retained,specification_digest,deployment_generation,core_url,enrollment_id) -VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id +INSERT INTO runtime_nodes(id,installation_id,name,backend_fingerprint,credential_sha256,max_active,max_retained,specification_digest,deployment_generation,core_url,enrollment_id,ready_generation) +VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$9) RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id, ready_generation ` type InsertRuntimeNodeParams struct { @@ -364,12 +367,13 @@ func (q *Queries) InsertRuntimeNode(ctx context.Context, arg InsertRuntimeNodePa &i.DeploymentGeneration, &i.CoreUrl, &i.EnrollmentID, + &i.ReadyGeneration, ) return i, err } const listLegacyRuntimeAllocations = `-- name: ListLegacyRuntimeAllocations :many -SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, e.session_id, s.tenant_id, s.deleted_at +SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, a.deployment_generation, e.session_id, s.tenant_id, s.deleted_at FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id WHERE a.node_id IS NULL AND a.state<>'released' AND a.id>$1 ORDER BY a.id LIMIT 32 FOR UPDATE OF a ` @@ -410,6 +414,7 @@ func (q *Queries) ListLegacyRuntimeAllocations(ctx context.Context, id pgtype.UU &i.RuntimeAllocation.NodeID, &i.RuntimeAllocation.ObservationError, &i.RuntimeAllocation.ComputePhaseChangedAt, + &i.RuntimeAllocation.DeploymentGeneration, &i.SessionID, &i.TenantID, &i.DeletedAt, @@ -425,7 +430,7 @@ func (q *Queries) ListLegacyRuntimeAllocations(ctx context.Context, id pgtype.UU } const listNodeRuntimeAllocations = `-- name: ListNodeRuntimeAllocations :many -SELECT a.id,a.node_id,a.observation_error,a.state,a.compute_phase,a.compute_phase_changed_at,a.initialization,a.created_at,a.environment_id,e.session_id,s.tenant_id +SELECT a.id,a.node_id,a.deployment_generation,a.observation_error,a.state,a.compute_phase,a.compute_phase_changed_at,a.initialization,a.created_at,a.environment_id,e.session_id,s.tenant_id FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id WHERE a.node_id=$1 AND a.state<>'released' ORDER BY a.created_at,a.id LIMIT 1000 ` @@ -433,6 +438,7 @@ WHERE a.node_id=$1 AND a.state<>'released' ORDER BY a.created_at,a.id LIMIT 1000 type ListNodeRuntimeAllocationsRow struct { ID pgtype.UUID `json:"id"` NodeID pgtype.UUID `json:"node_id"` + DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` ObservationError string `json:"observation_error"` State string `json:"state"` ComputePhase string `json:"compute_phase"` @@ -456,6 +462,7 @@ func (q *Queries) ListNodeRuntimeAllocations(ctx context.Context, nodeID pgtype. if err := rows.Scan( &i.ID, &i.NodeID, + &i.DeploymentGeneration, &i.ObservationError, &i.State, &i.ComputePhase, @@ -477,8 +484,8 @@ func (q *Queries) ListNodeRuntimeAllocations(ctx context.Context, nodeID pgtype. } const listRuntimeNodes = `-- name: ListRuntimeNodes :many -SELECT n.id, n.installation_id, n.name, n.backend_fingerprint, n.credential_sha256, n.max_active, n.max_retained, n.connection_id, n.provider_ready, n.health, n.connected_epoch, n.last_seen_at, n.created_at, n.removed_at, n.specification_digest, n.deployment_generation, n.core_url, n.enrollment_id, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at > clock_timestamp()-interval '45 seconds')::boolean AS online, - d.provider_kind, +SELECT n.id, n.installation_id, n.name, n.backend_fingerprint, n.credential_sha256, n.max_active, n.max_retained, n.connection_id, n.provider_ready, n.health, n.connected_epoch, n.last_seen_at, n.created_at, n.removed_at, n.specification_digest, n.deployment_generation, n.core_url, n.enrollment_id, n.ready_generation, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at > clock_timestamp()-interval '45 seconds')::boolean AS online, + d.provider_kind, d.generation AS target_generation, (SELECT count(*) FROM runtime_placements p LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id WHERE p.node_id=n.id AND p.released_at IS NULL AND (a.id IS NULL OR a.compute_phase <> 'suspended'))::bigint AS active, (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL)::bigint AS retained, (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=p.environment_id))::bigint AS reserved, @@ -509,8 +516,10 @@ type ListRuntimeNodesRow struct { DeploymentGeneration int64 `json:"deployment_generation"` CoreUrl string `json:"core_url"` EnrollmentID pgtype.UUID `json:"enrollment_id"` + ReadyGeneration pgtype.Int8 `json:"ready_generation"` Online bool `json:"online"` ProviderKind string `json:"provider_kind"` + TargetGeneration int64 `json:"target_generation"` Active int64 `json:"active"` Retained int64 `json:"retained"` Reserved int64 `json:"reserved"` @@ -547,8 +556,10 @@ func (q *Queries) ListRuntimeNodes(ctx context.Context, nodeID pgtype.UUID) ([]L &i.DeploymentGeneration, &i.CoreUrl, &i.EnrollmentID, + &i.ReadyGeneration, &i.Online, &i.ProviderKind, + &i.TargetGeneration, &i.Active, &i.Retained, &i.Reserved, @@ -597,7 +608,7 @@ func (q *Queries) ReleaseUnallocatedRuntimePlacement(ctx context.Context, sessio } const removeRuntimeNode = `-- name: RemoveRuntimeNode :exec -UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL WHERE id=$1 AND removed_at IS NULL +UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL,ready_generation=NULL WHERE id=$1 AND removed_at IS NULL ` func (q *Queries) RemoveRuntimeNode(ctx context.Context, id pgtype.UUID) error { @@ -641,7 +652,7 @@ func (q *Queries) SetRuntimeObservation(ctx context.Context, arg SetRuntimeObser } const updateRuntimeNode = `-- name: UpdateRuntimeNode :one -UPDATE runtime_nodes SET name=$2,max_active=$3,max_retained=$4 WHERE id=$1 AND removed_at IS NULL RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id +UPDATE runtime_nodes SET name=$2,max_active=$3,max_retained=$4 WHERE id=$1 AND removed_at IS NULL RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id, ready_generation ` type UpdateRuntimeNodeParams struct { @@ -678,6 +689,7 @@ func (q *Queries) UpdateRuntimeNode(ctx context.Context, arg UpdateRuntimeNodePa &i.DeploymentGeneration, &i.CoreUrl, &i.EnrollmentID, + &i.ReadyGeneration, ) return i, err } diff --git a/services/agents-api/internal/db/sqlc/runtime_suspension.sql.go b/services/agents-api/internal/db/sqlc/runtime_suspension.sql.go index f423b3d2d..20f273ecc 100644 --- a/services/agents-api/internal/db/sqlc/runtime_suspension.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_suspension.sql.go @@ -139,7 +139,7 @@ WHERE id = $4 AND compute_revision = $5 AND state = 'running' AND initialization = 'complete' AND ((compute_phase IN ('disabled','running') AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour')) OR (compute_phase NOT IN ('disabled','running') AND compute_retained_until > clock_timestamp())) -RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at +RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` type SetRuntimeComputeParams struct { @@ -179,6 +179,7 @@ func (q *Queries) SetRuntimeCompute(ctx context.Context, arg SetRuntimeComputePa &i.NodeID, &i.ObservationError, &i.ComputePhaseChangedAt, + &i.DeploymentGeneration, ) return i, err } diff --git a/services/agents-api/internal/db/sqlc/sandbox_deployment_setup.sql.go b/services/agents-api/internal/db/sqlc/sandbox_deployment_setup.sql.go index e48c7210b..82310a07e 100644 --- a/services/agents-api/internal/db/sqlc/sandbox_deployment_setup.sql.go +++ b/services/agents-api/internal/db/sqlc/sandbox_deployment_setup.sql.go @@ -106,7 +106,7 @@ func (q *Queries) RetireSandboxEnrollments(ctx context.Context) error { } const retireSandboxNodes = `-- name: RetireSandboxNodes :exec -UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL,provider_ready=false WHERE removed_at IS NULL +UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL,provider_ready=false,ready_generation=NULL WHERE removed_at IS NULL ` func (q *Queries) RetireSandboxNodes(ctx context.Context) error { diff --git a/services/agents-api/internal/db/sqlc/sandbox_generations.sql.go b/services/agents-api/internal/db/sqlc/sandbox_generations.sql.go new file mode 100644 index 000000000..39ca8a14a --- /dev/null +++ b/services/agents-api/internal/db/sqlc/sandbox_generations.sql.go @@ -0,0 +1,100 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: sandbox_generations.sql + +package sqlc + +import ( + "context" +) + +const clearSandboxGenerations = `-- name: ClearSandboxGenerations :exec +DELETE FROM runtime_deployment_generations +` + +func (q *Queries) ClearSandboxGenerations(ctx context.Context) error { + _, err := q.db.Exec(ctx, clearSandboxGenerations) + return err +} + +const collectSandboxGenerations = `-- name: CollectSandboxGenerations :exec +DELETE FROM runtime_deployment_generations WHERE generation IN (SELECT g.generation FROM runtime_deployment_generations g WHERE g.generation <> (SELECT generation FROM runtime_deployment) +AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.state <> 'released' AND a.deployment_generation=g.generation) +AND NOT EXISTS(SELECT 1 FROM runtime_placements p WHERE p.released_at IS NULL AND p.deployment_generation=g.generation) +AND NOT EXISTS(SELECT 1 FROM runtime_nodes n WHERE n.removed_at IS NULL AND n.ready_generation=g.generation) ORDER BY g.generation LIMIT 32) +` + +func (q *Queries) CollectSandboxGenerations(ctx context.Context) error { + _, err := q.db.Exec(ctx, collectSandboxGenerations) + return err +} + +const getSandboxGeneration = `-- name: GetSandboxGeneration :one +SELECT generation, provider_kind, specification, e2b_template, e2b_template_build_status, e2b_template_cpus, e2b_template_memory_mib, e2b_template_root_disk_mib, created_at FROM runtime_deployment_generations WHERE generation=$1 +` + +func (q *Queries) GetSandboxGeneration(ctx context.Context, generation int64) (RuntimeDeploymentGeneration, error) { + row := q.db.QueryRow(ctx, getSandboxGeneration, generation) + var i RuntimeDeploymentGeneration + err := row.Scan( + &i.Generation, + &i.ProviderKind, + &i.Specification, + &i.E2bTemplate, + &i.E2bTemplateBuildStatus, + &i.E2bTemplateCpus, + &i.E2bTemplateMemoryMib, + &i.E2bTemplateRootDiskMib, + &i.CreatedAt, + ) + return i, err +} + +const listSandboxGenerations = `-- name: ListSandboxGenerations :many +SELECT generation, provider_kind, specification, e2b_template, e2b_template_build_status, e2b_template_cpus, e2b_template_memory_mib, e2b_template_root_disk_mib, created_at FROM runtime_deployment_generations WHERE generation > $1 ORDER BY generation LIMIT 32 +` + +func (q *Queries) ListSandboxGenerations(ctx context.Context, generation int64) ([]RuntimeDeploymentGeneration, error) { + rows, err := q.db.Query(ctx, listSandboxGenerations, generation) + if err != nil { + return nil, err + } + defer rows.Close() + items := []RuntimeDeploymentGeneration{} + for rows.Next() { + var i RuntimeDeploymentGeneration + if err := rows.Scan( + &i.Generation, + &i.ProviderKind, + &i.Specification, + &i.E2bTemplate, + &i.E2bTemplateBuildStatus, + &i.E2bTemplateCpus, + &i.E2bTemplateMemoryMib, + &i.E2bTemplateRootDiskMib, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const retainSandboxGeneration = `-- name: RetainSandboxGeneration :exec +INSERT INTO runtime_deployment_generations(generation,provider_kind,specification,e2b_template,e2b_template_build_status,e2b_template_cpus,e2b_template_memory_mib,e2b_template_root_disk_mib) +SELECT generation,provider_kind,specification,e2b_template,e2b_template_build_status,e2b_template_cpus,e2b_template_memory_mib,e2b_template_root_disk_mib +FROM runtime_deployment d WHERE provider_kind <> '' AND ( + EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.state <> 'released' AND a.deployment_generation=d.generation) + OR EXISTS(SELECT 1 FROM runtime_placements p WHERE p.released_at IS NULL AND p.deployment_generation=d.generation) + OR EXISTS(SELECT 1 FROM runtime_nodes n WHERE n.removed_at IS NULL AND n.ready_generation=d.generation)) +` + +func (q *Queries) RetainSandboxGeneration(ctx context.Context) error { + _, err := q.db.Exec(ctx, retainSandboxGeneration) + return err +} diff --git a/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go b/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go index e3e21aa9a..976ebcf2a 100644 --- a/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go +++ b/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go @@ -53,7 +53,7 @@ func (q *Queries) ForceSandboxReset(ctx context.Context) error { const getSandboxDeploymentSnapshot = `-- name: GetSandboxDeploymentSnapshot :one WITH observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), held AS ( - SELECT a.node_id, s.id AS session_id, e.id AS environment_id, false AS pending, + SELECT a.deployment_generation, a.node_id, s.id AS session_id, e.id AS environment_id, false AS pending, (a.state = 'cleanup_pending' OR s.deleted_at IS NOT NULL OR e.status IN ('failed', 'expired') OR CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= observed.as_of @@ -62,14 +62,14 @@ held AS ( JOIN sessions s ON s.id = e.session_id CROSS JOIN runtime_deployment d CROSS JOIN observed WHERE a.state <> 'released' UNION ALL - SELECT p.node_id, s.id, e.id, true, false + SELECT p.deployment_generation, p.node_id, s.id, e.id, true, false FROM environments e JOIN sessions s ON s.id = e.session_id LEFT JOIN runtime_placements p ON p.environment_id = e.id AND p.released_at IS NULL WHERE s.deleted_at IS NULL AND e.status = 'pending' AND s.configuration->'environment'->>'type' = 'openai_hosted' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id) ), classified AS ( - SELECT h.node_id, h.session_id, h.environment_id, h.pending, h.cleanup, CASE WHEN h.cleanup THEN 'cleanup' + SELECT h.deployment_generation, h.node_id, h.session_id, h.environment_id, h.pending, h.cleanup, CASE WHEN h.cleanup THEN 'cleanup' WHEN EXISTS (SELECT 1 FROM turns t WHERE t.session_id = h.session_id AND t.status IN ('in_progress', 'waiting')) OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = h.session_id AND t.status IN ('in_progress', 'waiting')) OR EXISTS (SELECT 1 FROM environment_file_writes f WHERE f.environment_id = h.environment_id AND f.state = 'pending') @@ -77,6 +77,15 @@ held AS ( n.name, (h.node_id IS NOT NULL AND NOT COALESCE(n.removed_at IS NULL AND n.connection_id IS NOT NULL AND n.connected_epoch = d.owner_epoch AND n.last_seen_at > observed.as_of - interval '45 seconds', false)) AS offline FROM held h LEFT JOIN runtime_nodes n ON n.id = h.node_id CROSS JOIN runtime_deployment d CROSS JOIN observed +), rollout_nodes AS ( + SELECT CASE + WHEN n.connection_id IS NULL OR n.connected_epoch <> d.owner_epoch OR n.last_seen_at IS NULL OR n.last_seen_at <= observed.as_of - interval '45 seconds' THEN 'unknown' + WHEN n.deployment_generation <> d.generation THEN 'update_required' + WHEN n.provider_ready AND n.ready_generation = d.generation THEN 'ready' + WHEN COALESCE(n.health->>'diagnostic','') <> '' THEN 'failed' + ELSE 'unknown' END AS state + FROM runtime_nodes n CROSS JOIN runtime_deployment d CROSS JOIN observed + WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id ), offline AS ( SELECT node_id, name, count(*)::bigint AS resources FROM classified WHERE offline GROUP BY node_id, name ) @@ -89,7 +98,16 @@ SELECT d.singleton, d.installation_id, d.backend_fingerprint, d.admission_paused 'cleanup', (SELECT count(*) FROM classified WHERE category = 'cleanup'), 'on_offline_nodes', (SELECT count(*) FROM classified WHERE offline), 'offline_nodes', COALESCE((SELECT jsonb_agg(jsonb_build_object('node_id', node_id, 'name', name, 'resources', resources) ORDER BY node_id) FROM offline), '[]'::jsonb) - )::jsonb AS remaining + )::jsonb AS remaining, + jsonb_build_object('state','settled', + 'previous_generation_sandboxes',(SELECT count(*) FROM held h WHERE h.deployment_generation <> d.generation), + 'nodes',CASE WHEN d.mode='nodes' THEN jsonb_build_object( + 'ready',(SELECT count(*) FROM rollout_nodes WHERE state='ready'), + 'preparing',0, + 'failed',(SELECT count(*) FROM rollout_nodes WHERE state='failed'), + 'update_required',(SELECT count(*) FROM rollout_nodes WHERE state='update_required'), + 'unknown',(SELECT count(*) FROM rollout_nodes WHERE state='unknown')) ELSE NULL END + )::jsonb AS rollout FROM runtime_deployment d ` @@ -98,6 +116,7 @@ type GetSandboxDeploymentSnapshotRow struct { Allocations int64 `json:"allocations"` Pending int64 `json:"pending"` Remaining []byte `json:"remaining"` + Rollout []byte `json:"rollout"` } func (q *Queries) GetSandboxDeploymentSnapshot(ctx context.Context) (GetSandboxDeploymentSnapshotRow, error) { @@ -132,6 +151,7 @@ func (q *Queries) GetSandboxDeploymentSnapshot(ctx context.Context) (GetSandboxD &i.Allocations, &i.Pending, &i.Remaining, + &i.Rollout, ) return i, err } diff --git a/services/agents-api/internal/execution/runtime_manager.go b/services/agents-api/internal/execution/runtime_manager.go index 70671f69b..5c7299fad 100644 --- a/services/agents-api/internal/execution/runtime_manager.go +++ b/services/agents-api/internal/execution/runtime_manager.go @@ -224,6 +224,11 @@ func (m *runtimeManager) run(ctx context.Context) error { } m.running = true m.mu.Unlock() + if m.loadDeployment != nil { + if err := m.store.CollectSandboxGenerations(ctx); err != nil { + return err + } + } if err := m.resetStep(ctx); err != nil { return err } @@ -241,6 +246,11 @@ func (m *runtimeManager) run(ctx context.Context) error { case err := <-m.failed: return err case <-ticker.C: + if m.loadDeployment != nil { + if err := m.store.CollectSandboxGenerations(ctx); err != nil { + return err + } + } if err := m.resetStep(ctx); err != nil { return err } diff --git a/services/agents-api/internal/execution/sandbox_deployment_setup.go b/services/agents-api/internal/execution/sandbox_deployment_setup.go index eeeb8b922..d31c9ce0f 100644 --- a/services/agents-api/internal/execution/sandbox_deployment_setup.go +++ b/services/agents-api/internal/execution/sandbox_deployment_setup.go @@ -15,6 +15,8 @@ type PreparedRuntimeDeployment struct { Config *RuntimeProvider Publish func(*RuntimeProvider) E2BTemplateBuild *store.SandboxE2BTemplateBuild + VerifyCredential func(context.Context) error + FenceCredential func(context.Context) (func(), error) } // withTemplateBuild saves the validated build with the selection and fills diff --git a/services/agents-api/internal/execution/sandbox_deployment_switch.go b/services/agents-api/internal/execution/sandbox_deployment_switch.go index 966a80e14..31821d4af 100644 --- a/services/agents-api/internal/execution/sandbox_deployment_switch.go +++ b/services/agents-api/internal/execution/sandbox_deployment_switch.go @@ -123,20 +123,47 @@ func (w *Worker) UpdateSandboxDeployment(ctx context.Context, input store.Sandbo } defer unlock() m := w.runtimes - if err := m.store.CheckSandboxDeploymentSwitch(ctx, m.setupInstallationID, input); err != nil { + input, unchanged, err := m.store.ClassifySandboxDeploymentChange(ctx, m.setupInstallationID, input) + if err != nil { return store.RuntimeDeploymentView{}, err } + if unchanged { + return m.store.GetRuntimeDeployment(ctx) + } candidate, err := m.prepareCandidate(ctx, input.SandboxDeploymentSetupRequest) if err != nil { return store.RuntimeDeploymentView{}, err } - if err := m.pauseDeployment(ctx); err != nil { + if input.Provider == "e2b" { + if candidate.VerifyCredential == nil || candidate.FenceCredential == nil { + return store.RuntimeDeploymentView{}, ErrExecutionUnavailable + } + if err := candidate.VerifyCredential(ctx); err != nil { + return store.RuntimeDeploymentView{}, err + } + if input.E2B.ReplaceCredential { + fenceCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + release, err := candidate.FenceCredential(fenceCtx) + if err != nil { + return store.RuntimeDeploymentView{}, err + } + defer release() + // The final scan includes allocations admitted during preliminary verification. + if err := candidate.VerifyCredential(fenceCtx); err != nil { + return store.RuntimeDeploymentView{}, err + } + } + } else if err := m.pauseDeployment(ctx); err != nil { return store.RuntimeDeploymentView{}, errors.Join(err, m.restoreCommittedDeployment()) } input.SandboxDeploymentSetupRequest = withTemplateBuild(input.SandboxDeploymentSetupRequest, candidate) result, err := m.store.UpdateSandboxDeployment(ctx, m.setupInstallationID, input) if err != nil { - return store.RuntimeDeploymentView{}, errors.Join(err, m.restoreCommittedDeployment()) + if input.Provider != "e2b" { + err = errors.Join(err, m.restoreCommittedDeployment()) + } + return store.RuntimeDeploymentView{}, err } m.publishDeployment(candidate, result) return result, nil diff --git a/services/agents-api/internal/execution/sandbox_generations_test.go b/services/agents-api/internal/execution/sandbox_generations_test.go new file mode 100644 index 000000000..a674a2f6c --- /dev/null +++ b/services/agents-api/internal/execution/sandbox_generations_test.go @@ -0,0 +1,113 @@ +package execution + +import ( + "context" + "errors" + "testing" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/adminaudit" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/e2b" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/node" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" +) + +func TestE2BReplacementVerifiesTwiceAndNeverPublishesFailedCommit(t *testing.T) { + s, lease := resetManagerStore(t) + writer := lease.Store() + id := uuid.NewString() + if err := writer.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + t.Fatal(err) + } + input := store.SandboxDeploymentSetupRequest{Provider: "e2b", E2B: &store.SandboxE2BConfiguration{APIKey: "old-key", Template: "runtime:" + uuid.NewString()}} + input.Resources.CPUs = 2 + input.Resources.MemoryMiB = 2048 + if _, err := writer.InitializeSandboxDeployment(t.Context(), id, input); err != nil { + t.Fatal(err) + } + hub := node.NewHub(node.HubOptions{}) + defer hub.Close() + provider := hub.Proxy(uuid.NewString(), "docker") + verifyCalls, published, fenced, released := 0, 0, 0, 0 + var rejectAt int + config := NewDeferredRuntimeProvider(id, func(ctx context.Context) (*RuntimeProvider, error) { + setup, err := s.GetSandboxSetup(ctx) + if err != nil { + return nil, err + } + return &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", Generation: setup.Generation, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider}, nil + }, func(ctx context.Context, setup store.SandboxSetup) (PreparedRuntimeDeployment, error) { + return PreparedRuntimeDeployment{Config: &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider}, + VerifyCredential: func(context.Context) error { + verifyCalls++ + if verifyCalls == rejectAt { + return e2b.ErrTeamMismatch + } + return nil + }, + FenceCredential: func(context.Context) (func(), error) { fenced++; return func() { released++ }, nil }, Publish: func(*RuntimeProvider) { published++ }}, nil + }) + m, err := newRuntimeManager(writer, gateway.NewRegistry(), config) + if err != nil { + t.Fatal(err) + } + defer func() { m.stop(); m.drain() }() + if _, err = m.ensureDeployment(t.Context()); err != nil { + t.Fatal(err) + } + old, err := m.node("") + if err != nil { + t.Fatal(err) + } + worker := &Worker{runtimes: m} + input.E2B.APIKey = "candidate-key" + request := store.SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1} + audit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "test", RequestID: "test", TraceID: "test"}) + for _, failure := range []string{"preliminary", "final", "commit"} { + verifyCalls = 0 + rejectAt = 0 + ctx := audit + switch failure { + case "preliminary": + rejectAt = 1 + case "final": + rejectAt = 2 + case "commit": + ctx = t.Context() + } + if _, err := worker.UpdateSandboxDeployment(ctx, request); err == nil { + t.Fatal("failure published", failure) + } + committed, err := s.GetSandboxSetup(t.Context()) + if err != nil || committed.Generation != 1 || committed.E2B.APIKey != "old-key" || published != 0 || fenced != released { + t.Fatal("partial credential publication", failure, err) + } + current, err := m.node("") + if err != nil || current != old || m.switching { + t.Fatal("online failure drained an owned lifecycle", err) + } + } + verifyCalls = 0 + rejectAt = 0 + result, err := worker.UpdateSandboxDeployment(audit, request) + if err != nil || result.Generation != 2 || verifyCalls != 2 || published != 1 || fenced != released { + t.Fatal(result, verifyCalls, published, err) + } + current, err := m.node("") + if err != nil || current != old { + t.Fatal("online commit replaced lifecycle", err) + } + before := verifyCalls + if _, err := worker.UpdateSandboxDeployment(audit, request); err == nil { + t.Fatal("stale replay accepted") + } else { + var stale *store.SandboxGenerationStaleError + if !errors.As(err, &stale) { + t.Fatal(err) + } + } + if verifyCalls != before { + t.Fatal("stale request reached E2B") + } +} diff --git a/services/agents-api/internal/sandbox/e2b/credential.go b/services/agents-api/internal/sandbox/e2b/credential.go new file mode 100644 index 000000000..f7d795c6e --- /dev/null +++ b/services/agents-api/internal/sandbox/e2b/credential.go @@ -0,0 +1,44 @@ +package e2b + +import ( + "context" + "errors" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "time" +) + +var ErrRequestUnconfirmed = errors.New("E2B verification could not be confirmed") +var ErrTemplateInvalid = errors.New("E2B template build rejected") +var ErrCredentialInvalid = errors.New("E2B credential rejected") +var ErrTeamMismatch = errors.New("E2B team does not own the retained sandbox deployment") + +// VerifyCredential is read-only and bounded. A public readable template alone +// does not prove team ownership. References are one bounded Core-owned page. +func (p *Provider) VerifyCredential(ctx context.Context, refs []sandbox.Reference) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if len(refs) > 32 { + return sandbox.ErrInvalid + } + for _, r := range refs { + if !validReference(r) { + return sandbox.ErrInvalid + } + } + deadline, _ := ctx.Deadline() + out, err := p.caller.Call(ctx, Request{Version: ProtocolVersion, Operation: "verify_credential", Config: p.config, References: refs, Deadline: deadline}) + if err != nil || out.Version != ProtocolVersion { + return ErrRequestUnconfirmed + } + switch out.ErrorCode { + case "unauthorized": + return ErrCredentialInvalid + case "team_mismatch", "invalid": + return ErrTeamMismatch + case "": + if out.DeploymentValid && out.Info == nil && out.Command == nil && out.Observations == nil && out.TemplateBuild == nil { + return nil + } + } + return ErrRequestUnconfirmed +} diff --git a/services/agents-api/internal/sandbox/e2b/fence.go b/services/agents-api/internal/sandbox/e2b/fence.go new file mode 100644 index 000000000..89fa4a48a --- /dev/null +++ b/services/agents-api/internal/sandbox/e2b/fence.go @@ -0,0 +1,64 @@ +package e2b + +import ( + "context" + "math" + "sync" + + "golang.org/x/sync/semaphore" +) + +// CallFence excludes credential commits from provider calls, including helpers +// still completing after their HTTP caller's deadline. It never kills a helper +// or treats an unknown Create as absent. A timed-out fence leaves the key intact. +type CallFence struct { + gate *semaphore.Weighted + once sync.Once + mu sync.Mutex + active int + idle chan struct{} +} + +func (f *CallFence) init() { f.once.Do(func() { f.gate = semaphore.NewWeighted(math.MaxInt64) }) } +func (f *CallFence) Enter(ctx context.Context) (func(), error) { + f.init() + if err := f.gate.Acquire(ctx, 1); err != nil { + return nil, err + } + return func() { f.gate.Release(1) }, nil +} +func (f *CallFence) childStarted() func() { + f.mu.Lock() + if f.active == 0 { + f.idle = make(chan struct{}) + } + f.active++ + f.mu.Unlock() + return func() { + f.mu.Lock() + f.active-- + if f.active == 0 { + close(f.idle) + } + f.mu.Unlock() + } +} +func (f *CallFence) Fence(ctx context.Context) (func(), error) { + f.init() + if err := f.gate.Acquire(ctx, math.MaxInt64); err != nil { + return nil, err + } + release := func() { f.gate.Release(math.MaxInt64) } + f.mu.Lock() + idle, active := f.idle, f.active + f.mu.Unlock() + if active != 0 { + select { + case <-idle: + case <-ctx.Done(): + release() + return nil, ctx.Err() + } + } + return release, nil +} diff --git a/services/agents-api/internal/sandbox/e2b/process.go b/services/agents-api/internal/sandbox/e2b/process.go index 0a76dbfa3..eae28f4b3 100644 --- a/services/agents-api/internal/sandbox/e2b/process.go +++ b/services/agents-api/internal/sandbox/e2b/process.go @@ -15,9 +15,9 @@ import ( // The helper keeps its allocation lock until its bounded SDK operation settles. var errHelperNotStarted = errors.New("helper did not start") -type ProcessCaller struct{} +type ProcessCaller struct{ Fence *CallFence } -func (*ProcessCaller) Call(ctx context.Context, q Request) (Response, error) { +func (p *ProcessCaller) Call(ctx context.Context, q Request) (Response, error) { data, err := json.Marshal(q) if err != nil || len(data) > MaxRequestBytes { return Response{}, errHelperNotStarted @@ -39,8 +39,12 @@ func (*ProcessCaller) Call(ctx context.Context, q Request) (Response, error) { if cmd.Start() != nil { return Response{}, errHelperNotStarted } + finished := func() {} + if p.Fence != nil { + finished = p.Fence.childStarted() + } done := make(chan error, 1) - go func() { done <- cmd.Wait() }() + go func() { err := cmd.Wait(); finished(); done <- err }() select { case <-ctx.Done(): return Response{}, ctx.Err() diff --git a/services/agents-api/internal/sandbox/e2b/process_test.go b/services/agents-api/internal/sandbox/e2b/process_test.go index 2c46b245a..2cbedc943 100644 --- a/services/agents-api/internal/sandbox/e2b/process_test.go +++ b/services/agents-api/internal/sandbox/e2b/process_test.go @@ -59,3 +59,68 @@ func TestEnvironmentDropsProviderSelectorsAndCredentials(t *testing.T) { t.Fatal(err) } } + +func TestCredentialFenceWaitsForActualHelperExitAfterCancellation(t *testing.T) { + root := t.TempDir() + binary := filepath.Join(root, "helper") + marker := filepath.Join(root, "started") + finish := filepath.Join(root, "finish") + script := "#!/bin/sh\ntouch '" + marker + "'\nwhile ! test -f '" + finish + "'; do sleep 0.01; done\nprintf '%s' '{\"Version\":1}'\n" + if err := os.WriteFile(binary, []byte(script), 0700); err != nil { + t.Fatal(err) + } + fence := &CallFence{} + ctx, cancel := context.WithCancel(t.Context()) + entered, err := fence.Enter(ctx) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + _, err := (&ProcessCaller{Fence: fence}).Call(ctx, Request{Config: Config{Binary: binary}}) + entered() + done <- err + }() + t.Cleanup(func() { os.WriteFile(finish, nil, 0600); cancel() }) + deadline := time.Now().Add(3 * time.Second) + for { + if _, err := os.Stat(marker); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("helper did not start") + } + time.Sleep(time.Millisecond) + } + cancel() + if err := <-done; err == nil { + t.Fatal("caller cancellation hidden") + } + blocked, stop := context.WithTimeout(t.Context(), 30*time.Millisecond) + defer stop() + if unlock, err := fence.Fence(blocked); err == nil { + unlock() + t.Fatal("credential fence forgot a live helper") + } + // A failed exclusive waiter must return its permits immediately. + if release, err := fence.Enter(t.Context()); err != nil { + t.Fatal(err) + } else { + release() + } + if err := os.WriteFile(finish, nil, 0600); err != nil { + t.Fatal(err) + } + final, end := context.WithTimeout(t.Context(), 3*time.Second) + defer end() + release, err := fence.Fence(final) + if err != nil { + t.Fatal("late helper exit did not release fence", err) + } + release() + if release, err := fence.Enter(final); err != nil { + t.Fatal(err) + } else { + release() + } +} diff --git a/services/agents-api/internal/sandbox/e2b/provider.go b/services/agents-api/internal/sandbox/e2b/provider.go index af8cd91cb..5fe489980 100644 --- a/services/agents-api/internal/sandbox/e2b/provider.go +++ b/services/agents-api/internal/sandbox/e2b/provider.go @@ -141,6 +141,8 @@ func (p *Provider) call(ctx context.Context, operation string, r sandbox.Referen return out, nil case "legacy_template": return out, fmt.Errorf("%w: This E2B template was built before OpenAgentCore renamed its paths. Build a template with this release's build-template.py and replace it in the sandbox deployment.", sandbox.ErrInvalid) + case "unauthorized": + return out, ErrCredentialInvalid case "invalid": return out, sandbox.ErrInvalid case "ownership": diff --git a/services/agents-api/internal/store/runtime_allocations.go b/services/agents-api/internal/store/runtime_allocations.go index 71831b36c..7edd12ec8 100644 --- a/services/agents-api/internal/store/runtime_allocations.go +++ b/services/agents-api/internal/store/runtime_allocations.go @@ -16,6 +16,7 @@ import ( // RuntimeAllocation retains compute ownership, not public readiness. It survives // Session deletion until cleanup is confirmed. No bootstrap secret is retained. type RuntimeAllocation struct { + DeploymentGeneration uint64 NodeID string ObservationError string ComputePhase string @@ -100,6 +101,7 @@ func (s *Store) ReserveRuntimeAllocation(ctx context.Context, tenant, environmen if err != nil { return err } + generation := pgtype.Int8{Int64: deployment.Generation, Valid: true} if deployment.Mode == "nodes" { placement, err := q.GetRuntimePlacement(ctx, lookup.ID) if err != nil { @@ -109,13 +111,14 @@ func (s *Store) ReserveRuntimeAllocation(ctx context.Context, tenant, environmen return ErrRuntimeNodeUnavailable } nodeID = placement.NodeID + generation = placement.DeploymentGeneration } if err := createEnvironmentDevice(ctx, q, lookup, session, device); err != nil { return err } row, err := q.CreateRuntimeAllocation(ctx, sqlc.CreateRuntimeAllocationParams{ ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, EnvironmentID: lookup.ID, - DeviceID: device.ID, ProviderKey: provider, NodeID: nodeID, + DeviceID: device.ID, ProviderKey: provider, NodeID: nodeID, DeploymentGeneration: generation, }) if err == nil { result = runtimeAllocationFromRow(row, session, lookup.TenantID, pgtype.Timestamptz{}, false) @@ -208,7 +211,7 @@ func runtimeAllocationFromRow(row sqlc.RuntimeAllocation, session, tenant pgtype retainedUntil = &value } return RuntimeAllocation{ - NodeID: runtimeUUID(row.NodeID), ObservationError: row.ObservationError, + DeploymentGeneration: uint64(row.DeploymentGeneration.Int64), NodeID: runtimeUUID(row.NodeID), ObservationError: row.ObservationError, ComputePhase: row.ComputePhase, ComputeRevision: row.ComputeRevision, ComputeState: row.ComputeState, ComputeActivityAt: row.ComputeActivityAt.Time, ComputeWakeRequested: row.ComputeWakeRequested, ComputeRetainedUntil: retainedUntil, ID: uuid.UUID(row.ID.Bytes).String(), EnvironmentID: uuid.UUID(row.EnvironmentID.Bytes).String(), diff --git a/services/agents-api/internal/store/runtime_node_types.go b/services/agents-api/internal/store/runtime_node_types.go index 83decebc7..6cf19f7f3 100644 --- a/services/agents-api/internal/store/runtime_node_types.go +++ b/services/agents-api/internal/store/runtime_node_types.go @@ -61,6 +61,7 @@ type RuntimeNodeHealth struct { AvailableDiskBytes *int64 `json:"available_disk_bytes"` } type RuntimeNode struct { + Rollout SandboxNodeRollout `json:"rollout"` RuntimeNodeHealth Running int64 `json:"running"` Snapshots int64 `json:"snapshots"` @@ -125,7 +126,27 @@ type SandboxSuspensionView struct { IdleSeconds int64 `json:"idle_seconds"` RetentionSeconds int64 `json:"retention_seconds"` } +type SandboxNodeRollout struct { + // Target preparation, independent of an old pin's serving readiness. + State string `json:"state" enums:"ready,preparing,failed,update_required,unknown"` + // Durable serving-generation pin; online and provider_ready still gate placement. + ReadyGeneration *uint64 `json:"ready_generation" extensions:"x-nullable"` + Diagnostic string `json:"diagnostic,omitempty"` +} +type SandboxRolloutNodes struct { + Ready int64 `json:"ready"` + Preparing int64 `json:"preparing"` + Failed int64 `json:"failed"` + UpdateRequired int64 `json:"update_required"` + Unknown int64 `json:"unknown"` +} +type SandboxRollout struct { + State string `json:"state" enums:"settled,preparing"` + PreviousGenerationSandboxes int64 `json:"previous_generation_sandboxes"` + Nodes *SandboxRolloutNodes `json:"nodes" extensions:"x-nullable"` +} type RuntimeDeploymentView struct { + Rollout SandboxRollout `json:"rollout"` Specification *sandbox.DeploymentSpec `json:"specification,omitempty"` SpecificationDigest string `json:"specification_digest,omitempty"` Generation uint64 `json:"generation"` @@ -142,14 +163,15 @@ type RuntimeDeploymentView struct { CoreURL string `json:"core_url"` } type RuntimeNodeAllocation struct { - Diagnostic string `json:"diagnostic"` - ID string `json:"id"` - NodeID string `json:"node_id"` - TenantID string `json:"tenant_id"` - SessionID string `json:"session_id"` - EnvironmentID string `json:"environment_id"` - State string `json:"state"` - ComputePhase string `json:"compute_phase"` + DeploymentGeneration uint64 `json:"deployment_generation"` + Diagnostic string `json:"diagnostic"` + ID string `json:"id"` + NodeID string `json:"node_id"` + TenantID string `json:"tenant_id"` + SessionID string `json:"session_id"` + EnvironmentID string `json:"environment_id"` + State string `json:"state"` + ComputePhase string `json:"compute_phase"` // The time the allocation entered its current compute_phase, or null when unknown; an allocation that existed before Core recorded it reports null until its next phase change. For a suspended microsandbox allocation, this time plus the deployment's snapshot retention tells roughly when Core reclaims it. ComputePhaseChangedAt *time.Time `json:"compute_phase_changed_at" extensions:"x-nullable"` Initialization string `json:"initialization"` diff --git a/services/agents-api/internal/store/runtime_nodes.go b/services/agents-api/internal/store/runtime_nodes.go index d5bd9b3af..8df1179d9 100644 --- a/services/agents-api/internal/store/runtime_nodes.go +++ b/services/agents-api/internal/store/runtime_nodes.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" "strings" "time" @@ -77,6 +78,9 @@ func (s *Store) deploymentView(ctx context.Context, q *sqlc.Queries) (RuntimeDep } d := row.RuntimeDeployment result := runtimeDeploymentView(d, s.publicURL) + if err := json.Unmarshal(row.Rollout, &result.Rollout); err != nil { + return RuntimeDeploymentView{}, err + } result.Resources = SandboxDeploymentResources{Allocations: row.Allocations, Pending: row.Pending} if d.ResetClear.Valid { remaining := SandboxResetRemaining{} @@ -128,7 +132,7 @@ func runtimeNodeViews(rows []sqlc.ListRuntimeNodesRow) ([]RuntimeNode, error) { return nil, err } health.ProviderReady = n.ProviderReady - out = append(out, RuntimeNode{RuntimeNodeHealth: health, Running: n.Running, Snapshots: n.Snapshots, ID: runtimeUUID(n.ID), Name: n.Name, CoreURL: n.CoreUrl, EnrollmentID: optionalUUID(n.EnrollmentID), Provider: n.ProviderKind, Online: n.Online, LastSeenAt: seen, MaxActive: int(n.MaxActive), MaxRetained: retainedLimit(n.ProviderKind, int(n.MaxActive), int(n.MaxRetained)), Active: n.Active, Reserved: n.Reserved, Retained: n.Retained, CleanupPending: n.CleanupPending, CreatedAt: n.CreatedAt.Time}) + out = append(out, RuntimeNode{Rollout: nodeRollout(n), RuntimeNodeHealth: health, Running: n.Running, Snapshots: n.Snapshots, ID: runtimeUUID(n.ID), Name: n.Name, CoreURL: n.CoreUrl, EnrollmentID: optionalUUID(n.EnrollmentID), Provider: n.ProviderKind, Online: n.Online, LastSeenAt: seen, MaxActive: int(n.MaxActive), MaxRetained: retainedLimit(n.ProviderKind, int(n.MaxActive), int(n.MaxRetained)), Active: n.Active, Reserved: n.Reserved, Retained: n.Retained, CleanupPending: n.CleanupPending, CreatedAt: n.CreatedAt.Time}) } return out, nil } @@ -340,7 +344,32 @@ func (s *Store) ListNodeRuntimeAllocations(ctx context.Context, nodeID string) ( value := a.ComputePhaseChangedAt.Time phaseChanged = &value } - out = append(out, RuntimeNodeAllocation{Diagnostic: a.ObservationError, ID: runtimeUUID(a.ID), NodeID: runtimeUUID(a.NodeID), TenantID: runtimeUUID(a.TenantID), SessionID: runtimeUUID(a.SessionID), EnvironmentID: runtimeUUID(a.EnvironmentID), State: a.State, ComputePhase: a.ComputePhase, ComputePhaseChangedAt: phaseChanged, Initialization: a.Initialization, CreatedAt: a.CreatedAt.Time}) + out = append(out, RuntimeNodeAllocation{DeploymentGeneration: uint64(a.DeploymentGeneration.Int64), Diagnostic: a.ObservationError, ID: runtimeUUID(a.ID), NodeID: runtimeUUID(a.NodeID), TenantID: runtimeUUID(a.TenantID), SessionID: runtimeUUID(a.SessionID), EnvironmentID: runtimeUUID(a.EnvironmentID), State: a.State, ComputePhase: a.ComputePhase, ComputePhaseChangedAt: phaseChanged, Initialization: a.Initialization, CreatedAt: a.CreatedAt.Time}) } return out, nil } + +func nodeRollout(n sqlc.ListRuntimeNodesRow) SandboxNodeRollout { + out := SandboxNodeRollout{State: "unknown"} + if n.ReadyGeneration.Valid { + generation := uint64(n.ReadyGeneration.Int64) + out.ReadyGeneration = &generation + } + if !n.Online { + return out + } + if n.DeploymentGeneration != n.TargetGeneration { + out.State = "update_required" + return out + } + if n.ProviderReady && n.ReadyGeneration.Valid && n.ReadyGeneration.Int64 == n.TargetGeneration { + out.State = "ready" + return out + } + var health RuntimeNodeHealth + if json.Unmarshal(n.Health, &health) == nil && health.Diagnostic != "" { + out.State = "failed" + out.Diagnostic = sandbox.NormalizeNodeDiagnostic(health.Diagnostic) + } + return out +} diff --git a/services/agents-api/internal/store/runtime_worker_recovery_test.go b/services/agents-api/internal/store/runtime_worker_recovery_test.go index b99dbb6ed..601255d7f 100644 --- a/services/agents-api/internal/store/runtime_worker_recovery_test.go +++ b/services/agents-api/internal/store/runtime_worker_recovery_test.go @@ -16,8 +16,8 @@ import ( func insertWorkerRuntimeAllocation(t *testing.T, pool *pgxpool.Pool, h *dispatchHarness, phase string) { t.Helper() - _, err := pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,initialization,compute_phase,compute_retained_until) - VALUES($1,$2,$3,$4,'running',true,'complete',$5,clock_timestamp()+interval '1 hour')`, uuid.NewString(), h.device.EnvironmentID, h.device.ID, uuid.NewString(), phase) + _, err := pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,initialization,compute_phase,compute_retained_until,deployment_generation) + VALUES($1,$2,$3,$4,'running',true,'complete',$5,clock_timestamp()+interval '1 hour',(SELECT generation FROM runtime_deployment))`, uuid.NewString(), h.device.EnvironmentID, h.device.ID, uuid.NewString(), phase) if err != nil { t.Fatal(err) } diff --git a/services/agents-api/internal/store/sandbox_deployment_mutations.go b/services/agents-api/internal/store/sandbox_deployment_mutations.go index 9ddb287be..6d41dc3d6 100644 --- a/services/agents-api/internal/store/sandbox_deployment_mutations.go +++ b/services/agents-api/internal/store/sandbox_deployment_mutations.go @@ -22,9 +22,10 @@ var ErrSandboxCredentialUnavailable = errors.New("sandbox credential encryption // APIKey is internal configuration. HTTP requests use a write-only DTO. // TemplateBuild is set only by Core after it validates the candidate. type SandboxE2BConfiguration struct { - APIKey string `json:"-"` - Template string `json:"template"` - TemplateBuild *SandboxE2BTemplateBuild `json:"-"` + ReplaceCredential bool `json:"-"` + APIKey string `json:"-"` + Template string `json:"template"` + TemplateBuild *SandboxE2BTemplateBuild `json:"-"` } // SandboxE2BTemplateBuild is the fixed build as read by the validation that @@ -209,7 +210,7 @@ func (s *Store) InitializeSandboxDeployment(ctx context.Context, installationID } // CheckSandboxDeploymentSwitch is a preliminary check only. The mutation repeats -// it after execution has drained; no database lock spans provider work. +// it in the committing transaction; no database lock spans provider work. func (s *Store) CheckSandboxDeploymentSwitch(ctx context.Context, installation string, input SandboxDeploymentUpdateRequest) error { if s.executionLease == nil { return ErrInvalidInput @@ -241,6 +242,9 @@ func checkSandboxSwitch(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDepl if d.ProviderKind != input.Provider || unspecifiedNodeDeployment(d) { return &SandboxResetRequiredError{CurrentProvider: d.ProviderKind, RequestedProvider: input.Provider} } + if d.ProviderKind == "e2b" { + return nil + } resources, err := q.CountRuntimeDeploymentResources(ctx) if err != nil { return err @@ -274,19 +278,36 @@ func (s *Store) UpdateSandboxDeployment(ctx context.Context, installation string if err != nil { return err } - if !equal { - if err := s.saveSandboxSelection(ctx, q, d, input.SandboxDeploymentSetupRequest); err != nil { + if !equal || input.E2B != nil && input.E2B.ReplaceCredential { + if err := q.RetainSandboxGeneration(ctx); err != nil { return err } - if err := q.RetireSandboxNodes(ctx); err != nil { + if err := s.saveSandboxSelection(ctx, q, d, input.SandboxDeploymentSetupRequest); err != nil { return err } - if err := q.RetireSandboxEnrollments(ctx); err != nil { - return err + if input.Provider != "e2b" { + if err := q.RetireSandboxNodes(ctx); err != nil { + return err + } + if err := q.RetireSandboxEnrollments(ctx); err != nil { + return err + } + if err := q.AdvanceSandboxOwnerEpoch(ctx); err != nil { + return err + } } - if err := q.AdvanceSandboxOwnerEpoch(ctx); err != nil { + if err := q.CollectSandboxGenerations(ctx); err != nil { return err } + if input.Provider == "e2b" { + action := "change" + if input.E2B.ReplaceCredential { + action = "replace_credential" + } + if err := recordDeploymentMutation(ctx, q, action, "sandbox_deployment", installation); err != nil { + return err + } + } } else if err := recordTemplateBuild(ctx, q, input.SandboxDeploymentSetupRequest); err != nil { return err } diff --git a/services/agents-api/internal/store/sandbox_deployment_setup.go b/services/agents-api/internal/store/sandbox_deployment_setup.go index ea328cfb7..5c458f81c 100644 --- a/services/agents-api/internal/store/sandbox_deployment_setup.go +++ b/services/agents-api/internal/store/sandbox_deployment_setup.go @@ -43,6 +43,10 @@ func (s *Store) GetSandboxSetup(ctx context.Context) (SandboxSetup, error) { if err != nil { return SandboxSetup{}, err } + return s.sandboxSetup(d) +} + +func (s *Store) sandboxSetup(d sqlc.RuntimeDeployment) (SandboxSetup, error) { if !d.WebManaged { return SandboxSetup{}, ErrSandboxDeploymentConflict } diff --git a/services/agents-api/internal/store/sandbox_generations.go b/services/agents-api/internal/store/sandbox_generations.go new file mode 100644 index 000000000..afd81b7d0 --- /dev/null +++ b/services/agents-api/internal/store/sandbox_generations.go @@ -0,0 +1,151 @@ +package store + +import ( + "context" + "encoding/json" + "errors" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +// ClassifySandboxDeploymentChange resolves an omitted credential before validation. +// Explicit key submission is a verified mutation even when its bytes are unchanged. +func (s *Store) ClassifySandboxDeploymentChange(ctx context.Context, installation string, input SandboxDeploymentUpdateRequest) (SandboxDeploymentUpdateRequest, bool, error) { + var unchanged bool + err := s.resetTransaction(ctx, func(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment) error { + if err := checkSandboxSwitch(ctx, q, d, installation, input); err != nil { + return err + } + if input.E2B != nil { + copy := *input.E2B + copy.ReplaceCredential = copy.ReplaceCredential || copy.APIKey != "" + input.E2B = © + if copy.APIKey == "" && !copy.ReplaceCredential { + key, err := s.credentialCipher.OpenSandboxDeployment(d.E2bCredential, installation, uint64(d.Generation)) + if err != nil { + return ErrSandboxCredentialUnavailable + } + input.E2B.APIKey = string(key) + } + if input.E2B.Template == d.E2bTemplate && E2BResourcesPending(input.SandboxDeploymentSetupRequest) { + var saved sandbox.DeploymentSpec + if err := json.Unmarshal(d.Specification, &saved); err != nil { + return err + } + input.Resources = saved.Resources + } + } + if err := validateSandboxSelection(input.SandboxDeploymentSetupRequest); err != nil { + return err + } + equal, err := s.sandboxSelectionEqual(d, input.SandboxDeploymentSetupRequest) + unchanged = equal && (input.E2B == nil || !input.E2B.ReplaceCredential) + return err + }) + return input, unchanged, err +} + +// GetSandboxAllocationSetup reads immutable ownership and the current credential +// in one snapshot. A released receipt remains historical but is never rebound. +func (s *Store) GetSandboxAllocationSetup(ctx context.Context, ref sandbox.Reference) (SandboxSetup, error) { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) + if err != nil { + return SandboxSetup{}, err + } + defer tx.Rollback(ctx) + q := s.queries.WithTx(tx) + lookup, err := deviceLookup(ref.TenantID, ref.EnvironmentID) + if err != nil { + return SandboxSetup{}, err + } + a, err := q.GetRuntimeAllocation(ctx, sqlc.GetRuntimeAllocationParams{TenantID: lookup.TenantID, EnvironmentID: lookup.ID}) + if err != nil { + return SandboxSetup{}, err + } + if runtimeUUID(a.RuntimeAllocation.ID) != ref.AllocationID || !a.RuntimeAllocation.DeploymentGeneration.Valid || a.RuntimeAllocation.State == "released" { + return SandboxSetup{}, ErrInvalidInput + } + d, err := q.GetRuntimeDeployment(ctx) + if err != nil { + return SandboxSetup{}, err + } + if a.RuntimeAllocation.ProviderKey != d.InstallationID { + return SandboxSetup{}, sandbox.ErrOwnership + } + result, err := s.sandboxSetup(d) + if err != nil { + return SandboxSetup{}, err + } + generation := a.RuntimeAllocation.DeploymentGeneration.Int64 + if generation != d.Generation { + g, err := q.GetSandboxGeneration(ctx, generation) + if errors.Is(err, pgx.ErrNoRows) { + return SandboxSetup{}, ErrSandboxDeploymentConflict + } + if err != nil { + return SandboxSetup{}, err + } + if g.ProviderKind != d.ProviderKind { + return SandboxSetup{}, ErrSandboxDeploymentConflict + } + result.Generation = uint64(generation) + if err = json.Unmarshal(g.Specification, &result.Specification); err != nil { + return SandboxSetup{}, err + } + if result.E2B != nil { + result.E2B.Template = g.E2bTemplate + } + } + return result, tx.Commit(ctx) +} + +// SandboxGenerationPage is bounded; callers retain a deadline over the full scan. +func (s *Store) SandboxGenerationPage(ctx context.Context, after int64) ([]SandboxSetup, error) { + rows, err := s.queries.ListSandboxGenerations(ctx, after) + if err != nil { + return nil, err + } + result := make([]SandboxSetup, 0, len(rows)) + for _, r := range rows { + v := SandboxSetup{Generation: uint64(r.Generation), Provider: r.ProviderKind} + if err := json.Unmarshal(r.Specification, &v.Specification); err != nil { + return nil, err + } + if r.ProviderKind == "e2b" { + v.E2B = &SandboxE2BConfiguration{Template: r.E2bTemplate} + } + result = append(result, v) + } + return result, nil +} + +// CollectSandboxGenerations shares the same serialization as admission and PUT. +func (s *Store) CollectSandboxGenerations(ctx context.Context) error { + return s.resetTransaction(ctx, func(ctx context.Context, q *sqlc.Queries, _ sqlc.RuntimeDeployment) error { + return q.CollectSandboxGenerations(ctx) + }) +} + +// SandboxCredentialAllocationPage reads owned receipts without granting execution authority. +func (s *Store) SandboxCredentialAllocationPage(ctx context.Context, after string) ([]RuntimeAllocation, error) { + id := pgtype.UUID{Valid: true} + if after != "" { + var err error + id, err = parseID(after) + if err != nil { + return nil, err + } + } + rows, err := s.queries.ListRuntimeAllocations(ctx, id) + if err != nil { + return nil, err + } + out := make([]RuntimeAllocation, 0, len(rows)) + for _, r := range rows { + out = append(out, runtimeAllocationFromRow(r.RuntimeAllocation, r.SessionID, r.TenantID, r.DeletedAt, r.Expired)) + } + return out, nil +} diff --git a/services/agents-api/internal/store/sandbox_generations_test.go b/services/agents-api/internal/store/sandbox_generations_test.go new file mode 100644 index 000000000..559a68486 --- /dev/null +++ b/services/agents-api/internal/store/sandbox_generations_test.go @@ -0,0 +1,334 @@ +package store + +import ( + "database/sql" + "encoding/json" + "errors" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" + "os" + "testing" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/google/uuid" +) + +func TestE2BGenerationsRetainOwnershipAndUseCurrentCredential(t *testing.T) { + s, w, view, input := webSpecificationFixture(t, "e2b") + tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + owner := archiveAllocation(t, w, tenant, session, view.InstallationID) + ctx := SandboxResetTestContext(t.Context()) + oldTemplate := input.E2B.Template + input.E2B.Template = "next:" + uuid.NewString() + input.Resources.CPUs++ + changed, err := w.UpdateSandboxDeployment(ctx, view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}) + if err != nil || changed.Generation != 2 || changed.OwnerEpoch != view.OwnerEpoch || changed.Rollout.PreviousGenerationSandboxes != 1 || changed.Rollout.State != "settled" { + t.Fatal(changed, err) + } + ref := sandbox.Reference{TenantID: tenant, EnvironmentID: owner.EnvironmentID, AllocationID: owner.ID} + retained, err := s.GetSandboxAllocationSetup(t.Context(), ref) + if err != nil || retained.Generation != 1 || retained.E2B.Template != oldTemplate || retained.Specification.Resources.CPUs == input.Resources.CPUs { + t.Fatal(retained, err) + } + input.E2B.APIKey = "replacement-secret" + input.E2B.ReplaceCredential = true + changed, err = w.UpdateSandboxDeployment(ctx, view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 2}) + if err != nil || changed.Generation != 3 || changed.OwnerEpoch != view.OwnerEpoch { + t.Fatal(changed, err) + } + retained, err = s.GetSandboxAllocationSetup(t.Context(), ref) + if err != nil || retained.Generation != 1 || retained.E2B.APIKey != input.E2B.APIKey || retained.E2B.Template != oldTemplate { + t.Fatal("old generation did not use committed key", err) + } + if _, err = s.pool.Exec(t.Context(), `UPDATE runtime_allocations SET deployment_generation=3 WHERE id=$1`, owner.ID); err == nil { + t.Fatal("ownership generation was mutable") + } + if err = w.CollectSandboxGenerations(t.Context()); err != nil { + t.Fatal(err) + } + generations, err := s.SandboxGenerationPage(t.Context(), -1) + if err != nil || len(generations) != 1 || generations[0].Generation != 1 { + t.Fatal(generations, err) + } + if _, err = w.RequestRuntimeCleanup(t.Context(), owner); err != nil { + t.Fatal(err) + } + if _, err = w.SettleRuntimeCreation(t.Context(), owner); err != nil { + t.Fatal(err) + } + if _, err = w.ReleaseRuntimeAllocation(t.Context(), owner); err != nil { + t.Fatal(err) + } + if err = w.CollectSandboxGenerations(t.Context()); err != nil { + t.Fatal(err) + } + generations, err = s.SandboxGenerationPage(t.Context(), -1) + if err != nil || len(generations) != 0 { + t.Fatal(generations, err) + } + historical, err := s.GetRuntimeAllocation(t.Context(), tenant, owner.EnvironmentID) + if err != nil || historical.DeploymentGeneration != 1 { + t.Fatal("historical generation erased", err) + } +} + +func TestE2BChangeClassifierOmittedKeyAndExplicitSameKey(t *testing.T) { + _, w, view, input := webSpecificationFixture(t, "e2b") + key := input.E2B.APIKey + input.E2B.APIKey = "" + input.Resources = sandbox.Resources{} + resolved, noOp, err := w.ClassifySandboxDeploymentChange(t.Context(), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}) + if err != nil || !noOp || resolved.E2B.APIKey != key || resolved.Resources.CPUs == 0 { + t.Fatal(noOp, err) + } + input.E2B.APIKey = key + input.E2B.ReplaceCredential = true + _, noOp, err = w.ClassifySandboxDeploymentChange(t.Context(), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}) + if err != nil || noOp { + t.Fatal("explicit same key skipped verification", err) + } + invalid := input + invalid.Runtime = &sandbox.RuntimeRelease{} + if _, _, err := w.ClassifySandboxDeploymentChange(t.Context(), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: invalid, ExpectedGeneration: 1}); err == nil { + t.Fatal("omitted resources erased forbidden Runtime input") + } + _, _, err = w.ClassifySandboxDeploymentChange(t.Context(), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 0}) + var stale *SandboxGenerationStaleError + if !errors.As(err, &stale) { + t.Fatal("stale did not precede no-op", err) + } +} + +func TestGenerationPinRetainsOfflineZeroResourceFallback(t *testing.T) { + s, w, view, _ := webSpecificationFixture(t, "docker") + node := specificationNode(t, s, view) + if _, err := s.pool.Exec(t.Context(), `UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1`, node.NodeID); err != nil { + t.Fatal(err) + } + // PR-N will make this transition through its generation protocol. This fixture + // isolates the persistence invariant without enabling node online PUT in PR-G. + tx, err := s.pool.Begin(t.Context()) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(t.Context()) + q := s.queries.WithTx(tx) + if _, err = q.LockRuntimeDeployment(t.Context()); err != nil { + t.Fatal(err) + } + if err = q.RetainSandboxGeneration(t.Context()); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(t.Context(), `UPDATE runtime_deployment SET generation=2`); err != nil { + t.Fatal(err) + } + if err = tx.Commit(t.Context()); err != nil { + t.Fatal(err) + } + if err = w.CollectSandboxGenerations(t.Context()); err != nil { + t.Fatal(err) + } + rows, err := s.SandboxGenerationPage(t.Context(), -1) + if err != nil || len(rows) != 1 { + t.Fatal("offline pin was collected", rows, err) + } + nodes, err := s.ListRuntimeNodes(t.Context()) + if err != nil || len(nodes) != 1 || nodes[0].Rollout.State != "unknown" || nodes[0].Rollout.ReadyGeneration == nil || *nodes[0].Rollout.ReadyGeneration != 1 { + t.Fatal(nodes, err) + } + if _, err = s.pool.Exec(t.Context(), `UPDATE runtime_nodes SET removed_at=clock_timestamp(),ready_generation=NULL WHERE id=$1`, node.NodeID); err != nil { + t.Fatal(err) + } + if err = w.CollectSandboxGenerations(t.Context()); err != nil { + t.Fatal(err) + } + rows, err = s.SandboxGenerationPage(t.Context(), -1) + if err != nil || len(rows) != 0 { + t.Fatal(rows, err) + } +} + +func TestPendingPlacementGenerationSurvivesRepeatedUpdates(t *testing.T) { + s, w, view, _ := webSpecificationFixture(t, "docker") + node := specificationNode(t, s, view) + tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + for generation := uint64(1); generation <= 2; generation++ { + tx, err := s.pool.Begin(t.Context()) + if err != nil { + t.Fatal(err) + } + q := s.queries.WithTx(tx) + if _, err = q.LockRuntimeDeployment(t.Context()); err != nil { + t.Fatal(err) + } + if err = q.RetainSandboxGeneration(t.Context()); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(t.Context(), `UPDATE runtime_deployment SET generation=generation+1`); err != nil { + t.Fatal(err) + } + if err = tx.Commit(t.Context()); err != nil { + t.Fatal(err) + } + } + owner := archiveAllocation(t, w, tenant, session, view.InstallationID) + if owner.DeploymentGeneration != 1 || owner.NodeID != node.NodeID { + t.Fatal("pending allocation rebound to newest generation", owner) + } + if err := w.CollectSandboxGenerations(t.Context()); err != nil { + t.Fatal(err) + } + rows, err := s.SandboxGenerationPage(t.Context(), -1) + if err != nil || len(rows) != 1 || rows[0].Generation != 1 { + t.Fatal(rows, err) + } + if _, err = s.pool.Exec(t.Context(), `UPDATE runtime_placements SET deployment_generation=3 WHERE environment_id=$1`, owner.EnvironmentID); err == nil { + t.Fatal("placement generation changed") + } +} + +func TestGenerationMigrationBackfillsAndRejectsLossyDowngrade(t *testing.T) { + db, migrations := runtimeNamesMigrationSchema(t) + ctx := t.Context() + if _, err := migrations.UpTo(ctx, 79); err != nil { + t.Fatal(err) + } + installation, node := uuid.NewString(), uuid.NewString() + if _, err := db.ExecContext(ctx, `UPDATE runtime_deployment SET installation_id=$1,backend_fingerprint=repeat('a',64),provider_kind='docker',mode='nodes',generation=1`, installation); err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(ctx, `INSERT INTO runtime_nodes(id,installation_id,name,backend_fingerprint,credential_sha256,max_active,max_retained,deployment_generation) VALUES($1,$2,'old',repeat('a',64),repeat('b',64),1,1,1)`, node, installation); err != nil { + t.Fatal(err) + } + session, tenant, environment, device, allocation := uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString() + exec := func(query string, args ...any) { + t.Helper() + if _, err := db.ExecContext(ctx, query, args...); err != nil { + t.Fatal(err) + } + } + exec(`INSERT INTO sessions(id,tenant_id,engine,idempotency_key,request_hash,configuration) VALUES($1,$2,'codex','old','old','{}')`, session, tenant) + exec(`INSERT INTO environments(id,session_id) VALUES($1,$2)`, environment, session) + exec(`INSERT INTO devices(id,tenant_id,name,credential_hash) VALUES($1,$2,'old',repeat('b',64))`, device, tenant) + exec(`INSERT INTO runtime_placements(environment_id,node_id) VALUES($1,$2)`, environment, node) + exec(`INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,node_id) VALUES($1,$2,$3,$4,$5)`, allocation, environment, device, installation, node) + if _, err := migrations.UpTo(ctx, 80); err != nil { + t.Fatal(err) + } + for _, table := range []string{"runtime_allocations", "runtime_placements"} { + var generation int64 + if err := db.QueryRowContext(ctx, `SELECT deployment_generation FROM `+table).Scan(&generation); err != nil || generation != 1 { + t.Fatal("inexact migration backfill", table, generation, err) + } + } + var pin int64 + if err := db.QueryRowContext(ctx, `SELECT ready_generation FROM runtime_nodes WHERE id=$1`, node).Scan(&pin); err != nil || pin != 1 { + t.Fatal(pin, err) + } + if _, err := db.ExecContext(ctx, `UPDATE runtime_deployment SET generation=2`); err != nil { + t.Fatal(err) + } + if _, err := migrations.DownTo(ctx, 79); err == nil { + t.Fatal("downgrade erased offline serving pin") + } + if _, err := db.ExecContext(ctx, `UPDATE runtime_nodes SET removed_at=clock_timestamp(),ready_generation=NULL`); err != nil { + t.Fatal(err) + } + if _, err := migrations.DownTo(ctx, 79); err == nil { + t.Fatal("downgrade discarded held allocation/placement after pin removal") + } + exec(`UPDATE runtime_allocations SET state='released',released_at=clock_timestamp(),create_settled=true`) + if _, err := migrations.DownTo(ctx, 79); err == nil { + t.Fatal("downgrade discarded unreleased placement") + } + exec(`UPDATE runtime_placements SET released_at=clock_timestamp()`) + if _, err := migrations.DownTo(ctx, 79); err != nil { + t.Fatal("settled downgrade failed", err) + } +} + +func TestGenerationDowngradeRefusesOldAllocation(t *testing.T) { + s, w, view, input := webSpecificationFixture(t, "e2b") + tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + owner := archiveAllocation(t, w, tenant, session, view.InstallationID) + input.E2B.Template = "next:" + uuid.NewString() + if _, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}); err != nil { + t.Fatal(err) + } + db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) + defer db.Close() + migrations, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) + if err != nil { + t.Fatal(err) + } + if _, err = migrations.DownTo(t.Context(), 79); err == nil { + t.Fatal("downgrade erased old owned allocation") + } + if _, err = w.RequestRuntimeCleanup(t.Context(), owner); err != nil { + t.Fatal(err) + } + if _, err = w.SettleRuntimeCreation(t.Context(), owner); err != nil { + t.Fatal(err) + } + if _, err = w.ReleaseRuntimeAllocation(t.Context(), owner); err != nil { + t.Fatal(err) + } + if _, err = migrations.DownTo(t.Context(), 79); err != nil { + t.Fatal("released history prevented safe downgrade", err) + } +} + +func TestGenerationUpdateSerializesWithAllocationAdmission(t *testing.T) { + s, w, view, input := webSpecificationFixture(t, "e2b") + for generation := uint64(1); generation <= 6; generation++ { + tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + input.E2B.Template = "next:" + uuid.NewString() + start := make(chan struct{}) + changed := make(chan error, 1) + go func() { + <-start + _, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: generation}) + changed <- err + }() + close(start) + owner := archiveAllocation(t, w, tenant, session, view.InstallationID) + if err := <-changed; err != nil { + t.Fatal(err) + } + if owner.DeploymentGeneration != generation && owner.DeploymentGeneration != generation+1 { + t.Fatal("allocation bound unrelated generation", owner.DeploymentGeneration) + } + if err := w.CollectSandboxGenerations(t.Context()); err != nil { + t.Fatal(err) + } + setup, err := s.GetSandboxAllocationSetup(t.Context(), sandbox.Reference{TenantID: tenant, EnvironmentID: owner.EnvironmentID, AllocationID: owner.ID}) + if err != nil || setup.Generation != owner.DeploymentGeneration { + t.Fatal("committed allocation lost immutable routing", err) + } + } +} + +func TestNodeRolloutSeparatesOfflinePinAndTargetReadiness(t *testing.T) { + for _, tc := range []struct { + name string + online, ready bool + enrolled, pin, target int64 + diagnostic, state string + }{ + {"offline pin", false, true, 1, 1, 2, "", "unknown"}, + {"old serving", true, true, 1, 1, 2, "", "update_required"}, + {"current serving", true, true, 2, 2, 2, "", "ready"}, + {"current failed", true, false, 2, 2, 2, "kvm_unavailable", "failed"}, + {"current unknown", true, false, 2, 2, 2, "", "unknown"}, + } { + t.Run(tc.name, func(t *testing.T) { + raw, _ := json.Marshal(RuntimeNodeHealth{Diagnostic: tc.diagnostic}) + got := nodeRollout(sqlc.ListRuntimeNodesRow{Online: tc.online, ProviderReady: tc.ready, DeploymentGeneration: tc.enrolled, ReadyGeneration: pgtype.Int8{Int64: tc.pin, Valid: true}, TargetGeneration: tc.target, Health: raw}) + if got.State != tc.state || got.ReadyGeneration == nil || *got.ReadyGeneration != uint64(tc.pin) { + t.Fatal(got) + } + }) + } +} diff --git a/services/agents-api/internal/store/sandbox_reset.go b/services/agents-api/internal/store/sandbox_reset.go index 50ac5060d..9fcfe890c 100644 --- a/services/agents-api/internal/store/sandbox_reset.go +++ b/services/agents-api/internal/store/sandbox_reset.go @@ -207,6 +207,9 @@ func (s *Store) CompleteSandboxReset(ctx context.Context, installation string, g if err := q.RetireSandboxEnrollments(ctx); err != nil { return err } + if err := q.ClearSandboxGenerations(ctx); err != nil { + return err + } if err := recordDeploymentMutation(adminaudit.WithSource(ctx, source), q, "reset_complete", "sandbox_deployment", installation); err != nil { return err } diff --git a/services/agents-api/migrations/000080_sandbox_generations.sql b/services/agents-api/migrations/000080_sandbox_generations.sql new file mode 100644 index 000000000..e9655b3ff --- /dev/null +++ b/services/agents-api/migrations/000080_sandbox_generations.sql @@ -0,0 +1,67 @@ +-- +goose Up +CREATE TABLE runtime_deployment_generations ( + generation bigint PRIMARY KEY CHECK (generation >= 0), + provider_kind text NOT NULL CHECK (provider_kind IN ('docker','microsandbox','e2b')), + specification jsonb NOT NULL, + e2b_template text NOT NULL DEFAULT '', + e2b_template_build_status text, + e2b_template_cpus integer, + e2b_template_memory_mib integer, + e2b_template_root_disk_mib integer, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +ALTER TABLE runtime_allocations ADD COLUMN deployment_generation bigint; +ALTER TABLE runtime_placements ADD COLUMN deployment_generation bigint; +UPDATE runtime_allocations SET deployment_generation=(SELECT generation FROM runtime_deployment) WHERE state <> 'released'; +UPDATE runtime_placements SET deployment_generation=(SELECT generation FROM runtime_deployment) WHERE released_at IS NULL; +ALTER TABLE runtime_allocations ADD CHECK (deployment_generation >= 0), + ADD CHECK (state = 'released' OR deployment_generation IS NOT NULL); +ALTER TABLE runtime_placements ADD CHECK (deployment_generation >= 0), + ADD CHECK (released_at IS NOT NULL OR deployment_generation IS NOT NULL); +ALTER TABLE runtime_nodes ADD COLUMN ready_generation bigint CHECK (ready_generation >= 0); +-- The serving pin is durable, independent of current connection readiness. +UPDATE runtime_nodes SET ready_generation=deployment_generation WHERE removed_at IS NULL; +CREATE INDEX runtime_allocations_generation_live ON runtime_allocations(deployment_generation) WHERE state <> 'released'; +CREATE INDEX runtime_placements_generation_live ON runtime_placements(deployment_generation) WHERE released_at IS NULL; +CREATE INDEX runtime_nodes_generation_live ON runtime_nodes(ready_generation) WHERE removed_at IS NULL; +-- +goose StatementBegin +CREATE FUNCTION immutable_runtime_deployment_generation() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.deployment_generation IS DISTINCT FROM OLD.deployment_generation THEN + RAISE EXCEPTION 'Sandbox ownership generation is immutable'; + END IF; + RETURN NEW; +END $$; +-- +goose StatementEnd +CREATE TRIGGER immutable_runtime_allocation_generation BEFORE UPDATE ON runtime_allocations +FOR EACH ROW EXECUTE FUNCTION immutable_runtime_deployment_generation(); +CREATE TRIGGER immutable_runtime_placement_generation BEFORE UPDATE ON runtime_placements +FOR EACH ROW EXECUTE FUNCTION immutable_runtime_deployment_generation(); + +-- +goose StatementBegin +CREATE FUNCTION immutable_sandbox_specification() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION 'Retained sandbox specifications are immutable'; +END $$; +-- +goose StatementEnd +CREATE TRIGGER immutable_sandbox_specification BEFORE UPDATE ON runtime_deployment_generations +FOR EACH ROW EXECUTE FUNCTION immutable_sandbox_specification(); + +-- +goose Down +-- +goose StatementBegin +DO $$ BEGIN + IF EXISTS (SELECT 1 FROM runtime_allocations a CROSS JOIN runtime_deployment d WHERE a.state <> 'released' AND a.deployment_generation <> d.generation) + OR EXISTS (SELECT 1 FROM runtime_placements p CROSS JOIN runtime_deployment d WHERE p.released_at IS NULL AND p.deployment_generation <> d.generation) + OR EXISTS (SELECT 1 FROM runtime_nodes n CROSS JOIN runtime_deployment d WHERE n.removed_at IS NULL AND n.ready_generation <> d.generation) THEN + RAISE EXCEPTION 'Cannot downgrade while retained ownership or node serving pins require generation routing'; + END IF; +END $$; +-- +goose StatementEnd +DROP TRIGGER immutable_runtime_allocation_generation ON runtime_allocations; +DROP TRIGGER immutable_runtime_placement_generation ON runtime_placements; +DROP FUNCTION immutable_runtime_deployment_generation(); +ALTER TABLE runtime_nodes DROP COLUMN ready_generation; +ALTER TABLE runtime_placements DROP COLUMN deployment_generation; +ALTER TABLE runtime_allocations DROP COLUMN deployment_generation; +DROP TABLE runtime_deployment_generations; +DROP FUNCTION immutable_sandbox_specification(); diff --git a/services/agents-api/tools/e2b-provider/README.md b/services/agents-api/tools/e2b-provider/README.md index cdf997316..ff5659d25 100644 --- a/services/agents-api/tools/e2b-provider/README.md +++ b/services/agents-api/tools/e2b-provider/README.md @@ -12,7 +12,7 @@ selection without resources adopts the ready build's CPU and memory. It returns the build's status, CPU, memory and reported disk size for Core to record with the selection, and creates neither compute nor allocation receipts. -Runtime observation uses a third read-only request, `observe`, for at most 100 +Runtime observation uses a separate read-only request, `observe`, for at most 100 allocations. It reads each allocation's sandbox ID from its receipt without the allocation lock, then runs one `GET /sandboxes/metrics` request and one labelled listing of this installation's running sandboxes concurrently, within the @@ -27,11 +27,24 @@ resource drift still permits ownership-based cleanup. E2B disk capacity is not an independently configurable limit. Sandbox inspection does not expose a build UUID: build provenance comes from the validated immutable create selector. +Credential replacement uses `verify_credential`, a read-only request with at most +32 Core allocation references. It verifies the fixed build, the SDK's paginated +team-owned template listing, and each settled live receipt against the labelled +sandbox listing. Template and sandbox scans each stop at 100 pages or the caller's +30-second deadline. Missing/unsettled receipts, repeated template cursors and +unconfirmed reads never authorize replacement. No receipt is changed. Core scans +retained generations and allocation pages under one bounded verification context, +then repeats verification while provider calls are fenced before credential commit. +Authentication rejection, ownership mismatch and uncertainty remain distinct fixed +codes. A readable public template is not proof that the key owns it. + The private JSON boundary has version 1. Requests and credentials enter stdin; stdout contains one bounded response with sanitized error codes. API keys never enter arguments, inherited environment or receipts. The process retains its allocation lock when the Core caller times out, until the bounded SDK operation -returns. Core must serialize lifecycle requests and never replay Create. +returns. Core tracks actual child exit even after caller timeout. Credential fencing +waits for those children without killing them; child exit itself never proves remote +Create settled. Core must serialize lifecycle requests and never replay Create. `StateDir` must already exist, be owned by the service user and have mode 0700. Keep it on durable private storage through Core upgrades/restarts. Its receipts diff --git a/services/agents-api/tools/e2b-provider/credential_test.py b/services/agents-api/tools/e2b-provider/credential_test.py new file mode 100644 index 000000000..d546595c0 --- /dev/null +++ b/services/agents-api/tools/e2b-provider/credential_test.py @@ -0,0 +1,68 @@ +"""Credential verification never mutates receipts or infers absent ownership.""" +from types import SimpleNamespace +from unittest.mock import patch +import unittest + +from e2b.api.client.models.template import Template +from provider_test import ProviderTest +from sdk import verify_team_template +from state import Failure + + +class TeamTemplateTest(unittest.TestCase): + @patch('sdk.get_api_client') + @patch('e2b.api.client.api.templates.get_v2_templates.sync_detailed') + def test_paginated_owned_template_and_public_readability_are_distinct(self, listing, client): + template = object.__new__(Template) + template.template_id = 'owned' + config = {'Template': 'owned:build', 'APIKey': 'synthetic'} + listing.side_effect = [SimpleNamespace(status_code=200, parsed=[], headers={'x-next-token': 'page2'}), + SimpleNamespace(status_code=200, parsed=[template], headers={})] + verify_team_template(config, lambda: 5) + self.assertEqual(listing.call_args.kwargs['next_token'], 'page2') + listing.side_effect = None + listing.return_value = SimpleNamespace(status_code=200, parsed=[], headers={}) + with self.assertRaises(Failure) as error: + verify_team_template(config, lambda: 5) + self.assertEqual(error.exception.code, 'team_mismatch') + + @patch('sdk.get_api_client') + @patch('e2b.api.client.api.templates.get_v2_templates.sync_detailed') + def test_unauthorized_and_repeated_cursor_fail_closed(self, listing, client): + config = {'Template': 'owned:build', 'APIKey': 'synthetic'} + for response, code in [(SimpleNamespace(status_code=401), 'unauthorized'), + (SimpleNamespace(status_code=503), 'unconfirmed'), + (SimpleNamespace(status_code=200, parsed=[], headers={'x-next-token': 'same'}), 'unconfirmed')]: + listing.return_value = response + with self.assertRaises(Failure) as error: + verify_team_template(config, lambda: 5) + self.assertEqual(error.exception.code, code) + + +class CredentialReceiptTest(ProviderTest): + def verify(self): + self.request['References'] = [self.reference] + with patch('provider.verify_team_template'), patch('provider.validate_deployment'): + return self.call('verify_credential') + + def test_existing_receipt_is_verified_through_paginated_owned_listing(self): + self.assertEqual(self.call('create')['ErrorCode'], '') + before = self.record() + paginator = SimpleNamespace(has_next=True) + pages = iter([[], [self.cloud]]) + paginator.next_items = lambda **options: next(pages) + self.api.list.return_value = paginator + self.assertEqual(self.verify()['ErrorCode'], '') + self.assertEqual(self.record(), before) + self.api.kill.assert_not_called() + + def test_missing_unsettled_and_invisible_receipts_cannot_publish_a_key(self): + self.assertEqual(self.verify()['ErrorCode'], 'unconfirmed') + self.assertEqual(self.call('create')['ErrorCode'], '') + self.assertEqual(self.verify()['ErrorCode'], 'team_mismatch') + from state import Receipt + with Receipt(self.request, lambda: 5) as receipt: + receipt.save(settled=False) + before = self.record() + self.assertEqual(self.verify()['ErrorCode'], 'unconfirmed') + self.assertEqual(self.record(), before) diff --git a/services/agents-api/tools/e2b-provider/provider.py b/services/agents-api/tools/e2b-provider/provider.py index e46f092cf..99558e5e8 100644 --- a/services/agents-api/tools/e2b-provider/provider.py +++ b/services/agents-api/tools/e2b-provider/provider.py @@ -9,9 +9,9 @@ from e2b import Sandbox, SandboxQuery, SandboxState from e2b.api.client.models.sandbox_metric import SandboxMetric -from e2b.exceptions import FileNotFoundException, SandboxNotFoundException +from e2b.exceptions import AuthenticationException, FileNotFoundException, SandboxNotFoundException -from sdk import connection_material, definitely_rejected, read_metrics, restore, run, validate_deployment +from sdk import connection_material, definitely_rejected, read_metrics, restore, run, validate_deployment, verify_team_template from state import Failure, Receipt, private_root, read_receipt PREFIX = 'oac_' @@ -68,8 +68,8 @@ def __init__(self, request): self.reference = request['Reference'] self.references = request.get('References') or [] if (request['Version'] != 1 or request['Operation'] not in - ('create', 'inspect', 'renew', 'kill', 'command', 'validate_deployment', 'observe') or - (request['Operation'] not in ('validate_deployment', 'observe') and + ('create', 'inspect', 'renew', 'kill', 'command', 'validate_deployment', 'observe', 'verify_credential') or + (request['Operation'] not in ('validate_deployment', 'observe', 'verify_credential') and not valid_reference(self.reference)) or (request['Operation'] == 'observe' and (not 1 <= len(self.references) <= 100 or @@ -313,13 +313,62 @@ def metadata_for(self, reference): return {PREFIX + field.lower(): value for field, value in dict(reference, InstallationID=self.config['InstallationID']).items()} + def verify_credential(self): + verify_team_template(self.config, self.remaining) + validate_deployment(self.config, self.remaining) + if not isinstance(self.references, list) or len(self.references) > 32 or not all(valid_reference(r) for r in self.references): + raise Failure('invalid') + root = private_root(self.config) + wanted = {} + for reference in self.references: + record = read_receipt(self.config, root, reference) + if record is None: + # Missing durable evidence is not proof that Create never ran. + raise Failure('unconfirmed') + if record.get('status') in ('killed', 'rejected') and record.get('settled') is True: + continue + if record.get('settled') is not True: + raise Failure('unconfirmed') + ids = record.get('ids') + if not isinstance(ids, list) or not ids or any(not isinstance(i, str) or not i for i in ids): + raise Failure('unconfirmed') + for identity in ids: + if identity in wanted: + raise Failure('unconfirmed') + wanted[identity] = self.metadata_for(reference) + if not wanted: + return + paginator = Sandbox.list(query=SandboxQuery(metadata={PREFIX + 'installationid': self.config['InstallationID']}, + state=[SandboxState.RUNNING, SandboxState.PAUSED]), limit=100, **self.options()) + for _ in range(100): + self.remaining() + if not paginator.has_next: + raise Failure('team_mismatch') + page = paginator.next_items(**self.options()) + if len(page) > 100: + raise Failure('unconfirmed') + for cloud in page: + expected = wanted.get(cloud.sandbox_id) + if expected is not None: + if any(cloud.metadata.get(k) != v for k, v in expected.items()): + raise Failure('team_mismatch') + del wanted[cloud.sandbox_id] + if not wanted: + return + raise Failure('unconfirmed') + def execute(self): - if self.q['Operation'] in ('validate_deployment', 'observe'): + if self.q['Operation'] in ('validate_deployment', 'observe', 'verify_credential'): try: + if self.q['Operation'] == 'verify_credential': + self.verify_credential() + return {'Version': 1, 'DeploymentValid': True, 'ErrorCode': ''} if self.q['Operation'] == 'observe': return {'Version': 1, 'Observations': self.observe(), 'ErrorCode': ''} build = validate_deployment(self.config, self.remaining) return {'Version': 1, 'DeploymentValid': True, 'TemplateBuild': build, 'ErrorCode': ''} + except AuthenticationException: + return {'Version': 1, 'ErrorCode': 'unauthorized'} except Failure as error: return {'Version': 1, 'ErrorCode': error.code} except Exception: diff --git a/services/agents-api/tools/e2b-provider/sdk.py b/services/agents-api/tools/e2b-provider/sdk.py index b8610a901..9a3cb7ff3 100644 --- a/services/agents-api/tools/e2b-provider/sdk.py +++ b/services/agents-api/tools/e2b-provider/sdk.py @@ -42,7 +42,7 @@ def validate_deployment(config, remaining): response = get_templates_template_id.sync_detailed(template_id=template, client=client, next_token=cursor, limit=100) if response.status_code != 200 or not isinstance(response.parsed, TemplateWithBuilds): - raise Failure('invalid' if response.status_code in (400, 401, 403, 404, 422) else 'unconfirmed') + raise Failure('unauthorized' if response.status_code in (401, 403) else 'invalid' if response.status_code in (400, 404, 422) else 'unconfirmed') result = response.parsed if result.template_id != template: raise Failure('invalid') @@ -142,3 +142,30 @@ def bounded(index, text): raise except Exception: raise Failure('command_unconfirmed') from None + + +def verify_team_template(config, remaining): + """The pinned SDK's paginated team listing, not public template readability.""" + from e2b.api.client.api.templates import get_v2_templates + from e2b.api.client.models.template import Template + wanted = config['Template'].split(':', 1)[0] + cursor, seen = UNSET, set() + for _ in range(100): + client = get_api_client(ConnectionConfig(api_key=config['APIKey'], retries=0, + debug=False, request_timeout=remaining())) + response = get_v2_templates.sync_detailed(client=client, next_token=cursor, limit=100) + if response.status_code in (401, 403): + raise Failure('unauthorized') + if response.status_code != 200 or not isinstance(response.parsed, list): + raise Failure('unconfirmed') + if len(response.parsed) > 100 or any(not isinstance(item, Template) for item in response.parsed): + raise Failure('unconfirmed') + if any(item.template_id == wanted for item in response.parsed): + return + cursor = response.headers.get('x-next-token') + if not cursor: + raise Failure('team_mismatch') + if len(cursor) > 4096 or cursor in seen: + raise Failure('unconfirmed') + seen.add(cursor) + raise Failure('unconfirmed') From 272beb13f0fe626f84af7bfc18b66cd1307c09e9 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Sun, 27 Sep 2026 20:38:23 +0000 Subject: [PATCH 04/16] test(web): align existing fixtures with generation contracts --- apps/web/e2e/data/routes.mjs | 8 +++---- apps/web/e2e/fixture-console.mjs | 23 +++++++++++++++---- apps/web/e2e/nodes.spec.ts | 2 +- .../fleet/SandboxResetNotice.test.tsx | 2 +- .../src/features/fleet/fleet-queries.test.ts | 2 +- .../features/fleet/use-sandbox-fleet.test.tsx | 2 +- .../features/metrics/sandbox-runtime.test.ts | 2 +- .../features/overview/getting-started.test.ts | 2 +- .../src/features/overview/test-fixtures.ts | 1 + .../web/src/features/sandbox/NodeList.test.ts | 2 +- .../sandbox/SandboxResetControls.test.tsx | 2 +- .../sandbox/sandbox-deployment-write.test.ts | 2 +- .../features/sandbox/sandbox-queries.test.ts | 2 +- .../use-sandbox-manager-state.test.tsx | 2 +- 14 files changed, 34 insertions(+), 20 deletions(-) diff --git a/apps/web/e2e/data/routes.mjs b/apps/web/e2e/data/routes.mjs index e2cd59520..dd7572437 100644 --- a/apps/web/e2e/data/routes.mjs +++ b/apps/web/e2e/data/routes.mjs @@ -84,9 +84,9 @@ export function buildDemo(now = Math.floor(Date.now() / 1000), publicUrl = "http }); sessions.sort((a, b) => b.created_at - a.created_at); const nodes = [ - { id: "node-local", name: "core-01", provider: "docker", online: true, provider_ready: true, cpu_count: 16, available_memory_bytes: 38 * 2 ** 30, available_disk_bytes: 410 * 2 ** 30, running: 5, snapshots: 2, last_seen_at: new Date((now - 8) * 1000).toISOString(), max_active: 8, max_retained: 16, active: 5, reserved: 1, retained: 2, cleanup_pending: 0, created_at: new Date((now - 86400 * 30) * 1000).toISOString() }, - { id: "node-gpu", name: "gpu-worker-02", provider: "docker", online: true, provider_ready: false, diagnostic: "docker_limits_unsupported", cpu_count: 32, available_memory_bytes: 12 * 2 ** 30, available_disk_bytes: 96 * 2 ** 30, running: 7, snapshots: 5, last_seen_at: new Date((now - 12) * 1000).toISOString(), max_active: 8, max_retained: 16, active: 7, reserved: 0, retained: 5, cleanup_pending: 1, created_at: new Date((now - 86400 * 12) * 1000).toISOString() }, - { id: "node-edge", name: "edge-03", provider: "docker", online: false, provider_ready: false, cpu_count: 8, available_memory_bytes: null, available_disk_bytes: null, running: 0, snapshots: 0, last_seen_at: new Date((now - 5400) * 1000).toISOString(), max_active: 4, max_retained: 8, active: 0, reserved: 0, retained: 0, cleanup_pending: 0, created_at: new Date((now - 86400 * 3) * 1000).toISOString() }, + { rollout: { state: "ready", ready_generation: 1 }, id: "node-local", name: "core-01", provider: "docker", online: true, provider_ready: true, cpu_count: 16, available_memory_bytes: 38 * 2 ** 30, available_disk_bytes: 410 * 2 ** 30, running: 5, snapshots: 2, last_seen_at: new Date((now - 8) * 1000).toISOString(), max_active: 8, max_retained: 16, active: 5, reserved: 1, retained: 2, cleanup_pending: 0, created_at: new Date((now - 86400 * 30) * 1000).toISOString() }, + { rollout: { state: "failed", ready_generation: null, diagnostic: "docker_limits_unsupported" }, id: "node-gpu", name: "gpu-worker-02", provider: "docker", online: true, provider_ready: false, diagnostic: "docker_limits_unsupported", cpu_count: 32, available_memory_bytes: 12 * 2 ** 30, available_disk_bytes: 96 * 2 ** 30, running: 7, snapshots: 5, last_seen_at: new Date((now - 12) * 1000).toISOString(), max_active: 8, max_retained: 16, active: 7, reserved: 0, retained: 5, cleanup_pending: 1, created_at: new Date((now - 86400 * 12) * 1000).toISOString() }, + { rollout: { state: "unknown", ready_generation: 1 }, id: "node-edge", name: "edge-03", provider: "docker", online: false, provider_ready: false, cpu_count: 8, available_memory_bytes: null, available_disk_bytes: null, running: 0, snapshots: 0, last_seen_at: new Date((now - 5400) * 1000).toISOString(), max_active: 4, max_retained: 8, active: 0, reserved: 0, retained: 0, cleanup_pending: 0, created_at: new Date((now - 86400 * 3) * 1000).toISOString() }, ]; const hosted = sessions.filter((session) => session.environment.type === "openai_hosted"); const allocations = hosted.map((session, index) => { @@ -94,7 +94,7 @@ export function buildDemo(now = Math.floor(Date.now() / 1000), publicUrl = "http // Running since creation; suspended two hours ago, or at creation when that is later. const changed = suspended ? Math.max(session.created_at, now - 7_200) : session.created_at; return { - id: uuid(), node_id: index % 2 ? "node-gpu" : "node-local", tenant_id: "project", session_id: session.id, environment_id: session.environment.id, + deployment_generation: 1, id: uuid(), node_id: index % 2 ? "node-gpu" : "node-local", tenant_id: "project", session_id: session.id, environment_id: session.environment.id, state: "active", compute_phase: suspended ? "suspended" : "running", compute_phase_changed_at: new Date(changed * 1000).toISOString(), diagnostic: "", initialization: "ready", created_at: new Date(session.created_at * 1000).toISOString(), }; diff --git a/apps/web/e2e/fixture-console.mjs b/apps/web/e2e/fixture-console.mjs index 459334ed7..2c44ac18d 100644 --- a/apps/web/e2e/fixture-console.mjs +++ b/apps/web/e2e/fixture-console.mjs @@ -71,19 +71,26 @@ function installation() { }; } +/** Settled generation fields required by the strict client; this fixture does not simulate online preparation. */ +function settledRollout(mode) { + const nodes = mode === "nodes" ? { ready: 0, preparing: 0, failed: 0, update_required: 0, unknown: 0 } : null; + if (nodes) for (const node of state.nodes) nodes[node.rollout.state] += 1; + return { state: "settled", previous_generation_sandboxes: 0, nodes }; +} + function unconfiguredDeployment(generation = 0, ownerEpoch = 3) { - return { installation_id: INSTALLATION_ID, provider: "", core_url: publicUrl(), reset: null, owner_epoch: ownerEpoch, generation, mode: "", resources: { allocations: 0, pending: 0 }, suspension: null }; + return { installation_id: INSTALLATION_ID, provider: "", core_url: publicUrl(), rollout: settledRollout(""), reset: null, owner_epoch: ownerEpoch, generation, mode: "", resources: { allocations: 0, pending: 0 }, suspension: null }; } function configuredDeployment() { - return { installation_id: INSTALLATION_ID, provider: "docker", core_url: publicUrl(), reset: null, owner_epoch: 3, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, specification: { resources: { cpus: 2, memory_mib: 4096 }, runtime: release }, specification_digest: "fixture", suspension: null }; + return { installation_id: INSTALLATION_ID, provider: "docker", core_url: publicUrl(), rollout: settledRollout("nodes"), reset: null, owner_epoch: 3, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, specification: { resources: { cpus: 2, memory_mib: 4096 }, runtime: release }, specification_digest: "fixture", suspension: null }; } /** The E2B template build as Core read it when the selection was saved. */ const templateBuild = { status: "ready", resources: { cpus: 2, memory_mib: 2048, root_disk_mib: 10240 } }; // E2B runs sandboxes in its cloud: no nodes, only what Core holds there. function e2bDeployment() { - return { ...configuredDeployment(), provider: "e2b", mode: "direct", resources: { allocations: 3, pending: 1 }, specification: { resources: { cpus: 2, memory_mib: 2048 } }, e2b: { template: "oac-runtime:0f1e2d3c-4b5a-6978-8a9b-0c1d2e3f4a5b", credential_configured: true, template_build: templateBuild } }; + return { ...configuredDeployment(), provider: "e2b", mode: "direct", rollout: settledRollout("direct"), resources: { allocations: 3, pending: 1 }, specification: { resources: { cpus: 2, memory_mib: 2048 } }, e2b: { template: "oac-runtime:0f1e2d3c-4b5a-6978-8a9b-0c1d2e3f4a5b", credential_configured: true, template_build: templateBuild } }; } function reset(mode = "login", fresh = false, sandbox = "configured", nodes = "demo", address = "public", credentials = "configured", installers = true, artifacts = "docker,microsandbox") { @@ -359,7 +366,7 @@ async function sandboxRoute(request, response, path, url) { // As Core: E2B may omit resources and adopt its template build's CPU and memory; only microsandbox suspends. const resources = input.resources ?? { cpus: templateBuild.resources.cpus, memory_mib: templateBuild.resources.memory_mib }; state.deployment = { - ...configuredDeployment(), provider: input.provider, mode: e2b ? "direct" : "nodes", + ...configuredDeployment(), provider: input.provider, mode: e2b ? "direct" : "nodes", rollout: settledRollout(e2b ? "direct" : "nodes"), generation: state.deployment.generation + 1, owner_epoch: state.deployment.owner_epoch, specification: { resources, ...(input.runtime ? { runtime: input.runtime } : {}) }, ...(e2b ? { e2b: { template: input.e2b?.template ?? "", credential_configured: true, template_build: templateBuild } } : {}), @@ -401,6 +408,7 @@ async function sandboxRoute(request, response, path, url) { const index = state.nodes.findIndex((node) => node.id === m[1]); if (index < 0) return error(response, 404, "No such node."); state.nodes.splice(index, 1); + state.deployment.rollout = settledRollout(state.deployment.mode); return send(response, 200, { id: m[1], deleted: true }); } return error(response, 404, "Not found."); @@ -454,7 +462,7 @@ async function executorCredentialRoute(request, response, projectId, environment */ function registeredNode(nodeId) { const { enrollment_id = null, ...limits } = state.enrollment ?? { max_active: 1, max_retained: 1 }; - return { id: nodeId, name: nodeId, provider: state.deployment?.provider ?? "docker", core_url: publicUrl(), enrollment_id, online: false, provider_ready: false, cpu_count: null, available_memory_bytes: null, available_disk_bytes: null, running: 0, snapshots: 0, last_seen_at: null, ...limits, active: 0, reserved: 0, retained: 0, cleanup_pending: 0, created_at: new Date().toISOString() }; + return { rollout: { state: "unknown", ready_generation: null }, id: nodeId, name: nodeId, provider: state.deployment?.provider ?? "docker", core_url: publicUrl(), enrollment_id, online: false, provider_ready: false, cpu_count: null, available_memory_bytes: null, available_disk_bytes: null, running: 0, snapshots: 0, last_seen_at: null, ...limits, active: 0, reserved: 0, retained: 0, cleanup_pending: 0, created_at: new Date().toISOString() }; } const HARNESS_PROVIDER = /^\/harnesses\/([^/]+)\/model-provider$/; @@ -557,6 +565,11 @@ async function fixtureRoute(request, response, url) { let node = state.nodes.find((entry) => entry.id === nodeId); if (!node) state.nodes.push(node = registeredNode(nodeId)); Object.assign(node, fields); + if (!fields.rollout) node.rollout = !node.online ? { state: "unknown", ready_generation: node.rollout.ready_generation } + : node.provider_ready ? { state: "ready", ready_generation: state.deployment.generation } + : node.diagnostic ? { state: "failed", ready_generation: node.rollout.ready_generation, diagnostic: node.diagnostic } + : { state: "unknown", ready_generation: node.rollout.ready_generation }; + state.deployment.rollout = settledRollout(state.deployment.mode); if (!node.diagnostic) delete node.diagnostic; return send(response, 200, node); } diff --git a/apps/web/e2e/nodes.spec.ts b/apps/web/e2e/nodes.spec.ts index 9b56415b0..c41f9ba14 100644 --- a/apps/web/e2e/nodes.spec.ts +++ b/apps/web/e2e/nodes.spec.ts @@ -261,7 +261,7 @@ test("edits only the saved backend, preserving a custom size and Runtime", async const runtime = { source_commit: "0".repeat(40), image_id: `sha256:${"a".repeat(64)}`, image_manifest_digest: `sha256:${"b".repeat(64)}`, microsandbox_ref: `oac-runtime@sha256:${"b".repeat(64)}`, runtime_sha256: "c".repeat(64), firmware_sha256: "d".repeat(64) }; const current = { resources: { cpus: 7, memory_mib: 8192 }, runtime }; - let deployment = { installation_id: "94be54a1-138c-4f30-bc87-b13686272dbe", provider: "docker", core_url: "https://core.example", reset: null, + let deployment = { rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 0, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "94be54a1-138c-4f30-bc87-b13686272dbe", provider: "docker", core_url: "https://core.example", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, specification: current, specification_digest: "e".repeat(64), suspension: null }; let submitted: Record | null = null; diff --git a/apps/web/src/features/fleet/SandboxResetNotice.test.tsx b/apps/web/src/features/fleet/SandboxResetNotice.test.tsx index eeefdd1df..6ab8097f2 100644 --- a/apps/web/src/features/fleet/SandboxResetNotice.test.tsx +++ b/apps/web/src/features/fleet/SandboxResetNotice.test.tsx @@ -6,7 +6,7 @@ import { SandboxResetNotice } from "./SandboxResetNotice"; const reset: SandboxReset = { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 2, on_offline_nodes: 1, offline_nodes: [{ node_id: "n1", name: "Node 1", resources: 1 }] } }; -const deployment = (reset: SandboxReset | null): SandboxDeployment => ({ installation_id: "i", provider: "docker", core_url: "http://core", reset, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 3, pending: 0 }, suspension: null }); +const deployment = (reset: SandboxReset | null): SandboxDeployment => ({ rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 3, pending: 0 }, suspension: null }); const render = (value: SandboxDeployment | undefined, failed = false) => renderToStaticMarkup( {}} />); describe("reset notices on read-only surfaces", () => { diff --git a/apps/web/src/features/fleet/fleet-queries.test.ts b/apps/web/src/features/fleet/fleet-queries.test.ts index caa9f236d..25b8342b6 100644 --- a/apps/web/src/features/fleet/fleet-queries.test.ts +++ b/apps/web/src/features/fleet/fleet-queries.test.ts @@ -5,7 +5,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { sandboxAdmin, sandboxDeploymentQuery, sandboxScope } from "../sandbox/sandbox-queries"; import { fleetQuery } from "./fleet-queries"; -const deployment: SandboxDeployment = { installation_id: "i", provider: "docker", core_url: "http://core", owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null, +const deployment: SandboxDeployment = { rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null, reset: { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } }; afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); }); diff --git a/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx b/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx index c15a6dff3..38a4eba9d 100644 --- a/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx +++ b/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx @@ -10,7 +10,7 @@ import { sandboxDeploymentQuery } from "../sandbox/sandbox-queries"; import { consoleConfigQuery, fleetQuery, type FleetSnapshot } from "./fleet-queries"; import { fleetSnapshot, useSandboxFleet } from "./use-sandbox-fleet"; -const configured: SandboxDeployment = { installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null }; +const configured: SandboxDeployment = { rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null }; function Probe() { const { state, deployment } = useSandboxFleet(); diff --git a/apps/web/src/features/metrics/sandbox-runtime.test.ts b/apps/web/src/features/metrics/sandbox-runtime.test.ts index 3bfe9660d..abf38709f 100644 --- a/apps/web/src/features/metrics/sandbox-runtime.test.ts +++ b/apps/web/src/features/metrics/sandbox-runtime.test.ts @@ -65,7 +65,7 @@ describe("hosted Runtime projections", () => { }); it("places each Runtime on its host and derives uptime only from a current sample", () => { - const allocation = { id: "alloc_a", node_id: "n1" } as SandboxAllocation; + const allocation = { id: "alloc_a", node_id: "n1", deployment_generation: 1 } as SandboxAllocation; const rows = hostedRuntimeRows(load, "", { nodes: [node("n1", { name: "core-01" })], allocations: [allocation] }); expect(rows.map((row) => [row.observation.session_id, row.node?.name ?? null, row.uptimeSeconds])).toEqual([["a", "core-01", 600], ["b", null, 600], ["c", null, null]]); expect(hostedRuntimeRows(load, "", null)[0]?.node).toBeNull(); diff --git a/apps/web/src/features/overview/getting-started.test.ts b/apps/web/src/features/overview/getting-started.test.ts index 7accac861..0fcba8d80 100644 --- a/apps/web/src/features/overview/getting-started.test.ts +++ b/apps/web/src/features/overview/getting-started.test.ts @@ -6,7 +6,7 @@ import { checklistStorageKey, checklistView, gettingStartedSteps, nextStepAfterN import { node, project } from "./test-fixtures"; const deployment = (overrides: Partial = {}): SandboxDeployment => ({ - installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", + rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, suspension: null, ...overrides, }); const fleet = (value: SandboxDeployment, nodes = [node("n1")]): FleetState => ({ status: "ready", snapshot: { deployment: value, nodes, allocations: [], loadedAt: 0 }, refreshing: false, error: null }); diff --git a/apps/web/src/features/overview/test-fixtures.ts b/apps/web/src/features/overview/test-fixtures.ts index e7a8526ee..2ad47b8d1 100644 --- a/apps/web/src/features/overview/test-fixtures.ts +++ b/apps/web/src/features/overview/test-fixtures.ts @@ -36,6 +36,7 @@ export function session(id: string, overrides: Partial = {}): Agen export function node(id: string, overrides: Partial = {}): SandboxNode { return { id, name: id, provider: "docker", online: true, provider_ready: true, + rollout: { state: overrides.online === false ? "unknown" : "ready", ready_generation: 1 }, cpu_count: 8, available_memory_bytes: 1024, available_disk_bytes: 2048, running: 1, snapshots: 0, last_seen_at: "2026-09-24T00:00:00Z", max_active: 4, max_retained: 8, active: 1, reserved: 0, retained: 0, cleanup_pending: 0, diff --git a/apps/web/src/features/sandbox/NodeList.test.ts b/apps/web/src/features/sandbox/NodeList.test.ts index 493d9653e..6750fd415 100644 --- a/apps/web/src/features/sandbox/NodeList.test.ts +++ b/apps/web/src/features/sandbox/NodeList.test.ts @@ -4,7 +4,7 @@ import { describe, expect, it } from "vitest"; import { node } from "../overview/test-fixtures"; import { nodeState } from "./NodeList"; -const allocation = (nodeId: string, diagnostic: SandboxAllocation["diagnostic"]) => ({ id: `alloc_${nodeId}`, node_id: nodeId, diagnostic }) as SandboxAllocation; +const allocation = (nodeId: string, diagnostic: SandboxAllocation["diagnostic"]) => ({ id: `alloc_${nodeId}`, node_id: nodeId, deployment_generation: 1, diagnostic }) as SandboxAllocation; const core = "https://core.example"; diff --git a/apps/web/src/features/sandbox/SandboxResetControls.test.tsx b/apps/web/src/features/sandbox/SandboxResetControls.test.tsx index 5ba09fbef..128253eeb 100644 --- a/apps/web/src/features/sandbox/SandboxResetControls.test.tsx +++ b/apps/web/src/features/sandbox/SandboxResetControls.test.tsx @@ -5,7 +5,7 @@ import { SandboxResetControls } from "./SandboxResetControls"; const reset: SandboxReset = { clear: "auto", requested_at: "2020-01-01T10:00:00Z", deadline_at: "2020-01-01T11:00:00Z", forced_at: null, remaining: { busy: 2, idle: 1, cleanup: 3, on_offline_nodes: 2, offline_nodes: [{ node_id: "offline-a", name: "Offline A", resources: 2 }] } }; -const deployment = (value: SandboxReset | null): SandboxDeployment => ({ installation_id: "i", provider: "docker", core_url: "https://core.example", reset: value, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 5, pending: 1 }, suspension: null }); +const deployment = (value: SandboxReset | null): SandboxDeployment => ({ rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "https://core.example", reset: value, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 5, pending: 1 }, suspension: null }); const render = (value: SandboxReset | null, stale = false) => renderToStaticMarkup( true} onCancel={async () => true} />); describe("authoritative reset progress", () => { diff --git a/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts b/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts index 846641a4f..b9a2b54f7 100644 --- a/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts +++ b/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts @@ -5,7 +5,7 @@ import { sandboxAdmin, sandboxDeploymentQuery } from "./sandbox-queries"; import { writeSandboxDeployment } from "./sandbox-deployment-write"; import { sandboxWriteOwnershipQuery } from "./sandbox-write-ownership"; -const deployment: SandboxDeployment = { installation_id: "i", owner_epoch: 1, generation: 1, provider: "docker", core_url: "http://core", mode: "nodes", reset: null, resources: { allocations: 0, pending: 0 }, suspension: null }; +const deployment: SandboxDeployment = { rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "i", owner_epoch: 1, generation: 1, provider: "docker", core_url: "http://core", mode: "nodes", reset: null, resources: { allocations: 0, pending: 0 }, suspension: null }; function deferred() { let resolve!: (value: T) => void; let reject!: (error: Error) => void; diff --git a/apps/web/src/features/sandbox/sandbox-queries.test.ts b/apps/web/src/features/sandbox/sandbox-queries.test.ts index 7c271c407..7e55c70b9 100644 --- a/apps/web/src/features/sandbox/sandbox-queries.test.ts +++ b/apps/web/src/features/sandbox/sandbox-queries.test.ts @@ -9,7 +9,7 @@ const reset: SandboxReset = { remaining: { busy: 1, idle: 0, cleanup: 1, on_offline_nodes: 1, offline_nodes: [{ node_id: "node-a", name: "Offline host", resources: 1 }] }, }; const deployment = (overrides: Partial = {}): SandboxDeployment => ({ - installation_id: "installation-a", provider: "docker", core_url: "https://core.example", reset, + rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "installation-a", provider: "docker", core_url: "https://core.example", reset, owner_epoch: 1, generation: 2, mode: "nodes", resources: { allocations: 2, pending: 0 }, suspension: null, ...overrides, }); const clients: QueryClient[] = []; diff --git a/apps/web/src/features/sandbox/use-sandbox-manager-state.test.tsx b/apps/web/src/features/sandbox/use-sandbox-manager-state.test.tsx index ee59f67f1..c2bb89243 100644 --- a/apps/web/src/features/sandbox/use-sandbox-manager-state.test.tsx +++ b/apps/web/src/features/sandbox/use-sandbox-manager-state.test.tsx @@ -6,7 +6,7 @@ import { node } from "../overview/test-fixtures"; import { sandboxAdmin, sandboxDeploymentQuery, sandboxSnapshotQuery } from "./sandbox-queries"; import { useSandboxManagerState } from "./use-sandbox-manager-state"; -const configured: SandboxDeployment = { installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null }; +const configured: SandboxDeployment = { rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null }; const resetting: SandboxDeployment = { ...configured, reset: { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } }; const clients: QueryClient[] = []; function cache() { From e5f2ec0925443e33c6ef6e2622aa8543fbf1d470 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 20:47:19 +0000 Subject: [PATCH 05/16] Document generation-aware sandbox configuration and rollout facts --- apps/docs/FOLLOW-UP.md | 13 ++-- apps/docs/content/docs/admin-api.mdx | 14 ++++- .../api-reference/core/sandbox-manager.mdx | 16 ++--- apps/docs/content/docs/configure.mdx | 15 +++-- apps/docs/content/docs/hosted-providers.mdx | 9 +-- apps/docs/content/guide-sources.json | 12 ++-- apps/docs/openapi/core-api.yaml | 59 ++++++++++++++++++- apps/docs/openapi/sources.json | 6 +- docs/api/web-management.md | 14 ++++- docs/configuration.md | 15 +++-- docs/getting-started/nodes.md | 9 +-- docs/web/protocol-coverage.md | 2 +- 12 files changed, 137 insertions(+), 47 deletions(-) diff --git a/apps/docs/FOLLOW-UP.md b/apps/docs/FOLLOW-UP.md index e74b7e1a4..6da02fefd 100644 --- a/apps/docs/FOLLOW-UP.md +++ b/apps/docs/FOLLOW-UP.md @@ -1,14 +1,15 @@ # Feature integration follow-up -This candidate documents durable reset and the interim zero-resource same-provider -update rule at source `f36ac3592bfe9f77a2172680aa52b14c6f525062`. -The Core error-envelope foundation is included; online generations, later configuration -validators and diagnostic endpoints are not. Do not replace current procedures with -planned behavior before those features land. +This candidate documents durable reset, retained Core generations and verified +same-team E2B online changes at source `b9b13b186089b317db271ab83b10f86b48168f66`. +Node changes still require zero held resources until the node protocol is complete. +The Core error-envelope foundation is included; later configuration validators and +diagnostic endpoints are not. Do not replace current procedures with planned behavior +before those features land. | Feature dependency | Site pages to reconcile | Source and verification work | | --- | --- | --- | -| Online generations and E2B changes (design 50 PR-G) | configure, hosted-providers, troubleshooting, execution-model, API reference/core | Document actual generation/rollout states, the supported change matrix and existing-sandbox behavior. Verify PUT examples against the final contract. | +| Final Web generation controls (design 50 PR-W) | configure, hosted-providers, troubleshooting, execution-model | Reconcile the actual Web workflow after node online changes are qualified; keep target preparation separate from serving readiness. | | Node update protocol (design 50 PR-N) | hosted-providers, troubleshooting, API reference/machine | Explain the implemented update command, old-node handling and ownership preservation. Do not claim an update command now. | | Later configuration/error changes (design 40 and remaining installer work) | install, configure, troubleshooting, quickstart | Regenerate current configuration fields and revise diagnostics against merged code; keep secrets out of examples. | | Final feature assembly | every Chinese reading note, execution-model diagram, all generated references | Review translations against regenerated English, verify source provenance and links, run contract/route/copy/fact checks, typecheck/build, local browser review and the repository gate. | diff --git a/apps/docs/content/docs/admin-api.mdx b/apps/docs/content/docs/admin-api.mdx index d0b79bf04..351e5a328 100644 --- a/apps/docs/content/docs/admin-api.mdx +++ b/apps/docs/content/docs/admin-api.mdx @@ -99,13 +99,21 @@ may omit `resources` to adopt the validated build's CPU and memory; responses sh the build as read at selection time in `e2b.template_build`. Microsandbox responses return its idle `suspension` policy; other providers return null. -Same-provider updates currently require zero retained/pending resources and no -reset. Changing backend requires explicit durable reset before a new POST. +Same-team E2B updates apply online after verification. Existing sandboxes retain +their generation and use the committed credential for management; omitting the key +preserves it, while explicitly submitting even the same key verifies a replacement. +Node-provider updates still require zero retained/pending resources and no reset. +Changing backend or E2B team requires explicit durable reset before a new POST. Auto archives idle/queued/suspended hosted Sessions, waits for started work and file writes, and escalates at its persisted deadline; force requests cancellation and verified cleanup. The deployment response supplies the authoritative `reset.remaining` partition and offline-node subset; Web must not derive either -from independently loaded lists. No rollout projection is provided yet. +from independently loaded lists. The separate `rollout.state` describes target +preparation; old-generation resource counts alone do not imply active preparation. +Poll rapidly while reset is active or rollout is preparing. Node `ready_generation` +is a durable serving pin, not proof of current connectivity. Target unknown, failed +or update-required state does not by itself invalidate confirmed old-generation +service; consume Core's connection/provider facts separately. Administrators may [archive an individual hosted Session](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md#administrative-session-archive) at the current generation without reset. History and persisted Files/Artifacts diff --git a/apps/docs/content/docs/api-reference/core/sandbox-manager.mdx b/apps/docs/content/docs/api-reference/core/sandbox-manager.mdx index c0f819bcf..b391590f8 100644 --- a/apps/docs/content/docs/api-reference/core/sandbox-manager.mdx +++ b/apps/docs/content/docs/api-reference/core/sandbox-manager.mdx @@ -27,13 +27,15 @@ _openapi: no-op; differing selections and file-managed deployments reject. This does not create compute or execute work. - content: >- - Requires the current generation, the same backend type, no active - reset and verified cleanup of all old resources. A backend type change - requires reset. Credentials are write-only. E2B may omit resources to - adopt the validated template build's CPU and memory. A core_url member - is rejected with 400; the address comes from the installation public - URL. Historical records are retained; old node credentials and - enrollments are retired. Never automatically retry an uncertain write. + Requires the observed generation, the same backend type and no active + reset. E2B same-team changes apply online: allocations retain + immutable generation and current credentials; omitted api_key + preserves it, explicit submission including the same key verifies and + advances generation. Other teams require explicit reset. Node + providers retain the zero-resource guard and retire old nodes/tokens + on change. Core rejects core_url input. Never automatically replay an + uncertain write; rollout.state is the authoritative preparation + polling signal. - content: >- Archives hosted Sessions and waits for confirmed provider cleanup, preserving history and Files/Artifacts. Auto waits for started or diff --git a/apps/docs/content/docs/configure.mdx b/apps/docs/content/docs/configure.mdx index 33a9b1aaf..517c4c662 100644 --- a/apps/docs/content/docs/configure.mdx +++ b/apps/docs/content/docs/configure.mdx @@ -161,10 +161,17 @@ E2B takes no `runtime`, and Web sends no resources for it: Core adopts the CPU a memory of the ready template build `template-id:build-uuid`, and supplied values must match it. Docker has no separate disk quota. -Changing the provider, the size or the Runtime applies to the whole deployment; see the -**Nodes** page, and note that this flow will change in a coming release. A successful -change retires the old nodes and enrollment commands; history stays, and existing -Sessions never move between providers. See the +Changing backend type or E2B team requires an explicit reset and a new setup. +Same-team E2B template, resource and key changes apply online through the Core API: +new allocations use the new generation, while existing sandboxes retain their +original specification. Omit the key to preserve it; explicitly submitting a key, +even the same value, verifies the replacement and advances the generation. + +Docker and microsandbox size/Runtime edits still require zero held resources and +retire the old nodes and enrollment commands. Add those hosts again after cleaning +up their retired node installations. Node online preparation and the corresponding +Web controls are separate follow-up work. History stays, and existing Sessions never +move between providers. See the [operator reference](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-reset) and the [deployment contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/sandbox-deployment.md). diff --git a/apps/docs/content/docs/hosted-providers.mdx b/apps/docs/content/docs/hosted-providers.mdx index bad34441a..78f24ef83 100644 --- a/apps/docs/content/docs/hosted-providers.mdx +++ b/apps/docs/content/docs/hosted-providers.mdx @@ -266,13 +266,14 @@ WebSocket doesn't pass the reverse proxy. ## Change the sandbox backend or size -To change the sandbox backend, the sandbox size or the Runtime, see the **Nodes** page; -this flow will change in a coming release. A change applies to the whole deployment -and retires every node. Retired nodes drop out of the list, so there is nothing to +Changing backend type requires an explicit reset. Docker and microsandbox size or +Runtime edits currently require zero held resources; node online updates are not +available yet. These changes retire every node. Retired nodes drop out of the list, so there is nothing to remove in Web: on each host, run the [uninstall command](#remove-a-node) (Core no longer accepts the node, so it needs no `--force`), then add the host again with a new command. The [operator reference](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-reset) -describes today's procedure and API. +describes today's procedure and API. Same-team E2B edits have a separate online +Core API path and do not involve node installation. ## Remove a node added before the rename diff --git a/apps/docs/content/guide-sources.json b/apps/docs/content/guide-sources.json index 614a1706f..1816933ac 100644 --- a/apps/docs/content/guide-sources.json +++ b/apps/docs/content/guide-sources.json @@ -5,18 +5,18 @@ "docs/design-principles.md": "5e1bff3cdcc77cd02ad692f67977cb4e0a5654989982d722bac0e3c79dbf82cb", "docs/web/architecture.md": "80bdb8053c06c2b2030b193aae17d5434545f9c72af50e9d3df3ade522fe25ed", "docs/getting-started/install.md": "15cfe81d4005a7337ac82bf6c35a8b441e9f9b0f2a2299021ad6c539f3f336a3", - "docs/configuration.md": "401bf72003cd2eb03ae217d49591273ea0a5c42d68190d1a7ad2f12169c287ea", + "docs/configuration.md": "6114e2993607defea86f54181fb33824910e072d88ae1acf1c665c105d198d30", "docs/web/core-connection.md": "46f86520e70bf41079708e2c398655aa9087ad9488c8e661d9a73035d11098c4", "docs/getting-started/quickstart.md": "5effeffcfb343f7939910f0c1758e1492d49e7bd66c409b1819cefe67af592d7", "docs/api/README.md": "b2b4c7fdf9fbde62d2169014180698a77c671f85c1c68c2e5de0e73bbdf3b13c", "contracts/agents-api/execution-tools.md": "e0b61f6c0c236c362186c5f6d0ad69a16dd8a8d9afabe1329a1fe22e5f4a71e9", "docs/api/public-agent-api.md": "18829b366bd2e49fe5beb76dc8a709bc1aa8e35d4fbfd2484a0c52b46cf75417", "contracts/agents-api/environments.md": "89ffc903ee44d00ee87c16950202a3ca4c8345160780dcdd4ed2a71b25b1c2dc", - "docs/getting-started/nodes.md": "a79e0007c1712c457b40b7573423fd1fdb95d52e283567b4c3d732393fac5109", + "docs/getting-started/nodes.md": "71c422ba23b274b038a841dfdc960649c29a4409454f49f8916044fa88041458", "docs/getting-started/self-hosted.md": "3f1b402cf28bb320ba414eaf36110e9df4d792b81da0456a3c42d469357d8d99", "apps/web/public/onboarding/monitor-en.webp": "29dc220cb1250c7016b7c4bf7f30e9510b07c07b814a48c3aa9303b2d76f20b2", "docs/web/README.md": "9fce295f6f6ff04acefe338679c1e54723e123dd0027a778202adfa9ab95b6c4", - "docs/api/web-management.md": "ce248fbeb1ded0f3fe4c6021c800c268ebaf4bd37eb1292aecc3d919d138a391", + "docs/api/web-management.md": "dc79df33dade6139749752c9ce53d4e126354cfabccada3e7826de4dd20fae62", "contracts/agents-api/runtime-observability-api.md": "cd46e777fe716a7e76374f4655c96dfbad9d8be1bfca6203ed6118ab87874c2b", "docs/getting-started/operations.md": "b930724d4518621bd6bd8457c70e6aac8c041677ad532a2b63894ef0d20ba892", "apps/docs/scripts/guides.json": "9d1ba424f8fadd65ffa6175eaa9c7bc4878d3e1437971099812c984e1e593080" @@ -30,7 +30,7 @@ "content/docs/execution-model.zh.mdx": "9976a9203c6a83d49c2c753e3a8528a369078e87aeb88850654da948ac8c17e1", "content/docs/install.mdx": "cba5c14e55c6f474d810dc8e545fadbb4f35bcad50eeb6ceef0cbd967b3fb1ab", "content/docs/install.zh.mdx": "7ed0b726cf721b18e85b89145ef8ebd2f5d382afd0489633a7b63962aa2fc4ed", - "content/docs/configure.mdx": "ac8a8788881707287805811474aaabfc0d0f273aa2423b010b167443996d2530", + "content/docs/configure.mdx": "72b0dc1b4a862f3a69f776262f6e6235290dfed212a604b24b3bbdf4963c649b", "content/docs/configure.zh.mdx": "a0c3a1df3e9f75e672e1d0a8c68c82c24ac36a8479d10a951de441960d07927f", "content/docs/bootstrap-projects-keys.mdx": "fc2da0aa255278281d8a57c664d7d30e5a458add4e58402386e0374f23973da4", "content/docs/bootstrap-projects-keys.zh.mdx": "509b4dcea4c865d6834bcaef78baa681180d48274931230f958a39e46011c88f", @@ -44,14 +44,14 @@ "content/docs/sessions.zh.mdx": "b1b1a7f8aacd9a902320f9692173ede24036124f85761cf02f23ef3ef2bcc4c1", "content/docs/environments-and-files.mdx": "0fce7c8806d532d64eff6319bba7d0a8e44da46f888d224644b3ab5c23428c41", "content/docs/environments-and-files.zh.mdx": "ee64577220a798f023b330c65302c14438f25a34a78c4e36144bfdb4304a3ae5", - "content/docs/hosted-providers.mdx": "e73657db13cbde92c1291534d910eba76172781a58645916ec7bfa770c42a5cf", + "content/docs/hosted-providers.mdx": "02a39c9fd07c7adfc065ca9510136362503c791334d8b084b7b9dc0958eec81b", "content/docs/hosted-providers.zh.mdx": "f54d80d6e65596dca6b20262abaa3dcb5af245227ffad146fcaed4bd69454dee", "content/docs/self-hosted-execution.mdx": "34877624962b24c22e8371ebd2e2f83bbf5a0d004c74d5d9b5db57f5133b692c", "content/docs/self-hosted-execution.zh.mdx": "f11d34ce30d48039376256474bc9509a508a2f6b9906f49b5cf6e396707ae5c0", "public/images/source/apps/web/public/onboarding/monitor-en.webp": "29dc220cb1250c7016b7c4bf7f30e9510b07c07b814a48c3aa9303b2d76f20b2", "content/docs/console.mdx": "cf0e6a1f9dd690ffe74c5d83111bb18df7eebe287848497c01257692ab3edc6a", "content/docs/console.zh.mdx": "72f953ba46dcc95933773ac0c4dffdc0d0a499be9d9a1cb9467db2659f1fa10d", - "content/docs/admin-api.mdx": "1f4917510d3335ec1e9fe0dd7527d0e625b4b9257df39c1691d2564261603959", + "content/docs/admin-api.mdx": "22331a4aa1c720711861d93db7d207868bf815ccc32b900d07b3d48ef077afd9", "content/docs/admin-api.zh.mdx": "07006de30df3080e54c3cd79ed0ea5337dce557aa2f11afea84edb7f92884d38", "content/docs/observability.mdx": "c5e2963a4dd7abddc60869a3ecd3b6b32a97c2ba1ae28b88b7453136656c1201", "content/docs/observability.zh.mdx": "9574b3412a5754f41ab2d3a16062ac1cad06766be481a8e6bff1b84acbde7e7e", diff --git a/apps/docs/openapi/core-api.yaml b/apps/docs/openapi/core-api.yaml index 87bd862f1..5159dc6c2 100644 --- a/apps/docs/openapi/core-api.yaml +++ b/apps/docs/openapi/core-api.yaml @@ -3308,7 +3308,7 @@ paths: description: Deployment selection required: true put: - description: Requires the current generation, the same backend type, no active reset and verified cleanup of all old resources. A backend type change requires reset. Credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory. A core_url member is rejected with 400; the address comes from the installation public URL. Historical records are retained; old node credentials and enrollments are retired. Never automatically retry an uncertain write. + description: 'Requires the observed generation, the same backend type and no active reset. E2B same-team changes apply online: allocations retain immutable generation and current credentials; omitted api_key preserves it, explicit submission including the same key verifies and advances generation. Other teams require explicit reset. Node providers retain the zero-resource guard and retire old nodes/tokens on change. Core rejects core_url input. Never automatically replay an uncertain write; rollout.state is the authoritative preparation polling signal.' responses: '200': description: OK @@ -3348,7 +3348,7 @@ paths: $ref: '#/components/schemas/api.CoreErrorResponse' security: - DeploymentAdminAuth: [] - summary: Change a fully drained deployment's sandbox configuration + summary: Change the sandbox deployment configuration tags: - Sandbox Manager requestBody: @@ -4972,6 +4972,8 @@ components: - type: 'null' resources: $ref: '#/components/schemas/store.SandboxDeploymentResources' + rollout: + $ref: '#/components/schemas/store.SandboxRollout' specification: $ref: '#/components/schemas/sandbox.DeploymentSpec' specification_digest: @@ -5038,6 +5040,8 @@ components: type: integer retained: type: integer + rollout: + $ref: '#/components/schemas/store.SandboxNodeRollout' running: type: integer snapshots: @@ -5054,6 +5058,8 @@ components: description: The time the allocation entered its current compute_phase, or null when unknown; an allocation that existed before Core recorded it reports null until its next phase change. For a suspended microsandbox allocation, this time plus the deployment's snapshot retention tells roughly when Core reclaims it. created_at: type: string + deployment_generation: + type: integer diagnostic: type: string environment_id: @@ -5130,6 +5136,8 @@ components: type: integer retained: type: integer + rollout: + $ref: '#/components/schemas/store.SandboxNodeRollout' running: type: integer snapshots: @@ -5200,6 +5208,25 @@ components: - $ref: '#/components/schemas/store.SandboxE2BTemplateBuildView' description: The fixed template build as Core read it when this selection was saved. type: object + store.SandboxNodeRollout: + properties: + diagnostic: + type: string + ready_generation: + anyOf: + - type: integer + - type: 'null' + description: Durable serving-generation pin; online and provider_ready still gate placement. + state: + description: Target preparation, independent of an old pin's serving readiness. + enum: + - ready + - preparing + - failed + - update_required + - unknown + type: string + type: object store.SandboxResetOfflineNode: properties: name: @@ -5259,6 +5286,34 @@ components: requested_at: type: string type: object + store.SandboxRollout: + properties: + nodes: + anyOf: + - allOf: + - $ref: '#/components/schemas/store.SandboxRolloutNodes' + - type: 'null' + previous_generation_sandboxes: + type: integer + state: + enum: + - settled + - preparing + type: string + type: object + store.SandboxRolloutNodes: + properties: + failed: + type: integer + preparing: + type: integer + ready: + type: integer + unknown: + type: integer + update_required: + type: integer + type: object store.SandboxSuspensionView: properties: idle_seconds: diff --git a/apps/docs/openapi/sources.json b/apps/docs/openapi/sources.json index 3633bab31..a7a17bfbe 100644 --- a/apps/docs/openapi/sources.json +++ b/apps/docs/openapi/sources.json @@ -1,7 +1,7 @@ { "sources": { "contracts/agents-api/openapi.yaml": "a8896e8b67d78ed0608c7cc5636077ec33f4b4895a04bd6cfb8b038afe941e01", - "contracts/agents-api/core.openapi.yaml": "26500cd07c72e72bfda32dcc83594e8d182aeb08debf4b67ac3938b5a9a078fe", + "contracts/agents-api/core.openapi.yaml": "e9b159a84afef28d086f5c285d298834b9f647ae8e7c14457117572ee01cc3ab", "contracts/agents-api/runtime.openapi.yaml": "b3fb025b20a0bcf4635d60339479b2b07f6dbed80f51e72f40a63b31e42b5f4a" }, "outputs": { @@ -39,8 +39,8 @@ "content/docs/api-reference/core/skills.mdx": "ed899ade7b032b1732b3097d81419a8fdbb9ead0594fca9f2107da644b97031f", "content/docs/api-reference/core/vaults.mdx": "12d0aac96cd4a3472ef4eaa287f7945f78862017956e79bfc3ed0ef8e772e78a", "content/docs/api-reference/core/credentials.mdx": "27d586b9a8fd738b9947802f32c80458195370252886bc26abedccf2125ee981", - "content/docs/api-reference/core/sandbox-manager.mdx": "4e0d90b158d3cdb9632db53dfacaae8c2749596373422c554f81178c8a96e7e2", - "openapi/core-api.yaml": "38614bb103da6b89dceaedc8c82874bc92590a49da005f07a5e8a0147b291676", + "content/docs/api-reference/core/sandbox-manager.mdx": "6be7c52e057365949f0247c18c710561d680847a955f78f1bbe57825d4d5f549", + "openapi/core-api.yaml": "f6ba52edeb5c70c11f230da401e60cc442e7ac6760f63013b2293ba045786b5b", "content/docs/api-reference/core/meta.json": "588b163ed2de2100b0f8376688274f80698a6bf6ac037eef56d17c93fa8a5062", "content/docs/api-reference/core/index.mdx": "ddf0a8d87a6c21761beac703a32f2a40486a1c74abb341bd726f40c44a866405", "content/docs/api-reference/machine/sandbox-node.mdx": "5b367a08d43cea82686345d2ba1c75cc90627a7b2970fe42a0df9d77fb3e1e80", diff --git a/docs/api/web-management.md b/docs/api/web-management.md index 3311f5b54..a66c1556c 100644 --- a/docs/api/web-management.md +++ b/docs/api/web-management.md @@ -96,13 +96,21 @@ may omit `resources` to adopt the validated build's CPU and memory; responses sh the build as read at selection time in `e2b.template_build`. Microsandbox responses return its idle `suspension` policy; other providers return null. -Same-provider updates currently require zero retained/pending resources and no -reset. Changing backend requires explicit durable reset before a new POST. +Same-team E2B updates apply online after verification. Existing sandboxes retain +their generation and use the committed credential for management; omitting the key +preserves it, while explicitly submitting even the same key verifies a replacement. +Node-provider updates still require zero retained/pending resources and no reset. +Changing backend or E2B team requires explicit durable reset before a new POST. Auto archives idle/queued/suspended hosted Sessions, waits for started work and file writes, and escalates at its persisted deadline; force requests cancellation and verified cleanup. The deployment response supplies the authoritative `reset.remaining` partition and offline-node subset; Web must not derive either -from independently loaded lists. No rollout projection is provided yet. +from independently loaded lists. The separate `rollout.state` describes target +preparation; old-generation resource counts alone do not imply active preparation. +Poll rapidly while reset is active or rollout is preparing. Node `ready_generation` +is a durable serving pin, not proof of current connectivity. Target unknown, failed +or update-required state does not by itself invalidate confirmed old-generation +service; consume Core's connection/provider facts separately. Administrators may [archive an individual hosted Session](../../contracts/agents-api/admin-api.md#administrative-session-archive) at the current generation without reset. History and persisted Files/Artifacts diff --git a/docs/configuration.md b/docs/configuration.md index 019da0bad..bd7b0a7c5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -158,10 +158,17 @@ E2B takes no `runtime`, and Web sends no resources for it: Core adopts the CPU a memory of the ready template build `template-id:build-uuid`, and supplied values must match it. Docker has no separate disk quota. -Changing the provider, the size or the Runtime applies to the whole deployment; see the -**Nodes** page, and note that this flow will change in a coming release. A successful -change retires the old nodes and enrollment commands; history stays, and existing -Sessions never move between providers. See the +Changing backend type or E2B team requires an explicit reset and a new setup. +Same-team E2B template, resource and key changes apply online through the Core API: +new allocations use the new generation, while existing sandboxes retain their +original specification. Omit the key to preserve it; explicitly submitting a key, +even the same value, verifies the replacement and advances the generation. + +Docker and microsandbox size/Runtime edits still require zero held resources and +retire the old nodes and enrollment commands. Add those hosts again after cleaning +up their retired node installations. Node online preparation and the corresponding +Web controls are separate follow-up work. History stays, and existing Sessions never +move between providers. See the [operator reference](../services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-reset) and the [deployment contract](../contracts/agents-api/sandbox-deployment.md). diff --git a/docs/getting-started/nodes.md b/docs/getting-started/nodes.md index 60fef0bdc..4d500553d 100644 --- a/docs/getting-started/nodes.md +++ b/docs/getting-started/nodes.md @@ -263,13 +263,14 @@ WebSocket doesn't pass the reverse proxy. ## Change the sandbox backend or size -To change the sandbox backend, the sandbox size or the Runtime, see the **Nodes** page; -this flow will change in a coming release. A change applies to the whole deployment -and retires every node. Retired nodes drop out of the list, so there is nothing to +Changing backend type requires an explicit reset. Docker and microsandbox size or +Runtime edits currently require zero held resources; node online updates are not +available yet. These changes retire every node. Retired nodes drop out of the list, so there is nothing to remove in Web: on each host, run the [uninstall command](#remove-a-node) (Core no longer accepts the node, so it needs no `--force`), then add the host again with a new command. The [operator reference](../../services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-reset) -describes today's procedure and API. +describes today's procedure and API. Same-team E2B edits have a separate online +Core API path and do not involve node installation. ## Remove a node added before the rename diff --git a/docs/web/protocol-coverage.md b/docs/web/protocol-coverage.md index c14b5d3f0..11d12b547 100644 --- a/docs/web/protocol-coverage.md +++ b/docs/web/protocol-coverage.md @@ -141,7 +141,7 @@ consumed; the list carries each provider. | Operation | Route | Console use | | --- | --- | --- | -| Deployment | `GET`, `POST`, `PUT /core/v1/sandbox/deployment` | Read the provider, the read-only `core_url` (config.json's `public_url`, shown in the setup review and never sent), reset state, installation ID and specification; a 409 `sandbox_configuration_error` (E2B with a loopback `public_url`) shows Core's message in the setup wizard, with the installation's config file and apply command, and leaves nothing to confirm; initialize the deployment with `resources` and the Docker or microsandbox `runtime` release, or with the E2B account and no `resources` (Core adopts the template build's CPU and memory); change its settings with the expected generation. E2B's `e2b.template_build` (status, CPU, memory, disk) shows on System, the Sandbox backend summary and Sandbox metrics, and sizes each sandbox when `specification.resources` is missing; microsandbox's `suspension` (idle and retention seconds) shows on System and the Nodes summary | +| Deployment | `GET`, `POST`, `PUT /core/v1/sandbox/deployment` | Read the provider, the read-only `core_url` (config.json's `public_url`, shown in the setup review and never sent), reset state, installation ID and specification; a 409 `sandbox_configuration_error` (E2B with a loopback `public_url`) shows the shared client's fixed safe address-configuration message in the setup wizard, with the installation's config file and apply command, and leaves nothing to confirm; initialize the deployment with `resources` and the Docker or microsandbox `runtime` release, or with the E2B account and no `resources` (Core adopts the template build's CPU and memory); change its settings with the expected generation. E2B's `e2b.template_build` (status, CPU, memory, disk) shows on System, the Sandbox backend summary and Sandbox metrics, and sizes each sandbox when `specification.resources` is missing; microsandbox's `suspension` (idle and retention seconds) shows on System and the Nodes summary | | Reset | `POST/DELETE /core/v1/sandbox/deployment/reset` | Explicitly clear hosted resources or cancel the remaining clear at the observed generation; consume Core’s remaining/offline projection | | Nodes | `GET /core/v1/sandbox/nodes` | Nodes page; fleet on Overview; node capacity on Sandbox metrics. An online node's `diagnostic` (`docker_unavailable`, `docker_limits_unsupported`, `runtime_image_unavailable`, `kvm_unavailable`, `microsandbox_artifacts_unavailable`, `capacity_insufficient`, `provider_unavailable`; any other value reads as `provider_unavailable`) marks it degraded and names the reason and fix in the help tip beside its status on each of these and on the node's page. A node whose `core_url` (the address it enrolled with) differs from the deployment's `core_url` is named on the Nodes page as bound to an old address, to be removed and added again, and its status there and on its page reads Old address instead of its health; an empty `core_url` (a node Core did not enroll) is unknown, not old. **Add node** follows only the node whose `enrollment_id` equals its command's; a node enrolled before Core recorded it reports null and never matches | | Node detail | `GET /core/v1/sandbox/nodes/{node_id}?range=1h\|6h\|24h` | Sandbox metrics node dialog: the host's CPU busy share and memory from its last heartbeat, and their history over the page's range. **Edit node** reads `host.effective_cpu_cores` and `host.total_memory_bytes` to show the host beside each sandbox's size and at most how many of those fit | From d183e14005fb7f5cab56984d6f7a9be061914842 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 20:55:17 +0000 Subject: [PATCH 06/16] fix(e2b): anchor online replacement to committed team ownership --- CONTRIBUTING.md | 10 +- .../agents-client/src/sandbox-client.test.ts | 13 +++ packages/agents-client/src/sandbox-client.ts | 2 +- services/agents-api/HOSTED-SANDBOX-MANAGER.md | 10 +- .../cmd/server/managed_generations.go | 27 +++-- .../cmd/server/managed_generations_test.go | 100 ++++++++++++++++++ .../agents-api/cmd/server/managed_setup.go | 2 +- .../server/managed_setup_preflight_test.go | 19 ++++ services/agents-api/internal/api/errors.go | 2 +- .../api/sandbox_deployment_changes_test.go | 1 + .../execution/sandbox_generations_test.go | 11 +- .../internal/sandbox/e2b/provider.go | 4 +- .../tools/e2b-provider/credential_test.py | 10 ++ .../tools/e2b-provider/deployment_test.py | 3 +- .../agents-api/tools/e2b-provider/provider.py | 1 + 15 files changed, 200 insertions(+), 15 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8d7bd92ba..098a42935 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -809,7 +809,15 @@ Use one E2B classifier. Omitted key preserves the current key; identical selecti with omitted key is a no-op. Explicit key submission, including identical bytes, verifies and increments generation. Check the candidate and retained builds, paginated team-owned template membership and every settled live receipt under the candidate key. -Public template readability is not team ownership. Missing/unsettled receipts and +Initial E2B selection must also belong to the submitted key's team. Before online +replacement, verify the current exact template under the committed key, then under +the candidate key: shared membership is the ownership anchor, including for legacy +installations with no retained resources. The pinned SDK exposes team-owned template +listing, not an authenticated team ID; never invent or persist an inferred team ID. +Public readability cannot anchor ownership. A legacy selection outside the committed +key's team, or a revoked committed key, requires reset without blaming the candidate +key. Transport uncertainty remains unconfirmed. Keep the old key valid until the PUT +returns 200; revoke it only afterward. Missing/unsettled receipts and unconfirmed reads reject. Fence credential commits against all provider calls and actual helper subprocess completion after cancellation, then reverify. Helper exit never settles unknown remote Create. Bounded failure keeps the original key and diff --git a/packages/agents-client/src/sandbox-client.test.ts b/packages/agents-client/src/sandbox-client.test.ts index 1ec2a3f87..bb36580f4 100644 --- a/packages/agents-client/src/sandbox-client.test.ts +++ b/packages/agents-client/src/sandbox-client.test.ts @@ -255,6 +255,19 @@ describe("hosted provider configuration", () => { expect((shown as AgentCoreError).errorType).toBeUndefined(); await expect(client.initializeDeployment({ expected_generation: 0, provider: "e2b", e2b })).rejects.toMatchObject({ status: 409, code: "sandbox_configuration_error", message: rejection.message, param: null }); }); + it("keeps legacy E2B ownership reset actionable without reflecting either key", async () => { + const fetch = vi.fn().mockResolvedValue(response({ error: { + code: "sandbox_reset_required", message: "revoked-old-key " + e2b.api_key, + param: "revoked-old-key", details: { current_provider: "e2b", requested_provider: "e2b", secret: e2b.api_key }, + } }, 409)); + const client = new SandboxAdminClient({ fetch }); + const error = await client.updateDeployment({ provider: "e2b", e2b, expected_generation: 1 }).catch((error: unknown) => error); + expect(error).toMatchObject({ status: 409, code: "sandbox_reset_required", message: "Reset the sandbox deployment before changing this configuration.", param: null }); + expect((error as AgentCoreError).details).toBeUndefined(); + expect(JSON.stringify(error)).not.toContain("revoked-old-key"); + expect(JSON.stringify(error)).not.toContain(e2b.api_key); + expect(fetch).toHaveBeenCalledTimes(1); + }); it("never retries uncertain switch or reset writes", async () => { const fetch = vi.fn().mockRejectedValue(new TypeError("Connection lost")); const client = new SandboxAdminClient({ fetch }); diff --git a/packages/agents-client/src/sandbox-client.ts b/packages/agents-client/src/sandbox-client.ts index 8a6dbb4fd..3a8a8531f 100644 --- a/packages/agents-client/src/sandbox-client.ts +++ b/packages/agents-client/src/sandbox-client.ts @@ -342,7 +342,7 @@ export class SandboxAdminClient { if (error instanceof AgentCoreError && [400, 409, 503].includes(error.status)) { const messages: Record = { sandbox_generation_stale: "The sandbox configuration changed. Refresh before submitting again.", - sandbox_reset_required: "Reset the sandbox deployment before changing its backend.", + sandbox_reset_required: "Reset the sandbox deployment before changing this configuration.", sandbox_reset_in_progress: "A sandbox reset is in progress.", sandbox_not_configured: "The sandbox deployment is not configured.", sandbox_in_use: "Hosted sandbox resources still belong to this deployment.", diff --git a/services/agents-api/HOSTED-SANDBOX-MANAGER.md b/services/agents-api/HOSTED-SANDBOX-MANAGER.md index 76ec82a44..447b2a4bb 100644 --- a/services/agents-api/HOSTED-SANDBOX-MANAGER.md +++ b/services/agents-api/HOSTED-SANDBOX-MANAGER.md @@ -276,7 +276,15 @@ an unreachable Core or a 403 from a proxy in front of it, restart the service ev original identity. E2B same-team key/build/resource edits apply online; existing sandboxes retain their -original generation and new allocations use the new selection. Docker/microsandbox +original generation and new allocations use the new selection. Initial setup requires +the selected template to belong to the submitted key's team, even if another team's +public template is readable. Online replacement verifies the current exact template +in both the committed and candidate keys' team-owned listings. This is a shared +ownership anchor; Core does not invent a team ID from public readability. Legacy +selections without this anchor and revoked committed keys require reset +(`sandbox_reset_required`); transport or unsettled ownership remains unconfirmed (503). +Keep the old key valid until PUT returns 200, then revoke it. A rejected change keeps +the current key, generation and resource owners. Docker/microsandbox resource/Runtime edits still require zero unreleased allocations and pending hosted Environments, no reset and the observed generation. Their multi-generation preparation protocol remains future work. Submit once and read back after uncertain responses. diff --git a/services/agents-api/cmd/server/managed_generations.go b/services/agents-api/cmd/server/managed_generations.go index 484c3126b..310addfc2 100644 --- a/services/agents-api/cmd/server/managed_generations.go +++ b/services/agents-api/cmd/server/managed_generations.go @@ -112,24 +112,39 @@ func (s *managedSetup) routeGenerations(candidate execution.PreparedRuntimeDeplo candidate.VerifyCredential = func(ctx context.Context) error { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - // The candidate's template must itself belong to the same accessible team. + // Preserve the committed credential until its ownership anchor is verified. + // Public template readability cannot establish which team owns a deployment. verify := func(value store.SandboxSetup, refs []sandbox.Reference) error { value.InstallationID = setup.InstallationID - value.E2B = &store.SandboxE2BConfiguration{APIKey: setup.E2B.APIKey, Template: value.E2B.Template} provider, err := s.provider(value) if err != nil { return err } return provider.(*e2b.Provider).VerifyCredential(ctx, refs) } - if err := verify(setup, nil); err != nil { - return err - } current, err := db.GetSandboxSetup(ctx) if err != nil { return err } + if current.Provider != "e2b" || current.E2B == nil { + return &store.SandboxResetRequiredError{CurrentProvider: current.Provider, RequestedProvider: "e2b"} + } if err := verify(current, nil); err != nil { + if errors.Is(err, e2b.ErrCredentialInvalid) || errors.Is(err, e2b.ErrTeamMismatch) { + // A revoked legacy key or a public template outside its team cannot + // anchor ownership. This says nothing about the candidate key's validity. + return &store.SandboxResetRequiredError{CurrentProvider: "e2b", RequestedProvider: "e2b"} + } + return err + } + withCandidateKey := func(value store.SandboxSetup, refs []sandbox.Reference) error { + value.E2B = &store.SandboxE2BConfiguration{APIKey: setup.E2B.APIKey, Template: value.E2B.Template} + return verify(value, refs) + } + if err := withCandidateKey(current, nil); err != nil { + return err + } + if err := verify(setup, nil); err != nil { return err } for after := int64(-1); ; { @@ -138,7 +153,7 @@ func (s *managedSetup) routeGenerations(candidate execution.PreparedRuntimeDeplo return err } for _, g := range page { - if err := verify(g, nil); err != nil { + if err := withCandidateKey(g, nil); err != nil { return err } after = int64(g.Generation) diff --git a/services/agents-api/cmd/server/managed_generations_test.go b/services/agents-api/cmd/server/managed_generations_test.go index d5d3643e4..19cd0f48c 100644 --- a/services/agents-api/cmd/server/managed_generations_test.go +++ b/services/agents-api/cmd/server/managed_generations_test.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "os" "path/filepath" "strings" @@ -10,6 +11,7 @@ import ( "time" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/e2b" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" "github.com/google/uuid" ) @@ -103,3 +105,101 @@ print(json.dumps({'Version':1,'Info':info})) } } } + +// The empty resource set must not let a publicly readable template substitute +// for a team identity. In particular, the old key must be checked as itself. +func TestE2BReplacementRequiresCommittedOwnershipAnchor(t *testing.T) { + for _, tc := range []struct { + name, committedKey, committedTemplate, candidateKey, candidateTemplate string + want error + reset bool + }{ + {name: "legacy public template cross team", committedKey: "team-a", committedTemplate: "public-b", candidateKey: "team-b", candidateTemplate: "public-b", reset: true}, + {name: "revoked committed key", committedKey: "revoked", committedTemplate: "owned-a", candidateKey: "team-a", candidateTemplate: "owned-a", reset: true}, + {name: "unknown committed ownership", committedKey: "unconfirmed", committedTemplate: "owned-a", candidateKey: "team-a", candidateTemplate: "owned-a", want: e2b.ErrRequestUnconfirmed}, + {name: "proven different team", committedKey: "team-a", committedTemplate: "owned-a", candidateKey: "team-b", candidateTemplate: "public-b", want: e2b.ErrTeamMismatch}, + {name: "same team replacement", committedKey: "team-a", committedTemplate: "owned-a", candidateKey: "team-a-rotated", candidateTemplate: "new-a"}, + {name: "explicit same key", committedKey: "team-a", committedTemplate: "owned-a", candidateKey: "team-a", candidateTemplate: "owned-a"}, + } { + t.Run(tc.name, func(t *testing.T) { + state := t.TempDir() + if err := os.Chmod(state, 0700); err != nil { + t.Fatal(err) + } + helper := filepath.Join(t.TempDir(), "provider") + const script = `#!/usr/bin/env python3 +import json, pathlib, sys +q = json.load(sys.stdin) +k, template = q['Config']['APIKey'], q['Config']['Template'].split(':')[0] +with (pathlib.Path(q['Config']['StateDir']) / 'requests').open('a') as log: + log.write(json.dumps(q) + '\n') +code = '' +if k == 'revoked': code = 'unauthorized' +elif k == 'unconfirmed': code = 'unconfirmed' +elif not (k.startswith('team-a') and template in ('owned-a', 'new-a') or k == 'team-b' and template == 'public-b'): code = 'team_mismatch' +result = {'Version': 1, 'ErrorCode': code} +if not code: + result['DeploymentValid'] = True + if q['Operation'] == 'validate_deployment': + result['TemplateBuild'] = {'Status': 'ready', 'CPUs': 2, 'MemoryMiB': 2048} +print(json.dumps(result)) +` + if err := os.WriteFile(helper, []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("OAC_E2B_PROVIDER_BIN", helper) + t.Setenv("OAC_E2B_STATE_DIR", state) + id, build := uuid.NewString(), ":"+uuid.NewString() + current := store.SandboxSetup{InstallationID: id, Provider: "e2b", Mode: "direct", Generation: 1, + Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}}, + E2B: &store.SandboxE2BConfiguration{APIKey: tc.committedKey, Template: tc.committedTemplate + build}} + db := &setupStore{value: current} + s := &managedSetup{installationID: id, store: db} + loaded, err := s.load(t.Context()) + if err != nil { + t.Fatal(err) + } + next := current + next.E2B = &store.SandboxE2BConfiguration{APIKey: tc.candidateKey, Template: tc.candidateTemplate + build} + candidate, err := s.prepare(t.Context(), next) + if err != nil { + t.Fatal(err) + } + err = candidate.VerifyCredential(t.Context()) + var reset *store.SandboxResetRequiredError + if tc.reset { + if !errors.As(err, &reset) || errors.Is(err, e2b.ErrCredentialInvalid) || errors.Is(err, e2b.ErrTeamMismatch) { + t.Fatalf("unanchored ownership misattributed: %v", err) + } + } else if !errors.Is(err, tc.want) { + t.Fatalf("got %v; want %v", err, tc.want) + } + if db.value.Generation != 1 || db.value.E2B.APIKey != tc.committedKey || s.selected.Load().Config != loaded { + t.Fatal("verification mutated committed selection") + } + raw, err := os.ReadFile(filepath.Join(state, "requests")) + if err != nil { + t.Fatal(err) + } + var requests []e2b.Request + for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") { + var q e2b.Request + if err := json.Unmarshal([]byte(line), &q); err != nil { + t.Fatal(err) + } + requests = append(requests, q) + if q.Operation != "verify_credential" && q.Operation != "validate_deployment" { + t.Fatal("verification mutated provider", q.Operation) + } + } + if len(requests) < 2 || requests[1].Config.APIKey != tc.committedKey || requests[1].Config.Template != current.E2B.Template { + t.Fatal("committed key was replaced before establishing ownership") + } + if tc.reset || tc.want == e2b.ErrRequestUnconfirmed { + if len(requests) != 2 { + t.Fatal("unanchored current ownership reached candidate verification") + } + } + }) + } +} diff --git a/services/agents-api/cmd/server/managed_setup.go b/services/agents-api/cmd/server/managed_setup.go index 11e71a2c4..b362c09a1 100644 --- a/services/agents-api/cmd/server/managed_setup.go +++ b/services/agents-api/cmd/server/managed_setup.go @@ -94,7 +94,7 @@ func (s *managedSetup) prepare(ctx context.Context, setup store.SandboxSetup) (e if provider, ok := candidate.Config.Provider.(*e2b.Provider); ok { build, err := provider.ValidateDeployment(ctx) if err != nil { - if errors.Is(err, e2b.ErrCredentialInvalid) { + if errors.Is(err, e2b.ErrCredentialInvalid) || errors.Is(err, e2b.ErrTeamMismatch) { return execution.PreparedRuntimeDeployment{}, err } if errors.Is(err, sandbox.ErrInvalid) { diff --git a/services/agents-api/cmd/server/managed_setup_preflight_test.go b/services/agents-api/cmd/server/managed_setup_preflight_test.go index d75a2922a..e95d33fd6 100644 --- a/services/agents-api/cmd/server/managed_setup_preflight_test.go +++ b/services/agents-api/cmd/server/managed_setup_preflight_test.go @@ -129,3 +129,22 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { t.Fatalf("omitted resources were not adopted from the build: %s %v", logged, err) } } + +func TestInitialE2BPublicTemplateOutsideTeamIsRejected(t *testing.T) { + helper := filepath.Join(t.TempDir(), "provider") + if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":1,\"ErrorCode\":\"team_mismatch\"}'\n"), 0700); err != nil { + t.Fatal(err) + } + state := t.TempDir() + if err := os.Chmod(state, 0700); err != nil { + t.Fatal(err) + } + t.Setenv("OAC_E2B_PROVIDER_BIN", helper) + t.Setenv("OAC_E2B_STATE_DIR", state) + id := uuid.NewString() + s := &managedSetup{installationID: id, store: &setupStore{}} + selection := store.SandboxSetup{InstallationID: id, Provider: "e2b", E2B: &store.SandboxE2BConfiguration{APIKey: "synthetic-team-a", Template: "public-team-b:" + uuid.NewString()}} + if _, err := s.prepare(t.Context(), selection); !errors.Is(err, e2b.ErrTeamMismatch) || s.selected.Load() != nil { + t.Fatal("public readability accepted as team ownership", err) + } +} diff --git a/services/agents-api/internal/api/errors.go b/services/agents-api/internal/api/errors.go index d73197dbc..d3ddfa240 100644 --- a/services/agents-api/internal/api/errors.go +++ b/services/agents-api/internal/api/errors.go @@ -111,7 +111,7 @@ func writeStoreError(w http.ResponseWriter, r *http.Request, err error, notFound case errors.As(err, &stale): writeCoreError(w, http.StatusConflict, "sandbox_generation_stale", "The sandbox deployment generation changed. Refresh before submitting again.", CoreErrorDetails{"current_generation": CoreErrorNumber(float64(stale.CurrentGeneration))}) case errors.As(err, &resetRequired): - writeCoreError(w, http.StatusConflict, "sandbox_reset_required", "Reset the sandbox deployment before changing its backend.", CoreErrorDetails{"current_provider": CoreErrorString(resetRequired.CurrentProvider), "requested_provider": CoreErrorString(resetRequired.RequestedProvider)}) + writeCoreError(w, http.StatusConflict, "sandbox_reset_required", "Reset the sandbox deployment before changing this configuration.", CoreErrorDetails{"current_provider": CoreErrorString(resetRequired.CurrentProvider), "requested_provider": CoreErrorString(resetRequired.RequestedProvider)}) case errors.As(err, &inUse): writeCoreError(w, http.StatusConflict, "sandbox_in_use", "Hosted sandbox resources still belong to this deployment.", CoreErrorDetails{"allocations": CoreErrorNumber(float64(inUse.Resources.Allocations)), "pending": CoreErrorNumber(float64(inUse.Resources.Pending))}) case errors.Is(err, store.ErrSandboxResetAdmission): diff --git a/services/agents-api/internal/api/sandbox_deployment_changes_test.go b/services/agents-api/internal/api/sandbox_deployment_changes_test.go index 74a5c4d6a..5227157a1 100644 --- a/services/agents-api/internal/api/sandbox_deployment_changes_test.go +++ b/services/agents-api/internal/api/sandbox_deployment_changes_test.go @@ -97,6 +97,7 @@ func TestSandboxMutationErrorsExposeOnlyTypedCoreFacts(t *testing.T) { }{ {&store.SandboxGenerationStaleError{CurrentGeneration: 8}, "sandbox_generation_stale", 409, `"current_generation":8`}, {&store.SandboxResetRequiredError{CurrentProvider: "docker", RequestedProvider: "e2b"}, "sandbox_reset_required", 409, `"requested_provider":"e2b"`}, + {&store.SandboxResetRequiredError{CurrentProvider: "e2b", RequestedProvider: "e2b"}, "sandbox_reset_required", 409, `"current_provider":"e2b"`}, {&store.SandboxInUseError{Resources: store.SandboxDeploymentResources{Allocations: 2, Pending: 1}}, "sandbox_in_use", 409, `"allocations":2`}, {store.ErrSandboxResetInProgress, "sandbox_reset_in_progress", 409, ""}, {store.ErrSandboxResetAdmission, "sandbox_reset_in_progress", 503, ""}, diff --git a/services/agents-api/internal/execution/sandbox_generations_test.go b/services/agents-api/internal/execution/sandbox_generations_test.go index a674a2f6c..f322419b0 100644 --- a/services/agents-api/internal/execution/sandbox_generations_test.go +++ b/services/agents-api/internal/execution/sandbox_generations_test.go @@ -31,6 +31,7 @@ func TestE2BReplacementVerifiesTwiceAndNeverPublishesFailedCommit(t *testing.T) provider := hub.Proxy(uuid.NewString(), "docker") verifyCalls, published, fenced, released := 0, 0, 0, 0 var rejectAt int + var rejection error = e2b.ErrTeamMismatch config := NewDeferredRuntimeProvider(id, func(ctx context.Context) (*RuntimeProvider, error) { setup, err := s.GetSandboxSetup(ctx) if err != nil { @@ -42,7 +43,7 @@ func TestE2BReplacementVerifiesTwiceAndNeverPublishesFailedCommit(t *testing.T) VerifyCredential: func(context.Context) error { verifyCalls++ if verifyCalls == rejectAt { - return e2b.ErrTeamMismatch + return rejection } return nil }, @@ -64,7 +65,7 @@ func TestE2BReplacementVerifiesTwiceAndNeverPublishesFailedCommit(t *testing.T) input.E2B.APIKey = "candidate-key" request := store.SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1} audit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "test", RequestID: "test", TraceID: "test"}) - for _, failure := range []string{"preliminary", "final", "commit"} { + for _, failure := range []string{"preliminary", "final", "unanchored legacy or revoked committed key", "unknown ownership", "commit"} { verifyCalls = 0 rejectAt = 0 ctx := audit @@ -73,6 +74,12 @@ func TestE2BReplacementVerifiesTwiceAndNeverPublishesFailedCommit(t *testing.T) rejectAt = 1 case "final": rejectAt = 2 + case "unanchored legacy or revoked committed key": + rejectAt = 1 + rejection = &store.SandboxResetRequiredError{CurrentProvider: "e2b", RequestedProvider: "e2b"} + case "unknown ownership": + rejectAt = 1 + rejection = e2b.ErrRequestUnconfirmed case "commit": ctx = t.Context() } diff --git a/services/agents-api/internal/sandbox/e2b/provider.go b/services/agents-api/internal/sandbox/e2b/provider.go index 5fe489980..e1ec26507 100644 --- a/services/agents-api/internal/sandbox/e2b/provider.go +++ b/services/agents-api/internal/sandbox/e2b/provider.go @@ -141,6 +141,8 @@ func (p *Provider) call(ctx context.Context, operation string, r sandbox.Referen return out, nil case "legacy_template": return out, fmt.Errorf("%w: This E2B template was built before OpenAgentCore renamed its paths. Build a template with this release's build-template.py and replace it in the sandbox deployment.", sandbox.ErrInvalid) + case "team_mismatch": + return out, ErrTeamMismatch case "unauthorized": return out, ErrCredentialInvalid case "invalid": @@ -158,7 +160,7 @@ func (p *Provider) call(ctx context.Context, operation string, r sandbox.Referen } } -// ValidateDeployment reads the exact immutable build without creating compute or +// ValidateDeployment verifies team ownership and reads the exact immutable build without creating compute or // allocation receipts. Candidate configuration remains unpublished until it passes. // It returns the build as read. Without configured Resources it only requires a // ready build, whose CPU and memory the caller then adopts as the selection. diff --git a/services/agents-api/tools/e2b-provider/credential_test.py b/services/agents-api/tools/e2b-provider/credential_test.py index d546595c0..d35ec54d1 100644 --- a/services/agents-api/tools/e2b-provider/credential_test.py +++ b/services/agents-api/tools/e2b-provider/credential_test.py @@ -40,6 +40,16 @@ def test_unauthorized_and_repeated_cursor_fail_closed(self, listing, client): class CredentialReceiptTest(ProviderTest): + def test_initial_public_template_readability_does_not_establish_team(self): + with patch('provider.validate_deployment') as readable, patch('provider.verify_team_template', side_effect=Failure('team_mismatch')): + result = self.call('validate_deployment') + self.assertEqual(result['ErrorCode'], 'team_mismatch') + readable.assert_not_called() + self.api.create.assert_not_called() + from pathlib import Path + self.assertEqual(list(Path(self.temporary.name).iterdir()), []) + + def verify(self): self.request['References'] = [self.reference] with patch('provider.verify_team_template'), patch('provider.validate_deployment'): diff --git a/services/agents-api/tools/e2b-provider/deployment_test.py b/services/agents-api/tools/e2b-provider/deployment_test.py index 5e95d7474..11f12ed9e 100644 --- a/services/agents-api/tools/e2b-provider/deployment_test.py +++ b/services/agents-api/tools/e2b-provider/deployment_test.py @@ -42,9 +42,10 @@ def test_inspection_checks_current_limits(self): self.assertEqual(self.call('inspect')['ErrorCode'], 'invalid') def test_candidate_validation_never_opens_allocation_receipt(self): - with patch('provider.validate_deployment') as validate, patch('provider.Receipt') as receipt: + with patch('provider.verify_team_template') as ownership, patch('provider.validate_deployment') as validate, patch('provider.Receipt') as receipt: result = self.call('validate_deployment') self.assertTrue(result['DeploymentValid']) + ownership.assert_called_once() validate.assert_called_once() receipt.assert_not_called() self.assertEqual(list(Path(self.temporary.name).iterdir()), []) diff --git a/services/agents-api/tools/e2b-provider/provider.py b/services/agents-api/tools/e2b-provider/provider.py index 99558e5e8..14202940b 100644 --- a/services/agents-api/tools/e2b-provider/provider.py +++ b/services/agents-api/tools/e2b-provider/provider.py @@ -365,6 +365,7 @@ def execute(self): return {'Version': 1, 'DeploymentValid': True, 'ErrorCode': ''} if self.q['Operation'] == 'observe': return {'Version': 1, 'Observations': self.observe(), 'ErrorCode': ''} + verify_team_template(self.config, self.remaining) build = validate_deployment(self.config, self.remaining) return {'Version': 1, 'DeploymentValid': True, 'TemplateBuild': build, 'ErrorCode': ''} except AuthenticationException: From f94efa2d724c176e7b8eafbf8145ad4bb6af25b5 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 20:57:18 +0000 Subject: [PATCH 07/16] Explain committed E2B ownership and reset recovery requirements --- apps/docs/content/docs/configure.mdx | 5 ++++- apps/docs/content/guide-sources.json | 4 ++-- contracts/agents-api/sandbox-deployment.md | 23 ++++++++++++++-------- docs/configuration.md | 5 ++++- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/apps/docs/content/docs/configure.mdx b/apps/docs/content/docs/configure.mdx index 517c4c662..0ec141481 100644 --- a/apps/docs/content/docs/configure.mdx +++ b/apps/docs/content/docs/configure.mdx @@ -165,7 +165,10 @@ Changing backend type or E2B team requires an explicit reset and a new setup. Same-team E2B template, resource and key changes apply online through the Core API: new allocations use the new generation, while existing sandboxes retain their original specification. Omit the key to preserve it; explicitly submitting a key, -even the same value, verifies the replacement and advances the generation. +even the same value, verifies the replacement and advances the generation. Keep the +old key valid until the update succeeds. Initial setup requires a team-owned template; +a legacy public-template configuration or an already revoked old key requires reset +when Core cannot verify the committed ownership anchor. Docker and microsandbox size/Runtime edits still require zero held resources and retire the old nodes and enrollment commands. Add those hosts again after cleaning diff --git a/apps/docs/content/guide-sources.json b/apps/docs/content/guide-sources.json index 1816933ac..d6c333fe1 100644 --- a/apps/docs/content/guide-sources.json +++ b/apps/docs/content/guide-sources.json @@ -5,7 +5,7 @@ "docs/design-principles.md": "5e1bff3cdcc77cd02ad692f67977cb4e0a5654989982d722bac0e3c79dbf82cb", "docs/web/architecture.md": "80bdb8053c06c2b2030b193aae17d5434545f9c72af50e9d3df3ade522fe25ed", "docs/getting-started/install.md": "15cfe81d4005a7337ac82bf6c35a8b441e9f9b0f2a2299021ad6c539f3f336a3", - "docs/configuration.md": "6114e2993607defea86f54181fb33824910e072d88ae1acf1c665c105d198d30", + "docs/configuration.md": "e911757a03fbce4b803a284c8bb40bdb7000ca2f70e27938706dcdcd74bc7f9a", "docs/web/core-connection.md": "46f86520e70bf41079708e2c398655aa9087ad9488c8e661d9a73035d11098c4", "docs/getting-started/quickstart.md": "5effeffcfb343f7939910f0c1758e1492d49e7bd66c409b1819cefe67af592d7", "docs/api/README.md": "b2b4c7fdf9fbde62d2169014180698a77c671f85c1c68c2e5de0e73bbdf3b13c", @@ -30,7 +30,7 @@ "content/docs/execution-model.zh.mdx": "9976a9203c6a83d49c2c753e3a8528a369078e87aeb88850654da948ac8c17e1", "content/docs/install.mdx": "cba5c14e55c6f474d810dc8e545fadbb4f35bcad50eeb6ceef0cbd967b3fb1ab", "content/docs/install.zh.mdx": "7ed0b726cf721b18e85b89145ef8ebd2f5d382afd0489633a7b63962aa2fc4ed", - "content/docs/configure.mdx": "72b0dc1b4a862f3a69f776262f6e6235290dfed212a604b24b3bbdf4963c649b", + "content/docs/configure.mdx": "ec84259c766872eada11c1b0d9d9e3ec19acd8c867c6048cc0890a32b82d65f0", "content/docs/configure.zh.mdx": "a0c3a1df3e9f75e672e1d0a8c68c82c24ac36a8479d10a951de441960d07927f", "content/docs/bootstrap-projects-keys.mdx": "fc2da0aa255278281d8a57c664d7d30e5a458add4e58402386e0374f23973da4", "content/docs/bootstrap-projects-keys.zh.mdx": "509b4dcea4c865d6834bcaef78baa681180d48274931230f958a39e46011c88f", diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index 4bf263884..160c696a6 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -218,14 +218,21 @@ always verifies and advances generation. Null or empty keys are invalid. A key-o change uses the same full DTO: provider, existing template, optional resources and expected_generation, plus the new api_key. It has no separate route or implicit reset. -Verification reads the candidate and every retained build with the candidate key, -proves ownership using the SDK's paginated team-template listing, and confirms each -settled live receipt in the installation-labelled sandbox listing. Public template -readability alone is insufficient. Missing or unsettled receipts and unconfirmed -reads fail closed. Different-team ownership gives `409 e2b_team_mismatch`; explicitly -reset before initializing another team. Provider 401/403 gives `400 e2b_api_key_invalid`; -an invalid candidate build gives `400 e2b_template_build_invalid`; an unconfirmed -verification gives `503 e2b_request_unconfirmed`. No provider text or credential is +Initial setup requires the selected template to appear in the credential's team-owned +template listing; public readability alone is insufficient. Before an online change, +Core verifies that the committed key owns the current template, then requires the +candidate key to own that exact template as a shared ownership anchor. It also reads +the candidate and every retained build with the candidate key and confirms each +settled live receipt in the installation-labelled sandbox listing. + +A legacy public-template selection without this ownership anchor, or a committed key +that no longer authenticates, requires `409 sandbox_reset_required`; Core cannot +establish a safe online replacement from that state. Keep the old key valid until the +successful response. A candidate outside the verified team gives `409 e2b_team_mismatch`; +explicitly reset before initializing another team. Candidate-key 401/403 gives +`400 e2b_api_key_invalid`; an invalid candidate build gives +`400 e2b_template_build_invalid`. Missing or unsettled receipts and unconfirmed reads +fail closed with `503 e2b_request_unconfirmed`. No provider text or credential is returned. The write and `change` or `replace_credential` audit share one transaction. A credential replacement briefly fences provider calls, waits for actual helper diff --git a/docs/configuration.md b/docs/configuration.md index bd7b0a7c5..1575d3b3b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -162,7 +162,10 @@ Changing backend type or E2B team requires an explicit reset and a new setup. Same-team E2B template, resource and key changes apply online through the Core API: new allocations use the new generation, while existing sandboxes retain their original specification. Omit the key to preserve it; explicitly submitting a key, -even the same value, verifies the replacement and advances the generation. +even the same value, verifies the replacement and advances the generation. Keep the +old key valid until the update succeeds. Initial setup requires a team-owned template; +a legacy public-template configuration or an already revoked old key requires reset +when Core cannot verify the committed ownership anchor. Docker and microsandbox size/Runtime edits still require zero held resources and retire the old nodes and enrollment commands. Add those hosts again after cleaning From 0f2b036a4db48920743678fd8c75bca9e622c434 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 21:14:14 +0000 Subject: [PATCH 08/16] Allow bounded time for the Agents API integration suite --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a55303793..90c1b28c2 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,8 @@ build-agents-api-release: ./scripts/build-agents-api-release.sh check-agents-api: build-agents-api - go test ./services/agents-api/... ./packages/agents-client/... -count=1 + # Persistence integration tests include bounded lifecycle waits that together exceed Go's 10m default. + go test ./services/agents-api/... ./packages/agents-client/... -count=1 -timeout=20m PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s services/agents-api/tests -p 'official_diagnostics_test.py' PYTHONDONTWRITEBYTECODE=1 python3 services/agents-api/deploy/runtime/initialize_receipt_test.py PYTHONDONTWRITEBYTECODE=1 python3 services/agents-api/deploy/e2b/managed_init_test.py From 1a7e2fe45dad5be206fc416bfccdd7cf0040b5b1 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 21:48:25 +0000 Subject: [PATCH 09/16] Fence lifecycle cancellation between leased execution operations --- CONTRIBUTING.md | 11 +- .../execution/runtime_cancellation.go | 71 +++++ .../internal/execution/runtime_lifecycle.go | 35 +-- .../internal/execution/runtime_manager.go | 7 +- .../sandbox_deployment_drain_test.go | 287 ++++++++++++++++++ .../execution/sandbox_deployment_switch.go | 21 +- .../store/execution_cancellation_test.go | 87 ++++++ .../internal/store/execution_lease.go | 20 ++ 8 files changed, 513 insertions(+), 26 deletions(-) create mode 100644 services/agents-api/internal/execution/runtime_cancellation.go create mode 100644 services/agents-api/internal/execution/sandbox_deployment_drain_test.go create mode 100644 services/agents-api/internal/store/execution_cancellation_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 098a42935..72b4d00bb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -969,7 +969,16 @@ existing allocation must agree with the placement. Never choose another node. E2B direct allocations use one serial lifecycle without a node identity. The coordinator stops accepting work and cancels and drains all node workers and direct callers before releasing the sole execution lease. Lease loss is global; -ordinary provider failures stay within their node. Session locks, deployment +ordinary provider failures stay within their node. A planned deployment drain or +inventory retirement cancels lifecycle contexts synchronously between leased +operations, using the existing lease gate with its five-second bound. Include +an active manual reconcile's cancel function: an asynchronous `AfterFunc` alone +can cancel a later leased query after the gate reopens. Never cancel an in-flight +leased query merely to change deployment configuration. A failed cancellation +fence stops the owner and leaves the drain barrier closed; it cannot activate a +replacement. Release the lease gate before waiting for provider settlement or +lifecycle accounting. Ordinary caller deadlines and owner shutdown retain their +existing cancellation and fail-closed lease-loss behavior. Session locks, deployment capacity transactions and revision/one-shot receipts remain authoritative, with no external operation holding a database lock. diff --git a/services/agents-api/internal/execution/runtime_cancellation.go b/services/agents-api/internal/execution/runtime_cancellation.go new file mode 100644 index 000000000..66fc7ac65 --- /dev/null +++ b/services/agents-api/internal/execution/runtime_cancellation.go @@ -0,0 +1,71 @@ +package execution + +import "context" + +// cancelLifecycles leaves the manager mutex free while the bounded lease gate +// drains. The cancellation itself runs synchronously under that gate; waiting +// for provider operations and lifecycle accounting happens only after release. +func (m *runtimeManager) cancelLifecycles(nodes []*runtimeNode) error { + if len(nodes) == 0 { + return nil + } + cancel := func() { + for _, n := range nodes { + n.lifecycle.cancelOperations() + } + } + // Managers in pure lifecycle tests have no Store or execution connection. + if m.store == nil { + cancel() + return nil + } + err := m.store.CancelExecutionOperations(m.ctx, cancel) + if err != nil { + select { + case m.failed <- err: + default: + } + } + return err +} + +func (r *runtimeLifecycle) cancelOperations() { + r.cancelMu.Lock() + defer r.cancelMu.Unlock() + r.stop() + // A manual reconcile has an independently parented context. AfterFunc is + // asynchronous, so it cannot be the only cancellation under the lease gate. + if r.reconcileCancel != nil { + r.reconcileCancel() + } +} + +func (r *runtimeLifecycle) beginReconcile(parent context.Context) (context.Context, func(), error) { + ctx, cancel := context.WithCancel(parent) + detach := context.AfterFunc(r.ctx, cancel) + if err := r.lock(ctx); err != nil { + detach() + cancel() + return nil, nil, err + } + r.cancelMu.Lock() + if err := r.ctx.Err(); err != nil { + r.cancelMu.Unlock() + <-r.gate + detach() + cancel() + return nil, nil, err + } + // The lifecycle gate admits only one active reconcile. Register cancellation + // before its first leased query, and retain it until its last query returns. + r.reconcileCancel = cancel + r.cancelMu.Unlock() + return ctx, func() { + r.cancelMu.Lock() + r.reconcileCancel = nil + r.cancelMu.Unlock() + <-r.gate + detach() + cancel() + }, nil +} diff --git a/services/agents-api/internal/execution/runtime_lifecycle.go b/services/agents-api/internal/execution/runtime_lifecycle.go index 25bda3d67..a286c74e4 100644 --- a/services/agents-api/internal/execution/runtime_lifecycle.go +++ b/services/agents-api/internal/execution/runtime_lifecycle.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "net/url" + "sync" "time" "github.com/google/uuid" @@ -40,18 +41,20 @@ type RuntimeProvider struct { } type runtimeLifecycle struct { - store *store.Store - registry *gateway.Registry - config RuntimeProvider - nodeID string - gate chan struct{} - ctx context.Context - stop context.CancelFunc - cursor string - pendingCursor string - connections map[string]*runtimeConnection - initializing *runtimeInitialization - wakeHints chan struct{} + store *store.Store + registry *gateway.Registry + config RuntimeProvider + nodeID string + gate chan struct{} + ctx context.Context + stop context.CancelFunc + cancelMu sync.Mutex + reconcileCancel context.CancelFunc + cursor string + pendingCursor string + connections map[string]*runtimeConnection + initializing *runtimeInitialization + wakeHints chan struct{} } func newRuntimeManager(s *store.Store, registry *gateway.Registry, config *RuntimeProvider) (*runtimeManager, error) { @@ -262,13 +265,11 @@ func (w *Worker) ReconcileManagedRuntimes(ctx context.Context) error { } func (r *runtimeLifecycle) reconcile(ctx context.Context) error { - ctx, cancel := context.WithCancel(ctx) - detach := context.AfterFunc(r.ctx, cancel) - defer func() { detach(); cancel() }() - if err := r.lock(ctx); err != nil { + ctx, finish, err := r.beginReconcile(ctx) + if err != nil { return err } - defer func() { <-r.gate }() + defer finish() rows, err := r.store.ListRuntimeAllocationsForNode(ctx, r.nodeID, r.cursor) if err != nil { return err diff --git a/services/agents-api/internal/execution/runtime_manager.go b/services/agents-api/internal/execution/runtime_manager.go index 5c7299fad..5aabef290 100644 --- a/services/agents-api/internal/execution/runtime_manager.go +++ b/services/agents-api/internal/execution/runtime_manager.go @@ -28,7 +28,7 @@ type runtimeManager struct { setupGate chan struct{} mutationGate chan struct{} switching bool - switchDrained chan struct{} + switchDrained *deploymentDrain ctx context.Context cancel context.CancelFunc mu sync.Mutex @@ -179,10 +179,13 @@ func (m *runtimeManager) applyInventory(previous map[string]*runtimeNode, ids [] m.mu.Unlock() // Inventory includes offline nodes. Only explicit removal retires a lane; // the Store requires all retained resources to be cleaned before removal. + err := m.cancelLifecycles(retired) for _, n := range retired { - n.lifecycle.stop() go m.retire(n) } + if err != nil { + return nil, err + } return nodes, nil } diff --git a/services/agents-api/internal/execution/sandbox_deployment_drain_test.go b/services/agents-api/internal/execution/sandbox_deployment_drain_test.go new file mode 100644 index 000000000..c7fd9b219 --- /dev/null +++ b/services/agents-api/internal/execution/sandbox_deployment_drain_test.go @@ -0,0 +1,287 @@ +package execution + +import ( + "bytes" + "context" + "net" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/node" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// delayedLeaseRead holds one real PostgreSQL response while the deployment drain +// begins. Cancellation during this driver read closes the advisory-lock owner. +type delayedLeaseRead struct { + net.Conn + armed *atomic.Bool + held atomic.Bool + reading chan struct{} + release <-chan struct{} +} + +func (c *delayedLeaseRead) Write(p []byte) (int, error) { + if bytes.Contains(p, []byte("-- ping")) && c.armed.CompareAndSwap(true, false) { + c.held.Store(true) + } + return c.Conn.Write(p) +} +func (c *delayedLeaseRead) Read(p []byte) (int, error) { + if c.held.CompareAndSwap(true, false) { + close(c.reading) + <-c.release + } + return c.Conn.Read(p) +} + +func TestSandboxDeploymentDrainPreservesLeaseInFlightRead(t *testing.T) { + for _, mode := range []string{"deployment", "inventory", "manual"} { + t.Run(mode, func(t *testing.T) { + testLifecycleCancellationPreservesLease(t, mode) + }) + } +} + +func testLifecycleCancellationPreservesLease(t *testing.T, mode string) { + dsn := os.Getenv("OAC_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("dedicated PostgreSQL required") + } + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(cfg.ConnConfig.Database, "oac_") || !strings.HasSuffix(cfg.ConnConfig.Database, "_tests") { + t.Fatal("dedicated test database required") + } + admin, err := pgxpool.NewWithConfig(t.Context(), cfg.Copy()) + if err != nil { + t.Fatal(err) + } + defer admin.Close() + database := pgx.Identifier{"oac_drain_" + uuid.NewString()[:8] + "_tests"}.Sanitize() + if _, err := admin.Exec(t.Context(), "CREATE DATABASE "+database); err != nil { + t.Fatal(err) + } + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := admin.Exec(ctx, "DROP DATABASE "+database+" WITH (FORCE)"); err != nil { + t.Error(err) + } + }() + cfg.ConnConfig.Database = strings.Trim(database, `"`) + var armed atomic.Bool + reading, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + defer unblock() + dial := cfg.ConnConfig.DialFunc + cfg.ConnConfig.DialFunc = func(ctx context.Context, network, address string) (net.Conn, error) { + c, err := dial(ctx, network, address) + if err != nil { + return nil, err + } + return &delayedLeaseRead{Conn: c, armed: &armed, reading: reading, release: release}, nil + } + pool, err := pgxpool.NewWithConfig(t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + lease, err := store.New(pool).AcquireExecutionLease(t.Context()) + if err != nil { + t.Fatal(err) + } + defer lease.Close(context.Background()) + hub := node.NewHub(node.HubOptions{}) + defer hub.Close() + id := uuid.NewString() + configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker")} + m, err := newRuntimeManager(lease.Store(), gateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil })) + if err != nil { + t.Fatal(err) + } + defer func() { unblock(); m.stop(); m.drain() }() + if _, err := m.ensureDeployment(t.Context()); err != nil { + t.Fatal(err) + } + n, err := m.node(uuid.NewString()) + if err != nil { + t.Fatal(err) + } + var delayed *delayedCancellationContext + if mode == "manual" { + delayed = &delayedCancellationContext{Context: n.lifecycle.ctx, release: make(chan struct{}), finished: make(chan struct{})} + n.lifecycle.ctx = delayed + defer delayed.unblock() + } + m.active.Add(1) + queryDone := make(chan error, 1) + leaseFree := make(chan error, 1) + armed.Store(true) + go func() { + defer m.active.Done() + parent := n.lifecycle.ctx + if mode == "manual" { + parent = t.Context() + } + ctx, finish, err := n.lifecycle.beginReconcile(parent) + if err != nil { + queryDone <- err + return + } + defer finish() + queryDone <- lease.Store().CheckExecutionOwnership(ctx) + <-ctx.Done() + // Provider settlement remains outside the cancellation fence. A fresh owner + // read must proceed even before this tracked lifecycle operation returns. + leaseFree <- lease.Store().CheckExecutionOwnership(t.Context()) + }() + select { + case <-reading: + case <-time.After(2 * time.Second): + t.Fatal("lease query did not reach its driver read") + } + paused := make(chan error, 1) + go func() { + if mode == "inventory" { + _, err := m.applyInventory(map[string]*runtimeNode{n.lifecycle.nodeID: n}, nil) + paused <- err + } else { + paused <- m.pauseDeployment(t.Context()) + } + }() + // The cancellation is only legal after the leased query has returned. The + // small observation window is not a delay used to make the operation succeed. + select { + case <-n.lifecycle.ctx.Done(): + t.Log("drain cancelled a live leased operation") + case <-time.After(100 * time.Millisecond): + } + unblock() + queryErr := <-queryDone + select { + case err := <-leaseFree: + if err != nil { + t.Fatal("cancellation held or lost the lease during provider settlement", err) + } + case <-time.After(2 * time.Second): + t.Fatal("lifecycle cancellation or provider settlement did not proceed beyond the fence") + } + select { + case err := <-paused: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("cancellation depended on delayed AfterFunc or drain did not finish") + } + if mode == "inventory" { + settled := make(chan struct{}) + go func() { m.active.Wait(); close(settled) }() + select { + case <-settled: + case <-time.After(2 * time.Second): + t.Fatal("retired lifecycle did not settle") + } + } + if delayed != nil { + delayed.unblock() + } + if err := lease.Store().CheckExecutionOwnership(t.Context()); err != nil { + t.Fatalf("deployment drain destroyed the owner connection (in-flight query: %v): %v", queryErr, err) + } + if queryErr != nil { + t.Fatal("drain interrupted the leased read", queryErr) + } +} + +// Force the manual-reconcile AfterFunc to run after the synchronous fence. Value +// intentionally hides cancelCtx internals so context uses this AfterFunc method. +type delayedCancellationContext struct { + context.Context + release chan struct{} + finished chan struct{} + releaseOnce sync.Once + finishOnce sync.Once +} + +func (c *delayedCancellationContext) Value(any) any { return nil } +func (c *delayedCancellationContext) AfterFunc(f func()) func() bool { + stop := context.AfterFunc(c.Context, func() { + <-c.release + f() + c.finishOnce.Do(func() { close(c.finished) }) + }) + return func() bool { + stopped := stop() + if stopped { + c.finishOnce.Do(func() { close(c.finished) }) + } + return stopped + } +} +func (c *delayedCancellationContext) unblock() { + c.releaseOnce.Do(func() { close(c.release) }) + select { + case <-c.finished: + case <-time.After(time.Second): + } +} + +func TestSandboxDeploymentDrainFailureCannotReactivate(t *testing.T) { + _, lease := resetManagerStore(t) + hub := node.NewHub(node.HubOptions{}) + defer hub.Close() + id := uuid.NewString() + configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker")} + m, err := newRuntimeManager(lease.Store(), gateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil })) + if err != nil { + t.Fatal(err) + } + defer func() { m.stop(); m.drain() }() + if _, err := m.ensureDeployment(t.Context()); err != nil { + t.Fatal(err) + } + n, err := m.node(uuid.NewString()) + if err != nil { + t.Fatal(err) + } + if err := lease.Close(t.Context()); err != nil { + t.Fatal(err) + } + first := m.pauseDeployment(t.Context()) + if first == nil { + t.Fatal("drain accepted lost ownership") + } + if n.lifecycle.ctx.Err() != nil { + t.Fatal("failed fence canceled outside the lease gate") + } + if err := m.pauseDeployment(t.Context()); err != first { + t.Fatal("repeated drain forgot its failure", err, first) + } + if err := m.activateDeployment(t.Context(), store.RuntimeDeploymentView{InstallationID: id, Generation: 1, Mode: "nodes", Provider: "docker"}); err == nil { + t.Fatal("failed drain reopened the provider") + } + if _, _, err := m.enter(t.Context()); err == nil { + t.Fatal("failed drain admitted new work") + } + select { + case err := <-m.failed: + if err == nil { + t.Fatal("missing owner failure") + } + default: + t.Fatal("failed fence did not stop the owner") + } +} diff --git a/services/agents-api/internal/execution/sandbox_deployment_switch.go b/services/agents-api/internal/execution/sandbox_deployment_switch.go index 31821d4af..e06e20262 100644 --- a/services/agents-api/internal/execution/sandbox_deployment_switch.go +++ b/services/agents-api/internal/execution/sandbox_deployment_switch.go @@ -33,14 +33,14 @@ func (m *runtimeManager) pauseDeployment(ctx context.Context) error { } if !m.switching { m.switching = true - m.switchDrained = make(chan struct{}) + m.switchDrained = &deploymentDrain{done: make(chan struct{})} go m.drainDeployment(m.switchDrained) } drained := m.switchDrained m.mu.Unlock() select { - case <-drained: - return nil + case <-drained.done: + return drained.err case <-ctx.Done(): return ctx.Err() case <-m.ctx.Done(): @@ -50,8 +50,13 @@ func (m *runtimeManager) pauseDeployment(ctx context.Context) error { // One drain outlives the initiating HTTP request. A cancelled waiter cannot // permit a replacement generation while an old caller is still completing. -func (m *runtimeManager) drainDeployment(done chan struct{}) { - defer close(done) +type deploymentDrain struct { + done chan struct{} + err error // Published by closing done; immutable afterwards. +} + +func (m *runtimeManager) drainDeployment(result *deploymentDrain) { + defer close(result.done) for _, gate := range []chan struct{}{m.setupGate, m.inventory} { select { case gate <- struct{}{}: @@ -60,10 +65,14 @@ func (m *runtimeManager) drainDeployment(done chan struct{}) { } } m.mu.Lock() + nodes := make([]*runtimeNode, 0, len(m.nodes)) for _, n := range m.nodes { - n.lifecycle.stop() + nodes = append(nodes, n) } m.mu.Unlock() + if result.err = m.cancelLifecycles(nodes); result.err != nil { + return + } m.active.Wait() } diff --git a/services/agents-api/internal/store/execution_cancellation_test.go b/services/agents-api/internal/store/execution_cancellation_test.go new file mode 100644 index 000000000..695bfb59c --- /dev/null +++ b/services/agents-api/internal/store/execution_cancellation_test.go @@ -0,0 +1,87 @@ +package store + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestExecutionCancellationFenceHonorsBounds(t *testing.T) { + s, _ := testStore(t) + lease := executionLease(t, s) + for _, test := range []struct { + name string + limit, maximum time.Duration + }{ + {"caller", 25 * time.Millisecond, time.Second}, + {"operation", 8 * time.Second, 6 * time.Second}, + } { + t.Run(test.name, func(t *testing.T) { + if err := lease.lock(t.Context()); err != nil { + t.Fatal(err) + } + held := true + defer func() { + if held { + lease.unlock() + } + }() + operation, stopOperation := context.WithCancel(t.Context()) + defer stopOperation() + ctx, cancel := context.WithTimeout(t.Context(), test.limit) + defer cancel() + started := time.Now() + err := lease.Store().CancelExecutionOperations(ctx, stopOperation) + if !errors.Is(err, context.DeadlineExceeded) || time.Since(started) > test.maximum { + t.Fatal("cancellation fence did not preserve its deadline", err, time.Since(started)) + } + if operation.Err() != nil { + t.Fatal("timed-out fence canceled operations outside the lease gate") + } + lease.unlock() + held = false + if err := lease.Store().CheckExecutionOwnership(t.Context()); err != nil { + t.Fatal("gate timeout damaged the healthy owner", err) + } + if err := lease.Store().CancelExecutionOperations(t.Context(), stopOperation); err != nil { + t.Fatal(err) + } + if !errors.Is(operation.Err(), context.Canceled) { + t.Fatal("successful fence did not cancel synchronously") + } + if err := lease.Store().CheckExecutionOwnership(t.Context()); err != nil { + t.Fatal("cancellation damaged the healthy owner", err) + } + }) + } +} + +func TestExecutionCancellationFenceRejectsLostLease(t *testing.T) { + s, pool := testStore(t) + lease := executionLease(t, s) + var killed bool + if err := pool.QueryRow(t.Context(), "SELECT pg_terminate_backend($1, 1000)", lease.conn.Conn().PgConn().PID()).Scan(&killed); err != nil || !killed { + t.Fatal(killed, err) + } + operation, cancel := context.WithCancel(t.Context()) + defer cancel() + if err := lease.Store().CancelExecutionOperations(t.Context(), cancel); err == nil { + t.Fatal("lost lease accepted cancellation fence") + } + if operation.Err() != nil { + t.Fatal("lost lease ran the cancellation callback") + } + if err := lease.Store().CheckExecutionOwnership(t.Context()); err == nil { + t.Fatal("lost owner became writable") + } + if err := lease.Close(t.Context()); err != nil { + t.Fatal(err) + } + if err := lease.Store().CancelExecutionOperations(t.Context(), cancel); err == nil { + t.Fatal("closed lease accepted cancellation fence") + } + if err := s.CancelExecutionOperations(t.Context(), cancel); err == nil { + t.Fatal("pooled Store accepted execution cancellation") + } +} diff --git a/services/agents-api/internal/store/execution_lease.go b/services/agents-api/internal/store/execution_lease.go index 0dd91c559..f7d502c8c 100644 --- a/services/agents-api/internal/store/execution_lease.go +++ b/services/agents-api/internal/store/execution_lease.go @@ -56,6 +56,26 @@ func (s *Store) CheckExecutionOwnership(ctx context.Context) error { return s.executionLease.Ping(ctx) } +// CancelExecutionOperations cancels coordinator-owned contexts between leased +// operations. Canceling an in-flight pgx operation can close the connection that +// owns the execution advisory lock. The callback must only invoke synchronous +// context cancel functions; it must not perform database, provider or wait work. +// Caller cancellation and operation deadlines keep their existing semantics. +func (s *Store) CancelExecutionOperations(ctx context.Context, cancelOperations context.CancelFunc) error { + if s.executionLease == nil || cancelOperations == nil { + return ErrInvalidInput + } + ctx, cancel := context.WithTimeout(ctx, executionTransactionTimeout) + defer cancel() + return s.executionLease.withConn(ctx, func(conn *pgxpool.Conn) error { + if err := conn.Ping(ctx); err != nil { + return err + } + cancelOperations() + return nil + }) +} + func (l *ExecutionLease) lock(ctx context.Context) error { select { case l.gate <- struct{}{}: From 6d4bdf56347b0f5f2c582cc03b34528fb2066b2e Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 22:04:22 +0000 Subject: [PATCH 10/16] Close admission and retain lifecycle gates after failed retirement --- CONTRIBUTING.md | 9 +- .../execution/runtime_cancellation.go | 5 + .../internal/execution/runtime_manager.go | 14 +- .../runtime_retirement_failure_test.go | 268 ++++++++++++++++++ .../sandbox_deployment_drain_test.go | 4 +- .../execution/sandbox_deployment_switch.go | 13 + 6 files changed, 304 insertions(+), 9 deletions(-) create mode 100644 services/agents-api/internal/execution/runtime_retirement_failure_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 72b4d00bb..19fc0e569 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -975,9 +975,12 @@ operations, using the existing lease gate with its five-second bound. Include an active manual reconcile's cancel function: an asynchronous `AfterFunc` alone can cancel a later leased query after the gate reopens. Never cancel an in-flight leased query merely to change deployment configuration. A failed cancellation -fence stops the owner and leaves the drain barrier closed; it cannot activate a -replacement. Release the lease gate before waiting for provider settlement or -lifecycle accounting. Ordinary caller deadlines and owner shutdown retain their +fence synchronously closes manager admission and reports owner failure, even +before its coordinator starts. Failed inventory retirement retains the original +lifecycle identity and gate through owner shutdown; gate availability is not +proof that cancellation succeeded. The drain barrier stays closed and cannot +activate a replacement. Release the lease gate before waiting for provider +settlement or lifecycle accounting. Ordinary caller deadlines and owner shutdown retain their existing cancellation and fail-closed lease-loss behavior. Session locks, deployment capacity transactions and revision/one-shot receipts remain authoritative, with no external operation holding a database lock. diff --git a/services/agents-api/internal/execution/runtime_cancellation.go b/services/agents-api/internal/execution/runtime_cancellation.go index 66fc7ac65..dcf627782 100644 --- a/services/agents-api/internal/execution/runtime_cancellation.go +++ b/services/agents-api/internal/execution/runtime_cancellation.go @@ -21,6 +21,11 @@ func (m *runtimeManager) cancelLifecycles(nodes []*runtimeNode) error { } err := m.store.CancelExecutionOperations(m.ctx, cancel) if err != nil { + // A manual reconcile may have no running coordinator to consume failed. + // Close admission synchronously; Worker shutdown still owns cancellation. + m.mu.Lock() + m.closed = true + m.mu.Unlock() select { case m.failed <- err: default: diff --git a/services/agents-api/internal/execution/runtime_manager.go b/services/agents-api/internal/execution/runtime_manager.go index 5aabef290..3678c408c 100644 --- a/services/agents-api/internal/execution/runtime_manager.go +++ b/services/agents-api/internal/execution/runtime_manager.go @@ -179,13 +179,19 @@ func (m *runtimeManager) applyInventory(previous map[string]*runtimeNode, ids [] m.mu.Unlock() // Inventory includes offline nodes. Only explicit removal retires a lane; // the Store requires all retained resources to be cleaned before removal. - err := m.cancelLifecycles(retired) + if err := m.cancelLifecycles(retired); err != nil { + // No cancellation was authorized. Keep each retiring identity and gate; + // acquiring its gate later cannot substitute for successful cancellation. + // Undo only the retirement tasks that will not be started. Existing callers + // and workers retain their accounting until ordinary owner shutdown settles. + for range retired { + m.active.Done() + } + return nil, err + } for _, n := range retired { go m.retire(n) } - if err != nil { - return nil, err - } return nodes, nil } diff --git a/services/agents-api/internal/execution/runtime_retirement_failure_test.go b/services/agents-api/internal/execution/runtime_retirement_failure_test.go new file mode 100644 index 000000000..18096d7ee --- /dev/null +++ b/services/agents-api/internal/execution/runtime_retirement_failure_test.go @@ -0,0 +1,268 @@ +package execution + +import ( + "context" + "errors" + "net" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// Each case has its own real advisory-lock namespace. The delayed driver read +// lets the cancellation fence hit its own deadline without shortening production +// timeouts or depending on an unrelated SQL failure to exercise this branch. +func retirementFailureLease(t *testing.T, armed *atomic.Bool, reading chan struct{}, release <-chan struct{}) (*store.ExecutionLease, *pgxpool.Pool) { + t.Helper() + dsn := os.Getenv("OAC_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("dedicated PostgreSQL required") + } + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(cfg.ConnConfig.Database, "oac_") || !strings.HasSuffix(cfg.ConnConfig.Database, "_tests") { + t.Fatal("dedicated test database required") + } + admin, err := pgxpool.NewWithConfig(t.Context(), cfg.Copy()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(admin.Close) + name := "oac_retirement_" + uuid.NewString()[:8] + "_tests" + quoted := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(t.Context(), "CREATE DATABASE "+quoted); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := admin.Exec(ctx, "DROP DATABASE "+quoted+" WITH (FORCE)"); err != nil { + t.Error(err) + } + }) + cfg.ConnConfig.Database = name + dial := cfg.ConnConfig.DialFunc + cfg.ConnConfig.DialFunc = func(ctx context.Context, network, address string) (net.Conn, error) { + c, err := dial(ctx, network, address) + if err != nil { + return nil, err + } + return &delayedLeaseRead{Conn: c, armed: armed, reading: reading, release: release}, nil + } + pool, err := pgxpool.NewWithConfig(t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + lease, err := store.New(pool).AcquireExecutionLease(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := lease.Close(ctx); err != nil { + t.Error(err) + } + }) + return lease, pool +} + +func TestFailedInventoryRetirementClosesAdmissionAndRetainsGate(t *testing.T) { + for _, mode := range []string{"gate_timeout", "lease_loss"} { + t.Run(mode, func(t *testing.T) { + var armed atomic.Bool + reading, releaseRead := make(chan struct{}), make(chan struct{}) + var readOnce sync.Once + unblockRead := func() { readOnce.Do(func() { close(releaseRead) }) } + defer unblockRead() + lease, pool := retirementFailureLease(t, &armed, reading, releaseRead) + m := testRuntimeManager(t) + m.store = lease.Store() + m.loadDeployment = func(context.Context) (*RuntimeProvider, error) { return nil, nil } + m.mutationGate = make(chan struct{}, 1) + // This fixture models an already loaded node deployment; its provider is + // needed only for node admission, not for external sandbox operations. + m.config.Provider = &drainFixtureProvider{} + original, err := m.node("retiring") + if err != nil { + t.Fatal(err) + } + operation, finish, err := original.lifecycle.beginReconcile(t.Context()) + if err != nil { + t.Fatal(err) + } + releaseProvider := make(chan struct{}) + var providerOnce sync.Once + unblockProvider := func() { providerOnce.Do(func() { close(releaseProvider) }) } + t.Cleanup(unblockProvider) + m.active.Add(1) + go func() { defer m.active.Done(); defer finish(); <-operation.Done(); <-releaseProvider }() + var queryDone chan error + if mode == "gate_timeout" { + // This independent owner caller has its own bounded query context. Unlike + // a lifecycle scan, it is not canceled by the manager's shutdown context. + queryCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + queryDone = make(chan error, 1) + armed.Store(true) + go func() { queryDone <- lease.Ping(queryCtx) }() + select { + case <-reading: + case <-time.After(2 * time.Second): + t.Fatal("owner query did not reach delayed read") + } + defer func() { + if queryDone != nil { + unblockRead() + select { + case <-queryDone: + case <-time.After(2 * time.Second): + t.Error("owned query did not settle") + } + } + }() + } else { + var killed bool + err := pool.QueryRow(t.Context(), `SELECT pg_terminate_backend(pid,1000) FROM pg_locks WHERE locktype='advisory' AND database=(SELECT oid FROM pg_database WHERE datname=current_database())`).Scan(&killed) + if err != nil || !killed { + t.Fatal("could not terminate this test's owner", killed, err) + } + } + _, err = m.applyInventory(m.snapshotNodes(), nil) + if err == nil { + t.Fatal("failed owner accepted inventory retirement") + } + if mode == "gate_timeout" && !errors.Is(err, context.DeadlineExceeded) { + t.Fatal("did not exercise cancellation gate timeout", err) + } + if _, finish, err := m.enter(t.Context()); err == nil { + finish() + t.Error("manual caller admitted before coordinator consumed failure") + } + if _, err := m.node("retiring"); err == nil { + t.Error("failed retirement admitted a replacement gate") + } + if _, err := m.node("new-node"); err == nil { + t.Error("failed retirement admitted another lifecycle") + } + if unlock, err := m.lockMutation(t.Context()); err == nil { + unlock() + t.Error("failed retirement admitted a configuration mutation") + } + m.mu.Lock() + retained := m.nodes["retiring"] == original && original.retiring + m.mu.Unlock() + if !retained { + t.Error("failed retirement discarded its original lifecycle") + } + if original.lifecycle.ctx.Err() != nil { + t.Error("failed fence claimed successful lifecycle cancellation") + } + select { + case failure := <-m.failed: + if failure == nil { + t.Error("empty owner failure") + } + default: + t.Error("owner failure was not published") + } + if queryDone != nil { + unblockRead() + select { + case err := <-queryDone: + if err != nil { + t.Fatal("fence timeout damaged unrelated owner read", err) + } + case <-time.After(2 * time.Second): + t.Fatal("owner read did not settle") + } + queryDone = nil + } + // Shutdown can now cancel the original work, but cannot call it settled + // merely because cancellation was requested or a serial gate became free. + m.stop() + drained := make(chan struct{}) + go func() { m.drain(); close(drained) }() + select { + case <-drained: + t.Fatal("shutdown skipped retained provider settlement") + case <-time.After(25 * time.Millisecond): + } + unblockProvider() + select { + case <-drained: + case <-time.After(2 * time.Second): + t.Fatal("shutdown retirement accounting did not settle") + } + m.mu.Lock() + retained = m.nodes["retiring"] == original + m.mu.Unlock() + if !retained { + t.Fatal("failed-retirement entry was removed by a task that never canceled it") + } + }) + } +} + +// No provider method is called by the retirement ownership fixture. +type drainFixtureProvider struct{ sandbox.Provider } + +// Done is evaluated only after lockMutation's initial admission check, letting +// this test close admission while a request is already waiting for its gate. +type mutationWaitContext struct { + context.Context + waiting chan struct{} + once sync.Once +} + +func (c *mutationWaitContext) Done() <-chan struct{} { + c.once.Do(func() { close(c.waiting) }) + return c.Context.Done() +} + +func TestFailedManagerRejectsAlreadyWaitingMutation(t *testing.T) { + m := testRuntimeManager(t) + m.loadDeployment = func(context.Context) (*RuntimeProvider, error) { return nil, nil } + m.mutationGate = make(chan struct{}, 1) + m.mutationGate <- struct{}{} + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + waiting := &mutationWaitContext{Context: ctx, waiting: make(chan struct{})} + result := make(chan error, 1) + go func() { + unlock, err := m.lockMutation(waiting) + if unlock != nil { + unlock() + } + result <- err + }() + select { + case <-waiting.waiting: + case <-time.After(time.Second): + t.Fatal("mutation never reached gate wait") + } + m.mu.Lock() + m.closed = true + m.mu.Unlock() + <-m.mutationGate + select { + case err := <-result: + if !errors.Is(err, ErrExecutionUnavailable) { + t.Fatal("waiting mutation bypassed synchronous admission closure", err) + } + case <-time.After(time.Second): + t.Fatal("waiting mutation did not settle") + } +} diff --git a/services/agents-api/internal/execution/sandbox_deployment_drain_test.go b/services/agents-api/internal/execution/sandbox_deployment_drain_test.go index c7fd9b219..f7d7cfff1 100644 --- a/services/agents-api/internal/execution/sandbox_deployment_drain_test.go +++ b/services/agents-api/internal/execution/sandbox_deployment_drain_test.go @@ -267,8 +267,8 @@ func TestSandboxDeploymentDrainFailureCannotReactivate(t *testing.T) { if n.lifecycle.ctx.Err() != nil { t.Fatal("failed fence canceled outside the lease gate") } - if err := m.pauseDeployment(t.Context()); err != first { - t.Fatal("repeated drain forgot its failure", err, first) + if err := m.pauseDeployment(t.Context()); err == nil { + t.Fatal("repeated drain forgot its failure") } if err := m.activateDeployment(t.Context(), store.RuntimeDeploymentView{InstallationID: id, Generation: 1, Mode: "nodes", Provider: "docker"}); err == nil { t.Fatal("failed drain reopened the provider") diff --git a/services/agents-api/internal/execution/sandbox_deployment_switch.go b/services/agents-api/internal/execution/sandbox_deployment_switch.go index e06e20262..6ea7dbf9b 100644 --- a/services/agents-api/internal/execution/sandbox_deployment_switch.go +++ b/services/agents-api/internal/execution/sandbox_deployment_switch.go @@ -13,8 +13,21 @@ func (m *runtimeManager) lockMutation(ctx context.Context) (func(), error) { if m == nil || m.loadDeployment == nil { return nil, store.ErrSandboxDeploymentConflict } + m.mu.Lock() + closed := m.closed + m.mu.Unlock() + if closed { + return nil, ErrExecutionUnavailable + } select { case m.mutationGate <- struct{}{}: + m.mu.Lock() + closed := m.closed + m.mu.Unlock() + if closed { + <-m.mutationGate + return nil, ErrExecutionUnavailable + } return func() { <-m.mutationGate }, nil case <-ctx.Done(): return nil, ctx.Err() From e8459870d9813c86df4ddf8145cdfd321d24c1e0 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 22:07:58 +0000 Subject: [PATCH 11/16] test: reproduce archived waiting cancellation loss --- .../forced_waiting_cleanup_diagnosis_test.go | 131 +++++++++++ .../store/forced_waiting_diagnosis_test.go | 214 ++++++++++++++++++ 2 files changed, 345 insertions(+) create mode 100644 services/agents-api/internal/execution/forced_waiting_cleanup_diagnosis_test.go create mode 100644 services/agents-api/internal/store/forced_waiting_diagnosis_test.go diff --git a/services/agents-api/internal/execution/forced_waiting_cleanup_diagnosis_test.go b/services/agents-api/internal/execution/forced_waiting_cleanup_diagnosis_test.go new file mode 100644 index 000000000..e5efbf933 --- /dev/null +++ b/services/agents-api/internal/execution/forced_waiting_cleanup_diagnosis_test.go @@ -0,0 +1,131 @@ +package execution + +import ( + "context" + "encoding/json" + "testing" + + v1 "github.com/MiniMax-AI-Dev/parsar/contracts/agents-api/v1" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/adminaudit" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/identity" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" +) + +// Provider callbacks inspect the real database at the instant destructive +// cleanup is invoked. They do not create containers or claim native evidence. +type waitingCleanupProvider struct { + sandbox.Provider + beforeKill func() +} + +func (p waitingCleanupProvider) GetInfo(_ context.Context, r sandbox.Reference) (sandbox.Info, error) { + return sandbox.Info{Reference: r, ProviderID: r.AllocationID, State: "running", BootstrapComplete: true, CreateSettled: true}, nil +} +func (p waitingCleanupProvider) Kill(context.Context, sandbox.Reference) error { + p.beforeKill() + return nil +} + +type waitingCleanupCheckpoint struct { + sandbox.CheckpointProvider + beforeKill func() +} + +func (p waitingCleanupCheckpoint) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { + p.beforeKill() + return nil +} + +func TestArchiveWaitingDestructiveCleanupPrecedesReceiptDiagnosis(t *testing.T) { + for _, checkpoint := range []bool{false, true} { + t.Run(map[bool]string{false: "Kill", true: "KillCompute"}[checkpoint], func(t *testing.T) { + s, lease := resetManagerStore(t) + writer := lease.Store() + installation := uuid.NewString() + if err := writer.ClaimWebSandboxDeployment(t.Context(), installation); err != nil { + t.Fatal(err) + } + selection := store.SandboxDeploymentSetupRequest{Provider: "e2b", E2B: &store.SandboxE2BConfiguration{APIKey: "fixture-key", Template: "runtime:" + uuid.NewString()}} + selection.Resources.CPUs = 2 + selection.Resources.MemoryMiB = 2048 + if _, err := writer.InitializeSandboxDeployment(t.Context(), installation, selection); err != nil { + t.Fatal(err) + } + projectID := uuid.NewString() + audit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", ProjectID: projectID, RequestID: uuid.NewString(), TraceID: uuid.NewString()}) + project, err := s.CreateProject(audit, projectID, "Cleanup diagnosis") + if err != nil { + t.Fatal(err) + } + session, err := s.CreateSession(t.Context(), project.TenantID, store.CreateSessionInput{Creator: identity.Subject{Kind: "service_account", ID: "fixture"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"disabled"}}}`), ModelProvider: &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://model.fixture.example/v1", APIKey: "fixture-key"}, ModelProviderSource: v1.ModelProviderSourceSession}) + if err != nil { + t.Fatal(err) + } + owner, err := writer.ReserveRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID, installation, device.HashCredential(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + owner, err = writer.ObserveRuntimeRunning(t.Context(), owner) + if err != nil { + t.Fatal(err) + } + input, err := s.SubmitMessage(t.Context(), project.TenantID, session.ID, "start", json.RawMessage(`{"text":"run"}`)) + if err != nil { + t.Fatal(err) + } + // This fixture isolates lifecycle ordering. Protocol-driven waiting is + // independently exercised in TestArchiveWaitingHeartbeatCancellationDiagnosis. + for _, transition := range []store.TurnTransition{{ExpectedStatus: store.TurnQueued, Status: store.TurnInProgress}, {ExpectedStatus: store.TurnInProgress, Status: store.TurnWaiting}} { + if _, err := writer.TransitionTurn(t.Context(), project.TenantID, session.ID, input.TurnID, transition); err != nil { + t.Fatal(err) + } + } + currentCompute := sandbox.Compute{ID: uuid.NewString(), Name: owner.ID + "-g0"} + if checkpoint { + state, _ := json.Marshal(runtimeCompute{Current: currentCompute}) + owner, err = writer.SetRuntimeCompute(t.Context(), owner, "running", state, nil, 0) + if err != nil { + t.Fatal(err) + } + } + if _, err := writer.ArchiveManagedSession(audit, project.TenantID, session.ID, 1); err != nil { + t.Fatal(err) + } + owner, err = s.GetRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID) + if err != nil { + t.Fatal(err) + } + kills := 0 + provider := waitingCleanupProvider{beforeKill: func() { + kills++ + turn, err := s.GetTurn(t.Context(), project.TenantID, session.ID, input.TurnID) + if err != nil || turn.Status != store.TurnWaiting || turn.CancelRequestedAt.IsZero() || !turn.CompletedAt.IsZero() { + t.Fatal("Kill did not precede terminal receipt", turn, err) + } + allocation, err := s.GetRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID) + if err != nil || allocation.State != "cleanup_pending" { + t.Fatal("Kill bypassed durable cleanup ownership", allocation, err) + } + }} + lifecycle := &runtimeLifecycle{store: writer, registry: gateway.NewRegistry(), config: RuntimeProvider{InstallationID: installation, Provider: provider}, connections: map[string]*runtimeConnection{}} + if checkpoint { + lifecycle.config.Provider = waitingCleanupCheckpoint{beforeKill: provider.beforeKill} + } + if err := lifecycle.observe(t.Context(), owner); err != nil { + t.Fatal(err) + } + after, err := s.GetRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID) + if err != nil || after.State != "released" || kills != 1 { + t.Fatal("cleanup did not release", after, kills, err) + } + turn, err := s.GetTurn(t.Context(), project.TenantID, session.ID, input.TurnID) + if err != nil || turn.Status != store.TurnWaiting || !turn.CompletedAt.IsZero() { + t.Fatal("cleanup should not fabricate cancellation", turn, err) + } + }) + } +} diff --git a/services/agents-api/internal/store/forced_waiting_diagnosis_test.go b/services/agents-api/internal/store/forced_waiting_diagnosis_test.go new file mode 100644 index 000000000..030c6dc48 --- /dev/null +++ b/services/agents-api/internal/store/forced_waiting_diagnosis_test.go @@ -0,0 +1,214 @@ +package store_test + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/adminaudit" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/runtime" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" + "github.com/gorilla/websocket" +) + +// Controlled protocol fixtures, not native/model acceptance. Allocation setup +// uses the real execution lease; the real dispatcher consumes a function request +// through the real authenticated WebSocket and persists waiting before archive. +func TestArchiveWaitingHeartbeatCancellationDiagnosis(t *testing.T) { + for _, heartbeat := range []bool{false, true} { + t.Run(map[bool]string{false: "receipt_without_heartbeat", true: "heartbeat_before_receipt"}[heartbeat], func(t *testing.T) { + s, pool := store.NewManagedTestStore(t) + lease, err := s.AcquireExecutionLease(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := lease.Close(context.Background()); err != nil { + t.Error(err) + } + }) + writer := lease.Store() + installation := uuid.NewString() + if err := writer.ClaimWebSandboxDeployment(t.Context(), installation); err != nil { + t.Fatal(err) + } + if _, err := writer.InitializeSandboxDeployment(t.Context(), installation, store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("e2b"), Provider: "e2b", E2B: &store.SandboxE2BConfiguration{APIKey: "fixture", Template: "runtime:" + uuid.NewString()}}); err != nil { + t.Fatal(err) + } + projectID := uuid.NewString() + auditCtx := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", ProjectID: projectID, RequestID: uuid.NewString(), TraceID: uuid.NewString()}) + project, err := s.CreateProject(auditCtx, projectID, "Archive diagnosis") + if err != nil { + t.Fatal(err) + } + configuration := strings.Replace(functionConfiguration, `"type":"none"`, `"type":"openai_hosted","network":{"access":"disabled"}`, 1) + session, err := s.CreateSession(t.Context(), project.TenantID, store.WithFixtureModelProvider(store.CreateSessionInput{Creator: store.FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(configuration)})) + if err != nil { + t.Fatal(err) + } + secret := uuid.NewString() + owner, err := writer.ReserveRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID, installation, device.HashCredential(secret)) + if err != nil { + t.Fatal(err) + } + for _, step := range []func(context.Context, store.RuntimeAllocation) (store.RuntimeAllocation, error){writer.ObserveRuntimeRunning, writer.SettleRuntimeCreation} { + owner, err = step(t.Context(), owner) + if err != nil { + t.Fatal(err) + } + } + if err := writer.BindSessionDevice(t.Context(), project.TenantID, session.ID, owner.DeviceID); err != nil { + t.Fatal(err) + } + generation := uuid.NewString() + if err := writer.ReplaceEnvironmentConnection(t.Context(), project.TenantID, session.Environment.ID, generation); err != nil { + t.Fatal(err) + } + if err := writer.ObserveEnvironmentConnection(t.Context(), project.TenantID, session.Environment.ID, generation, 1, true); err != nil { + t.Fatal(err) + } + server := httptest.NewUnstartedServer(nil) + wsURL := "ws://" + server.Listener.Addr().String() + "/api/v1/agent-daemon/ws" + handler, registry, err := runtime.NewGateway(s, wsURL) + if err != nil { + t.Fatal(err) + } + server.Config.Handler = handler + server.Start() + t.Cleanup(func() { runtime.CloseConnections(registry); server.Close() }) + u, _ := url.Parse(wsURL) + u.RawQuery = url.Values{"device_id": {owner.DeviceID}, "version": {proto.Version}}.Encode() + conn, _, err := websocket.DefaultDialer.Dial(u.String(), http.Header{"Authorization": {"Bearer " + secret}}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + h := &dispatchHarness{t: t, s: s, tenant: project.TenantID, session: session, conn: conn, registry: registry, d: &execution.Dispatcher{Store: writer, Registry: registry}} + capabilities := workerEnvironmentCapabilities() + capabilities.FunctionTools = true + h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: capabilities}}}) + var peer *gateway.Session + for deadline := time.Now().Add(3 * time.Second); ; { + peer, err = registry.LookupDevice(owner.DeviceID) + if err == nil { + info, _, known := peer.AgentKindStatus("codex") + if known && info.Capabilities.FunctionTools { + break + } + } + if time.Now().After(deadline) { + t.Fatal("initial heartbeat missing") + } + time.Sleep(time.Millisecond) + } + pending, err := s.ReserveEnvironmentInput(t.Context(), h.tenant, session.ID, "pending", []store.Input{{Kind: "message", Payload: json.RawMessage(`{"text":"first"}`)}, {Kind: "message", Payload: json.RawMessage(`{"text":"second"}`)}}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + result := runPreparedDispatch(h, ctx, pending) + frame := h.read(proto.TypeExecutionPrepare) + handle := acknowledgePreparation(h, frame.ID) + start := readyPreparedDispatch(t, h, frame.ID, handle) + h.write(frame.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 3, State: "started", RunID: start.RunID}) + input := store.InputReceipt{TurnID: start.RunID} + h.write(input.TurnID, proto.TypeFunctionCall, proto.FunctionCallPayload{CallID: "pending", Name: "lookup_ticket", Arguments: json.RawMessage(`{"ticket":"42"}`)}) + state := functionState(t, h, 1) + if state.LastTurn.Status != store.TurnWaiting { + t.Fatal(state.LastTurn) + } + + archived, err := writer.ArchiveManagedSession(auditCtx, h.tenant, session.ID, 1) + if err != nil || archived.State != "cleanup_pending" { + t.Fatal(archived, err) + } + current, err := s.GetTurn(t.Context(), h.tenant, session.ID, input.TurnID) + if err != nil || current.Status != store.TurnWaiting || current.CancelRequestedAt.IsZero() { + t.Fatal("archive must request rather than invent cancellation", current, err) + } + if _, err := gateway.NewAuthenticator(s).AuthenticateBearer(t.Context(), owner.DeviceID, secret); !errors.Is(err, gateway.ErrAuthUnknownDevice) { + t.Fatal("archive allowed renewed authority", err) + } + rejected, response, dialErr := websocket.DefaultDialer.Dial(u.String(), http.Header{"Authorization": {"Bearer " + secret}}) + if rejected != nil { + rejected.Close() + } + if response != nil { + response.Body.Close() + } + if dialErr == nil || response == nil || response.StatusCode != http.StatusUnauthorized { + t.Fatal("revoked Runtime reconnected") + } + // Explicitly observe cancel delivery before inducing transport loss. This + // proves even a sent cancellation can lose its receipt; no ticker timing guess. + var request proto.PromptCancelPayload + if err := h.read(proto.TypePromptCancel).DecodePayload(&request); err != nil { + t.Fatal(err) + } + if request.DeliveryID == "" { + t.Fatal("missing cancel delivery identity") + } + if heartbeat { + h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{}) + select { + case <-peer.Closed(): + case <-time.After(3 * time.Second): + t.Fatal("revoked heartbeat did not close peer") + } + got := awaitPreparedDispatch(t, result) + if got.err != nil || got.run.Turn.Status != store.TurnFailed { + t.Fatal(got.run.Turn.Status, got.err) + } + done := got.run.Turn + var outcome execution.Result + if err := json.Unmarshal(done.Outcome, &outcome); err != nil || outcome.ErrorCode != "event_stream_incomplete" { + t.Fatal(string(done.Outcome), err) + } + } else { + h.write(input.TurnID, proto.TypeInteractionDecisionAck, proto.InteractionDecisionAckPayload{DeliveryID: request.DeliveryID, Applied: true, Outcome: &proto.DonePayload{Usage: proto.Usage{InputTokens: 17}, Metadata: map[string]any{proto.DoneMetaAgentSessionID: "cancelled-native"}}}) + got := awaitPreparedDispatch(t, result) + if got.err != nil || got.run.Turn.Status != store.TurnCancelled { + t.Fatal(got.run.Turn.Status, got.err) + } + done := got.run.Turn + var outcome execution.Result + if err := json.Unmarshal(done.Outcome, &outcome); err != nil || outcome.Done.Usage.InputTokens != 17 { + t.Fatal("receipt lost usage", string(done.Outcome), err) + } + } + var receipts int + if err := pool.QueryRow(t.Context(), "SELECT count(*) FROM turn_events WHERE turn_id=$1 AND kind='cancel_receipt'", input.TurnID).Scan(&receipts); err != nil { + t.Fatal(err) + } + wantReceipts := 1 + if heartbeat { + wantReceipts = 0 + } + if receipts != wantReceipts { + t.Fatal("durable cancellation receipt mismatch", receipts, wantReceipts) + } + // No provider was invoked in either case; the failure therefore does not + // require compute destruction. The original durable cleanup owner survives. + allocation, err := s.GetRuntimeAllocation(t.Context(), h.tenant, session.Environment.ID) + if err != nil || allocation.State != "cleanup_pending" { + t.Fatal(allocation, err) + } + var revoked bool + if err := pool.QueryRow(t.Context(), "SELECT revoked_at IS NOT NULL FROM devices WHERE id=$1", owner.DeviceID).Scan(&revoked); err != nil || !revoked { + t.Fatal(revoked, err) + } + }) + } +} From 097a02f2212a88e15df08bb9c0d49e6362ea93a6 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 22:26:37 +0000 Subject: [PATCH 12/16] fix: retain archived Runtime cancellation receipts before cleanup --- CONTRIBUTING.md | 13 ++ contracts/agents-api/sandbox-deployment.md | 11 ++ internal/agentdaemon/device/cancellation.go | 14 ++ internal/agentdaemon/gateway/cancellation.go | 122 +++++++++++++ .../agentdaemon/gateway/cancellation_test.go | 146 ++++++++++++++++ internal/agentdaemon/gateway/session.go | 32 +++- internal/agentdaemon/gateway/session_test.go | 4 + .../internal/db/queries/devices.sql | 2 +- .../db/queries/runtime_cancellation.sql | 32 ++++ .../internal/db/sqlc/devices.sql.go | 2 +- .../agents-api/internal/db/sqlc/models.go | 19 ++- .../db/sqlc/runtime_cancellation.sql.go | 99 +++++++++++ ...o => archive_cancellation_cleanup_test.go} | 84 ++++++++- .../internal/execution/device_authority.go | 6 +- .../internal/execution/dispatcher.go | 5 + .../internal/execution/prepared_dispatch.go | 6 + .../internal/execution/runtime_lifecycle.go | 9 + .../internal/store/admin_session_archive.go | 2 +- ...s_test.go => archive_cancellation_test.go} | 160 +++++++++++++++--- .../archived_cancellation_migration_test.go | 48 ++++++ .../store/runtime_allocation_state.go | 2 +- .../internal/store/runtime_cancellation.go | 35 ++++ .../000080_archived_cancellation_receipts.sql | 17 ++ 23 files changed, 812 insertions(+), 58 deletions(-) create mode 100644 internal/agentdaemon/device/cancellation.go create mode 100644 internal/agentdaemon/gateway/cancellation.go create mode 100644 internal/agentdaemon/gateway/cancellation_test.go create mode 100644 services/agents-api/internal/db/queries/runtime_cancellation.sql create mode 100644 services/agents-api/internal/db/sqlc/runtime_cancellation.sql.go rename services/agents-api/internal/execution/{forced_waiting_cleanup_diagnosis_test.go => archive_cancellation_cleanup_test.go} (61%) rename services/agents-api/internal/store/{forced_waiting_diagnosis_test.go => archive_cancellation_test.go} (58%) create mode 100644 services/agents-api/internal/store/archived_cancellation_migration_test.go create mode 100644 services/agents-api/internal/store/runtime_cancellation.go create mode 100644 services/agents-api/migrations/000080_archived_cancellation_receipts.sql diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 19fc0e569..f66461e8c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -871,6 +871,19 @@ The ordinary provider lifecycle owns compute/snapshot release. Retain history an persisted Files/Artifacts; archived Sessions cannot resume unpersisted workspace. The archive GET reports resource disposition, not provenance or Turn finalization. +Archive cancellation must not destroy a healthy receipt path before its terminal +commit. Only the archive that first revokes a device may persist its exact +`archive_cancel_turn_id`; ordinary revocation clears it, and repeated cleanup +preserves rather than recreates it. The existing authenticated delivery can drain +its cancellation for at most 20 seconds from the Turn's original +`cancel_requested_at`. Track that delivery through Done, cancellation ACK and +terminal commit, independently of subscription removal. Heartbeats and both normal +and checkpoint cleanup use the same identity/deadline check; no new connection, +input, file/MCP authority or lease renewal is granted. Do not hold a transaction or +lifecycle gate waiting for the receipt. Lost peers, expiry and restart retain the +ordinary failure/cleanup fallback, never a fabricated cancelled outcome. Preserve +unknown Create ownership and reject schema downgrade with unsettled markers. + Core rejects `AGENTS_API_MANAGED_RUNTIMES_FILE`; there is no file-managed startup path or embedded local node. An older file-managed database is not automatically adopted after its environment variable is removed. Settle and drain that deployment diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index 160c696a6..4cafb14fc 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -351,6 +351,17 @@ without reset. It preserves history and persisted Files/Artifacts but discards unpersisted workspace and prevents that Session from resuming. Poll its archive GET for actual release. Reset never manufactures a release receipt. +A force archive fences credentials and new work immediately. If its original +hosted delivery is still connected, Core preserves only that delivery's native +cancellation/terminal receipt path until terminal commit or a fixed 20-second +bound from the original cancellation request. Done does not end this bound while +a cancellation acknowledgment or terminal commit is pending. This internal drain +never authorizes reconnect, workspace/MCP access or renewed execution. Explicit +credential revocation ends the exception; repeats do not extend it. Missing or +failed receipts retain honest failure outcomes, and disconnected, expired or +restarted owners fall back to ordinary provider cleanup. There is no new public +state, request field or model/tool timeout. + One mutation gate serializes setup, PUT, reset, cancel and finalization. Archive locks Session before deployment; finalization never reverses that order or waits for itself inside counted manager work. Candidates bind to the reset request time, diff --git a/internal/agentdaemon/device/cancellation.go b/internal/agentdaemon/device/cancellation.go new file mode 100644 index 000000000..791f8b3b2 --- /dev/null +++ b/internal/agentdaemon/device/cancellation.go @@ -0,0 +1,14 @@ +package device + +import "time" + +// ArchivedCancellationReceiptLimit bounds receipt draining after an administrator +// archive. It is not a model/tool timeout and never restarts on heartbeat or retry. +const ArchivedCancellationReceiptLimit = 20 * time.Second + +// ArchivedCancellationReceipt authorizes only the original delivery's receipt +// drain. It grants neither Runtime credentials nor permission to start work. +type ArchivedCancellationReceipt struct { + RunID string + Deadline time.Time +} diff --git a/internal/agentdaemon/gateway/cancellation.go b/internal/agentdaemon/gateway/cancellation.go new file mode 100644 index 000000000..83937c0bb --- /dev/null +++ b/internal/agentdaemon/gateway/cancellation.go @@ -0,0 +1,122 @@ +package gateway + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +// ArchivedCancellationStore is deliberately separate from RuntimeStore: a +// receipt opportunity cannot authenticate a new connection or authorize work. +type ArchivedCancellationStore interface { + ArchivedCancellationReceipt(context.Context, string, string, []string) (device.ArchivedCancellationReceipt, error) +} + +// TrackExecutionDelivery retains the exact existing Core delivery through its +// terminal database commit. Durable subscriptions may already have seen Done or +// been removed while cancellation ACK/commit is still pending. +func (s *Session) TrackExecutionDelivery(runID string) (func(), error) { + s.receiptMu.Lock() + defer s.receiptMu.Unlock() + if runID == "" || s.IsClosed() || s.receiptDrain.RunID != "" { + return nil, ErrSessionClosed + } + if s.deliveries == nil { + s.deliveries = make(map[string]struct{}) + } + if _, exists := s.deliveries[runID]; exists { + return nil, errors.New("execution delivery already tracked") + } + s.deliveries[runID] = struct{}{} + var once sync.Once + return func() { + once.Do(func() { + s.receiptMu.Lock() + delete(s.deliveries, runID) + closePeer := s.receiptDrain.RunID == runID + s.receiptMu.Unlock() + if closePeer { + s.CloseWithCode(CloseRuntimeDeleted, "archived cancellation settled") + } + }) + }, nil +} + +// DrainArchivedCancellation checks both durable archive identity and this exact +// connection's unfinished delivery. It performs no liveness or credential write. +func (s *Session) DrainArchivedCancellation(ctx context.Context) (bool, error) { + store, ok := s.heartbeat.(ArchivedCancellationStore) + if !ok || s.IsClosed() { + return false, nil + } + s.receiptMu.Lock() + ids := make([]string, 0, len(s.deliveries)) + for id := range s.deliveries { + ids = append(ids, id) + } + s.receiptMu.Unlock() + if len(ids) == 0 { + return false, nil + } + receipt, err := store.ArchivedCancellationReceipt(ctx, s.DeviceID, s.credentialHash, ids) + if err != nil || receipt.RunID == "" { + return false, err + } + s.receiptMu.Lock() + defer s.receiptMu.Unlock() + if _, exists := s.deliveries[receipt.RunID]; !exists || s.IsClosed() || !time.Now().Before(receipt.Deadline) { + return false, nil + } + if s.receiptDrain.RunID != "" { + // Once fenced, neither a later cancel nor a changed response extends it. + return s.receiptDrain.RunID == receipt.RunID && s.receiptDrain.Deadline.Equal(receipt.Deadline), nil + } + s.receiptDrain = receipt + s.receiptTimer = time.AfterFunc(time.Until(receipt.Deadline), func() { s.CloseWithCode(CloseRuntimeDeleted, "archived cancellation deadline") }) + return true, nil +} + +func (s *Session) receiptDraining() bool { + s.receiptMu.Lock() + defer s.receiptMu.Unlock() + return s.receiptDrain.RunID != "" +} + +func (s *Session) allowsReceiptFrame(env proto.Envelope, outbound bool) bool { + s.receiptMu.Lock() + defer s.receiptMu.Unlock() + if s.receiptDrain.RunID == "" { + return true + } + if !time.Now().Before(s.receiptDrain.Deadline) { + return false + } + if !outbound && env.Type == proto.TypeHeartbeat { + return true + } + if env.ID != s.receiptDrain.RunID { + return false + } + if outbound { + return env.Type == proto.TypePromptCancel + } + if env.Type == proto.TypeInteractionDecisionAck { + var ack proto.InteractionDecisionAckPayload + return env.DecodePayload(&ack) == nil && ack.DeliveryID == "cancel:"+env.ID + } + // Existing run events must remain lossless, including child observations and + // Done before ACK. Workspace/preparation replies have other correlation IDs. + return true +} + +func (s *Session) stopReceiptTimer() { + s.receiptMu.Lock() + defer s.receiptMu.Unlock() + if s.receiptTimer != nil { + s.receiptTimer.Stop() + } +} diff --git a/internal/agentdaemon/gateway/cancellation_test.go b/internal/agentdaemon/gateway/cancellation_test.go new file mode 100644 index 000000000..1f0caa264 --- /dev/null +++ b/internal/agentdaemon/gateway/cancellation_test.go @@ -0,0 +1,146 @@ +package gateway + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +type receiptStore struct { + *fakeHeartbeatStore + mu sync.Mutex + receipt device.ArchivedCancellationReceipt +} + +func (r *receiptStore) ArchivedCancellationReceipt(_ context.Context, id, hash string, runs []string) (device.ArchivedCancellationReceipt, error) { + r.mu.Lock() + defer r.mu.Unlock() + if id != "device" || hash != "original-hash" { + return device.ArchivedCancellationReceipt{}, nil + } + for _, run := range runs { + if run == r.receipt.RunID { + return r.receipt, nil + } + } + return device.ArchivedCancellationReceipt{}, nil +} + +func TestArchivedReceiptTracksDeliveryBeyondDoneAndRejectsNewWork(t *testing.T) { + conn := newFakeConn() + peer := NewSession(conn, "device", "", "", NewRegistry(), nil) + defer peer.Close("test complete") + peer.credentialHash = "original-hash" + receipt := device.ArchivedCancellationReceipt{RunID: "run", Deadline: time.Now().Add(time.Second)} + peer.heartbeat = &receiptStore{fakeHeartbeatStore: newFakeHeartbeatStore(), receipt: receipt} + if draining, err := peer.DrainArchivedCancellation(t.Context()); err != nil || draining { + t.Fatal("unowned delivery got drain", draining, err) + } + release, err := peer.TrackExecutionDelivery("run") + if err != nil { + t.Fatal(err) + } + subscription, err := peer.SubscribeDurable("run") + if err != nil { + t.Fatal(err) + } + done, _ := proto.NewEnvelope(proto.TypeDone, "run", proto.DonePayload{}) + peer.dispatch(done) + for range subscription.Events { + } + if draining, err := peer.DrainArchivedCancellation(t.Context()); err != nil || !draining { + t.Fatal("Done discarded in-flight ACK/commit owner", draining, err) + } + if _, err := peer.TrackExecutionDelivery("new-run"); err == nil { + t.Fatal("new delivery admitted while draining") + } + for _, kind := range []string{proto.TypePromptRequest, proto.TypePromptSteer, proto.TypeExecutionPrepare, proto.TypeWorkspaceWrite, proto.TypeWorkspaceRead} { + env, _ := proto.NewEnvelope(kind, "run", nil) + if err := peer.Send(t.Context(), env); err == nil { + t.Fatal("drain permitted new operation", kind) + } + } + cancel, _ := proto.NewEnvelope(proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: "cancel:run"}) + if err := peer.Send(t.Context(), cancel); err != nil { + t.Fatal("drain blocked cancellation", err) + } + release() // The caller has now committed its terminal result. + if !peer.IsClosed() { + t.Fatal("finished delivery retained revoked connection") + } +} + +type blockedControlConn struct { + *fakeConn + entered chan struct{} + resume chan struct{} + wrote chan struct{} + once sync.Once +} + +func (c *blockedControlConn) WriteMessage(kind int, data []byte) error { + c.once.Do(func() { close(c.entered) }) + <-c.resume + defer close(c.wrote) + return c.fakeConn.WriteMessage(kind, data) +} +func (c *blockedControlConn) WriteControl(kind int, data []byte, _ time.Time) error { + return c.fakeConn.WriteMessage(kind, data) +} + +func TestCloseWithCodeUsesConcurrentControlWriter(t *testing.T) { + conn := &blockedControlConn{fakeConn: newFakeConn(), entered: make(chan struct{}), resume: make(chan struct{}), wrote: make(chan struct{})} + peer := NewSession(conn, "device", "", "", NewRegistry(), nil) + peer.Start() + env, _ := proto.NewEnvelope(proto.TypePromptCancel, "run", nil) + if err := peer.Send(t.Context(), env); err != nil { + t.Fatal(err) + } + select { + case <-conn.entered: + case <-time.After(time.Second): + t.Fatal("data writer never entered") + } + finished := make(chan struct{}) + go func() { peer.CloseWithCode(CloseRuntimeDeleted, "retired"); close(finished) }() + select { + case <-finished: + case <-time.After(time.Second): + close(conn.resume) + <-finished + t.Fatal("close used the blocked data writer") + } + close(conn.resume) + select { + case <-conn.wrote: + case <-time.After(time.Second): + t.Fatal("data writer did not exit") + } +} + +func TestArchivedReceiptDeadlineDoesNotRenew(t *testing.T) { + peer := NewSession(newFakeConn(), "device", "", "", NewRegistry(), nil) + defer peer.Close("test complete") + peer.credentialHash = "original-hash" + receipt := device.ArchivedCancellationReceipt{RunID: "run", Deadline: time.Now().Add(100 * time.Millisecond)} + peer.heartbeat = &receiptStore{fakeHeartbeatStore: newFakeHeartbeatStore(), receipt: receipt} + release, err := peer.TrackExecutionDelivery("run") + if err != nil { + t.Fatal(err) + } + defer release() + for range 10 { + if draining, err := peer.DrainArchivedCancellation(t.Context()); err != nil || !draining { + t.Fatal(draining, err) + } + } + select { + case <-peer.Closed(): + case <-time.After(time.Second): + t.Fatal("receipt retries renewed deadline") + } +} diff --git a/internal/agentdaemon/gateway/session.go b/internal/agentdaemon/gateway/session.go index c0cd7572f..7e79ddfdb 100644 --- a/internal/agentdaemon/gateway/session.go +++ b/internal/agentdaemon/gateway/session.go @@ -57,6 +57,7 @@ var ErrSessionClosed = errors.New("agentdaemon gateway: session closed") type WSConn interface { ReadMessage() (int, []byte, error) WriteMessage(messageType int, data []byte) error + WriteControl(messageType int, data []byte, deadline time.Time) error SetReadLimit(limit int64) SetReadDeadline(t time.Time) error SetWriteDeadline(t time.Time) error @@ -125,6 +126,11 @@ type Session struct { // closeOnce guards the shutdown path so concurrent Close calls // collapse into one. + receiptMu sync.Mutex + deliveries map[string]struct{} + receiptDrain device.ArchivedCancellationReceipt + receiptTimer *time.Timer + closeOnce sync.Once closed chan struct{} } @@ -249,6 +255,7 @@ func legacyClaudeCodeKind() device.SupportedAgentKind { // with a synthetic error+done pair, and deregisters. Idempotent. func (s *Session) Close(reason string) { s.closeOnce.Do(func() { + s.stopReceiptTimer() close(s.closed) _ = s.conn.Close() // Synthetic error + done so the connector's translation loop @@ -276,9 +283,8 @@ func (s *Session) Close(reason string) { // permanent conditions (e.g. runtime deleted) can be distinguished // from transient disconnects. func (s *Session) CloseWithCode(code int, reason string) { - _ = s.conn.SetWriteDeadline(time.Now().Add(WriteTimeout)) - _ = s.conn.WriteMessage(websocket.CloseMessage, - websocket.FormatCloseMessage(code, reason)) + // gorilla permits WriteControl concurrently with the sole data writer. + _ = s.conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(code, reason), time.Now().Add(WriteTimeout)) s.Close(reason) } @@ -308,7 +314,7 @@ func (s *Session) deliverSynthetic(runID string, ch chan proto.Envelope, reason // daemon frame inherits the caller's trace_id. Callers that explicitly // set env.Trace win. func (s *Session) Send(ctx context.Context, env proto.Envelope) error { - if s.IsClosed() { + if s.IsClosed() || !s.allowsReceiptFrame(env, true) { return ErrSessionClosed } if env.Trace == "" { @@ -381,6 +387,9 @@ func (s *Session) writeLoop() { if !ok { return } + if !s.allowsReceiptFrame(env, true) { + continue + } raw, err := json.Marshal(env) if err != nil { s.log("agentdaemon gateway: marshal outbound envelope: %v", err) @@ -412,9 +421,11 @@ func (s *Session) readLoop() { s.log("agentdaemon gateway: read frame: %v", err) return } - s.markSeen() - if !s.renewOwnerLease() { - return + if !s.receiptDraining() { + s.markSeen() + if !s.renewOwnerLease() { + return + } } var env proto.Envelope @@ -512,6 +523,10 @@ func (s *Session) handleHeartbeat(env proto.Envelope) { return } if status.Deleted { + draining, drainErr := s.DrainArchivedCancellation(ctx) + if drainErr == nil && draining { + return + } s.log("agentdaemon gateway: runtime retired, closing session device=%s", s.DeviceID) // "retired" rather than "deleted by admin": the row may have // been soft-deleted by sandbox stale-row cleanup or by an @@ -583,6 +598,9 @@ func deviceKindsFromHeartbeat(p proto.HeartbeatPayload) []device.SupportedAgentK } func (s *Session) dispatch(env proto.Envelope) { + if !s.allowsReceiptFrame(env, false) { + return + } switch env.Type { case proto.TypeWorkspaceExportResult: s.dispatchWorkspaceExport(env) diff --git a/internal/agentdaemon/gateway/session_test.go b/internal/agentdaemon/gateway/session_test.go index 778a8eb15..30fc4396f 100644 --- a/internal/agentdaemon/gateway/session_test.go +++ b/internal/agentdaemon/gateway/session_test.go @@ -477,3 +477,7 @@ func TestSession_HeartbeatInfersClaudeCodeFromLegacyFlag(t *testing.T) { func jsonMarshal(v any) ([]byte, error) { return json.Marshal(v) } + +func (c *fakeConn) WriteControl(kind int, data []byte, _ time.Time) error { + return c.WriteMessage(kind, data) +} diff --git a/services/agents-api/internal/db/queries/devices.sql b/services/agents-api/internal/db/queries/devices.sql index d7379b580..80262106e 100644 --- a/services/agents-api/internal/db/queries/devices.sql +++ b/services/agents-api/internal/db/queries/devices.sql @@ -12,7 +12,7 @@ LEFT JOIN runtime_allocations a ON a.device_id = d.id WHERE d.id = $1; -- name: RevokeDevice :execrows -UPDATE devices SET revoked_at = COALESCE(revoked_at, clock_timestamp()) +UPDATE devices SET revoked_at = COALESCE(revoked_at, clock_timestamp()), archive_cancel_turn_id = NULL WHERE tenant_id = $1 AND id = $2; -- name: TouchDevice :execrows diff --git a/services/agents-api/internal/db/queries/runtime_cancellation.sql b/services/agents-api/internal/db/queries/runtime_cancellation.sql new file mode 100644 index 000000000..600b246b7 --- /dev/null +++ b/services/agents-api/internal/db/queries/runtime_cancellation.sql @@ -0,0 +1,32 @@ +-- name: GetArchivedCancellationReceipt :one +SELECT t.id, t.cancel_requested_at +FROM devices d +JOIN runtime_allocations a ON a.device_id = d.id +JOIN environments e ON e.id = a.environment_id AND e.id = d.environment_id +JOIN sessions s ON s.id = e.session_id AND s.tenant_id = d.tenant_id +JOIN turns t ON t.session_id = s.id AND t.id = d.archive_cancel_turn_id +WHERE d.id = sqlc.arg(device_id) + AND d.credential_hash = sqlc.arg(credential_hash) + AND d.executor_key_id IS NULL AND d.revoked_at IS NOT NULL + AND a.state = 'cleanup_pending' AND a.released_at IS NULL + AND e.status = 'expired' AND s.deleted_at IS NULL + AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND t.status IN ('in_progress', 'waiting') + AND t.id::text = ANY(sqlc.arg(run_ids)::text[]) + AND t.cancel_requested_at IS NOT NULL + AND d.revoked_at >= t.cancel_requested_at + AND t.cancel_requested_at > clock_timestamp() - sqlc.arg(limit_seconds)::int * interval '1 second'; + +-- name: RevokeArchivedRuntimeDevice :execrows +UPDATE devices d +SET archive_cancel_turn_id = CASE WHEN d.revoked_at IS NULL THEN ( + SELECT t.id FROM turns t + WHERE t.session_id = sqlc.arg(session_id) + AND t.status IN ('in_progress', 'waiting') AND t.cancel_requested_at IS NOT NULL +) ELSE d.archive_cancel_turn_id END, + revoked_at = COALESCE(d.revoked_at, clock_timestamp()) +WHERE d.tenant_id = sqlc.arg(tenant_id) AND d.id = sqlc.arg(device_id); + +-- name: RevokeRuntimeCleanupDevice :execrows +UPDATE devices SET revoked_at = COALESCE(revoked_at, clock_timestamp()) +WHERE tenant_id = sqlc.arg(tenant_id) AND id = sqlc.arg(device_id); diff --git a/services/agents-api/internal/db/sqlc/devices.sql.go b/services/agents-api/internal/db/sqlc/devices.sql.go index 7a2ff38a6..fbe8cbd2c 100644 --- a/services/agents-api/internal/db/sqlc/devices.sql.go +++ b/services/agents-api/internal/db/sqlc/devices.sql.go @@ -195,7 +195,7 @@ func (q *Queries) RememberNativeSession(ctx context.Context, arg RememberNativeS } const revokeDevice = `-- name: RevokeDevice :execrows -UPDATE devices SET revoked_at = COALESCE(revoked_at, clock_timestamp()) +UPDATE devices SET revoked_at = COALESCE(revoked_at, clock_timestamp()), archive_cancel_turn_id = NULL WHERE tenant_id = $1 AND id = $2 ` diff --git a/services/agents-api/internal/db/sqlc/models.go b/services/agents-api/internal/db/sqlc/models.go index c1ff3d716..90f61391c 100644 --- a/services/agents-api/internal/db/sqlc/models.go +++ b/services/agents-api/internal/db/sqlc/models.go @@ -64,15 +64,16 @@ type DeploymentModelProvider struct { } type Device struct { - ID pgtype.UUID `json:"id"` - TenantID pgtype.UUID `json:"tenant_id"` - Name string `json:"name"` - CredentialHash pgtype.Text `json:"credential_hash"` - CreatedAt pgtype.Timestamptz `json:"created_at"` - LastSeenAt pgtype.Timestamptz `json:"last_seen_at"` - RevokedAt pgtype.Timestamptz `json:"revoked_at"` - EnvironmentID pgtype.UUID `json:"environment_id"` - ExecutorKeyID pgtype.UUID `json:"executor_key_id"` + ID pgtype.UUID `json:"id"` + TenantID pgtype.UUID `json:"tenant_id"` + Name string `json:"name"` + CredentialHash pgtype.Text `json:"credential_hash"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + LastSeenAt pgtype.Timestamptz `json:"last_seen_at"` + RevokedAt pgtype.Timestamptz `json:"revoked_at"` + EnvironmentID pgtype.UUID `json:"environment_id"` + ExecutorKeyID pgtype.UUID `json:"executor_key_id"` + ArchiveCancelTurnID pgtype.UUID `json:"archive_cancel_turn_id"` } type Environment struct { diff --git a/services/agents-api/internal/db/sqlc/runtime_cancellation.sql.go b/services/agents-api/internal/db/sqlc/runtime_cancellation.sql.go new file mode 100644 index 000000000..dbe9b5ba2 --- /dev/null +++ b/services/agents-api/internal/db/sqlc/runtime_cancellation.sql.go @@ -0,0 +1,99 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: runtime_cancellation.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getArchivedCancellationReceipt = `-- name: GetArchivedCancellationReceipt :one +SELECT t.id, t.cancel_requested_at +FROM devices d +JOIN runtime_allocations a ON a.device_id = d.id +JOIN environments e ON e.id = a.environment_id AND e.id = d.environment_id +JOIN sessions s ON s.id = e.session_id AND s.tenant_id = d.tenant_id +JOIN turns t ON t.session_id = s.id AND t.id = d.archive_cancel_turn_id +WHERE d.id = $1 + AND d.credential_hash = $2 + AND d.executor_key_id IS NULL AND d.revoked_at IS NOT NULL + AND a.state = 'cleanup_pending' AND a.released_at IS NULL + AND e.status = 'expired' AND s.deleted_at IS NULL + AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND t.status IN ('in_progress', 'waiting') + AND t.id::text = ANY($3::text[]) + AND t.cancel_requested_at IS NOT NULL + AND d.revoked_at >= t.cancel_requested_at + AND t.cancel_requested_at > clock_timestamp() - $4::int * interval '1 second' +` + +type GetArchivedCancellationReceiptParams struct { + DeviceID pgtype.UUID `json:"device_id"` + CredentialHash pgtype.Text `json:"credential_hash"` + RunIds []string `json:"run_ids"` + LimitSeconds int32 `json:"limit_seconds"` +} + +type GetArchivedCancellationReceiptRow struct { + ID pgtype.UUID `json:"id"` + CancelRequestedAt pgtype.Timestamptz `json:"cancel_requested_at"` +} + +func (q *Queries) GetArchivedCancellationReceipt(ctx context.Context, arg GetArchivedCancellationReceiptParams) (GetArchivedCancellationReceiptRow, error) { + row := q.db.QueryRow(ctx, getArchivedCancellationReceipt, + arg.DeviceID, + arg.CredentialHash, + arg.RunIds, + arg.LimitSeconds, + ) + var i GetArchivedCancellationReceiptRow + err := row.Scan(&i.ID, &i.CancelRequestedAt) + return i, err +} + +const revokeArchivedRuntimeDevice = `-- name: RevokeArchivedRuntimeDevice :execrows +UPDATE devices d +SET archive_cancel_turn_id = CASE WHEN d.revoked_at IS NULL THEN ( + SELECT t.id FROM turns t + WHERE t.session_id = $1 + AND t.status IN ('in_progress', 'waiting') AND t.cancel_requested_at IS NOT NULL +) ELSE d.archive_cancel_turn_id END, + revoked_at = COALESCE(d.revoked_at, clock_timestamp()) +WHERE d.tenant_id = $2 AND d.id = $3 +` + +type RevokeArchivedRuntimeDeviceParams struct { + SessionID pgtype.UUID `json:"session_id"` + TenantID pgtype.UUID `json:"tenant_id"` + DeviceID pgtype.UUID `json:"device_id"` +} + +func (q *Queries) RevokeArchivedRuntimeDevice(ctx context.Context, arg RevokeArchivedRuntimeDeviceParams) (int64, error) { + result, err := q.db.Exec(ctx, revokeArchivedRuntimeDevice, arg.SessionID, arg.TenantID, arg.DeviceID) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const revokeRuntimeCleanupDevice = `-- name: RevokeRuntimeCleanupDevice :execrows +UPDATE devices SET revoked_at = COALESCE(revoked_at, clock_timestamp()) +WHERE tenant_id = $1 AND id = $2 +` + +type RevokeRuntimeCleanupDeviceParams struct { + TenantID pgtype.UUID `json:"tenant_id"` + DeviceID pgtype.UUID `json:"device_id"` +} + +func (q *Queries) RevokeRuntimeCleanupDevice(ctx context.Context, arg RevokeRuntimeCleanupDeviceParams) (int64, error) { + result, err := q.db.Exec(ctx, revokeRuntimeCleanupDevice, arg.TenantID, arg.DeviceID) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/services/agents-api/internal/execution/forced_waiting_cleanup_diagnosis_test.go b/services/agents-api/internal/execution/archive_cancellation_cleanup_test.go similarity index 61% rename from services/agents-api/internal/execution/forced_waiting_cleanup_diagnosis_test.go rename to services/agents-api/internal/execution/archive_cancellation_cleanup_test.go index e5efbf933..4ccfa7374 100644 --- a/services/agents-api/internal/execution/forced_waiting_cleanup_diagnosis_test.go +++ b/services/agents-api/internal/execution/archive_cancellation_cleanup_test.go @@ -3,11 +3,18 @@ package execution import ( "context" "encoding/json" + runtimegateway "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/runtime" + "github.com/gorilla/websocket" + "net/http" + "net/http/httptest" + "net/url" "testing" + "time" v1 "github.com/MiniMax-AI-Dev/parsar/contracts/agents-api/v1" "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/adminaudit" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/identity" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" @@ -40,9 +47,13 @@ func (p waitingCleanupCheckpoint) KillCompute(context.Context, sandbox.Reference return nil } -func TestArchiveWaitingDestructiveCleanupPrecedesReceiptDiagnosis(t *testing.T) { - for _, checkpoint := range []bool{false, true} { - t.Run(map[bool]string{false: "Kill", true: "KillCompute"}[checkpoint], func(t *testing.T) { +func TestArchiveWaitingCleanupReceiptBarrier(t *testing.T) { + for _, scenario := range []struct { + name string + checkpoint, delivery bool + }{{"Kill_no_delivery", false, false}, {"KillCompute_no_delivery", true, false}, {"Kill_live_delivery", false, true}, {"KillCompute_live_delivery", true, true}} { + t.Run(scenario.name, func(t *testing.T) { + checkpoint := scenario.checkpoint s, lease := resetManagerStore(t) writer := lease.Store() installation := uuid.NewString() @@ -65,7 +76,8 @@ func TestArchiveWaitingDestructiveCleanupPrecedesReceiptDiagnosis(t *testing.T) if err != nil { t.Fatal(err) } - owner, err := writer.ReserveRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID, installation, device.HashCredential(uuid.NewString())) + secret := uuid.NewString() + owner, err := writer.ReserveRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID, installation, device.HashCredential(secret)) if err != nil { t.Fatal(err) } @@ -78,7 +90,7 @@ func TestArchiveWaitingDestructiveCleanupPrecedesReceiptDiagnosis(t *testing.T) t.Fatal(err) } // This fixture isolates lifecycle ordering. Protocol-driven waiting is - // independently exercised in TestArchiveWaitingHeartbeatCancellationDiagnosis. + // independently exercised in TestArchiveWaitingCancellationReceipts. for _, transition := range []store.TurnTransition{{ExpectedStatus: store.TurnQueued, Status: store.TurnInProgress}, {ExpectedStatus: store.TurnInProgress, Status: store.TurnWaiting}} { if _, err := writer.TransitionTurn(t.Context(), project.TenantID, session.ID, input.TurnID, transition); err != nil { t.Fatal(err) @@ -92,6 +104,42 @@ func TestArchiveWaitingDestructiveCleanupPrecedesReceiptDiagnosis(t *testing.T) t.Fatal(err) } } + registry := gateway.NewRegistry() + if scenario.delivery { + server := httptest.NewUnstartedServer(nil) + wsURL := "ws://" + server.Listener.Addr().String() + "/api/v1/agent-daemon/ws" + handler, liveRegistry, err := runtimegateway.NewGateway(s, wsURL) + if err != nil { + t.Fatal(err) + } + registry = liveRegistry + server.Config.Handler = handler + server.Start() + t.Cleanup(func() { runtimegateway.CloseConnections(registry); server.Close() }) + u, _ := url.Parse(wsURL) + u.RawQuery = url.Values{"device_id": {owner.DeviceID}, "version": {proto.Version}}.Encode() + conn, _, err := websocket.DefaultDialer.Dial(u.String(), http.Header{"Authorization": {"Bearer " + secret}}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + var peer *gateway.Session + for end := time.Now().Add(3 * time.Second); ; { + peer, err = registry.LookupDevice(owner.DeviceID) + if err == nil { + break + } + if time.Now().After(end) { + t.Fatal(err) + } + time.Sleep(time.Millisecond) + } + release, err := peer.TrackExecutionDelivery(input.TurnID) + if err != nil { + t.Fatal(err) + } + defer release() + } if _, err := writer.ArchiveManagedSession(audit, project.TenantID, session.ID, 1); err != nil { t.Fatal(err) } @@ -100,30 +148,48 @@ func TestArchiveWaitingDestructiveCleanupPrecedesReceiptDiagnosis(t *testing.T) t.Fatal(err) } kills := 0 + expectedStatus := store.TurnWaiting provider := waitingCleanupProvider{beforeKill: func() { kills++ turn, err := s.GetTurn(t.Context(), project.TenantID, session.ID, input.TurnID) - if err != nil || turn.Status != store.TurnWaiting || turn.CancelRequestedAt.IsZero() || !turn.CompletedAt.IsZero() { - t.Fatal("Kill did not precede terminal receipt", turn, err) + if err != nil || turn.Status != expectedStatus || turn.CancelRequestedAt.IsZero() || (turn.CompletedAt.IsZero() != (expectedStatus == store.TurnWaiting)) { + t.Fatal("cleanup observed unexpected terminal state", turn, err) } allocation, err := s.GetRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID) if err != nil || allocation.State != "cleanup_pending" { t.Fatal("Kill bypassed durable cleanup ownership", allocation, err) } }} - lifecycle := &runtimeLifecycle{store: writer, registry: gateway.NewRegistry(), config: RuntimeProvider{InstallationID: installation, Provider: provider}, connections: map[string]*runtimeConnection{}} + lifecycle := &runtimeLifecycle{store: writer, registry: registry, config: RuntimeProvider{InstallationID: installation, Provider: provider}, connections: map[string]*runtimeConnection{}} if checkpoint { lifecycle.config.Provider = waitingCleanupCheckpoint{beforeKill: provider.beforeKill} } if err := lifecycle.observe(t.Context(), owner); err != nil { t.Fatal(err) } + if scenario.delivery { + if kills != 0 { + t.Fatal("destroyed compute before cancellation committed") + } + pending, err := s.GetRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID) + if err != nil || pending.State != "cleanup_pending" { + t.Fatal(pending, err) + } + // Controlled terminal receipt fixture; no native cancellation claim. + if _, err := writer.TransitionTurn(t.Context(), project.TenantID, session.ID, input.TurnID, store.TurnTransition{ExpectedStatus: store.TurnWaiting, Status: store.TurnCancelled}); err != nil { + t.Fatal(err) + } + expectedStatus = store.TurnCancelled + if err := lifecycle.observe(t.Context(), owner); err != nil { + t.Fatal(err) + } + } after, err := s.GetRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID) if err != nil || after.State != "released" || kills != 1 { t.Fatal("cleanup did not release", after, kills, err) } turn, err := s.GetTurn(t.Context(), project.TenantID, session.ID, input.TurnID) - if err != nil || turn.Status != store.TurnWaiting || !turn.CompletedAt.IsZero() { + if err != nil || turn.Status != expectedStatus || (turn.CompletedAt.IsZero() != (expectedStatus == store.TurnWaiting)) { t.Fatal("cleanup should not fabricate cancellation", turn, err) } }) diff --git a/services/agents-api/internal/execution/device_authority.go b/services/agents-api/internal/execution/device_authority.go index 8cd0ff502..b5cc877af 100644 --- a/services/agents-api/internal/execution/device_authority.go +++ b/services/agents-api/internal/execution/device_authority.go @@ -19,7 +19,11 @@ func authorizedRuntimePeer(ctx context.Context, s *store.Store, registry *gatewa return nil, err } if !found || !peer.AuthenticatedWith(credential.CredentialHash) { - peer.Close("Runtime authorization changed") + // Reject new work even while this exact old delivery drains its receipt. + draining, drainErr := peer.DrainArchivedCancellation(ctx) + if drainErr != nil || !draining { + peer.Close("Runtime authorization changed") + } return nil, store.ErrNotFound } if peer.IsClosed() { diff --git a/services/agents-api/internal/execution/dispatcher.go b/services/agents-api/internal/execution/dispatcher.go index 11ec43c75..ba92fc92c 100644 --- a/services/agents-api/internal/execution/dispatcher.go +++ b/services/agents-api/internal/execution/dispatcher.go @@ -94,6 +94,11 @@ func (d *Dispatcher) Run(ctx context.Context, tenantID, sessionID, turnID string } req.ConversationID, req.RunID, req.Input = sessionID, turnID, text req.WorkDir, req.DisableExecutionEnvironment = workDir, noEnvironment + release, err := peer.TrackExecutionDelivery(req.RunID) + if err != nil { + return d.finishRun(tenantID, sessionID, turnID, snapshot.Agent.Model, Result{ErrorCode: "delivery_unknown", AppliedThrough: through}, store.TurnFailed) + } + defer release() result, status := d.deliver(ctx, tenantID, sessionID, peer, req, through, nil) return d.finishRun(tenantID, sessionID, turnID, snapshot.Agent.Model, result, status) } diff --git a/services/agents-api/internal/execution/prepared_dispatch.go b/services/agents-api/internal/execution/prepared_dispatch.go index 453d85745..f13d3f48d 100644 --- a/services/agents-api/internal/execution/prepared_dispatch.go +++ b/services/agents-api/internal/execution/prepared_dispatch.go @@ -106,6 +106,12 @@ func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, tenantID, sessionI req.RunID = run.Reservation.Receipts[0].TurnID req.Input = messages through := run.Reservation.Receipts[len(run.Reservation.Receipts)-1].Sequence + releaseDelivery, err := peer.TrackExecutionDelivery(req.RunID) + if err != nil { + run.Turn, err = d.finishRun(tenantID, sessionID, req.RunID, snapshot.Agent.Model, Result{ErrorCode: "delivery_unknown", AppliedThrough: through}, store.TurnFailed) + return run, err + } + defer releaseDelivery() result, status := d.deliver(owner, tenantID, sessionID, peer, req, through, prepared) result, status = d.captureCompletedArtifacts(owner, peer, session, environment, bound.Device, req.RunID, result, status) run.Turn, err = d.finishRun(tenantID, sessionID, req.RunID, snapshot.Agent.Model, result, status) diff --git a/services/agents-api/internal/execution/runtime_lifecycle.go b/services/agents-api/internal/execution/runtime_lifecycle.go index a286c74e4..cf00b5d29 100644 --- a/services/agents-api/internal/execution/runtime_lifecycle.go +++ b/services/agents-api/internal/execution/runtime_lifecycle.go @@ -327,6 +327,15 @@ func (r *runtimeLifecycle) observe(ctx context.Context, owner store.RuntimeAlloc if err != nil { return err } + if peer, err := r.registry.LookupDevice(owner.DeviceID); err == nil { + draining, err := peer.DrainArchivedCancellation(ctx) + if err != nil { + return err + } + if draining { + return nil + } + } r.clearRuntimeState(owner) } else if owner.ComputePhase == "disabled" || owner.ComputePhase == "running" { if err := r.observeConnection(ctx, owner); err != nil { diff --git a/services/agents-api/internal/store/admin_session_archive.go b/services/agents-api/internal/store/admin_session_archive.go index fb50ac101..3235c8bdc 100644 --- a/services/agents-api/internal/store/admin_session_archive.go +++ b/services/agents-api/internal/store/admin_session_archive.go @@ -116,7 +116,7 @@ func (s *Store) archiveManagedSession(ctx context.Context, tenantID, sessionID s return err } if allocated && allocation.RuntimeAllocation.State != "released" { - if _, err := q.RevokeDevice(ctx, sqlc.RevokeDeviceParams{TenantID: tenant, ID: allocation.RuntimeAllocation.DeviceID}); err != nil { + if _, err := q.RevokeArchivedRuntimeDevice(ctx, sqlc.RevokeArchivedRuntimeDeviceParams{TenantID: tenant, DeviceID: allocation.RuntimeAllocation.DeviceID, SessionID: session}); err != nil { return err } if _, err := q.RequestRuntimeCleanup(ctx, allocation.RuntimeAllocation.ID); err != nil { diff --git a/services/agents-api/internal/store/forced_waiting_diagnosis_test.go b/services/agents-api/internal/store/archive_cancellation_test.go similarity index 58% rename from services/agents-api/internal/store/forced_waiting_diagnosis_test.go rename to services/agents-api/internal/store/archive_cancellation_test.go index 030c6dc48..ad4044573 100644 --- a/services/agents-api/internal/store/forced_waiting_diagnosis_test.go +++ b/services/agents-api/internal/store/archive_cancellation_test.go @@ -25,9 +25,10 @@ import ( // Controlled protocol fixtures, not native/model acceptance. Allocation setup // uses the real execution lease; the real dispatcher consumes a function request // through the real authenticated WebSocket and persists waiting before archive. -func TestArchiveWaitingHeartbeatCancellationDiagnosis(t *testing.T) { - for _, heartbeat := range []bool{false, true} { - t.Run(map[bool]string{false: "receipt_without_heartbeat", true: "heartbeat_before_receipt"}[heartbeat], func(t *testing.T) { +func TestArchiveWaitingCancellationReceipts(t *testing.T) { + for _, scenario := range []string{"receipt_without_heartbeat", "heartbeat_before_receipt", "done_heartbeat_ack", "ack_commit_blocked", "rotated", "expired", "transport_lost", "negative_ack", "missing_outcome", "revoke_before_archive", "cancel_revoke_archive", "revoke_after_archive", "revoke_concurrent_archive"} { + t.Run(scenario, func(t *testing.T) { + heartbeat := scenario != "receipt_without_heartbeat" s, pool := store.NewManagedTestStore(t) lease, err := s.AcquireExecutionLease(t.Context()) if err != nil { @@ -130,10 +131,46 @@ func TestArchiveWaitingHeartbeatCancellationDiagnosis(t *testing.T) { t.Fatal(state.LastTurn) } + if scenario == "cancel_revoke_archive" { + if _, err := s.RequestCancel(t.Context(), h.tenant, session.ID, "ordinary-cancel"); err != nil { + t.Fatal(err) + } + } + if scenario == "revoke_before_archive" || scenario == "cancel_revoke_archive" { + if err := s.RevokeDevice(t.Context(), h.tenant, owner.DeviceID); err != nil { + t.Fatal(err) + } + } + var revokeDone chan error + if scenario == "revoke_concurrent_archive" { + revokeDone = make(chan error, 1) + go func() { revokeDone <- s.RevokeDevice(t.Context(), h.tenant, owner.DeviceID) }() + } + archived, err := writer.ArchiveManagedSession(auditCtx, h.tenant, session.ID, 1) if err != nil || archived.State != "cleanup_pending" { t.Fatal(archived, err) } + if revokeDone != nil { + if err := <-revokeDone; err != nil { + t.Fatal(err) + } + } + if scenario == "revoke_after_archive" { + if err := s.RevokeDevice(t.Context(), h.tenant, owner.DeviceID); err != nil { + t.Fatal(err) + } + } + // A repeat archive and cleanup must not recreate an explicitly + // cleared marker, nor erase the marker from a fresh archive. + repeatAudit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", ProjectID: projectID, RequestID: uuid.NewString(), TraceID: uuid.NewString()}) + if _, err := writer.ArchiveManagedSession(repeatAudit, h.tenant, session.ID, 1); err != nil { + t.Fatal(err) + } + if _, err := writer.RequestRuntimeCleanup(t.Context(), owner); err != nil { + t.Fatal(err) + } + current, err := s.GetTurn(t.Context(), h.tenant, session.ID, input.TurnID) if err != nil || current.Status != store.TurnWaiting || current.CancelRequestedAt.IsZero() { t.Fatal("archive must request rather than invent cancellation", current, err) @@ -151,6 +188,17 @@ func TestArchiveWaitingHeartbeatCancellationDiagnosis(t *testing.T) { if dialErr == nil || response == nil || response.StatusCode != http.StatusUnauthorized { t.Fatal("revoked Runtime reconnected") } + drain, err := s.ArchivedCancellationReceipt(t.Context(), owner.DeviceID, secret, nil) + if err != nil || drain.RunID != "" { + t.Fatal("unowned delivery got receipt permission", drain, err) + } + drain, err = s.ArchivedCancellationReceipt(t.Context(), owner.DeviceID, device.HashCredential(secret), []string{input.TurnID}) + if err != nil || (drain.RunID == input.TurnID) == strings.Contains(scenario, "revoke") { + t.Fatal("archive revocation causality lost", drain, err) + } + if drain.RunID != "" && !drain.Deadline.Equal(current.CancelRequestedAt.Add(device.ArchivedCancellationReceiptLimit)) { + t.Fatal("archive renewed cancellation deadline") + } // Explicitly observe cancel delivery before inducing transport loss. This // proves even a sent cancellation can lose its receipt; no ticker timing guess. var request proto.PromptCancelPayload @@ -160,47 +208,103 @@ func TestArchiveWaitingHeartbeatCancellationDiagnosis(t *testing.T) { if request.DeliveryID == "" { t.Fatal("missing cancel delivery identity") } - if heartbeat { - h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{}) - select { - case <-peer.Closed(): - case <-time.After(3 * time.Second): - t.Fatal("revoked heartbeat did not close peer") + if scenario == "rotated" { + if _, err := pool.Exec(t.Context(), "UPDATE devices SET credential_hash=$2 WHERE id=$1", owner.DeviceID, device.HashCredential(uuid.NewString())); err != nil { + t.Fatal(err) } - got := awaitPreparedDispatch(t, result) - if got.err != nil || got.run.Turn.Status != store.TurnFailed { - t.Fatal(got.run.Turn.Status, got.err) + } + if scenario == "expired" { + if _, err := pool.Exec(t.Context(), "UPDATE turns SET cancel_requested_at=clock_timestamp()-interval '21 seconds' WHERE id=$1", input.TurnID); err != nil { + t.Fatal(err) } - done := got.run.Turn - var outcome execution.Result - if err := json.Unmarshal(done.Outcome, &outcome); err != nil || outcome.ErrorCode != "event_stream_incomplete" { - t.Fatal(string(done.Outcome), err) + } + var unlockCommit func() + if scenario == "ack_commit_blocked" { + tx, err := pool.Begin(t.Context()) + if err != nil { + t.Fatal(err) } - } else { - h.write(input.TurnID, proto.TypeInteractionDecisionAck, proto.InteractionDecisionAckPayload{DeliveryID: request.DeliveryID, Applied: true, Outcome: &proto.DonePayload{Usage: proto.Usage{InputTokens: 17}, Metadata: map[string]any{proto.DoneMetaAgentSessionID: "cancelled-native"}}}) - got := awaitPreparedDispatch(t, result) - if got.err != nil || got.run.Turn.Status != store.TurnCancelled { - t.Fatal(got.run.Turn.Status, got.err) + if _, err := tx.Exec(t.Context(), "SELECT session_id FROM session_devices WHERE session_id=$1 FOR UPDATE", session.ID); err != nil { + t.Fatal(err) } - done := got.run.Turn - var outcome execution.Result - if err := json.Unmarshal(done.Outcome, &outcome); err != nil || outcome.Done.Usage.InputTokens != 17 { - t.Fatal("receipt lost usage", string(done.Outcome), err) + unlockCommit = func() { + if err := tx.Rollback(context.Background()); err != nil { + t.Fatal(err) + } } + defer func() { _ = tx.Rollback(context.Background()) }() + } + + if scenario == "done_heartbeat_ack" { + h.write(input.TurnID, proto.TypeDone, proto.DonePayload{}) } + closedCase := scenario == "rotated" || scenario == "expired" || scenario == "transport_lost" || strings.Contains(scenario, "revoke") + if scenario == "transport_lost" { + h.conn.Close() + } else if heartbeat && scenario != "ack_commit_blocked" { + h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{}) + } + if !closedCase { + ack := proto.InteractionDecisionAckPayload{DeliveryID: request.DeliveryID, Applied: true, Outcome: &proto.DonePayload{Usage: proto.Usage{InputTokens: 17}, Metadata: map[string]any{proto.DoneMetaAgentSessionID: "cancelled-native"}}} + if scenario == "negative_ack" { + ack.Applied = false + ack.ErrorCode = "cancel_failed" + ack.Outcome = nil + } + if scenario == "missing_outcome" { + ack.Outcome = nil + } + h.write(input.TurnID, proto.TypeInteractionDecisionAck, ack) + } + if unlockCommit != nil { + // Observe actual SQL lock contention, not an assumed timing delay. + for deadline := time.Now().Add(3 * time.Second); ; { + var blocked bool + if err := pool.QueryRow(t.Context(), "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock' AND query ILIKE '%session_devices%')").Scan(&blocked); err != nil { + t.Fatal(err) + } + if blocked { + break + } + if time.Now().After(deadline) { + t.Fatal("terminal commit never blocked") + } + time.Sleep(time.Millisecond) + } + h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{}) + draining, err := peer.DrainArchivedCancellation(t.Context()) + if err != nil || !draining || peer.IsClosed() { + t.Fatal("ACK lost drain before terminal commit", draining, err) + } + unlockCommit() + } + + got := awaitPreparedDispatch(t, result) + wantStatus := store.TurnCancelled + if closedCase || scenario == "negative_ack" || scenario == "missing_outcome" { + wantStatus = store.TurnFailed + } + if got.err != nil || got.run.Turn.Status != wantStatus { + t.Fatal(got.run.Turn.Status, got.err, string(got.run.Turn.Outcome)) + } + var outcome execution.Result + if err := json.Unmarshal(got.run.Turn.Outcome, &outcome); err != nil || (wantStatus == store.TurnCancelled && outcome.Done.Usage.InputTokens != 17) { + t.Fatal("receipt lost usage", string(got.run.Turn.Outcome), err) + } + var receipts int if err := pool.QueryRow(t.Context(), "SELECT count(*) FROM turn_events WHERE turn_id=$1 AND kind='cancel_receipt'", input.TurnID).Scan(&receipts); err != nil { t.Fatal(err) } wantReceipts := 1 - if heartbeat { + if closedCase { wantReceipts = 0 } if receipts != wantReceipts { t.Fatal("durable cancellation receipt mismatch", receipts, wantReceipts) } - // No provider was invoked in either case; the failure therefore does not - // require compute destruction. The original durable cleanup owner survives. + // The original cleanup owner survives every delivery outcome; only + // provider receipts can release its resources. allocation, err := s.GetRuntimeAllocation(t.Context(), h.tenant, session.Environment.ID) if err != nil || allocation.State != "cleanup_pending" { t.Fatal(allocation, err) diff --git a/services/agents-api/internal/store/archived_cancellation_migration_test.go b/services/agents-api/internal/store/archived_cancellation_migration_test.go new file mode 100644 index 000000000..dbf72c05e --- /dev/null +++ b/services/agents-api/internal/store/archived_cancellation_migration_test.go @@ -0,0 +1,48 @@ +package store + +import ( + "database/sql" + "os" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" +) + +func TestArchivedCancellationMigrationDoesNotAdoptOldRevocations(t *testing.T) { + s, w, installation := managedArchiveFixture(t) + tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + owner := archiveAllocation(t, w, tenant, session, installation) + input := submitMessage(t, s, tenant, session.ID, "waiting") + transition(t, w, tenant, session.ID, input.TurnID, TurnQueued, TurnInProgress) + transition(t, w, tenant, session.ID, input.TurnID, TurnInProgress, TurnWaiting) + if _, err := w.ArchiveManagedSession(adminDeleteContext(t.Context(), tenant, uuid.NewString()), tenant, session.ID, 1); err != nil { + t.Fatal(err) + } + db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) + defer db.Close() + provider, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) + if err != nil { + t.Fatal(err) + } + if _, err := provider.DownTo(t.Context(), 79); err == nil { + t.Fatal("downgrade discarded unsettled cancellation marker") + } + if _, err := w.SettleRuntimeCreation(t.Context(), owner); err != nil { + t.Fatal(err) + } + if _, err := w.ReleaseRuntimeAllocation(t.Context(), owner); err != nil { + t.Fatal(err) + } + if _, err := provider.DownTo(t.Context(), 79); err != nil { + t.Fatal(err) + } + if _, err := provider.Up(t.Context()); err != nil { + t.Fatal(err) + } + var marker *string + if err := s.pool.QueryRow(t.Context(), "SELECT archive_cancel_turn_id::text FROM devices WHERE id=$1", owner.DeviceID).Scan(&marker); err != nil || marker != nil { + t.Fatal("upgrade adopted historical revoked device", marker, err) + } +} diff --git a/services/agents-api/internal/store/runtime_allocation_state.go b/services/agents-api/internal/store/runtime_allocation_state.go index 50535992a..10d7e390a 100644 --- a/services/agents-api/internal/store/runtime_allocation_state.go +++ b/services/agents-api/internal/store/runtime_allocation_state.go @@ -63,7 +63,7 @@ func (s *Store) requestRuntimeCleanup(ctx context.Context, owner RuntimeAllocati return row, nil } tenant, _ := parseID(owner.TenantID) - if _, err := q.RevokeDevice(ctx, sqlc.RevokeDeviceParams{TenantID: tenant, ID: row.DeviceID}); err != nil { + if _, err := q.RevokeRuntimeCleanupDevice(ctx, sqlc.RevokeRuntimeCleanupDeviceParams{TenantID: tenant, DeviceID: row.DeviceID}); err != nil { return sqlc.RuntimeAllocation{}, err } current, err := q.GetRuntimeAllocation(ctx, sqlc.GetRuntimeAllocationParams{TenantID: tenant, EnvironmentID: row.EnvironmentID}) diff --git a/services/agents-api/internal/store/runtime_cancellation.go b/services/agents-api/internal/store/runtime_cancellation.go new file mode 100644 index 000000000..7a64eb08a --- /dev/null +++ b/services/agents-api/internal/store/runtime_cancellation.go @@ -0,0 +1,35 @@ +package store + +import ( + "context" + "errors" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +// ArchivedCancellationReceipt is a read-only exception for the exact already +// authenticated delivery. The ordinary credential view remains revoked; this +// cannot authorize bootstrap, reconnect, dispatch, workspace access or renewal. +// A marker records that archive caused the first revocation; timestamps alone +// cannot distinguish an earlier ordinary cancel/revoke followed by archive. +func (s *Store) ArchivedCancellationReceipt(ctx context.Context, deviceID, credentialHash string, runIDs []string) (device.ArchivedCancellationReceipt, error) { + if len(runIDs) == 0 || credentialHash == "" { + return device.ArchivedCancellationReceipt{}, nil + } + id, err := parseID(deviceID) + if err != nil { + return device.ArchivedCancellationReceipt{}, err + } + row, err := s.queries.GetArchivedCancellationReceipt(ctx, sqlc.GetArchivedCancellationReceiptParams{DeviceID: id, CredentialHash: pgtype.Text{String: credentialHash, Valid: true}, RunIds: runIDs, LimitSeconds: int32(device.ArchivedCancellationReceiptLimit.Seconds())}) + if errors.Is(err, pgx.ErrNoRows) { + return device.ArchivedCancellationReceipt{}, nil + } + if err != nil { + return device.ArchivedCancellationReceipt{}, err + } + return device.ArchivedCancellationReceipt{RunID: uuid.UUID(row.ID.Bytes).String(), Deadline: row.CancelRequestedAt.Time.Add(device.ArchivedCancellationReceiptLimit)}, nil +} diff --git a/services/agents-api/migrations/000080_archived_cancellation_receipts.sql b/services/agents-api/migrations/000080_archived_cancellation_receipts.sql new file mode 100644 index 000000000..97e0ff974 --- /dev/null +++ b/services/agents-api/migrations/000080_archived_cancellation_receipts.sql @@ -0,0 +1,17 @@ +-- +goose Up +-- Only the archive transaction that first revokes the device may retain the +-- existing delivery's cancellation receipt. Old revocations are not adopted. +ALTER TABLE devices + ADD COLUMN archive_cancel_turn_id uuid REFERENCES turns(id) ON DELETE SET NULL; + +-- +goose Down +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM devices d JOIN runtime_allocations a ON a.device_id = d.id + WHERE d.archive_cancel_turn_id IS NOT NULL AND a.state <> 'released') THEN + RAISE EXCEPTION 'Cannot remove archived cancellation receipts while cleanup is unsettled'; + END IF; +END $$; +-- +goose StatementEnd +ALTER TABLE devices DROP COLUMN archive_cancel_turn_id; From 596cce46c04c699a06a45f97bafbf9f187f66ea5 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 22:33:27 +0000 Subject: [PATCH 13/16] Order generation migration after archived cancellation receipts --- .../internal/store/sandbox_generations_test.go | 16 ++++++++-------- ...ations.sql => 000081_sandbox_generations.sql} | 0 2 files changed, 8 insertions(+), 8 deletions(-) rename services/agents-api/migrations/{000080_sandbox_generations.sql => 000081_sandbox_generations.sql} (100%) diff --git a/services/agents-api/internal/store/sandbox_generations_test.go b/services/agents-api/internal/store/sandbox_generations_test.go index 559a68486..05e7b6221 100644 --- a/services/agents-api/internal/store/sandbox_generations_test.go +++ b/services/agents-api/internal/store/sandbox_generations_test.go @@ -192,7 +192,7 @@ func TestPendingPlacementGenerationSurvivesRepeatedUpdates(t *testing.T) { func TestGenerationMigrationBackfillsAndRejectsLossyDowngrade(t *testing.T) { db, migrations := runtimeNamesMigrationSchema(t) ctx := t.Context() - if _, err := migrations.UpTo(ctx, 79); err != nil { + if _, err := migrations.UpTo(ctx, 80); err != nil { t.Fatal(err) } installation, node := uuid.NewString(), uuid.NewString() @@ -214,7 +214,7 @@ func TestGenerationMigrationBackfillsAndRejectsLossyDowngrade(t *testing.T) { exec(`INSERT INTO devices(id,tenant_id,name,credential_hash) VALUES($1,$2,'old',repeat('b',64))`, device, tenant) exec(`INSERT INTO runtime_placements(environment_id,node_id) VALUES($1,$2)`, environment, node) exec(`INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,node_id) VALUES($1,$2,$3,$4,$5)`, allocation, environment, device, installation, node) - if _, err := migrations.UpTo(ctx, 80); err != nil { + if _, err := migrations.UpTo(ctx, 81); err != nil { t.Fatal(err) } for _, table := range []string{"runtime_allocations", "runtime_placements"} { @@ -230,21 +230,21 @@ func TestGenerationMigrationBackfillsAndRejectsLossyDowngrade(t *testing.T) { if _, err := db.ExecContext(ctx, `UPDATE runtime_deployment SET generation=2`); err != nil { t.Fatal(err) } - if _, err := migrations.DownTo(ctx, 79); err == nil { + if _, err := migrations.DownTo(ctx, 80); err == nil { t.Fatal("downgrade erased offline serving pin") } if _, err := db.ExecContext(ctx, `UPDATE runtime_nodes SET removed_at=clock_timestamp(),ready_generation=NULL`); err != nil { t.Fatal(err) } - if _, err := migrations.DownTo(ctx, 79); err == nil { + if _, err := migrations.DownTo(ctx, 80); err == nil { t.Fatal("downgrade discarded held allocation/placement after pin removal") } exec(`UPDATE runtime_allocations SET state='released',released_at=clock_timestamp(),create_settled=true`) - if _, err := migrations.DownTo(ctx, 79); err == nil { + if _, err := migrations.DownTo(ctx, 80); err == nil { t.Fatal("downgrade discarded unreleased placement") } exec(`UPDATE runtime_placements SET released_at=clock_timestamp()`) - if _, err := migrations.DownTo(ctx, 79); err != nil { + if _, err := migrations.DownTo(ctx, 80); err != nil { t.Fatal("settled downgrade failed", err) } } @@ -263,7 +263,7 @@ func TestGenerationDowngradeRefusesOldAllocation(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err = migrations.DownTo(t.Context(), 79); err == nil { + if _, err = migrations.DownTo(t.Context(), 80); err == nil { t.Fatal("downgrade erased old owned allocation") } if _, err = w.RequestRuntimeCleanup(t.Context(), owner); err != nil { @@ -275,7 +275,7 @@ func TestGenerationDowngradeRefusesOldAllocation(t *testing.T) { if _, err = w.ReleaseRuntimeAllocation(t.Context(), owner); err != nil { t.Fatal(err) } - if _, err = migrations.DownTo(t.Context(), 79); err != nil { + if _, err = migrations.DownTo(t.Context(), 80); err != nil { t.Fatal("released history prevented safe downgrade", err) } } diff --git a/services/agents-api/migrations/000080_sandbox_generations.sql b/services/agents-api/migrations/000081_sandbox_generations.sql similarity index 100% rename from services/agents-api/migrations/000080_sandbox_generations.sql rename to services/agents-api/migrations/000081_sandbox_generations.sql From c47ac60f47345644d1aebe9e191eb688544793b4 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 22:39:14 +0000 Subject: [PATCH 14/16] Restore current schema after expected partial downgrade refusal --- .../store/archived_cancellation_migration_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/services/agents-api/internal/store/archived_cancellation_migration_test.go b/services/agents-api/internal/store/archived_cancellation_migration_test.go index dbf72c05e..0ffa52a2d 100644 --- a/services/agents-api/internal/store/archived_cancellation_migration_test.go +++ b/services/agents-api/internal/store/archived_cancellation_migration_test.go @@ -3,6 +3,7 @@ package store import ( "database/sql" "os" + "strings" "testing" "github.com/google/uuid" @@ -28,6 +29,13 @@ func TestArchivedCancellationMigrationDoesNotAdoptOldRevocations(t *testing.T) { } if _, err := provider.DownTo(t.Context(), 79); err == nil { t.Fatal("downgrade discarded unsettled cancellation marker") + } else if !strings.Contains(err.Error(), "Cannot remove archived cancellation receipts while cleanup is unsettled") { + t.Fatal("downgrade failed before checking the cancellation marker", err) + } + // Later migrations can already have been reverted before migration 80 + // refuses. Restore the current schema before using current generated queries. + if _, err := provider.Up(t.Context()); err != nil { + t.Fatal(err) } if _, err := w.SettleRuntimeCreation(t.Context(), owner); err != nil { t.Fatal(err) From 38e0d6a152eec37bd4c7b7fc979c9ae6a32b79ef Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 23:15:10 +0000 Subject: [PATCH 15/16] test(node): await peer publication before responsiveness checks --- services/agents-api/internal/sandbox/node/hub_lifetime_test.go | 1 + services/agents-api/internal/sandbox/node/hub_postgres_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/services/agents-api/internal/sandbox/node/hub_lifetime_test.go b/services/agents-api/internal/sandbox/node/hub_lifetime_test.go index 283614a93..d72021c21 100644 --- a/services/agents-api/internal/sandbox/node/hub_lifetime_test.go +++ b/services/agents-api/internal/sandbox/node/hub_lifetime_test.go @@ -194,6 +194,7 @@ func TestSlowOpeningAndClosingLeaveOtherNodesResponsive(t *testing.T) { live := connectRawNode(t, server.URL, fast) defer live.Close() go serveRawInfo(live) + wait(t, func() bool { return hub.Online(fast.NodeID) }) beginRawNode(t, server.URL, slow) select { case <-opening: diff --git a/services/agents-api/internal/sandbox/node/hub_postgres_test.go b/services/agents-api/internal/sandbox/node/hub_postgres_test.go index ac29125e1..38a751663 100644 --- a/services/agents-api/internal/sandbox/node/hub_postgres_test.go +++ b/services/agents-api/internal/sandbox/node/hub_postgres_test.go @@ -113,6 +113,7 @@ func TestHubPostgresBlockedOpeningIsBounded(t *testing.T) { live := connectRawNode(t, server.URL, fast) defer live.Close() go serveRawInfo(live) + wait(t, func() bool { return hub.Online(fast.NodeID) }) start := time.Now() beginRawNode(t, server.URL, slow) var pid uint32 From 54d32b16831872c8af53a8160148a4be9a12b7b7 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 23:28:54 +0000 Subject: [PATCH 16/16] Bound generation snapshot singleton planning within reset deadline --- CONTRIBUTING.md | 8 +- .../internal/db/queries/sandbox_reset.sql | 13 +- .../internal/db/sqlc/sandbox_reset.sql.go | 11 +- .../internal/execution/sandbox_reset_test.go | 8 ++ .../execution/sandbox_snapshot_budget_test.go | 114 ++++++++++++++++++ .../store/sandbox_generations_test.go | 34 ++++++ .../internal/store/sandbox_reset_test.go | 2 + .../store/sandbox_snapshot_plan_test.go | 106 ++++++++++++++++ 8 files changed, 285 insertions(+), 11 deletions(-) create mode 100644 services/agents-api/internal/execution/sandbox_snapshot_budget_test.go create mode 100644 services/agents-api/internal/store/sandbox_snapshot_plan_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f66461e8c..5a1d0cca1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3008,8 +3008,12 @@ replaced; do not carry obsolete compatibility code forward to satisfy this secti every Session transaction: binding, claim/reconciliation, journal/Items/Usage, function callbacks/application receipts and terminal/native continuity. Serialize these short transactions and lease pings; execution transactions have a five-second - deadline including gate and Session-lock waits. Never hold a transaction across - daemon/model work, reconnect the writer or fall back to the pool after lease loss. + deadline including gate and Session-lock waits. Sandbox reset snapshots bound the + deployment relation explicitly to its legal singleton row before resource joins. + Keep this cardinality visible even on fresh databases without statistics: inflated + join estimates can trigger expensive JIT compilation inside the lease deadline. + Never hold a transaction across daemon/model work, reconnect the writer or fall + back to the pool after lease loss. The original Store handles public admission and device/auth maintenance on pooled connections. Execution reads may also use the pool; a read grants no write authority. At startup, reconcile previously claimed work as failed, preserve queued inputs diff --git a/services/agents-api/internal/db/queries/sandbox_reset.sql b/services/agents-api/internal/db/queries/sandbox_reset.sql index 77f30e4a2..a5d2161a7 100644 --- a/services/agents-api/internal/db/queries/sandbox_reset.sql +++ b/services/agents-api/internal/db/queries/sandbox_reset.sql @@ -55,7 +55,8 @@ ORDER BY s.id LIMIT 32; SELECT id FROM projects WHERE tenant_id = $1; -- name: GetSandboxDeploymentSnapshot :one -WITH observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), +WITH deployment AS MATERIALIZED (SELECT * FROM runtime_deployment WHERE singleton = true LIMIT 1), +observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), held AS ( SELECT a.deployment_generation, a.node_id, s.id AS session_id, e.id AS environment_id, false AS pending, (a.state = 'cleanup_pending' OR s.deleted_at IS NOT NULL OR e.status IN ('failed', 'expired') @@ -63,7 +64,7 @@ held AS ( THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= observed.as_of ELSE a.node_id IS NULL AND d.mode <> 'direct' AND a.kept_at <= observed.as_of - interval '1 hour' END) AS cleanup FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id - JOIN sessions s ON s.id = e.session_id CROSS JOIN runtime_deployment d CROSS JOIN observed + JOIN sessions s ON s.id = e.session_id CROSS JOIN deployment d CROSS JOIN observed WHERE a.state <> 'released' UNION ALL SELECT p.deployment_generation, p.node_id, s.id, e.id, true, false @@ -80,7 +81,7 @@ held AS ( THEN 'busy' ELSE 'idle' END AS category, n.name, (h.node_id IS NOT NULL AND NOT COALESCE(n.removed_at IS NULL AND n.connection_id IS NOT NULL AND n.connected_epoch = d.owner_epoch AND n.last_seen_at > observed.as_of - interval '45 seconds', false)) AS offline - FROM held h LEFT JOIN runtime_nodes n ON n.id = h.node_id CROSS JOIN runtime_deployment d CROSS JOIN observed + FROM held h LEFT JOIN runtime_nodes n ON n.id = h.node_id CROSS JOIN deployment d CROSS JOIN observed ), rollout_nodes AS ( SELECT CASE WHEN n.connection_id IS NULL OR n.connected_epoch <> d.owner_epoch OR n.last_seen_at IS NULL OR n.last_seen_at <= observed.as_of - interval '45 seconds' THEN 'unknown' @@ -88,7 +89,7 @@ held AS ( WHEN n.provider_ready AND n.ready_generation = d.generation THEN 'ready' WHEN COALESCE(n.health->>'diagnostic','') <> '' THEN 'failed' ELSE 'unknown' END AS state - FROM runtime_nodes n CROSS JOIN runtime_deployment d CROSS JOIN observed + FROM runtime_nodes n CROSS JOIN deployment d CROSS JOIN observed WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id ), offline AS ( SELECT node_id, name, count(*)::bigint AS resources FROM classified WHERE offline GROUP BY node_id, name @@ -112,4 +113,6 @@ SELECT sqlc.embed(d), 'update_required',(SELECT count(*) FROM rollout_nodes WHERE state='update_required'), 'unknown',(SELECT count(*) FROM rollout_nodes WHERE state='unknown')) ELSE NULL END )::jsonb AS rollout -FROM runtime_deployment d; +FROM runtime_deployment d +WHERE d.singleton = true +LIMIT 1; diff --git a/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go b/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go index 976ebcf2a..a3e0fd7cd 100644 --- a/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go +++ b/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go @@ -51,7 +51,8 @@ func (q *Queries) ForceSandboxReset(ctx context.Context) error { } const getSandboxDeploymentSnapshot = `-- name: GetSandboxDeploymentSnapshot :one -WITH observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), +WITH deployment AS MATERIALIZED (SELECT singleton, installation_id, backend_fingerprint, admission_paused, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, idle_seconds, retention_seconds, generation, mode, e2b_template, e2b_credential, specification, e2b_template_build_status, e2b_template_cpus, e2b_template_memory_mib, e2b_template_root_disk_mib, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit FROM runtime_deployment WHERE singleton = true LIMIT 1), +observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), held AS ( SELECT a.deployment_generation, a.node_id, s.id AS session_id, e.id AS environment_id, false AS pending, (a.state = 'cleanup_pending' OR s.deleted_at IS NOT NULL OR e.status IN ('failed', 'expired') @@ -59,7 +60,7 @@ held AS ( THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= observed.as_of ELSE a.node_id IS NULL AND d.mode <> 'direct' AND a.kept_at <= observed.as_of - interval '1 hour' END) AS cleanup FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id - JOIN sessions s ON s.id = e.session_id CROSS JOIN runtime_deployment d CROSS JOIN observed + JOIN sessions s ON s.id = e.session_id CROSS JOIN deployment d CROSS JOIN observed WHERE a.state <> 'released' UNION ALL SELECT p.deployment_generation, p.node_id, s.id, e.id, true, false @@ -76,7 +77,7 @@ held AS ( THEN 'busy' ELSE 'idle' END AS category, n.name, (h.node_id IS NOT NULL AND NOT COALESCE(n.removed_at IS NULL AND n.connection_id IS NOT NULL AND n.connected_epoch = d.owner_epoch AND n.last_seen_at > observed.as_of - interval '45 seconds', false)) AS offline - FROM held h LEFT JOIN runtime_nodes n ON n.id = h.node_id CROSS JOIN runtime_deployment d CROSS JOIN observed + FROM held h LEFT JOIN runtime_nodes n ON n.id = h.node_id CROSS JOIN deployment d CROSS JOIN observed ), rollout_nodes AS ( SELECT CASE WHEN n.connection_id IS NULL OR n.connected_epoch <> d.owner_epoch OR n.last_seen_at IS NULL OR n.last_seen_at <= observed.as_of - interval '45 seconds' THEN 'unknown' @@ -84,7 +85,7 @@ held AS ( WHEN n.provider_ready AND n.ready_generation = d.generation THEN 'ready' WHEN COALESCE(n.health->>'diagnostic','') <> '' THEN 'failed' ELSE 'unknown' END AS state - FROM runtime_nodes n CROSS JOIN runtime_deployment d CROSS JOIN observed + FROM runtime_nodes n CROSS JOIN deployment d CROSS JOIN observed WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id ), offline AS ( SELECT node_id, name, count(*)::bigint AS resources FROM classified WHERE offline GROUP BY node_id, name @@ -109,6 +110,8 @@ SELECT d.singleton, d.installation_id, d.backend_fingerprint, d.admission_paused 'unknown',(SELECT count(*) FROM rollout_nodes WHERE state='unknown')) ELSE NULL END )::jsonb AS rollout FROM runtime_deployment d +WHERE d.singleton = true +LIMIT 1 ` type GetSandboxDeploymentSnapshotRow struct { diff --git a/services/agents-api/internal/execution/sandbox_reset_test.go b/services/agents-api/internal/execution/sandbox_reset_test.go index c90038f4e..f2fcca3b5 100644 --- a/services/agents-api/internal/execution/sandbox_reset_test.go +++ b/services/agents-api/internal/execution/sandbox_reset_test.go @@ -22,6 +22,11 @@ import ( // The execution lease is database-scoped, so this manager test owns a database. func resetManagerStore(t *testing.T) (*store.Store, *store.ExecutionLease) { + t.Helper() + return resetManagerStoreConfig(t, nil) +} + +func resetManagerStoreConfig(t *testing.T, configure func(*pgxpool.Config)) (*store.Store, *store.ExecutionLease) { t.Helper() url := os.Getenv("OAC_TEST_DATABASE_URL") if url == "" { @@ -56,6 +61,9 @@ func resetManagerStore(t *testing.T) (*store.Store, *store.ExecutionLease) { if err != nil { t.Fatal(err) } + if configure != nil { + configure(cfg) + } pool, err := pgxpool.NewWithConfig(t.Context(), cfg) if err != nil { t.Fatal(err) diff --git a/services/agents-api/internal/execution/sandbox_snapshot_budget_test.go b/services/agents-api/internal/execution/sandbox_snapshot_budget_test.go new file mode 100644 index 000000000..339f18f5c --- /dev/null +++ b/services/agents-api/internal/execution/sandbox_snapshot_budget_test.go @@ -0,0 +1,114 @@ +package execution + +import ( + "context" + "runtime/debug" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/adminaudit" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/node" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +type snapshotBudgetKey struct{} +type snapshotBudgetQuery struct { + started time.Time + ordinal int64 +} +type snapshotBudget struct { + t *testing.T + armed atomic.Bool + reads atomic.Int64 +} + +func (d *snapshotBudget) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context { + q := snapshotBudgetQuery{started: time.Now()} + if d.armed.Load() && strings.Contains(data.SQL, "GetSandboxDeploymentSnapshot") && strings.Contains(string(debug.Stack()), "runtimeManager).resetPage") { + q.ordinal = d.reads.Add(1) + } + return context.WithValue(ctx, snapshotBudgetKey{}, q) +} +func (d *snapshotBudget) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + q, _ := ctx.Value(snapshotBudgetKey{}).(snapshotBudgetQuery) + if q.ordinal == 0 { + return + } + d.t.Logf("snapshot=%d query_elapsed=%s err=%v ctx=%v", q.ordinal, time.Since(q.started), data.Err, ctx.Err()) + // Model scheduling pressure on the first two reads without changing the real + // five-second page deadline. The final snapshot uses the lease connection. + if q.ordinal <= 2 && data.Err == nil { + delay := 2200*time.Millisecond - time.Since(q.started) + if delay > 0 { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-timer.C: + case <-ctx.Done(): + } + } + } +} + +func TestSandboxResetSnapshotFitsPageBudget(t *testing.T) { + budget := &snapshotBudget{t: t} + s, lease := resetManagerStoreConfig(t, func(cfg *pgxpool.Config) { + cfg.ConnConfig.RuntimeParams["jit"] = "on" + cfg.ConnConfig.Tracer = budget + }) + w := lease.Store() + id := uuid.NewString() + if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + t.Fatal(err) + } + selection := store.SandboxDeploymentSetupRequest{Provider: "e2b", E2B: &store.SandboxE2BConfiguration{APIKey: "fixture-key", Template: "runtime:" + uuid.NewString()}} + selection.Resources.CPUs = 2 + selection.Resources.MemoryMiB = 2048 + if _, err := w.InitializeSandboxDeployment(t.Context(), id, selection); err != nil { + t.Fatal(err) + } + hub := node.NewHub(node.HubOptions{}) + defer hub.Close() + configuration := NewDeferredRuntimeProvider(id, func(ctx context.Context) (*RuntimeProvider, error) { + setup, err := s.GetSandboxSetup(ctx) + if err != nil || setup.Provider == "" { + return nil, err + } + return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker")}, nil + }) + m, err := newRuntimeManager(w, gateway.NewRegistry(), configuration) + if err != nil { + t.Fatal(err) + } + defer func() { m.stop(); m.drain() }() + if _, err = m.ensureDeployment(t.Context()); err != nil { + t.Fatal(err) + } + if _, err = w.StartSandboxReset(adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "reset-test", ActorLabel: "operator", RequestID: "request", TraceID: "trace"}), id, store.SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err != nil { + t.Fatal(err) + } + budget.armed.Store(true) + started := time.Now() + err = m.resetStep(t.Context()) + ping := lease.Ping(t.Context()) + t.Logf("reset_elapsed=%s reset_error=%v lease_ping=%v", time.Since(started), err, ping) + if err != nil || ping != nil { + t.Fatal("bounded snapshot lost execution ownership", err, ping) + } + if budget.reads.Load() < 3 { + t.Fatal("did not exercise final leased snapshot") + } + if err := w.CollectSandboxGenerations(t.Context()); err != nil { + t.Fatal("next generation collection lost ownership", err) + } + view, err := s.GetRuntimeDeployment(t.Context()) + if err != nil || view.Generation != 2 || view.Provider != "" || view.Reset != nil { + t.Fatal("reset did not commit", view, err) + } +} diff --git a/services/agents-api/internal/store/sandbox_generations_test.go b/services/agents-api/internal/store/sandbox_generations_test.go index 05e7b6221..714b3ebf2 100644 --- a/services/agents-api/internal/store/sandbox_generations_test.go +++ b/services/agents-api/internal/store/sandbox_generations_test.go @@ -27,6 +27,7 @@ func TestE2BGenerationsRetainOwnershipAndUseCurrentCredential(t *testing.T) { if err != nil || changed.Generation != 2 || changed.OwnerEpoch != view.OwnerEpoch || changed.Rollout.PreviousGenerationSandboxes != 1 || changed.Rollout.State != "settled" { t.Fatal(changed, err) } + assertSandboxSnapshotEquivalent(t, s.pool) ref := sandbox.Reference{TenantID: tenant, EnvironmentID: owner.EnvironmentID, AllocationID: owner.ID} retained, err := s.GetSandboxAllocationSetup(t.Context(), ref) if err != nil || retained.Generation != 1 || retained.E2B.Template != oldTemplate || retained.Specification.Resources.CPUs == input.Resources.CPUs { @@ -173,6 +174,7 @@ func TestPendingPlacementGenerationSurvivesRepeatedUpdates(t *testing.T) { t.Fatal(err) } } + assertSandboxSnapshotEquivalent(t, s.pool) owner := archiveAllocation(t, w, tenant, session, view.InstallationID) if owner.DeploymentGeneration != 1 || owner.NodeID != node.NodeID { t.Fatal("pending allocation rebound to newest generation", owner) @@ -332,3 +334,35 @@ func TestNodeRolloutSeparatesOfflinePinAndTargetReadiness(t *testing.T) { }) } } + +func TestSandboxSnapshotRolloutEquivalence(t *testing.T) { + s, _, view, _ := webSpecificationFixture(t, "docker") + node := specificationNode(t, s, view) + for _, state := range []string{"ready", "failed", "unconfirmed", "offline", "update_required"} { + t.Run(state, func(t *testing.T) { + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET provider_ready=true,ready_generation=1,health='{}',last_seen_at=clock_timestamp(),connected_epoch=(SELECT owner_epoch FROM runtime_deployment) WHERE id=$1`, node.NodeID) + want := SandboxRolloutNodes{} + switch state { + case "ready": + want.Ready = 1 + case "failed": + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET provider_ready=false,health='{"diagnostic":"provider_unavailable"}' WHERE id=$1`, node.NodeID) + want.Failed = 1 + case "unconfirmed": + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET provider_ready=false WHERE id=$1`, node.NodeID) + want.Unknown = 1 + case "offline": + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET last_seen_at=clock_timestamp()-interval '46 seconds',health='{"diagnostic":"provider_unavailable"}' WHERE id=$1`, node.NodeID) + want.Unknown = 1 + case "update_required": + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_deployment SET generation=2`) + want.UpdateRequired = 1 + } + assertSandboxSnapshotEquivalent(t, s.pool) + got, err := s.GetRuntimeDeployment(t.Context()) + if err != nil || got.Rollout.Nodes == nil || *got.Rollout.Nodes != want || got.Rollout.State != "settled" { + t.Fatal("rollout classification changed", got.Rollout, err) + } + }) + } +} diff --git a/services/agents-api/internal/store/sandbox_reset_test.go b/services/agents-api/internal/store/sandbox_reset_test.go index 317d19be5..83e903403 100644 --- a/services/agents-api/internal/store/sandbox_reset_test.go +++ b/services/agents-api/internal/store/sandbox_reset_test.go @@ -78,6 +78,7 @@ func TestSandboxResetAutoUsesStartedWorkAndLockedRecheck(t *testing.T) { t.Fatal(err) } assertResetPartition(t, reset) + assertSandboxSnapshotEquivalent(t, s.pool) if (reset.Reset.Remaining.Busy == 1) != busy { t.Fatalf("wrong busy classification: %+v", reset.Reset.Remaining) } @@ -268,6 +269,7 @@ func TestSandboxResetSnapshotCountsOfflineOwnershipOnce(t *testing.T) { t.Fatal(err) } assertResetPartition(t, view) + assertSandboxSnapshotEquivalent(t, s.pool) want := int64(3) if state == "preparing" { want = 0 diff --git a/services/agents-api/internal/store/sandbox_snapshot_plan_test.go b/services/agents-api/internal/store/sandbox_snapshot_plan_test.go new file mode 100644 index 000000000..feb3e86df --- /dev/null +++ b/services/agents-api/internal/store/sandbox_snapshot_plan_test.go @@ -0,0 +1,106 @@ +package store + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func sandboxSnapshotSQL(t *testing.T) string { + t.Helper() + raw, err := os.ReadFile("../db/queries/sandbox_reset.sql") + if err != nil { + t.Fatal(err) + } + query := strings.Split(string(raw), "-- name: GetSandboxDeploymentSnapshot :one\n")[1] + return strings.TrimSuffix(strings.TrimSpace(strings.ReplaceAll(strings.Split(query, "\n-- name:")[0], "sqlc.embed(d)", "d.*")), ";") +} + +func TestSandboxSnapshotFreshPlan(t *testing.T) { + _, pool := newManagedTestStore(t) + tx, err := pool.Begin(t.Context()) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(t.Context()) + if _, err = tx.Exec(t.Context(), "SET LOCAL jit=on"); err != nil { + t.Fatal(err) + } + var stats int + if err = tx.QueryRow(t.Context(), "SELECT count(*) FROM pg_stats WHERE tablename='runtime_deployment'").Scan(&stats); err != nil || stats != 0 { + t.Fatal("fixture must exercise fresh cardinality", stats, err) + } + query := sandboxSnapshotSQL(t) + var raw []byte + if err = tx.QueryRow(t.Context(), "EXPLAIN (ANALYZE, FORMAT JSON) "+query).Scan(&raw); err != nil { + t.Fatal(err) + } + var plan []map[string]any + if err = json.Unmarshal(raw, &plan); err != nil { + t.Fatal(err) + } + if _, jit := plan[0]["JIT"]; jit { + t.Fatal("fresh singleton snapshot unexpectedly requires JIT", string(raw)) + } + bounded := false + var visit func(any) + visit = func(v any) { + switch v := v.(type) { + case map[string]any: + if v["Subplan Name"] == "CTE deployment" { + bounded = v["Plan Rows"] == float64(1) + } + for _, child := range v { + visit(child) + } + case []any: + for _, child := range v { + visit(child) + } + } + } + visit(plan[0]) + if !bounded { + t.Fatal("planner lost singleton bound", string(raw)) + } + t.Logf("fresh snapshot execution_ms=%v planning_ms=%v; singleton rows=1, no JIT", plan[0]["Execution Time"], plan[0]["Planning Time"]) + // A legal empty singleton table must not fabricate a deployment. + if _, err = tx.Exec(t.Context(), "DELETE FROM runtime_deployment"); err != nil { + t.Fatal(err) + } + var empty any + if err = tx.QueryRow(t.Context(), query).Scan(&empty); err != pgx.ErrNoRows { + t.Fatal("empty deployment produced a row", err) + } +} + +// Compare both complete projections in one MVCC statement and at one as_of, +// including every resource/rollout field. Only the cardinality annotation differs. +func assertSandboxSnapshotEquivalent(t *testing.T, pool *pgxpool.Pool) { + t.Helper() + query := sandboxSnapshotSQL(t) + legacy := strings.Replace(query, "WITH deployment AS MATERIALIZED (SELECT * FROM runtime_deployment WHERE singleton = true LIMIT 1),\nobserved AS", "WITH observed AS", 1) + legacy = strings.ReplaceAll(legacy, "CROSS JOIN deployment d", "CROSS JOIN runtime_deployment d") + legacy = strings.TrimSuffix(legacy, "\nWHERE d.singleton = true\nLIMIT 1") + query = strings.Replace(query, "clock_timestamp() AS as_of", "(SELECT as_of FROM snapshot_time) AS as_of", 1) + legacy = strings.Replace(legacy, "clock_timestamp() AS as_of", "(SELECT as_of FROM snapshot_time) AS as_of", 1) + tx, err := pool.Begin(t.Context()) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(t.Context()) + // Avoid paying the legacy planner's JIT cost in this result-equivalence check. + // The separate fresh-plan test keeps JIT enabled. + if _, err = tx.Exec(t.Context(), "SET LOCAL jit=off"); err != nil { + t.Fatal(err) + } + var same bool + err = tx.QueryRow(t.Context(), "WITH snapshot_time AS MATERIALIZED (SELECT clock_timestamp() AS as_of) SELECT (SELECT to_jsonb(s) FROM ("+query+") s) IS NOT DISTINCT FROM (SELECT to_jsonb(s) FROM ("+legacy+") s)").Scan(&same) + if err != nil || !same { + t.Fatal("singleton annotation changed snapshot", same, err) + } +}