diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7adab337e..5de334571 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -613,8 +613,7 @@ batches. Resource reads need only tenant authorization, not a live Runtime. See the [Template coverage and unresolved semantics](contracts/agents-api/environment-templates.md). SandboxProvider has five operations: Create, GetInfo, Renew, Kill and RunCommand. -Use maintained provider SDKs and thin adapters. Hosted deployments select Docker or -the optional microsandbox profile on the assigned node. +Use maintained provider SDKs and thin adapters. Hosted deployments select one deployment-wide Provider: E2B cloud, or Docker/microsandbox on administrator-owned nodes. Provider initialization creates the sandbox and starts its daemon/harness; RunCommand is for initialization only. Daily execution and Files use Runtime and native or bounded local capabilities. Docker's lack of a native renewable lease @@ -631,13 +630,27 @@ executor process or additional execution architecture is required. Qualify principal/tenant ownership, credentials and connection lifecycle using the pinned client and actual execution; document our transport boundary explicitly. -The Core-managed E2B Provider, including its custom HTTP/Connect and envd protocol -implementation, is retired. User-side E2B tooling uses the official SDK and the -shared Runtime, not an additional execution architecture. The -[E2B guide](services/agents-api/deploy/e2b/README.md) owns packaging and user-managed -allocation, renewal and cleanup. Historical Core-managed E2B acceptance retains -only its original scope; it does not qualify the new enrollment path. Current -public Template acceptance uses Docker. +Core-managed E2B is a separate hosted deployment choice, using the official pinned +Python SDK through a packaged private helper. Do not restore the retired custom +HTTP/Connect or envd implementation. The adapter implements the same five operations; +cloud allocations use direct placement with no synthetic node, while Runtime execution +and file access keep the shared daemon contract. Its immutable Runtime template build +is deployment configuration, not a public Environment Template. Keep the account API +key encrypted in the database, write-only through admin input and absent from helper +arguments, logs, metadata and receipts. SDK connection materials and attempted-create +receipts belong in the private durable provider state directory; never replace missing +state to make cleanup appear successful. Create runs once. Unknown control-plane +outcomes remain blockers even if a listing is empty. Explicit matching-reference +CreateSettled evidence proves that the original initialization cannot mutate further; +confirmed absent compute may then be released. Ordinary 404 responses do not prove it. +The helper's pinned SDK, dependencies and licenses ship with Core; users do not install +Python packages after selecting E2B in Web. Application-managed self_hosted tooling +remains independent and uses the same Runtime. Qualify each changed path using actual +provider and model execution before claiming acceptance. Run `make check-e2b-provider` +with `PARSAR_E2B_SDK_PYTHON` pointing to the pinned SDK environment; the packaged +helper build runs the provider tests as well. The SDK gate also covers the +application-managed launch tests. `make check` covers shared initialization and +managed initialization using only the Python standard library. The independent Docker Provider consumes an immutable Runtime image and retains one caller-owned allocation reference through partial creation and cleanup. Persist @@ -667,14 +680,25 @@ of non-secret matched distribution artifacts for its node installation command; never serve private installation files or arbitrary paths. Node installation reuses the existing node process and Provider configuration, verifies downloaded files, retains private identity and uses a user service. It performs no SSH -installation, Session creation or model call. PostgreSQL owns this immutable selection under -the existing execution lease and deployment lock. Exact retries are idempotent; -a changed selection conflicts. No provider migration or hot reload is implied. +installation, Session creation or model call. PostgreSQL owns a generation-tagged +selection under the existing execution lease and deployment lock. E2B setup needs an +account key and a qualified immutable Runtime template, not node enrollment. Changing +selection requires maintenance, the current generation and verified zero retained or +pending resources. Retire old nodes and enrollment credentials in the same transaction; +keep history. Do not automatically delete resources or retry uncertain writes. Web-managed startup claims the stable installation identity and a new owner epoch even before provider selection. The existing runtime manager stays present and -loads one immutable configuration when selection becomes available, before any -node lifecycle is created. Admission refuses uninitialized hosted work without +loads an immutable configuration for the selected generation. A clean switch pauses +new manager operations, drains old lifecycle calls and loops, commits the new selection +and activates it through the existing loader. Keep the Worker and runtime manager as +single owners. Failed replacement activation stays in maintenance and can be retried; resume only +when the committed generation is active. A locally unavailable provider dependency +keeps hosted admission closed while the existing scan waits for repair; it must not +take the administrator recovery API offline, including on restart. Database and +ownership failures still stop the execution owner. Mutation responses read actual +resource counts in their transaction; omitted accounting must never imply cleanup. +Observation and bootstrap share this selection. Admission refuses uninitialized hosted work without creating Session state. File-managed and Web-managed configuration are mutually exclusive. Node registration, observation and daemon bootstrap reuse existing contracts. Derive Runtime bootstrap and daemon WebSocket addresses from the saved @@ -683,23 +707,23 @@ configuration API a startup snapshot; use the live deployment endpoint in setup. A Core deployment may run without a sandbox provider. When enabled, exactly one -sandbox provider is selected at setup: Docker or microsandbox. Keep both adapters but reject multiple provider entries, +sandbox provider is selected at setup: E2B, Docker or microsandbox. Keep both adapters but reject multiple provider entries, legacy default-provider maps and engine-based placement. Harness selection is independent. The configuration has one installation UUID and one provider kind. A local node has one explicit backend object; a remote-only Core has none. No mixed-provider or engine-based provider route is supported. The execution database pins the selected installation and backend namespace. Under the existing execution lease, startup validates that identity before -reconciling work. Changing an installation or backend requires a previous startup -of the old configuration in maintenance, with new configuration still in -maintenance. Maintenance prevents fresh hosted Sessions and fresh allocations, +reconciling work. File-managed backend changes require starting the old and new configurations in +maintenance. Web-managed changes use the generation-checked maintenance and deployment +endpoints; the Core origin remains unchanged within this operation. Maintenance prevents fresh hosted Sessions and fresh allocations, while retaining known receipts, existing Session use and explicit cleanup. Unreleased allocation receipts include live/stopped compute, snapshots, uncertain operations and pending cleanup; all must be released before a switch. Pending hosted Environments that have not yet received an allocation also block a switch. A failed check reports why and performs no resource deletion or provider change. -After a successful switch, restart the same configuration with maintenance off to -admit new sandboxes. Retain immutable historical allocation ownership; never +After a successful Web switch, explicitly resume through the maintenance endpoint. +File-managed deployments restart the same configuration with maintenance off. Retain immutable historical allocation ownership; never migrate an existing Session to another provider or recreate a released allocation. Fresh adoption of a deployment with unverified retained allocations fails closed. diff --git a/Makefile b/Makefile index e57fb6309..f3a5cf108 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,7 @@ build-agents-api-release: check-agents-api: build-agents-api go test ./services/agents-api/... ./packages/agents-client/... -count=1 PYTHONDONTWRITEBYTECODE=1 python3 services/agents-api/deploy/runtime/initialize_receipt_test.py + PYTHONDONTWRITEBYTECODE=1 python3 services/agents-api/deploy/e2b/managed_init_test.py docker-build-agents-api: ./scripts/build-agents-api-image.sh @@ -126,3 +127,12 @@ check-distribution: build-core-distribution: ./scripts/build-core-distribution.sh + +.PHONY: build-e2b-provider check-e2b-provider +build-e2b-provider: + ./scripts/build-e2b-provider.sh + +# The pinned SDK environment is also tested when building the shipped helper. +check-e2b-provider: + PYTHONDONTWRITEBYTECODE=1 $${PARSAR_E2B_SDK_PYTHON:-python3} -m unittest discover -s services/agents-api/deploy/e2b -p '*_test.py' + PYTHONDONTWRITEBYTECODE=1 $${PARSAR_E2B_SDK_PYTHON:-python3} -m unittest discover -s services/agents-api/tools/e2b-provider -p '*_test.py' diff --git a/apps/web/e2e/fixture-sandbox.mjs b/apps/web/e2e/fixture-sandbox.mjs index 6c6527f7a..58aac8cf0 100644 --- a/apps/web/e2e/fixture-sandbox.mjs +++ b/apps/web/e2e/fixture-sandbox.mjs @@ -38,7 +38,7 @@ export function handleSandboxFixture(request, response, url, sendJson, sendError calls.push({ path, method: request.method, authorization: request.headers.authorization ?? null }); // This fixture represents authenticated console routes, not direct Core administration. if (request.headers.authorization) { sendError(response, 400, "Browser admin credentials are not accepted.", "unexpected_authorization"); return true; } - const deployment = () => ({ installation_id: "fixture-installation", provider, core_url: coreUrl, maintenance: false, owner_epoch: 1 }); + const deployment = () => ({ installation_id: "fixture-installation", provider, core_url: coreUrl, maintenance: false, owner_epoch: 1, generation: provider ? 1 : 0, mode: provider ? "nodes" : "", resources: { allocations: nodes.some((entry) => entry.id === "node-local") ? 1 : 0, pending: 0 } }); if (path.endsWith("/deployment") && request.method === "POST") { let body = ""; request.on("data", (chunk) => { body += chunk; }); diff --git a/apps/web/e2e/sandbox-manager.spec.ts b/apps/web/e2e/sandbox-manager.spec.ts index 1c14c90f9..4b92f6b1d 100644 --- a/apps/web/e2e/sandbox-manager.spec.ts +++ b/apps/web/e2e/sandbox-manager.spec.ts @@ -100,7 +100,8 @@ for (const value of ["node_unavailable", "resource_missing"]) { } test("Chinese actions, diagnostics, and enrollment are translated and language persists", async ({ page, request }) => { - await page.getByLabel("Language / 语言").selectOption("zh"); + await page.getByRole("button", { name: "Language and appearance" }).click(); + await page.getByRole("menuitemradio", { name: "简体中文" }).click(); await request.post(`${fixture}/__fixture/sandbox-diagnostic?value=resource_missing`); await page.getByRole("button", { name: "托管沙箱管理", exact: true }).click(); await page.locator(".sandbox-topology-node").first().click(); @@ -109,9 +110,10 @@ test("Chinese actions, diagnostics, and enrollment are translated and language p await expect(page.getByLabel("一次性注册命令")).toHaveValue(/fixture-once-token/); await expect(page.getByRole("dialog")).toContainText("等待节点连接"); await page.keyboard.press("Escape"); - await page.getByLabel("Language / 语言").selectOption("en"); + await page.getByRole("button", { name: "语言和外观" }).click(); + await page.getByRole("menuitemradio", { name: "English" }).click(); await page.reload(); - await expect(page.getByLabel("Language / 语言")).toHaveValue("en"); + await expect(page.getByRole("button", { name: "Language and appearance" })).toContainText("EN"); await expect(page.getByLabel("One-time enrollment command")).toHaveCount(0); }); @@ -290,7 +292,8 @@ for (const theme of ["light", "dark"]) { test(`Chinese topology and node details render on desktop in ${theme}`, async ({ page }, testInfo) => { await page.setViewportSize({ width: 1440, height: 1000 }); await page.evaluate((theme) => document.documentElement.dataset.theme = theme, theme); - await page.getByLabel("Language / 语言").selectOption("zh"); + await page.getByRole("button", { name: "Language and appearance" }).click(); + await page.getByRole("menuitemradio", { name: "简体中文" }).click(); await page.getByRole("button", { name: "托管沙箱管理", exact: true }).click(); await expect(page.getByRole("region", { name: "沙箱节点", exact: true })).toContainText("可用"); await page.screenshot({ path: testInfo.outputPath(`topology-zh-${theme}.png`), animations: "disabled" }); diff --git a/apps/web/e2e/sandbox-provider.spec.ts b/apps/web/e2e/sandbox-provider.spec.ts new file mode 100644 index 000000000..2bdb523f8 --- /dev/null +++ b/apps/web/e2e/sandbox-provider.spec.ts @@ -0,0 +1,162 @@ +import { expect, test, type Page } from "@playwright/test"; +import type { SandboxDeployment } from "../../../packages/agents-client/src/sandbox-client"; + +const fixture = `http://127.0.0.1:${process.env.AGENTS_FIXTURE_PORT ?? 18092}`; +const template = "runtime:00000000-0000-0000-0000-000000000001"; +function deployment(overrides: Partial = {}): SandboxDeployment { + return { installation_id: "installation", provider: "docker", core_url: "https://core.example", maintenance: false, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 2, pending: 1 }, ...overrides }; +} +async function openManager(page: Page) { + await page.goto("/#sandbox"); + await expect(page.getByRole("heading", { name: "Hosted Sandbox Manager", exact: true })).toBeVisible(); +} +async function e2bForm(page: Page) { + await page.getByLabel("Where to run sandboxes").selectOption("direct"); + await page.getByLabel("E2B API key", { exact: true }).fill("fixture-private-e2b-key"); + await page.getByLabel("Immutable Runtime template", { exact: true }).fill(template); +} +test.beforeEach(async ({ page, request }) => { + await request.post(`${fixture}/__fixture/reset`); + await page.route("**/console/config", (route) => route.fulfill({ json: { sandbox_admin: true, node_installer: true, node_installer_sha256: "a".repeat(64) } })); +}); + +test("E2B setup has no physical node or enrollment and never persists the API key", async ({ page }) => { + let current = deployment({ provider: "", generation: 0, mode: "", resources: { allocations: 0, pending: 0 } }); + const writes: unknown[] = []; + await page.route("**/core/v1/sandbox/deployment", (route) => { + if (route.request().method() === "POST") { + const input = route.request().postDataJSON(); writes.push(input); + current = deployment({ provider: "e2b", generation: 1, mode: "direct", resources: { allocations: 0, pending: 0 }, e2b: { template: input.e2b.template, credential_configured: true } }); + } + return route.fulfill({ json: current }); + }); + await openManager(page); + await e2bForm(page); + await page.getByLabel("Core origin reachable from nodes and guests").fill("https://core.example"); + await page.getByRole("button", { name: "Initialize sandbox deployment" }).click(); + await expect(page.getByText("Saved configuration does not confirm execution readiness.", { exact: false })).toBeVisible(); + await expect(page.getByRole("button", { name: "Add node", exact: true })).toHaveCount(0); + await expect(page.getByRole("heading", { name: "Nodes", exact: true })).toHaveCount(0); + await expect(page.getByLabel("E2B API key", { exact: true })).toHaveCount(0); + expect(writes).toEqual([{ provider: "e2b", core_url: "https://core.example", e2b: { api_key: "fixture-private-e2b-key", template } }]); + expect(await page.evaluate(() => JSON.stringify({ local: { ...localStorage }, session: { ...sessionStorage } }))).not.toContain("fixture-private-e2b-key"); + let nodeReads = 0; + await page.route("**/core/v1/sandbox/nodes", (route) => { nodeReads++; return route.fulfill({ status: 500 }); }); + await page.getByRole("button", { name: "Refresh sandbox state" }).click(); + await expect(page.getByRole("button", { name: "Enter maintenance to change provider" })).toBeEnabled(); + expect(nodeReads).toBe(0); +}); + +test("switching waits for maintenance and verified cleanup, keeps Core origin, then requires explicit resume", async ({ page }) => { + let current = deployment(); + const writes: Array<{ method: string; body: Record }> = []; + await page.route("**/core/v1/sandbox/deployment{,/maintenance}", async (route) => { + const method = route.request().method(); + if (method !== "GET") { + const body = route.request().postDataJSON(); writes.push({ method, body }); + expect(body.expected_generation).toBe(current.generation); + if (method === "PATCH") current = { ...current, maintenance: body.maintenance }; + if (method === "PUT") { + expect(current.maintenance).toBe(true); + expect(current.resources).toEqual({ allocations: 0, pending: 0 }); + current = deployment({ provider: body.provider, generation: 2, mode: "direct", maintenance: true, resources: { allocations: 0, pending: 0 }, e2b: { template: body.e2b.template, credential_configured: true } }); + } + } + await route.fulfill({ json: current }); + }); + await openManager(page); + await expect(page.getByRole("button", { name: "Change provider", exact: true })).toHaveCount(0); + await page.getByRole("button", { name: "Enter maintenance to change provider" }).click(); + await expect(page.getByRole("button", { name: "Change provider", exact: true })).toBeDisabled(); + await expect(page.getByRole("link", { name: "Open Sessions" })).toHaveAttribute("href", "#sessions"); + current = { ...current, resources: { allocations: 0, pending: 1 } }; + await page.getByRole("button", { name: "Check cleanup" }).click(); + await expect(page.getByRole("button", { name: "Change provider", exact: true })).toBeDisabled(); + current = { ...current, resources: { allocations: 0, pending: 0 } }; + await page.getByRole("button", { name: "Check cleanup" }).click(); + await page.getByRole("button", { name: "Change provider", exact: true }).click(); + await e2bForm(page); + await expect(page.getByLabel("Core origin reachable from nodes and guests")).toHaveCount(0); + await page.getByRole("button", { name: "Save provider and stay in maintenance" }).click(); + await expect(page.getByRole("button", { name: "Resume hosted placement" })).toBeEnabled(); + expect(writes).toHaveLength(2); + expect(writes[1]?.body).toMatchObject({ core_url: "https://core.example", expected_generation: 1 }); + expect(current.maintenance).toBe(true); + await page.getByRole("button", { name: "Resume hosted placement" }).click(); + expect(writes[2]).toEqual({ method: "PATCH", body: { maintenance: false, expected_generation: 2 } }); + await expect(page.getByRole("button", { name: "Enter maintenance to change provider" })).toBeVisible(); +}); + +test("uncertain E2B writes clear secrets and require a successful refresh before another write", async ({ page }) => { + let readsFail = false, attempts = 0; + await page.route("**/core/v1/sandbox/deployment", (route) => { + if (route.request().method() === "PUT") { attempts++; return route.fulfill({ status: 503, json: { error: { message: "fixture-private-e2b-key", code: "fixture-private-e2b-key" } } }); } + return readsFail ? route.fulfill({ status: 503 }) : route.fulfill({ json: deployment({ maintenance: true, resources: { allocations: 0, pending: 0 } }) }); + }); + await openManager(page); + await page.getByRole("button", { name: "Change provider", exact: true }).click(); + await e2bForm(page); + await page.getByRole("button", { name: "Save provider and stay in maintenance" }).click(); + await expect(page.getByRole("alert")).toContainText("Refresh sandbox state to confirm"); + await expect(page.getByRole("alert")).not.toContainText("fixture-private-e2b-key"); + await expect(page.getByLabel("E2B API key", { exact: true })).toHaveValue(""); + await expect(page.getByRole("button", { name: "Save provider and stay in maintenance" })).toBeDisabled(); + readsFail = true; + await page.getByRole("button", { name: "Refresh sandbox state" }).click(); + await expect(page.getByRole("button", { name: "Change provider", exact: true })).toBeDisabled(); + readsFail = false; + await page.getByRole("button", { name: "Refresh sandbox state" }).click(); + await expect(page.getByRole("button", { name: "Change provider", exact: true })).toBeEnabled(); + expect(attempts).toBe(1); +}); + +for (const theme of ["light", "dark"]) { + test(`E2B configuration and maintenance fit desktop in ${theme}`, async ({ page }, testInfo) => { + await page.setViewportSize({ width: 1440, height: 1000 }); + await page.route("**/core/v1/sandbox/deployment", (route) => route.fulfill({ json: deployment({ provider: "e2b", mode: "direct", maintenance: true, e2b: { template, credential_configured: true } }) })); + await openManager(page); + await page.evaluate((theme) => document.documentElement.dataset.theme = theme, theme); + await page.getByRole("button", { name: "Language and appearance" }).click(); + await page.getByRole("menuitemradio", { name: "简体中文" }).click(); + await expect(page.getByRole("heading", { name: "部署运行后端", exact: true })).toBeVisible(); + await expect(page.getByText("已配置", { exact: true })).toBeVisible(); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await page.screenshot({ path: testInfo.outputPath(`e2b-provider-${theme}.png`), animations: "disabled" }); + }); +} + +test("failed activation stays in maintenance and requires refresh before an explicit resume retry", async ({ page }) => { + const current = deployment({ provider: "e2b", mode: "direct", maintenance: true, e2b: { template, credential_configured: true }, resources: { allocations: 0, pending: 0 } }); + let attempts = 0; + await page.route("**/core/v1/sandbox/deployment", (route) => route.fulfill({ json: current })); + await page.route("**/core/v1/sandbox/deployment/maintenance", (route) => { + attempts++; + return route.fulfill({ status: 503, json: { error: { message: "Activation failed", code: "sandbox_provider_unavailable" } } }); + }); + await openManager(page); + await page.getByRole("button", { name: "Resume hosted placement" }).click(); + await expect(page.getByRole("alert")).toBeVisible(); + await expect(page.getByRole("button", { name: "Resume hosted placement" })).toBeDisabled(); + await expect(page.getByText("Maintenance is enabled. New sandbox placement is paused.")).toBeVisible(); + await page.getByRole("button", { name: "Refresh sandbox state" }).click(); + await expect(page.getByRole("button", { name: "Resume hosted placement" })).toBeEnabled(); + expect(attempts).toBe(1); +}); + +test("changing location and leaving the configuration clear the transient E2B key", async ({ page }) => { + await page.route("**/core/v1/sandbox/deployment", (route) => route.fulfill({ json: deployment({ maintenance: true, resources: { allocations: 0, pending: 0 } }) })); + await openManager(page); + await page.getByRole("button", { name: "Change provider", exact: true }).click(); + await e2bForm(page); + await page.getByLabel("Immutable Runtime template", { exact: true }).fill("mutable-alias"); + await expect(page.getByText("Enter a template ID and build UUID separated by a colon.")).toBeVisible(); + await expect(page.getByRole("button", { name: "Save provider and stay in maintenance" })).toBeDisabled(); + await page.getByLabel("Where to run sandboxes").selectOption("nodes"); + await page.getByLabel("Where to run sandboxes").selectOption("direct"); + await expect(page.getByLabel("E2B API key", { exact: true })).toHaveValue(""); + await e2bForm(page); + await page.getByRole("button", { name: "Cancel", exact: true }).click(); + await page.getByRole("button", { name: "Change provider", exact: true }).click(); + await page.getByLabel("Where to run sandboxes").selectOption("direct"); + await expect(page.getByLabel("E2B API key", { exact: true })).toHaveValue(""); +}); diff --git a/apps/web/e2e/sandbox-setup.spec.ts b/apps/web/e2e/sandbox-setup.spec.ts index aae90e191..c58cd027d 100644 --- a/apps/web/e2e/sandbox-setup.spec.ts +++ b/apps/web/e2e/sandbox-setup.spec.ts @@ -27,6 +27,7 @@ test("bundled console needs no extra admin key and provides one install command await expect(page.getByLabel("Deployment admin key")).toHaveCount(0); await expect(page.getByLabel("Core origin reachable from nodes and guests")).toBeVisible(); await page.getByLabel("Core origin reachable from nodes and guests").fill("https://core.example"); + await page.getByLabel("Where to run sandboxes").selectOption("nodes"); await page.getByLabel("Sandbox provider").selectOption("docker"); await page.getByRole("button", { name: "Initialize sandbox deployment" }).click(); await page.getByRole("button", { name: "Add node", exact: true }).click(); @@ -45,13 +46,14 @@ test("bundled console needs no extra admin key and provides one install command }); for (const provider of ["docker", "microsandbox"]) { - test(`initial ${provider} setup, enrollment, refresh and immutable selection`, async ({ page, request }) => { + test(`initial ${provider} setup, enrollment and refresh`, async ({ page, request }) => { await page.setViewportSize({ width: 390, height: 844 }); await openSetup(page); const submit = page.getByRole("button", { name: "Initialize sandbox deployment" }); await expect(submit).toBeDisabled(); - await expect(page.getByLabel("Sandbox provider")).toHaveValue(""); + await expect(page.getByLabel("Where to run sandboxes")).toHaveValue(""); await expect(page.getByRole("button", { name: "Add node", exact: true })).toHaveCount(0); + await page.getByLabel("Where to run sandboxes").selectOption("nodes"); await page.getByLabel("Sandbox provider").selectOption(provider); const origin = page.getByLabel("Core origin reachable from nodes and guests"); for (const invalid of ["http://127.0.0.1:8080", "https://localhost", "https://[::1]", "http://core.example", "https://core.example/v1", "https://user:secret@core.example", "https://core.example?key=secret"]) { @@ -82,6 +84,7 @@ for (const provider of ["docker", "microsandbox"]) { test("concurrent setup conflict requires refresh and displays the committed provider", async ({ page, request }) => { await openSetup(page); + await page.getByLabel("Where to run sandboxes").selectOption("nodes"); await page.getByLabel("Sandbox provider").selectOption("docker"); await page.getByLabel("Core origin reachable from nodes and guests").fill("https://core.example"); const winner = { provider: "microsandbox", core_url: "https://other-core.example" }; @@ -99,6 +102,7 @@ test("concurrent setup conflict requires refresh and displays the committed prov test("a lost setup response is not retried and refresh recovers the saved deployment", async ({ page, request }) => { await openSetup(page); + await page.getByLabel("Where to run sandboxes").selectOption("nodes"); await page.getByLabel("Sandbox provider").selectOption("docker"); await page.getByLabel("Core origin reachable from nodes and guests").fill("https://core.example"); let writes = 0; @@ -121,6 +125,7 @@ test("a lost setup response is not retried and refresh recovers the saved deploy test("a failed setup refresh keeps setup disabled until a successful read", async ({ page }) => { await openSetup(page); + await page.getByLabel("Where to run sandboxes").selectOption("nodes"); await page.getByLabel("Sandbox provider").selectOption("docker"); await page.getByLabel("Core origin reachable from nodes and guests").fill("https://core.example"); await page.route("**/core/v1/sandbox/deployment", (route) => route.fulfill({ status: 503, contentType: "application/json", body: JSON.stringify({ error: { message: "Deployment unavailable" } }) })); @@ -128,11 +133,11 @@ test("a failed setup refresh keeps setup disabled until a successful read", asyn await expect(page.getByRole("alert")).toContainText("Refresh sandbox state to confirm"); await page.getByRole("button", { name: "Refresh sandbox state" }).click(); await expect(page.getByRole("alert")).toContainText("The sandbox service is unavailable"); - await expect(page.getByLabel("Sandbox provider")).toBeDisabled(); + await expect(page.getByLabel("Where to run sandboxes")).toBeDisabled(); await page.unroute("**/core/v1/sandbox/deployment"); await page.getByRole("button", { name: "Refresh sandbox state" }).click(); - await expect(page.getByLabel("Sandbox provider")).toBeEnabled(); - await expect(page.getByLabel("Sandbox provider")).toHaveValue(""); + await expect(page.getByLabel("Where to run sandboxes")).toBeEnabled(); + await expect(page.getByLabel("Where to run sandboxes")).toHaveValue(""); }); for (const operation of ["setup", "enrollment"] as const) { @@ -154,6 +159,7 @@ for (const operation of ["setup", "enrollment"] as const) { await page.reload(); await openSetup(page); if (operation === "setup") { + await page.getByLabel("Where to run sandboxes").selectOption("nodes"); await page.getByLabel("Sandbox provider").selectOption("docker"); await page.getByLabel("Core origin reachable from nodes and guests").fill("https://core.example"); await page.getByRole("button", { name: "Initialize sandbox deployment" }).click(); @@ -202,6 +208,7 @@ test("loopback console setup exposes the address and cannot save the automatic d await expect(origin).toBeVisible(); await expect(origin).toHaveValue(new URL(page.url()).origin); await expect(page.getByText("This console address cannot be used by sandbox guests.", { exact: false })).toBeVisible(); + await page.getByLabel("Where to run sandboxes").selectOption("nodes"); await page.getByLabel("Sandbox provider").selectOption("docker"); const submit = page.getByRole("button", { name: "Initialize sandbox deployment" }); await expect(submit).toBeDisabled(); @@ -224,6 +231,7 @@ test("a usable HTTPS console origin initializes without exposing the network fie await page.goto("https://core.example/"); await openSetup(page); await expect(page.getByLabel("Core origin reachable from nodes and guests")).toBeHidden(); + await page.getByLabel("Where to run sandboxes").selectOption("nodes"); await page.getByLabel("Sandbox provider").selectOption("docker"); await page.getByRole("button", { name: "Initialize sandbox deployment" }).click(); await expect(page.getByRole("button", { name: "Add node", exact: true })).toBeVisible(); diff --git a/apps/web/src/features/sandbox/SandboxDeploymentSettings.tsx b/apps/web/src/features/sandbox/SandboxDeploymentSettings.tsx new file mode 100644 index 000000000..ec5fce3b9 --- /dev/null +++ b/apps/web/src/features/sandbox/SandboxDeploymentSettings.tsx @@ -0,0 +1,48 @@ +import { useState } from "react"; +import type { InitializeSandboxDeployment, SandboxDeployment } from "@agents-core-web/agents-client"; +import { useTranslation } from "react-i18next"; +import { sandboxProviderLabel } from "../../lib/sandbox-labels"; +import { SandboxSetup } from "./SandboxSetup"; + +export function SandboxDeploymentSettings({ deployment, disabled, fresh, error, onMaintenance, onUpdate, onRefresh }: { + deployment: SandboxDeployment; + disabled: boolean; + fresh: boolean; + error: string | null; + onMaintenance: (maintenance: boolean) => Promise; + onUpdate: (input: InitializeSandboxDeployment) => Promise; + onRefresh: () => void; +}) { + const { t, i18n } = useTranslation("sandbox"); + const locale = i18n.resolvedLanguage?.startsWith("zh") ? "zh" : "en"; + const [changing, setChanging] = useState(false); + const clean = fresh && deployment.resources?.allocations === 0 && deployment.resources?.pending === 0; + return
+

{t("Deployment provider")}

{t("One provider serves this entire Core deployment. Switching requires maintenance and verified cleanup of all existing execution resources.")}

+
+
{t("Provider")}
{sandboxProviderLabel(deployment.provider, locale)}
+
{t("Allocated resources")}
{deployment.resources?.allocations ?? t("Unknown state")}
+
{t("Pending environments")}
{deployment.resources?.pending ?? t("Unknown state")}
+
+ {deployment.provider === "e2b" ?
+

{t("Core creates E2B sandboxes directly. No node enrollment is needed.")}

+
{t("Immutable Runtime template")}
{deployment.e2b?.template || t("Unknown state")}
{t("E2B credential")}
{t(deployment.e2b?.credential_configured ? "Configured" : "Not configured")}
+

{t("Saved configuration does not confirm execution readiness. Session and Environment state report actual execution.")}

+
: null} + {!fresh ?

{t("Previously loaded state is shown below.")}

: null} + {error ?

{error}

: null} + {!deployment.maintenance ? : <> +

{t("Maintenance pauses new hosted placement. Clean up existing execution resources before changing provider; historical Sessions and results are preserved by the switch.")}

+

{t("Stopped or offline resources, snapshots, uncertain creates and pending environments still block switching. Deleting a Session alone does not prove cleanup; Core must confirm both counts are zero.")}

+
{t("Open Sessions")}
+

{t(clean ? "Core reports no remaining execution resources. You can choose the next provider." : "Provider changes are blocked until Core confirms complete cleanup.")}

+ {!changing ? : <> +

{t("Saving retires old node identities and enrollment credentials. It does not migrate Sessions or resume placement automatically.")}

+ + + } + +

{t("Resume is explicit and succeeds only when Core activates the saved configuration. A failed activation leaves maintenance enabled.")}

+ } +
; +} diff --git a/apps/web/src/features/sandbox/SandboxManagerView.css b/apps/web/src/features/sandbox/SandboxManagerView.css index dca095be8..4ae7ff692 100644 --- a/apps/web/src/features/sandbox/SandboxManagerView.css +++ b/apps/web/src/features/sandbox/SandboxManagerView.css @@ -80,3 +80,10 @@ } .sandbox-add-node .sr-only { position: absolute; width: 1px; height: 1px; padding: 0; margin: -1px; overflow: hidden; clip-path: inset(50%); white-space: nowrap; border: 0; } + +.sandbox-provider-settings { padding: 24px; border: 1px solid var(--line); border-radius: 12px; gap: 16px; } +.sandbox-provider-settings h2 { margin: 0 0 8px; } +.sandbox-provider-settings p, .sandbox-enrollment > p { font-size: 13px; line-height: 1.6; color: var(--fg-muted); } +.sandbox-provider-settings > .button, .sandbox-enrollment > .button { align-self: flex-start; } +.sandbox-cloud-summary { display: grid; gap: 16px; } +.sandbox-provider-settings .sandbox-enrollment { max-width: none; } diff --git a/apps/web/src/features/sandbox/SandboxManagerView.tsx b/apps/web/src/features/sandbox/SandboxManagerView.tsx index 0c8abe9ef..0713a0a4f 100644 --- a/apps/web/src/features/sandbox/SandboxManagerView.tsx +++ b/apps/web/src/features/sandbox/SandboxManagerView.tsx @@ -3,11 +3,12 @@ import { SandboxAdminClient, type InitializeSandboxDeployment, type SandboxAlloc import { RefreshCw, Server } from "lucide-react"; import { useTranslation } from "react-i18next"; import { isLocalProxyBaseUrl } from "../../lib/connection"; -import { sandboxRequestError } from "../../lib/sandbox-labels"; +import { sandboxProviderLabel, sandboxRequestError } from "../../lib/sandbox-labels"; import { SandboxTopology } from "./SandboxTopology"; import { SandboxNodeCard } from "./SandboxNodeCard"; import { sandboxConsoleConfig, type SandboxConsoleConfig } from "./console-config"; import { SandboxSetup } from "./SandboxSetup"; +import { SandboxDeploymentSettings } from "./SandboxDeploymentSettings"; import { NodeEnrollment } from "./NodeEnrollment"; import "./SandboxManagerView.css"; @@ -54,6 +55,7 @@ function SandboxManager({ consoleConfig, presentation }: { consoleConfig: Sandbo const revealNode = useCallback((id: string) => { setSelectedId(id); setRemoveId(null); }, []); const initialCoreUrl = window.location.origin; const [setupNeedsRefresh, setSetupNeedsRefresh] = useState(false); + const [mutationError, setMutationError] = useState(null); const lifetime = useRef(null); useEffect(() => { const controller = new AbortController(); lifetime.current = controller; @@ -61,9 +63,10 @@ function SandboxManager({ consoleConfig, presentation }: { consoleConfig: Sandbo }, []); useEffect(() => { const controller = new AbortController(); - setLoading(true); setError(null); + setLoading(true); setError(null); setMutationError(null); setFresh(false); void (async () => { - const [deployment, nodes] = await Promise.all([client.retrieveDeployment({ signal: controller.signal }), client.listNodes({ signal: controller.signal })]); + const deployment = await client.retrieveDeployment({ signal: controller.signal }); + const nodes = deployment.provider === "e2b" ? { data: [] } : await client.listNodes({ signal: controller.signal }); const allocations = await Promise.all(nodes.data.map((node) => client.listAllocations(node.id, { signal: controller.signal }))); if (!controller.signal.aborted) { setSnapshot({ deployment, nodes: nodes.data, allocations: allocations.flatMap((page) => page.data) }); @@ -73,17 +76,30 @@ function SandboxManager({ consoleConfig, presentation }: { consoleConfig: Sandbo .finally(() => { if (!controller.signal.aborted) setLoading(false); }); return () => controller.abort(); }, [client, revision]); - async function initialize(input: InitializeSandboxDeployment) { + async function changeDeployment(operation: (signal: AbortSignal) => Promise) { const controller = lifetime.current; - if (!controller || busy || loading || setupNeedsRefresh) return; - setBusy(true); setError(null); + if (!controller || busy || loading || setupNeedsRefresh || !fresh) return; + setBusy(true); setMutationError(null); try { - const deployment = await client.initializeDeployment(input, { signal: controller.signal }); - if (!controller.signal.aborted) setSnapshot({ deployment, nodes: [], allocations: [] }); + const deployment = await operation(controller.signal); + if (!controller.signal.aborted) { + setSnapshot((current) => ({ deployment, nodes: deployment.generation === current?.deployment.generation ? current.nodes : [], allocations: deployment.generation === current?.deployment.generation ? current.allocations : [] })); + setSelectedId(null); setRemoveId(null); + } } catch (error) { - if (!controller.signal.aborted) { setSetupNeedsRefresh(true); setError(error); } + if (!controller.signal.aborted) { setSetupNeedsRefresh(true); setMutationError(error); setFresh(false); } } finally { if (!controller.signal.aborted) setBusy(false); } } + function initialize(input: InitializeSandboxDeployment) { + return changeDeployment((signal) => client.initializeDeployment(input, { signal })); + } + function update(input: InitializeSandboxDeployment) { + return changeDeployment((signal) => client.updateDeployment({ ...input, core_url: snapshot!.deployment.core_url, expected_generation: snapshot!.deployment.generation }, { signal })); + } + function maintenance(maintenance: boolean) { + return changeDeployment((signal) => client.setMaintenance({ maintenance, expected_generation: snapshot!.deployment.generation }, { signal })); + } + const writeError = mutationError === null ? null : `${sandboxRequestError(mutationError, locale)} ${t("Refresh sandbox state to confirm whether the change was saved before submitting again.")}`; async function remove() { const controller = lifetime.current; if (!controller || !removeId || busy) return; @@ -96,23 +112,25 @@ function SandboxManager({ consoleConfig, presentation }: { consoleConfig: Sandbo finally { if (!controller.signal.aborted) setBusy(false); } } return
-
{presentation === "home" ?

{t("Node network")}

: <>

{t("Hosted Sandbox Manager")}

{t("Your hosts for running sandboxes.")}

}
+
{presentation === "home" ?

{t(snapshot?.deployment.provider === "e2b" ? "Hosted sandboxes" : "Node network")}

: <>

{t("Hosted Sandbox Manager")}

{t("Manage hosted execution for this Core deployment.")}

}
- {snapshot?.deployment.provider ? : null} + {snapshot?.deployment.provider && snapshot.deployment.provider !== "e2b" ? : null}
{loading && !snapshot ?

{t("Loading sandbox state…")}

: null} {busy ? {t("Saving sandbox change…")} : null} - {error !== null ?

{sandboxRequestError(error, locale)}{setupNeedsRefresh ? ` ${t("Refresh sandbox state to confirm whether setup was saved before submitting again.")}` : ""}{snapshot ? ` ${t("Previously loaded state is shown below.")}` : ""}

: null} + {error !== null ?

{sandboxRequestError(error, locale)}{snapshot ? ` ${t("Previously loaded state is shown below.")}` : ""}

: null} + {snapshot && !snapshot.deployment.provider && writeError ?

{writeError}

: null} {snapshot && !snapshot.deployment.provider ? : null} {presentation === "home" && snapshot && !snapshot.deployment.provider ? : null} {snapshot?.deployment.provider ? <> {snapshot.deployment.maintenance ?

{t("Maintenance is enabled. New sandbox placement is paused.")}

: null} -

{t("Nodes")}

{snapshot.nodes.length}
+ + {snapshot.deployment.provider !== "e2b" ?

{t("Nodes")}

{snapshot.nodes.length}
{snapshot.nodes.length || presentation === "home" ? :

{t("Add your first node")}

{t("No nodes registered. Add a node to provide hosted capacity.")}

} {selectedNode ?
allocation.node_id === selectedNode.id)} stale={!fresh} disabled={busy || loading} confirming={removeId === selectedNode.id} onRemove={() => setRemoveId(selectedNode.id)} onConfirm={() => void remove()} onCancel={() => setRemoveId(null)} />
: null} -
+
: null}
{t("Deployment details")}
-
{t("Provider")}
{snapshot.deployment.provider === "docker" ? "Docker" : "microsandbox"}
+
{t("Provider")}
{sandboxProviderLabel(snapshot.deployment.provider, locale)}
{t("Maintenance")}
{snapshot.deployment.maintenance ? t("Enabled") : t("Off")}
{t("Installation")}
{snapshot.deployment.installation_id}
{t("Core origin")}
{snapshot.deployment.core_url || initialCoreUrl}
diff --git a/apps/web/src/features/sandbox/SandboxSetup.tsx b/apps/web/src/features/sandbox/SandboxSetup.tsx index 291743b41..673f252c7 100644 --- a/apps/web/src/features/sandbox/SandboxSetup.tsx +++ b/apps/web/src/features/sandbox/SandboxSetup.tsx @@ -3,32 +3,54 @@ import type { InitializeSandboxDeployment, SandboxProvider } from "@agents-core- import { useTranslation } from "react-i18next"; import { sandboxSetupOrigin } from "./core-origin"; -export function SandboxSetup({ initialCoreUrl, disabled, onInitialize }: { +export function SandboxSetup({ initialCoreUrl, disabled, onInitialize, switching = false }: { initialCoreUrl: string; disabled: boolean; onInitialize: (input: InitializeSandboxDeployment) => Promise; + switching?: boolean; }) { const { t } = useTranslation("sandbox"); + const [location, setLocation] = useState<"" | "nodes" | "direct">(""); const [provider, setProvider] = useState(""); const [coreUrl, setCoreUrl] = useState(initialCoreUrl); + const [apiKey, setApiKey] = useState(""); + const [template, setTemplate] = useState(""); const origin = sandboxSetupOrigin(coreUrl); const needsNetworkAddress = !sandboxSetupOrigin(initialCoreUrl); - function submit(event: FormEvent) { + const validTemplate = /^[a-zA-Z0-9_-]+:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(template.trim()); + const validE2B = provider !== "e2b" || (apiKey.trim() && validTemplate); + async function submit(event: FormEvent) { event.preventDefault(); - if (provider && origin && !disabled) void onInitialize({ provider, core_url: origin }); + if (!provider || !origin || !validE2B || disabled) return; + try { + await onInitialize({ provider, core_url: switching ? initialCoreUrl : origin, ...(provider === "e2b" ? { e2b: { api_key: apiKey.trim(), template: template.trim() } } : {}) }); + } finally { setApiKey(""); } } - return
-

{t("Set up hosted sandboxes")}

-

{t("Choose a provider for your nodes. This choice cannot be changed later.")}

- -
{t("Advanced network settings")} - {needsNetworkAddress ?

{t("This console address cannot be used by sandbox guests. Enter the HTTPS Core address that your nodes and guests can reach.")}

: null} - -

{t("Use an HTTPS Core origin reachable from every node and sandbox guest, without a path or credentials. The console URL may be different.")}

- {coreUrl && !origin ?

{t("Enter a non-loopback HTTPS origin, such as https://core.example.")}

: null} -
- + {location === "nodes" ? : null} + {location === "direct" ? <> +

{t("Core creates E2B sandboxes directly. No node enrollment is needed.")}

+ +

{t("The key is write-only. It is cleared from this form after submission and never saved in browser storage.")}

+ +

{template && !validTemplate ? t("Enter a template ID and build UUID separated by a colon.") : null}

+

{t("Use the qualified managed Runtime template ID followed by its build UUID. A template alias or a generic E2B template cannot provide Core execution.")}

+ : null} + {switching ?

{t("Core origin stays unchanged")}: {initialCoreUrl}

:
{t("Advanced network settings")} + {needsNetworkAddress ?

{t("This console address cannot be used by sandbox guests. Enter the HTTPS Core address that your nodes and guests can reach.")}

: null} + +

{t("Use an HTTPS Core origin reachable from every node and sandbox guest, without a path or credentials. The console URL may be different.")}

+ {coreUrl && !origin ?

{t("Enter a non-loopback HTTPS origin, such as https://core.example.")}

: null} +
} +
; } diff --git a/apps/web/src/lib/locale-strings.ts b/apps/web/src/lib/locale-strings.ts index 5405e77b5..52dcb91d8 100644 --- a/apps/web/src/lib/locale-strings.ts +++ b/apps/web/src/lib/locale-strings.ts @@ -35,7 +35,6 @@ export const chinese = { "Connection status unavailable. Refresh to check your node.": "连接状态暂不可用,请刷新以查看节点。", "Generate a new command to continue.": "请生成新命令后继续。", "Generate new command": "生成新命令", - "Choose a provider for your nodes. This choice cannot be changed later.": "为节点选择运行后端,此选项之后无法更改。", "Advanced network settings": "高级网络设置", "Hosted Sandbox Manager": "托管沙箱管理", @@ -155,6 +154,42 @@ export const chinese = { "Initialize sandbox deployment": "初始化沙箱部署", "Copied": "已复制", "Copy node command": "复制节点命令", - "Select the command above and copy it manually.": "请选择上方命令并手动复制。" + "Select the command above and copy it manually.": "请选择上方命令并手动复制。", + "Choose the next provider": "选择下一个运行后端", + "One provider serves this entire Core deployment. Switching requires maintenance and verified cleanup of all existing execution resources.": "整个 Core 部署使用一个运行后端。切换前需进入维护模式,并确认所有现有执行资源已清理。", + "Where to run sandboxes": "沙箱运行位置", + "Choose where to run": "选择运行位置", + "E2B cloud": "E2B 云端", + "Own machines": "自有机器", + "Core creates E2B sandboxes directly. No node enrollment is needed.": "Core 直接创建 E2B 沙箱,无需注册节点。", + "E2B API key": "E2B API 密钥", + "The key is write-only. It is cleared from this form after submission and never saved in browser storage.": "密钥仅供写入,提交后会从表单清除,不会保存在浏览器存储中。", + "Enter a template ID and build UUID separated by a colon.": "请输入模板 ID 和构建 UUID,并用冒号分隔。", + "Immutable Runtime template": "固定版本 Runtime 模板", + "Use the qualified managed Runtime template ID followed by its build UUID. A template alias or a generic E2B template cannot provide Core execution.": "请填写已验证的托管 Runtime 模板 ID 和构建 UUID。模板别名或通用 E2B 模板无法提供 Core 执行能力。", + "Core origin stays unchanged": "Core 地址保持不变", + "Save provider and stay in maintenance": "保存运行后端并保持维护模式", + "Deployment provider": "部署运行后端", + "Allocated resources": "已分配资源", + "Pending environments": "待处理环境", + "E2B credential": "E2B 凭据", + "Configured": "已配置", + "Not configured": "未配置", + "Saved configuration does not confirm execution readiness. Session and Environment state report actual execution.": "配置已保存不代表执行已就绪。实际执行情况以会话和环境状态为准。", + "Enter maintenance to change provider": "进入维护模式以更改运行后端", + "Maintenance pauses new hosted placement. Clean up existing execution resources before changing provider; historical Sessions and results are preserved by the switch.": "维护模式会暂停新的托管分配。更改运行后端前请清理现有执行资源;切换会保留历史会话和结果。", + "Stopped or offline resources, snapshots, uncertain creates and pending environments still block switching. Deleting a Session alone does not prove cleanup; Core must confirm both counts are zero.": "已停止或离线的资源、快照、创建结果未确认的资源和待处理环境均会阻止切换。删除会话并不代表清理完成,需由 Core 确认两项计数均为零。", + "Open Sessions": "打开会话", + "Check cleanup": "检查清理状态", + "Core reports no remaining execution resources. You can choose the next provider.": "Core 确认已无剩余执行资源,可以选择下一个运行后端。", + "Provider changes are blocked until Core confirms complete cleanup.": "Core 确认清理完成前无法更改运行后端。", + "Change provider": "更改运行后端", + "Saving retires old node identities and enrollment credentials. It does not migrate Sessions or resume placement automatically.": "保存后旧节点身份和注册凭据将失效。不会迁移会话或自动恢复分配。", + "Cancel": "取消", + "Resume hosted placement": "恢复托管分配", + "Resume is explicit and succeeds only when Core activates the saved configuration. A failed activation leaves maintenance enabled.": "需手动恢复分配,且仅在 Core 成功激活已保存配置后生效。激活失败时保持维护模式。", + "Refresh sandbox state to confirm whether the change was saved before submitting again.": "再次提交前,请刷新沙箱状态以确认更改是否已保存。", + "Hosted sandboxes": "托管沙箱", + "Manage hosted execution for this Core deployment.": "管理此 Core 部署的托管执行。", } as const; export type MessageKey = keyof typeof chinese; diff --git a/apps/web/src/lib/sandbox-labels.ts b/apps/web/src/lib/sandbox-labels.ts index f0297773c..04e318770 100644 --- a/apps/web/src/lib/sandbox-labels.ts +++ b/apps/web/src/lib/sandbox-labels.ts @@ -1,4 +1,4 @@ -import { AgentCoreError, type SandboxNode } from "@agents-core-web/agents-client"; +import { AgentCoreError, type SandboxNode, type SandboxProvider } from "@agents-core-web/agents-client"; import { translate, type Locale } from "./locale"; import type { MessageKey } from "./locale-strings"; @@ -13,7 +13,8 @@ export function sandboxStateLabel(state: string, locale: Locale): string { export function sandboxRequestError(error: unknown, locale: Locale): string { let key: MessageKey = "The sandbox request failed. Refresh to check the current state before trying again."; if (error instanceof AgentCoreError) { - if (error.code === "runtime_node_in_use") key = "The node has active allocations or retained resources. Clear allocations, snapshots, reservations and pending cleanup before removal."; + if (error.code === "sandbox_configuration_unconfirmed") key = "The sandbox request failed. Refresh to check the current state before trying again."; + else if (error.code === "runtime_node_in_use") key = "The node has active allocations or retained resources. Clear allocations, snapshots, reservations and pending cleanup before removal."; else if (error.code === "runtime_node_unavailable") key = "The selected sandbox node is unavailable or has no capacity."; else if (error.code === "sandbox_deployment_conflict") key = "Sandbox deployment is already configured. Refresh to inspect the saved provider and Core origin."; else if (error.status === 401) key = "Sign in to the console again to access sandbox management."; @@ -27,3 +28,9 @@ export function sandboxRequestError(error: unknown, locale: Locale): string { export function sandboxNodeStatus(node: SandboxNode, stale: boolean, locale: Locale): string { return translate(locale, stale ? "Status unconfirmed" : !node.online ? "Offline" : node.provider_ready ? "Available" : "Unavailable"); } + +export function sandboxProviderLabel(provider: SandboxProvider | "", locale: Locale): string { + if (provider === "docker") return "Docker"; + if (provider === "microsandbox") return "microsandbox"; + return translate(locale, provider === "e2b" ? "E2B cloud" : "Unknown state"); +} diff --git a/contracts/agents-api/sandbox-manager.openapi.yaml b/contracts/agents-api/sandbox-manager.openapi.yaml index 46c436c92..26ae534b8 100644 --- a/contracts/agents-api/sandbox-manager.openapi.yaml +++ b/contracts/agents-api/sandbox-manager.openapi.yaml @@ -30,6 +30,33 @@ definitions: $ref: '#/definitions/store.RuntimeNodeAllocation' type: array type: object + api.SandboxDeploymentChangeInput: + properties: + core_url: + type: string + e2b: + $ref: '#/definitions/api.SandboxE2BInput' + expected_generation: + type: integer + provider: + type: string + type: object + api.SandboxDeploymentInput: + properties: + core_url: + type: string + e2b: + $ref: '#/definitions/api.SandboxE2BInput' + provider: + type: string + type: object + api.SandboxE2BInput: + properties: + api_key: + type: string + template: + type: string + type: object api.SandboxEnrollmentToken: properties: expires_at: @@ -96,14 +123,22 @@ definitions: properties: core_url: type: string + e2b: + $ref: '#/definitions/store.SandboxE2BView' + generation: + type: integer installation_id: type: string maintenance: type: boolean + mode: + type: string owner_epoch: type: integer provider: type: string + resources: + $ref: '#/definitions/store.SandboxDeploymentResources' type: object store.RuntimeNode: properties: @@ -217,13 +252,27 @@ definitions: name: type: string type: object - store.SandboxDeploymentSetupRequest: + store.SandboxDeploymentResources: properties: - core_url: - type: string - provider: + allocations: + type: integer + pending: + type: integer + type: object + store.SandboxE2BView: + properties: + credential_configured: + type: boolean + template: type: string type: object + store.SandboxMaintenanceRequest: + properties: + expected_generation: + type: integer + maintenance: + type: boolean + type: object v1.APIError: properties: code: @@ -512,14 +561,101 @@ paths: post: consumes: - application/json - description: Selects one immutable provider and public Core origin. Exact retries return the existing selection; differing selections and file-managed deployments reject. This does not create a node or execute work. + description: Selects a provider and public Core origin. E2B credentials are write-only. Exact retries return the existing selection; differing selections and file-managed deployments reject. This does not create compute or execute work. parameters: - description: Deployment selection in: body name: body required: true schema: - $ref: '#/definitions/store.SandboxDeploymentSetupRequest' + $ref: '#/definitions/api.SandboxDeploymentInput' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/store.RuntimeDeploymentView' + "400": + description: Bad Request + schema: + $ref: '#/definitions/v1.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/v1.ErrorResponse' + "409": + description: Conflict + schema: + $ref: '#/definitions/v1.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/v1.ErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/v1.ErrorResponse' + security: + - DeploymentAdminAuth: [] + summary: Initialize the deployment sandbox provider + tags: + - Sandbox Manager + put: + consumes: + - application/json + description: Requires maintenance, the current generation and verified cleanup of all old resources. Credentials are write-only. The public Core origin stays unchanged. Historical records are retained; old node credentials and enrollments are retired. Explicitly resume after success. Never automatically retry an uncertain write. + parameters: + - description: Replacement deployment selection + in: body + name: body + required: true + schema: + $ref: '#/definitions/api.SandboxDeploymentChangeInput' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/store.RuntimeDeploymentView' + "400": + description: Bad Request + schema: + $ref: '#/definitions/v1.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/v1.ErrorResponse' + "409": + description: Conflict + schema: + $ref: '#/definitions/v1.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/v1.ErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/v1.ErrorResponse' + security: + - DeploymentAdminAuth: [] + summary: Change a fully drained deployment's sandbox provider + tags: + - Sandbox Manager + /core/v1/sandbox/deployment/maintenance: + patch: + consumes: + - application/json + description: Requires the current generation. Maintenance preserves execution, cleanup and history. Resume requires successfully activated provider configuration. + parameters: + - description: Maintenance state + in: body + name: body + required: true + schema: + $ref: '#/definitions/store.SandboxMaintenanceRequest' produces: - application/json responses: @@ -549,7 +685,7 @@ paths: $ref: '#/definitions/v1.ErrorResponse' security: - DeploymentAdminAuth: [] - summary: Initialize the deployment sandbox provider once + summary: Pause or resume new hosted allocations tags: - Sandbox Manager /core/v1/sandbox/enroll: diff --git a/deploy/distribution/Dockerfile b/deploy/distribution/Dockerfile index 9da812463..8125b3506 100644 --- a/deploy/distribution/Dockerfile +++ b/deploy/distribution/Dockerfile @@ -1,5 +1,5 @@ -# Core is independent of execution payloads. Optional native Provider artifacts -# are downloaded separately; the Docker Provider is compiled into the API. +# Core carries its cloud control-plane SDK helper. Node and Runtime payloads +# remain independently downloaded artifacts. FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates libgcc-s1 \ @@ -7,7 +7,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates COPY --chmod=0555 bin/agents-api bin/agents-api-migrate bin/agents-api-device bin/agents-api-environment-key /usr/local/bin/ +COPY e2b/ /opt/parsar/e2b/ + ENV AGENTS_API_ADDR=:8091 +ENV AGENTS_API_E2B_PROVIDER_BIN=/opt/parsar/e2b/agents-api-e2b-provider EXPOSE 8091 USER 65532:65532 CMD ["/usr/local/bin/agents-api"] diff --git a/deploy/install/configuration.py b/deploy/install/configuration.py index 56e278e21..55bada124 100644 --- a/deploy/install/configuration.py +++ b/deploy/install/configuration.py @@ -55,6 +55,9 @@ def core_environment(root, state, database_password): "AGENTS_API_ADDR": f'127.0.0.1:{state["core_port"]}' if native else ":8091", "AGENTS_API_ENGINE": "codex", "AGENTS_API_HARNESSES": "codex,claude_sdk,mcode", "AGENTS_API_DAEMON_WS_URL": daemon_url, + "AGENTS_API_E2B_PROVIDER_BIN": (str(Path(root) / "native/e2b/agents-api-e2b-provider") + if native else "/opt/parsar/e2b/agents-api-e2b-provider"), + "AGENTS_API_E2B_STATE_DIR": str(Path(root) / "state/e2b") if native else "/state/e2b", } result["AGENTS_API_SANDBOX_ADMIN_DIGESTS_FILE"] = ( str(Path(root) / "admin/digests.json") if native else "/admin/digests.json") @@ -100,6 +103,7 @@ def compose_config(root, state, manifest, database_password): services.pop("migrate") else: core["volumes"].append(bind(root / "admin/digests.json", "/admin/digests.json")) + core["volumes"].append(bind(root / "state/e2b", "/state/e2b", False)) if state["provider"] == "docker": core["volumes"].append(bind("/var/run/docker.sock", "/var/run/docker.sock", False)) core["volumes"].append(bind(root / "state/sandbox-node", "/state/sandbox-node", False)) diff --git a/deploy/install/install.py b/deploy/install/install.py index 2bd987b30..5f88ff4c5 100644 --- a/deploy/install/install.py +++ b/deploy/install/install.py @@ -67,6 +67,7 @@ def verify_bundle(bundle): required = {"manifest.json", "install.sh", "install.py", "configuration.py", "native_service.py", "node-install.pyz", "self-hosted-install.pyz", "distribution.py", "runtime/seccomp.json"} required.update(f"images/{name}.tar" for name in ("core", "web", "database")) required.update("native/bin/" + name for name in ("agents-api", "agents-api-migrate")) + required.add("native/e2b/agents-api-e2b-provider") if not required.issubset(covered): raise InstallError("Distribution checksum list is incomplete") manifest = json.loads((bundle / "manifest.json").read_text()) @@ -213,6 +214,11 @@ def initialize(root, args, manifest): managed = json.loads((root / "config/managed-runtimes.json").read_text()) if managed.get("docker", {}).get("image") != manifest["images"]["runtime"]: raise InstallError("Retained Runtime image differs; preserve the installation and inspect its configuration") + if mode != "web-only": + directory = root / "state/e2b" + if (not directory.is_dir() or directory.is_symlink() or + stat.S_IMODE(directory.stat().st_mode) & 0o077): + raise InstallError("Private provider receipts are missing or unsafe; restore the retained installation") if state.get("console_auth") == "account": # Never let Compose create replacement bind sources for lost auth # state. An existing install must retain its account directory. @@ -247,8 +253,9 @@ def initialize(root, args, manifest): directories = ["config"] if mode != "web-only": directories.append("admin") - if args.provider or mode != "core-only": - directories.append("state") + directories.append("state") + if mode != "web-only": + directories.append("state/e2b") if args.provider: directories.append("state/sandbox-node") if mode != "core-only": diff --git a/deploy/install/native_service.py b/deploy/install/native_service.py index dd6801e7e..69ddbf96e 100644 --- a/deploy/install/native_service.py +++ b/deploy/install/native_service.py @@ -11,7 +11,8 @@ REQUIRED = ("bin/agents-api", "bin/agents-api-microsandbox-provider", - "microsandbox/msb", "microsandbox/libkrunfw.so.5.6.1") + "microsandbox/msb", "microsandbox/libkrunfw.so.5.6.1", + "e2b/agents-api-e2b-provider") def is_native(state): @@ -149,7 +150,7 @@ def prepare(root, state, bundle, environment): shutil.copytree(source, staged) os.replace(staged, target) for path in [target, *target.rglob("*")]: - executable = path.is_dir() or path.parent == target / "bin" or path == target / "microsandbox/msb" + executable = path.is_dir() or path.parent == target / "bin" or path == target / "microsandbox/msb" or path == target / "e2b/agents-api-e2b-provider" os.chmod(path, 0o700 if executable else 0o600) config = root / "config" if config.is_symlink(): diff --git a/deploy/install/test_install.py b/deploy/install/test_install.py index 7b31bb535..6761902c4 100644 --- a/deploy/install/test_install.py +++ b/deploy/install/test_install.py @@ -118,10 +118,10 @@ def bundle(self): for name in self.manifest["images"]: (bundle / "images" / (name + ".tar")).write_bytes(("synthetic " + name).encode()) for name in ("bin/agents-api", "bin/agents-api-migrate", "bin/agents-api-microsandbox-provider", - "bin/parsar-sandbox-node", "microsandbox/msb", "microsandbox/libkrunfw.so.5.6.1"): + "bin/parsar-sandbox-node", "microsandbox/msb", "microsandbox/libkrunfw.so.5.6.1", "e2b/agents-api-e2b-provider"): path = bundle / "native" / name path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(b"synthetic native file") + path.write_bytes(b"\x7fELFsynthetic native file") self.write_checksums(bundle) return bundle @@ -132,6 +132,19 @@ def write_checksums(bundle): hashlib.sha256(path.read_bytes()).hexdigest() + " " + str(path.relative_to(bundle)) + "\n" for path in files)) + def test_provider_receipts_are_private_durable_core_state(self): + self.initialize() + receipt_dir = self.root / "state/e2b" + self.assertEqual(stat.S_IMODE(receipt_dir.stat().st_mode), 0o700) + services = self.document("compose.json")["services"] + mounts = {item["target"]: item for item in services["core"]["volumes"]} + self.assertEqual(mounts["/state/e2b"]["source"], str(receipt_dir)) + self.assertFalse(mounts["/state/e2b"]["read_only"]) + self.assertFalse(any(item["target"] == "/state/e2b" for item in services["web"]["volumes"])) + receipt_dir.rmdir() + with self.assertRaisesRegex(install.InstallError, "provider receipts"): + self.initialize() + def test_thin_bundle_verifies_without_downloading_runtime(self): bundle = self.bundle() for name in ("images/runtime.tar", "native/bin/parsar-sandbox-node", @@ -219,7 +232,7 @@ def test_legacy_installation_keeps_basic_credentials_and_compose(self): state = self.initialize() state.pop("console_auth") install.private_write(self.root / "config/console.password", "retained-console-password") - shutil.rmtree(self.root / "state") + shutil.rmtree(self.root / "state/console") (self.root / "installation.json").write_text(json.dumps(state)) compose = install.compose_config(self.root, state, self.manifest, "retained-database-password") (self.root / "compose.json").write_text(json.dumps(compose)) diff --git a/deploy/install/test_native_service.py b/deploy/install/test_native_service.py index 5ee445155..00c0096c7 100644 --- a/deploy/install/test_native_service.py +++ b/deploy/install/test_native_service.py @@ -65,6 +65,11 @@ def test_only_core_microsandbox_uses_native_service(self): self.assertFalse(service.active(state)) self.run.assert_not_called() + def test_cloud_helper_remains_executable_but_private(self): + self.prepare() + helper = self.root / "native/e2b/agents-api-e2b-provider" + self.assertEqual(stat.S_IMODE(helper.stat().st_mode), 0o700) + def test_prepare_preserves_direct_core_and_runtime_process_lifetime(self): self.prepare() unit = configparser.ConfigParser(interpolation=None) diff --git a/docs/getting-started/install.md b/docs/getting-started/install.md index e494f2c79..fd7d1d232 100644 --- a/docs/getting-started/install.md +++ b/docs/getting-started/install.md @@ -310,12 +310,23 @@ localhost download command to a different machine. Running plain `./install.sh` is suitable for local console/API inspection; prepare the shared endpoint before installing a deployment that will enroll nodes. +The distribution includes the pinned E2B SDK helper, so selecting E2B does not +require Python or pip installation on the Core host. Its private receipts live in +`state/e2b` and are mounted only into Core. Preserve them together with the database +and credential key when backing up or moving this installation. E2B template +preparation is described in the [E2B guide](../../services/agents-api/deploy/e2b/README.md). + ## After installation Start with an optional [API request](quickstart.md). The read-only example works with the default installation. The execution example requires an installation -with a connected, ready sandbox node, either installed locally or added through -Web. Session creation supplies the model, +with either E2B configured through Web or a connected, ready Docker/microsandbox node. +The Hosted Sandbox Manager selects one scheme for the whole deployment. E2B uses +an account API key and a qualified immutable Runtime template; Core provisions +sandboxes without a node installer. Own machines use the existing node command. +Enter maintenance and finish resource cleanup before switching schemes; see +[provider switching](../../services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-maintenance). +Session creation supplies the model, harness and write-only model credentials. Core owns sandbox preparation and Runtime startup. Configuration is never injected into a public Agent instruction or baked into a Runtime image. diff --git a/internal/agentdaemon/device/credential.go b/internal/agentdaemon/device/credential.go index e2a0377c9..cefa69303 100644 --- a/internal/agentdaemon/device/credential.go +++ b/internal/agentdaemon/device/credential.go @@ -16,6 +16,8 @@ type Credential struct { CredentialHash string // RuntimeNodeID is the persisted managed allocation binding, never caller input. RuntimeNodeID string + // RuntimeAllocationID distinguishes managed cloud compute from self-hosted devices. + RuntimeAllocationID string } // HashCredential preserves the paired runtime bearer format, including trimming diff --git a/internal/agentdaemon/gateway/auth.go b/internal/agentdaemon/gateway/auth.go index bb9c53819..9f0a03c23 100644 --- a/internal/agentdaemon/gateway/auth.go +++ b/internal/agentdaemon/gateway/auth.go @@ -36,10 +36,11 @@ type RuntimeStore interface { // AuthenticatedRuntime is the result of a successful credential check. type AuthenticatedRuntime struct { - DeviceID string - WorkspaceID string - Name string - RuntimeNodeID string + DeviceID string + WorkspaceID string + Name string + RuntimeNodeID string + RuntimeAllocationID string } // Authenticator validates the (device_id, token, version) trio that @@ -85,9 +86,10 @@ func (a *Authenticator) AuthenticateBearer(ctx context.Context, deviceID, bearer return AuthenticatedRuntime{}, ErrAuthBadCredential } return AuthenticatedRuntime{ - DeviceID: rt.ID, - WorkspaceID: rt.WorkspaceID, - Name: rt.Name, - RuntimeNodeID: rt.RuntimeNodeID, + DeviceID: rt.ID, + WorkspaceID: rt.WorkspaceID, + Name: rt.Name, + RuntimeNodeID: rt.RuntimeNodeID, + RuntimeAllocationID: rt.RuntimeAllocationID, }, nil } diff --git a/packages/agents-client/src/sandbox-client.test.ts b/packages/agents-client/src/sandbox-client.test.ts index e9f9f9702..d6bc96cc1 100644 --- a/packages/agents-client/src/sandbox-client.test.ts +++ b/packages/agents-client/src/sandbox-client.test.ts @@ -83,3 +83,42 @@ describe("Core sandbox credential boundaries", () => { expect(fetch).toHaveBeenCalledTimes(1); }); }); + +describe("hosted provider configuration", () => { + const e2b = { api_key: "test-only-secret", template: "runtime:00000000-0000-0000-0000-000000000001" }; + it("writes E2B configuration and generation without beta headers or browser credentials", async () => { + const fetch = vi.fn().mockImplementation(async () => response({ generation: 2, mode: "direct", resources: { allocations: 0, pending: 0 }, e2b: { template: e2b.template, credential_configured: true } })); + const client = new SandboxAdminClient({ baseUrl: "/core/v1/sandbox", fetch }); + const controller = new AbortController(); + await client.initializeDeployment({ provider: "e2b", core_url: "https://core.example", e2b }); + await client.updateDeployment({ provider: "e2b", core_url: "https://core.example", e2b, expected_generation: 1 }, { signal: controller.signal }); + await client.setMaintenance({ maintenance: false, expected_generation: 2 }); + expect(fetch.mock.calls.map(([url, init]) => [url, init?.method])).toEqual([ + ["/core/v1/sandbox/deployment", "POST"], ["/core/v1/sandbox/deployment", "PUT"], ["/core/v1/sandbox/deployment/maintenance", "PATCH"], + ]); + expect(JSON.parse(String(fetch.mock.calls[1]?.[1]?.body))).toEqual({ provider: "e2b", core_url: "https://core.example", e2b, expected_generation: 1 }); + expect(fetch.mock.calls[1]?.[1]?.signal).toBe(controller.signal); + expect(JSON.parse(String(fetch.mock.calls[2]?.[1]?.body))).toEqual({ maintenance: false, expected_generation: 2 }); + for (const [, init] of fetch.mock.calls) { + expect(new Headers(init?.headers).has("Authorization")).toBe(false); + expect(new Headers(init?.headers).has("OpenAI-Beta")).toBe(false); + } + }); + it.each([409, 503])("does not expose reflected E2B keys or retry configuration after HTTP %s", async (status) => { + const fetch = vi.fn().mockResolvedValue(response({ error: { message: e2b.api_key, code: e2b.api_key, param: e2b.api_key } }, status)); + const client = new SandboxAdminClient({ fetch }); + await expect(client.updateDeployment({ provider: "e2b", core_url: "https://core.example", e2b, expected_generation: 1 })).rejects.toMatchObject({ code: "sandbox_configuration_unconfirmed", status }); + await client.initializeDeployment({ provider: "e2b", core_url: "https://core.example", e2b }).catch((error) => { + expect(JSON.stringify(error)).not.toContain(e2b.api_key); + expect(error.message).not.toContain(e2b.api_key); + }); + expect(fetch).toHaveBeenCalledTimes(2); + }); + it("never retries uncertain switch or maintenance writes", async () => { + const fetch = vi.fn().mockRejectedValue(new TypeError("Connection lost")); + const client = new SandboxAdminClient({ fetch }); + await expect(client.updateDeployment({ provider: "docker", core_url: "https://core.example", expected_generation: 1 })).rejects.toThrow(); + await expect(client.setMaintenance({ maintenance: true, expected_generation: 1 })).rejects.toThrow(); + expect(fetch).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/agents-client/src/sandbox-client.ts b/packages/agents-client/src/sandbox-client.ts index 6caf8d0c8..46365aa80 100644 --- a/packages/agents-client/src/sandbox-client.ts +++ b/packages/agents-client/src/sandbox-client.ts @@ -1,10 +1,16 @@ -import { OpenAIAgentsClient } from "./client"; +import { AgentCoreError, OpenAIAgentsClient } from "./client"; import type { ReadOptions } from "./types"; export type SandboxDiagnostic = "" | "node_unavailable" | "resource_missing" | "compute_unconfirmed" | "ownership_mismatch" | "provider_unavailable"; -export type SandboxProvider = "docker" | "microsandbox"; -export interface InitializeSandboxDeployment { provider: SandboxProvider; core_url: string } +export type SandboxProvider = "docker" | "microsandbox" | "e2b"; +export interface InitializeSandboxDeployment { + provider: SandboxProvider; + core_url: string; + e2b?: { api_key: string; template: string }; +} +export interface UpdateSandboxDeployment extends InitializeSandboxDeployment { expected_generation: number } +export interface SetSandboxMaintenance { maintenance: boolean; expected_generation: number } export interface SandboxDeployment { installation_id: string; @@ -12,6 +18,10 @@ export interface SandboxDeployment { core_url: string; maintenance: boolean; owner_epoch: number; + generation: number; + mode: "nodes" | "direct" | ""; + resources: { allocations: number; pending: number }; + e2b?: { template: string; credential_configured: boolean }; } export interface SandboxNode { id: string; @@ -70,7 +80,22 @@ export class SandboxAdminClient extends OpenAIAgentsClient { return this.request("/deployment", { signal: options?.signal }, undefined, false); } initializeDeployment(input: InitializeSandboxDeployment, options?: ReadOptions): Promise { - return this.request("/deployment", { method: "POST", body: JSON.stringify(input), signal: options?.signal }, undefined, false); + return this.writeDeployment("POST", input, options); + } + updateDeployment(input: UpdateSandboxDeployment, options?: ReadOptions): Promise { + return this.writeDeployment("PUT", input, options); + } + setMaintenance(input: SetSandboxMaintenance, options?: ReadOptions): Promise { + return this.request("/deployment/maintenance", { method: "PATCH", body: JSON.stringify(input), signal: options?.signal }, undefined, false); + } + private async writeDeployment(method: "POST" | "PUT", input: InitializeSandboxDeployment | UpdateSandboxDeployment, options?: ReadOptions): Promise { + try { + return await this.request("/deployment", { method, body: JSON.stringify(input), signal: options?.signal }, undefined, false); + } catch (error) { + if (!input.e2b) throw error; + // A credential-bearing rejection may reflect the key in any error field. + throw new AgentCoreError("Sandbox configuration could not be confirmed. Refresh before submitting again.", error instanceof AgentCoreError ? error.status : 0, "sandbox_configuration_unconfirmed"); + } } listNodes(options?: ReadOptions): Promise<{ data: SandboxNode[] }> { return this.request("/nodes", { signal: options?.signal }, undefined, false); diff --git a/scripts/build-agents-api-image.sh b/scripts/build-agents-api-image.sh index 941b3362e..037578b4e 100755 --- a/scripts/build-agents-api-image.sh +++ b/scripts/build-agents-api-image.sh @@ -15,5 +15,10 @@ trap 'rm -rf "$image_context"' EXIT # Reuse the source boundary; never send the repository or runtime keys to Docker. GOOS=linux GOARCH=amd64 AGENTS_API_BUILD_DIR="$image_context" \ "$repo_root/scripts/build-agents-api.sh" +E2B_PROVIDER_BUILD_DIR="$image_context/e2b-build" "$repo_root/scripts/build-e2b-provider.sh" +mkdir -p "$image_context/e2b" +tar -xzf "$image_context/e2b-build/agents-api-e2b-provider-linux-amd64.tar.gz" \ + --strip-components=1 -C "$image_context/e2b" +rm -rf "$image_context/e2b-build" cp "$repo_root/services/agents-api/Dockerfile" "$image_context/Dockerfile" docker build --platform linux/amd64 --tag "$image" "$image_context" diff --git a/scripts/build-core-distribution.sh b/scripts/build-core-distribution.sh index 8d4fc7466..ad6ede28f 100755 --- a/scripts/build-core-distribution.sh +++ b/scripts/build-core-distribution.sh @@ -120,6 +120,11 @@ else fi mkdir -p "$bundle/native" cp -R "$stage/core/bin" "$stage/core/microsandbox" "$bundle/native/" +E2B_SOURCE_REVISION="$revision" E2B_PROVIDER_BUILD_DIR="$stage/e2b-build" scripts/build-e2b-provider.sh +mkdir -p "$stage/core/e2b" +tar -xzf "$stage/e2b-build/agents-api-e2b-provider-linux-amd64.tar.gz" \ + --strip-components=1 -C "$stage/core/e2b" +cp -R "$stage/core/e2b" "$bundle/native/e2b" cp deploy/distribution/Dockerfile "$stage/core/Dockerfile" build_image --platform linux/amd64 --iidfile "$stage/core.id" \ --label "org.opencontainers.image.revision=$revision" "$stage/core" diff --git a/scripts/build-e2b-provider.sh b/scripts/build-e2b-provider.sh new file mode 100755 index 000000000..c987f02d6 --- /dev/null +++ b/scripts/build-e2b-provider.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 022 +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +output_dir="${E2B_PROVIDER_BUILD_DIR:-${PARSAR_HOME:-$HOME/.parsar}/build/e2b-provider}" +case "$output_dir" in + /*) ;; + *) printf 'E2B_PROVIDER_BUILD_DIR must be absolute\n' >&2; exit 1 ;; +esac +mkdir -p "$output_dir" +source_revision="${E2B_SOURCE_REVISION:-$(git -C "$repo_root" rev-parse HEAD)}" +image="parsar-e2b-provider-build:${source_revision:0:12}" +# Proxy values are build-only operator settings; no account key is needed. +docker build --platform linux/amd64 --build-arg HTTP_PROXY --build-arg HTTPS_PROXY \ + --build-arg ALL_PROXY --build-arg NO_PROXY \ + --file "$repo_root/services/agents-api/tools/e2b-provider/Build.Dockerfile" \ + --tag "$image" "$repo_root/services/agents-api/tools/e2b-provider" +docker run --rm --platform linux/amd64 \ + --env HTTP_PROXY --env HTTPS_PROXY --env ALL_PROXY --env NO_PROXY \ + --env "E2B_SOURCE_REVISION=$source_revision" \ + --mount "type=bind,src=$repo_root,dst=/source,readonly" \ + --mount "type=bind,src=$output_dir,dst=/output" "$image" diff --git a/services/agents-api/Dockerfile b/services/agents-api/Dockerfile index 870279b00..d2d8eb1d6 100644 --- a/services/agents-api/Dockerfile +++ b/services/agents-api/Dockerfile @@ -1,9 +1,13 @@ # Linux amd64 runtime. Build with make docker-build-agents-api. -# The context contains only the independently built static executables. -FROM gcr.io/distroless/static-debian13:nonroot@sha256:e754765ad9e167b0677b41c617fd44afb7b9818a477f48f17bda08e12cfb98cb +# The context contains independently built executables and the pinned cloud SDK helper. +FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates libgcc-s1 \ + && rm -rf /var/lib/apt/lists/* COPY --chmod=0555 agents-api agents-api-migrate agents-api-device agents-api-environment-key parsar-sandbox-node /usr/local/bin/ +COPY e2b/ /opt/parsar/e2b/ ENV AGENTS_API_ADDR=:8091 +ENV AGENTS_API_E2B_PROVIDER_BIN=/opt/parsar/e2b/agents-api-e2b-provider EXPOSE 8091 USER 65532:65532 CMD ["/usr/local/bin/agents-api"] diff --git a/services/agents-api/HOSTED-SANDBOX-MANAGER.md b/services/agents-api/HOSTED-SANDBOX-MANAGER.md index a28e85d4e..67a074f46 100644 --- a/services/agents-api/HOSTED-SANDBOX-MANAGER.md +++ b/services/agents-api/HOSTED-SANDBOX-MANAGER.md @@ -1,8 +1,9 @@ # Hosted Sandbox Manager One Core execution owner can manage sandbox nodes on its own host and on other -Linux hosts. The deployment selects exactly one provider: `docker` or -`microsandbox`. All nodes must use that provider and installation ID. These nodes +Linux hosts, or provision cloud sandboxes directly through E2B. A deployment selects +exactly one provider: `e2b`, `docker` or `microsandbox`. Docker/microsandbox nodes +must all use the selected provider and installation ID. These nodes are distinct from user-managed `self_hosted` Environments, whose provisioning remains the user's responsibility. @@ -33,7 +34,10 @@ node installation payload. Project keys cannot register, edit or remove nodes. ## Start with zero nodes Default installation starts Core, Web and PostgreSQL without local compute. -Hosted Sandbox Manager first asks for Docker or microsandbox. It defaults to the +Hosted Sandbox Manager first asks for **E2B cloud** or **Own machines**. Own machines +then choose Docker or microsandbox; E2B takes an account API key and a qualified +immutable Runtime template build (`template-id:build-uuid`). The key is write-only, +encrypted by Core and never returned to the browser. E2B needs no node installation. It defaults to the paired console origin, which forwards the required Core API and WebSocket routes. Use advanced network settings only when nodes and guests need a different public HTTPS origin. When the inferred address is loopback or is not HTTPS, setup @@ -41,14 +45,32 @@ opens the network field and requires a non-loopback HTTPS origin before saving. The API still accepts HTTP loopback for explicit local development; a guest's loopback address cannot reach its Core host. -Saving initializes the deployment once. An identical request may be retried; -a different provider or origin returns a conflict. Refresh after an uncertain -response before trying again. Selection persists in PostgreSQL, activates without -a restart and applies to every node. Removing all nodes does not reset it. +Saving initializes the deployment. Refresh after an uncertain response before +trying again. Selection persists in PostgreSQL and activates without a restart. +Removing all nodes does not reset it. E2B uses the same daemon, harness and workspace +Runtime as node-backed hosting. Configuration alone does not prove provider or model +readiness. To prepare the qualified E2B Runtime build, use +[the E2B build guide](deploy/e2b/README.md); it is not a public Environment Template. Microsandbox suspends eligible idle Sessions after 300 seconds and retains their -snapshots for 86400 seconds. Docker has no memory snapshot policy. - -Click **Add node**, copy the installation command from the dialog, and run it on +snapshots for 86400 seconds. Docker and E2B have no memory snapshot policy. + +For source/manual deployments using E2B, install the packaged helper and set +`AGENTS_API_E2B_PROVIDER_BIN` to its absolute executable path. Set +`AGENTS_API_E2B_STATE_DIR` to a persistent directory owned by the Core service user, +mode `0700`. The standard distribution prepares both. Back up this private state +with the database and credential-encryption key; losing it can leave an uncertain +allocation that cannot safely be reclaimed. Do not mount it into Web or Runtime. + +Drain and confirm cleanup before revoking the configured E2B account key. Replacing +that key currently uses the same zero-resource configuration guard; in-place key +rotation with retained resources is not supported. If the key is revoked early, +Core keeps unverifiable allocations and blocks switching, even if compute was +removed through the provider console. Do not clear database allocations or private +receipts to bypass this check. A future credential-repair operation must verify +account/resource ownership before accepting a replacement key; an inaccessible +sandbox or empty listing from another account is not proof of cleanup. + +For own machines, click **Add node**, copy the installation command from the dialog, and run it on the target Linux amd64 host. The command uses the saved deployment origin, or the paired console origin for file-managed deployments, without a routine URL field. Closing the dialog discards its one-time command. An expired command requires @@ -57,7 +79,7 @@ checks its hashes, prepares provider configuration and starts the existing node program as a systemd user service. Web polls readiness and capacity while waiting. It does not install software through SSH. Registration itself does not create a Session, sandbox or model request. Hosted Session admission -fails until setup is complete and a ready node has capacity. +fails until setup is complete and a ready node has capacity. E2B allocates directly without this node requirement. For manual zero-node deployments, set `AGENTS_API_SANDBOX_INSTALLATION_ID` to a stable UUID, configure `AGENTS_API_SANDBOX_ADMIN_DIGESTS_FILE`, and enable the daemon @@ -243,14 +265,30 @@ maintenance transition. That transition currently changes the deployment backend configuration and requires resources on all nodes to be cleared; it is not a per-node drain operation. -Provider changes remain an explicit deployment maintenance operation: start the -old configuration in maintenance, resolve all old resources, then start the new -provider configuration in maintenance before reopening admission. Merely changing -a configuration file or disconnecting hosts does not clear resources. Preserve -old node state and backend storage until cleanup is confirmed. A new backend -requires its own node identity. Existing Sessions do not migrate across nodes or -providers. Switching local participation also changes backend configuration and -is subject to the same resource guard; it is not an automatic migration. +For Web-managed deployments, use **Change provider**: + +1. Enter maintenance. This blocks new hosted Sessions and allocations while allowing + existing work, queries and cleanup. +2. Delete no-longer-needed Sessions through their existing project-authorized flow. + Check the deployment's remaining allocation and pending-environment counts. + Stopped compute, snapshots, unknown creates and pending cleanup still block switching. +3. Once cleanup is verified, save the new deployment-wide selection. Core rechecks + resources and the configuration generation under its execution lock, retires old + nodes and unused enrollments, then activates the new provider. History is retained. +4. Explicitly resume. If activation fails, maintenance remains enabled; inspect the + configuration and refresh before retrying. Saving never automatically deletes compute. + +`GET /core/v1/sandbox/deployment` reports the safe configuration, `generation` and +`resources` counts. `PATCH /core/v1/sandbox/deployment/maintenance` takes +`maintenance` and `expected_generation`; `PUT /core/v1/sandbox/deployment` takes +the replacement selection and `expected_generation`. Both require deployment admin +authority. The public Core origin cannot change in this operation. E2B is not +combined with own-machine nodes, and existing Sessions never migrate providers. + +File-managed deployments retain their explicit maintenance/restart process. Merely +changing a file or disconnecting a host does not clear resources. Preserve old node +state and backend storage until cleanup is confirmed. A new backend requires a fresh +node identity. Changing local participation also requires full cleanup. This release has one Core execution owner. It does not add Core multi-active, cross-node snapshot restore, automatic failover, Kubernetes or autoscaling. diff --git a/services/agents-api/cmd/server/daemon_bootstrap.go b/services/agents-api/cmd/server/daemon_bootstrap.go index 29669f0b5..71f2f6025 100644 --- a/services/agents-api/cmd/server/daemon_bootstrap.go +++ b/services/agents-api/cmd/server/daemon_bootstrap.go @@ -10,10 +10,10 @@ import ( func (m *managedNodes) webSocketURL(publicURL string) func(context.Context, gateway.AuthenticatedRuntime) (string, error) { return func(ctx context.Context, auth gateway.AuthenticatedRuntime) (string, error) { - if m == nil || auth.RuntimeNodeID == "" { + if m == nil || (auth.RuntimeNodeID == "" && auth.RuntimeAllocationID == "") { return publicURL, nil } - if m.runtime != nil && auth.RuntimeNodeID == m.runtime.LocalNodeID { + if m.runtime != nil && auth.RuntimeNodeID != "" && auth.RuntimeNodeID == m.runtime.LocalNodeID { return runtimeWebSocketURL(m.runtime.CoreURL) } if m.setup != nil { diff --git a/services/agents-api/cmd/server/daemon_bootstrap_test.go b/services/agents-api/cmd/server/daemon_bootstrap_test.go index 06118bd16..83fa75d18 100644 --- a/services/agents-api/cmd/server/daemon_bootstrap_test.go +++ b/services/agents-api/cmd/server/daemon_bootstrap_test.go @@ -11,21 +11,25 @@ import ( "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" ) -type bootstrapCredentialStore struct{ nodeID string } +type bootstrapCredentialStore struct { + nodeID string + allocationID string +} func (s bootstrapCredentialStore) GetDeviceCredential(context.Context, string) (device.Credential, bool, error) { return device.Credential{ID: "runtime", Type: gateway.RuntimeTypeAgentDaemon, - CredentialHash: device.HashCredential("synthetic-token"), RuntimeNodeID: s.nodeID}, true, nil + CredentialHash: device.HashCredential("synthetic-token"), RuntimeNodeID: s.nodeID, RuntimeAllocationID: s.allocationID}, true, nil } func TestBootstrapAddressFollowsAuthenticatedAllocation(t *testing.T) { const publicURL = "wss://private-proxy.example/api/v1/agent-daemon/ws" const selectedURL = "wss://selected-node-entry.example/api/v1/agent-daemon/ws" local := &managedNodes{runtime: &execution.RuntimeProvider{LocalNodeID: "local-node", CoreURL: "http://host.microsandbox.internal:8091/api/v1"}} - remote := &managedNodes{setup: &managedSetup{}} - remote.setup.selected.Store(&execution.RuntimeProvider{CoreURL: "https://selected-node-entry.example/api/v1"}) + remote := &managedNodes{setup: &managedSetup{store: &setupStore{value: store.SandboxSetup{Provider: "docker", Generation: 1}}}} + remote.setup.selected.Store(&execution.RuntimeProvider{CoreURL: "https://selected-node-entry.example/api/v1", Generation: 1}) zero := &managedNodes{setup: &managedSetup{}} for _, tc := range []struct { name, node, want string @@ -73,3 +77,13 @@ func TestEmbeddedBootstrapDoesNotFallbackWhenInternalAddressIsInvalid(t *testing t.Fatalf("bootstrap fell back after route failure: %d %s", response.Code, response.Body.String()) } } + +func TestCloudBootstrapUsesAuthenticatedAllocationWithoutNode(t *testing.T) { + s := &managedSetup{store: &setupStore{value: store.SandboxSetup{Provider: "e2b", Generation: 1}}} + s.selected.Store(&execution.RuntimeProvider{ProviderKind: "e2b", Generation: 1, CoreURL: "https://cloud-entry.example/api/v1"}) + m := &managedNodes{setup: s} + got, err := m.webSocketURL("wss://original.example/api/v1/agent-daemon/ws")(t.Context(), gateway.AuthenticatedRuntime{DeviceID: "device", RuntimeAllocationID: "allocation"}) + if err != nil || got != "wss://cloud-entry.example/api/v1/agent-daemon/ws" { + t.Fatal("cloud allocation lost configured bootstrap address", err) + } +} diff --git a/services/agents-api/cmd/server/main.go b/services/agents-api/cmd/server/main.go index e9b31b9be..b678f6167 100644 --- a/services/agents-api/cmd/server/main.go +++ b/services/agents-api/cmd/server/main.go @@ -206,7 +206,7 @@ func run() error { return err } if managedNodes != nil && managedNodes.setup != nil { - options = append(options, api.WithSandboxDeploymentSetup(worker.InitializeSandboxDeployment)) + options = append(options, api.WithSandboxDeploymentSetup(worker.InitializeSandboxDeployment), api.WithSandboxDeploymentChanges(worker.UpdateSandboxDeployment, worker.SetSandboxMaintenance)) } workerDone = make(chan error, 1) go func() { workerDone <- worker.Run(ctx) }() diff --git a/services/agents-api/cmd/server/managed_setup.go b/services/agents-api/cmd/server/managed_setup.go index fcc9aba61..fcd809354 100644 --- a/services/agents-api/cmd/server/managed_setup.go +++ b/services/agents-api/cmd/server/managed_setup.go @@ -3,12 +3,17 @@ package main import ( "context" "errors" + "fmt" + "os" + "sync" "sync/atomic" "time" + "github.com/MiniMax-AI-Dev/parsar/internal/obs/log" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/runtimeobs" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/e2b" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/node" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" ) @@ -16,16 +21,19 @@ import ( // managedSetup publishes one immutable selection to execution, bootstrap and // observation. The database owns the selection; this cache is never a writer. type managedSetup struct { - store *store.Store + store interface { + GetSandboxSetup(context.Context) (store.SandboxSetup, error) + ResolveRuntimeNode(context.Context, string, string) (string, error) + } + mu sync.Mutex hub *node.Hub installationID string selected atomic.Pointer[execution.RuntimeProvider] } func (s *managedSetup) load(ctx context.Context) (*execution.RuntimeProvider, error) { - if selected := s.selected.Load(); selected != nil { - return selected, nil - } + s.mu.Lock() + defer s.mu.Unlock() setup, err := s.store.GetSandboxSetup(ctx) if err != nil { return nil, err @@ -36,17 +44,22 @@ func (s *managedSetup) load(ctx context.Context) (*execution.RuntimeProvider, er if setup.Provider == "" { return nil, nil } - provider := s.hub.Provider(setup.Provider, func(ctx context.Context, ref sandbox.Reference) (string, error) { - return s.store.ResolveRuntimeNode(ctx, ref.TenantID, ref.EnvironmentID) - }) - selected := &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, + if selected := s.selected.Load(); selected != nil && selected.Generation == setup.Generation { + return selected, nil + } + provider, err := s.provider(setup) + if err != nil { + log.Warn(ctx, "Hosted provider is unavailable; administrator recovery remains available", "provider", setup.Provider, "error", err) + return nil, fmt.Errorf("%w: %v", execution.ErrExecutionUnavailable, err) + } + selected := &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, Maintenance: setup.Maintenance, CoreURL: setup.CoreURL + "/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider} if setup.Provider == "microsandbox" { selected.Suspension = &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Duration(setup.IdleSeconds) * time.Second, Retention: time.Duration(setup.RetentionSeconds) * time.Second, MaxActive: 4, MaxRetained: 16} } - s.selected.CompareAndSwap(nil, selected) - return s.selected.Load(), nil + s.selected.Store(selected) + return selected, nil } func (s *managedSetup) webSocketURL(fallback string) func(context.Context) (string, error) { @@ -83,3 +96,31 @@ func (s *managedSetup) Observe(ctx context.Context, target runtimeobs.Target) (r } return source.Observe(ctx, target) } + +func (s *managedSetup) provider(setup store.SandboxSetup) (sandbox.Provider, error) { + switch setup.Provider { + case "docker", "microsandbox": + if s.hub == nil { + return nil, errors.New("sandbox node transport is unavailable") + } + return s.hub.Provider(setup.Provider, func(ctx context.Context, ref sandbox.Reference) (string, error) { + return s.store.ResolveRuntimeNode(ctx, ref.TenantID, ref.EnvironmentID) + }), nil + case "e2b": + if setup.E2B == nil { + return nil, errors.New("E2B deployment configuration is unavailable") + } + binary := os.Getenv("AGENTS_API_E2B_PROVIDER_BIN") + if binary == "" { + binary = "/opt/parsar/e2b/agents-api-e2b-provider" + } + provider, err := e2b.New(e2b.Config{Binary: binary, StateDir: os.Getenv("AGENTS_API_E2B_STATE_DIR"), + InstallationID: setup.InstallationID, APIKey: setup.E2B.APIKey, Template: setup.E2B.Template, TimeoutSeconds: 3600}) + if err != nil { + return nil, errors.New("E2B provider cannot load; check the installed helper and private state directory") + } + return provider, nil + default: + return nil, errors.New("sandbox provider is unavailable") + } +} diff --git a/services/agents-api/cmd/server/managed_setup_test.go b/services/agents-api/cmd/server/managed_setup_test.go index 8d5000d23..06f081e20 100644 --- a/services/agents-api/cmd/server/managed_setup_test.go +++ b/services/agents-api/cmd/server/managed_setup_test.go @@ -8,7 +8,9 @@ import ( "path/filepath" "testing" + "errors" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" "github.com/google/uuid" ) @@ -41,8 +43,8 @@ func TestSelectedSetupPublishesSameOriginForDaemonBootstrap(t *testing.T) { {"https://core.example:8443/api/v1", "wss://core.example:8443/api/v1/agent-daemon/ws"}, {"http://127.0.0.1:8091/api/v1", "ws://127.0.0.1:8091/api/v1/agent-daemon/ws"}, } { - s := &managedSetup{} - s.selected.Store(&execution.RuntimeProvider{CoreURL: tc.origin, ProviderKind: "docker"}) + s := &managedSetup{store: &setupStore{value: store.SandboxSetup{Provider: "docker", Generation: 1}}} + s.selected.Store(&execution.RuntimeProvider{CoreURL: tc.origin, ProviderKind: "docker", Generation: 1}) got, err := s.webSocketURL("ws://private-core:8091/api/v1/agent-daemon/ws")(context.Background()) if err != nil || got != tc.want { t.Fatalf("bootstrap URL=%s error=%v", got, err) @@ -52,3 +54,56 @@ func TestSelectedSetupPublishesSameOriginForDaemonBootstrap(t *testing.T) { } } } + +type setupStore struct { + value store.SandboxSetup + err error +} + +func (s *setupStore) GetSandboxSetup(context.Context) (store.SandboxSetup, error) { + return s.value, s.err +} +func (*setupStore) ResolveRuntimeNode(context.Context, string, string) (string, error) { + return "", errors.New("unexpected node lookup") +} + +func TestManagedSetupNeverReusesAnotherGenerationOrUnverifiedState(t *testing.T) { + db := &setupStore{value: store.SandboxSetup{InstallationID: "installation", Provider: "docker", Generation: 1}} + s := &managedSetup{store: db, installationID: "installation"} + cached := &execution.RuntimeProvider{InstallationID: "installation", ProviderKind: "docker", Generation: 1} + s.selected.Store(cached) + if got, err := s.load(t.Context()); err != nil || got != cached { + t.Fatal("matching immutable selection was not reused") + } + db.err = errors.New("database unavailable") + if _, err := s.load(t.Context()); err == nil { + t.Fatal("stale cached selection hid storage failure") + } + db.err = nil + db.value.Generation = 2 + // No Hub is installed; a changed generation must construct again and fail. + if _, err := s.load(t.Context()); !errors.Is(err, execution.ErrExecutionUnavailable) { + t.Fatal("changed provider availability must block execution without losing recovery", err) + } + db.value.InstallationID = "other-installation" + db.value.Generation = 1 + if _, err := s.load(t.Context()); err == nil { + t.Fatal("cache ignored installation identity") + } +} + +func TestMissingE2BHelperReportsProviderUnavailable(t *testing.T) { + id := uuid.NewString() + t.Setenv("AGENTS_API_E2B_PROVIDER_BIN", filepath.Join(t.TempDir(), "missing-helper")) + t.Setenv("AGENTS_API_E2B_STATE_DIR", t.TempDir()) + s := &managedSetup{installationID: id, store: &setupStore{value: store.SandboxSetup{ + InstallationID: id, Provider: "e2b", Mode: "direct", Generation: 1, + E2B: &store.SandboxE2BConfiguration{APIKey: "synthetic-key", Template: "runtime:" + uuid.NewString()}, + }}} + if _, err := s.load(t.Context()); !errors.Is(err, execution.ErrExecutionUnavailable) { + t.Fatal("missing local helper must leave administrative recovery available", err) + } + if s.selected.Load() != nil { + t.Fatal("unavailable provider was published") + } +} diff --git a/services/agents-api/deploy/e2b/README.md b/services/agents-api/deploy/e2b/README.md index fc1937338..b0bcf354b 100644 --- a/services/agents-api/deploy/e2b/README.md +++ b/services/agents-api/deploy/e2b/README.md @@ -1,15 +1,48 @@ -# User-managed E2B Runtime +# E2B Runtime deployment -The application creates, renews and destroys its own E2B sandbox using the -maintained E2B SDK. Core receives neither the E2B API key nor an allocation +E2B supports two separate ownership choices. Core-managed `openai_hosted` uses +the deployment-wide E2B Provider. Application-managed `self_hosted` uses the +startup example below; in that path the application creates, renews and destroys +its own sandbox, and Core receives neither the E2B account key nor an allocation request. The sandbox runs the existing V1 daemon, selected native harness, tools and workspace together. Codex, Claude Code and MiniMax Code use the same startup contract and their respective qualified Runtime images. This deployment uses Parsar daemon enrollment, not Codex `exec-server` or Noise. Public execution and Files/Artifacts continue through Core and the daemon; -E2B commands/files are used only for application-controlled deployment and -inspection. A public Session deletion does not destroy the user-owned VM. +E2B commands/files are used only for deployment, initialization and inspection. +A public Session deletion does not destroy an application-owned VM; managed +compute cleanup remains Core's responsibility. + +## Core-managed hosted deployment + +Select E2B in Core's hosted deployment setup and supply the account key and an +immutable `templateID:build_UUID`. One deployment uses one managed Provider; +E2B needs no physical node enrollment. Switching Providers requires maintenance +and confirmed cleanup of every owned allocation, pending creation and snapshot. +Do not infer execution readiness from saved configuration or running compute. + +The pure-Go adapter invokes the packaged official Python SDK helper. The Core +image and native installation include its runtime dependencies; configuring E2B +does not require installing Python or pip on the server. The account key is +write-only server configuration, passed to the helper over stdin. It never enters +a template, command argument, inherited environment, receipt or public response. + +Keep the helper's private receipt directory on durable storage, owned by the Core +service user with mode 0700. Manual deployments using the default non-root Core +image must give its service UID ownership of that directory. SDK connection +credentials and one-shot allocation claims are stored there; they are not Core +execution state. Preserve them across upgrades and failures. An empty cloud +lookup cannot settle an unknown Create, and inspection never creates, resumes +or reboots compute. Initialization uncertainty requires reclaiming the original +allocation rather than replaying startup. See the [helper contract](../../tools/e2b-provider/README.md). + +Rebuild old templates with this directory's `build-template.py` before managed +use: it includes protected `init.py` and `managed_init.py`. The latter consumes +Core's existing managed daemon auth profile, while application-managed startup +continues to use the executor credential flow below. Both reuse the same daemon, +native harnesses and colocated workspace. A qualified combined Runtime image can +support several harnesses; provider selection is independent of harness choice. ## Build the packaged Runtime @@ -29,11 +62,11 @@ python -m venv "$HOME/.parsar/build/e2b-sdk" The builder preserves the existing image's binaries, native configuration and private workspace layout. Its `template` output is an immutable -`templateID:build_UUID`; use that exact value. Each engine needs its qualified -image/build. No E2B account key, executor key or model credential belongs in a +`templateID:build_UUID`; use that exact value. The build must qualify every harness it advertises; a combined Runtime image +can include several harnesses. No E2B account key, executor key or model credential belongs in a build, template environment, metadata, command argument or log. -## Start an existing self-hosted Environment +## Application-managed: start an existing self-hosted Environment Create a public `self_hosted` Environment through Core and retain its ID and exact returned `remote_url`. Obtain an authorized connect-only executor key scoped to diff --git a/services/agents-api/deploy/e2b/build-template.py b/services/agents-api/deploy/e2b/build-template.py index 3b930d922..aa5e72b00 100644 --- a/services/agents-api/deploy/e2b/build-template.py +++ b/services/agents-api/deploy/e2b/build-template.py @@ -55,7 +55,8 @@ for entry in tree.iterdir(): archive.add(entry, arcname=entry.name) (context / 'runtime-env.json').write_text(json.dumps(environment)) - (context / 'init.py').write_bytes(Path(__file__).with_name('init.py').read_bytes()) + for name in ['init.py', 'managed_init.py']: + (context / name).write_bytes(Path(__file__).with_name(name).read_bytes()) template = (Template(file_context_path=context).from_image(BASE) .run_cmd('apt-get update && apt-get install -y --no-install-recommends ' 'ca-certificates bash git python3 python3-pip ripgrep bubblewrap socat util-linux ' @@ -63,13 +64,14 @@ .copy('runtime.tar.gz', '/root/runtime.tar.gz', user='root') .copy('runtime-env.json', '/etc/parsar-runtime-env.json', user='root') .copy('init.py', '/opt/parsar-e2b/init.py', user='root') + .copy('managed_init.py', '/opt/parsar-e2b/managed_init.py', user='root') .run_cmd('tar --no-same-owner -xzf /root/runtime.tar.gz -C / && rm /root/runtime.tar.gz ' '&& usermod -l runtime -d /home/runtime node ' '&& mkdir -p /home/runtime/.parsar /environment/workspace /environment/staging /environment/initialization /environment/packages /workspace ' '&& chown -R 1000:1000 /home/runtime /environment ' '&& chmod 0700 /home/runtime/.parsar /environment/staging ' '&& chmod 0444 /etc/parsar-runtime-env.json ' - '&& chmod 0555 /opt/parsar-e2b /opt/parsar-e2b/init.py', user='root') + '&& chmod 0555 /opt/parsar-e2b /opt/parsar-e2b/init.py /opt/parsar-e2b/managed_init.py', user='root') .set_user('runtime').set_workdir('/environment/workspace')) result = Template.build(template, name=args.name, cpu_count=2, memory_mb=2048, on_build_logs=lambda entry: print(entry.message, flush=True), diff --git a/services/agents-api/deploy/e2b/init.py b/services/agents-api/deploy/e2b/init.py index 11e249ed6..29ddc7af3 100644 --- a/services/agents-api/deploy/e2b/init.py +++ b/services/agents-api/deploy/e2b/init.py @@ -63,19 +63,8 @@ def write_private(path, value, owner=None): sync_directory(path.parent) -def initialize(): - ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) - ROOT.chmod(0o700) - source = ROOT / 'bootstrap.json' - receipt = ROOT / 'ready.json' - if receipt.exists() or (ROOT / 'launch.json').exists(): - raise RuntimeError('Runtime startup cannot be replayed; inspect or destroy this sandbox') - if source.stat().st_size > 32768: - raise ValueError('Runtime startup input too large') - payload = json.loads(source.read_text()) - identity = launch_identity(payload) - # Claim before any side effect. An interrupted attempt must never start twice. - write_private(ROOT / 'launch.json', identity) +def prepare_runtime(): + """Restore the protected image and shared Runtime layout before any daemon starts.""" # E2B finalization makes /usr/local world-writable after template commands. subprocess.run(['chown', '-R', 'root:root', '/usr/local'], check=True) subprocess.run(['chmod', '-R', 'go-w', '/usr/local'], check=True) @@ -101,6 +90,23 @@ def initialize(): Path('/environment/initialization'), Path('/environment/packages')]: os.chown(directory, 1000, 1000) directory.chmod(0o700) + return environment + + +def initialize(): + ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) + ROOT.chmod(0o700) + source = ROOT / 'bootstrap.json' + receipt = ROOT / 'ready.json' + if any((ROOT / name).exists() for name in ['ready.json', 'launch.json', 'managed-launch.json', 'managed-ready.json']): + raise RuntimeError('Runtime startup cannot be replayed; inspect or destroy this sandbox') + if source.stat().st_size > 32768: + raise ValueError('Runtime startup input too large') + payload = json.loads(source.read_text()) + identity = launch_identity(payload) + # Claim before any side effect. An interrupted attempt must never start twice. + write_private(ROOT / 'launch.json', identity) + environment = prepare_runtime() credential = PROFILE.parent / 'executor-key.json' write_private(credential, payload['executor_key'], owner=1000) source.unlink() diff --git a/services/agents-api/deploy/e2b/managed_init.py b/services/agents-api/deploy/e2b/managed_init.py new file mode 100644 index 000000000..136d43be3 --- /dev/null +++ b/services/agents-api/deploy/e2b/managed_init.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""One-shot managed bootstrap using the existing daemon authentication profile.""" +import importlib.util +import json +import os +from pathlib import Path +import subprocess +from urllib.parse import urlsplit +from uuid import UUID + +# -I excludes the script directory from sys.path; load only its protected sibling. +_spec = importlib.util.spec_from_file_location('runtime_init', Path(__file__).with_name('init.py')) +shared = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(shared) + + +def identity(payload): + fields = ['InstallationID', 'TenantID', 'EnvironmentID', 'AllocationID', 'SessionID', 'DeviceID'] + if not isinstance(payload, dict) or set(payload) != set(fields + ['CoreURL', 'Credential', 'NetworkAccess', 'AllowedDomains']): + raise ValueError('Invalid managed bootstrap fields') + for field in fields: + value = payload[field] + if not isinstance(value, str) or str(UUID(value)) != value or UUID(value).int == 0: + raise ValueError('Invalid managed bootstrap identity') + address = urlsplit(payload['CoreURL']) + if (address.scheme not in ('http', 'https') or not address.hostname or address.username is not None or + address.query or address.fragment or any(char.isspace() for char in payload['CoreURL'])): + raise ValueError('Invalid managed bootstrap address') + if (not isinstance(payload['Credential'], str) or not payload['Credential'] or + any(char.isspace() or char == '\0' for char in payload['Credential']) or + payload['NetworkAccess'] not in ('enabled', 'disabled', 'restricted') or + payload['AllowedDomains'] is not None and + (not isinstance(payload['AllowedDomains'], list) or + any(not isinstance(domain, str) for domain in payload['AllowedDomains']))): + raise ValueError('Invalid managed bootstrap configuration') + return {field: payload[field] for field in fields} + + +def initialize(): + root = shared.ROOT + root.mkdir(mode=0o700, parents=True, exist_ok=True) + root.chmod(0o700) + source = root / 'managed-bootstrap.json' + if any((root / name).exists() for name in ['launch.json', 'ready.json', 'managed-launch.json', 'managed-ready.json']): + raise RuntimeError('Runtime bootstrap cannot be replayed') + if source.stat().st_size > 65536: + raise ValueError('Managed bootstrap input too large') + payload = json.loads(source.read_text()) + binding = identity(payload) + shared.write_private(root / 'managed-launch.json', binding) + environment = shared.prepare_runtime() + environment.update(PARSAR_RUNTIME_ENVIRONMENT_ID=payload['EnvironmentID'], + PARSAR_RUNTIME_SESSION_ID=payload['SessionID'], + PARSAR_RUNTIME_NETWORK_ACCESS=payload['NetworkAccess'], + PARSAR_RUNTIME_ALLOWED_DOMAINS=json.dumps(payload['AllowedDomains'] or [])) + shared.write_private(shared.PROFILE / 'auth.json', + {'server_url': payload['CoreURL'], 'runtime_id': payload['DeviceID'], + 'runner_credential': payload['Credential']}, owner=1000) + source.unlink() + with (shared.PROFILE / 'daemon.log').open('xb') as stream: + os.fchmod(stream.fileno(), 0o600) + os.fchown(stream.fileno(), 1000, 1000) + child = subprocess.Popen(['/usr/local/bin/parsar-daemon', 'connect', '--profile', 'default'], + cwd='/environment/workspace', env=environment, user=1000, group=1000, + extra_groups=[], start_new_session=True, stdin=subprocess.DEVNULL, + stdout=stream, stderr=subprocess.STDOUT, umask=0o077) + shared.write_private(root / 'managed-ready.tmp', + {'identity': binding, 'status': 'daemon_started', 'daemon_pid': child.pid}) + os.replace(root / 'managed-ready.tmp', root / 'managed-ready.json') + shared.sync_directory(root) + + +if __name__ == '__main__': + try: + initialize() + except Exception: + raise SystemExit('Managed Runtime startup failed; retain and reclaim the owned sandbox') from None diff --git a/services/agents-api/deploy/e2b/managed_init_test.py b/services/agents-api/deploy/e2b/managed_init_test.py new file mode 100644 index 000000000..d9ead6862 --- /dev/null +++ b/services/agents-api/deploy/e2b/managed_init_test.py @@ -0,0 +1,85 @@ +"""Managed bootstrap reuses image protection without changing self-hosted enrollment.""" +import copy +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch +from uuid import uuid4 + +import managed_init + + +def payload(): + value = {key: str(uuid4()) for key in ['InstallationID', 'TenantID', 'EnvironmentID', + 'AllocationID', 'SessionID', 'DeviceID']} + return dict(value, CoreURL='https://core.example/api/v1', Credential='private-managed-token', + NetworkAccess='restricted', AllowedDomains=['example.com']) + + +class ManagedStartupTest(unittest.TestCase): + def test_invalid_binding_rejected(self): + source = payload() + for key, value in [('DeviceID', 'other'), ('CoreURL', 'https://secret@host/api/v1'), + ('Credential', 'x\n'), ('NetworkAccess', 'unknown')]: + with self.subTest(key=key), self.assertRaises(ValueError): + managed_init.identity(dict(source, **{key: value})) + + def exercise(self, failed=False): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / 'receipt' + root.mkdir() + profile = Path(temporary) / 'profile' + profile.mkdir() + source = root / 'managed-bootstrap.json' + data = payload() + source.write_text(json.dumps(data)) + process = Mock(return_value=Mock(pid=456)) + if failed: + process.side_effect = RuntimeError('private process diagnostic') + image_env = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'PARSAR_HOME': '/home/runtime/.parsar', + 'PARSAR_RUNTIME_WORKSPACE': '/environment/workspace'} + with patch.object(managed_init.shared, 'ROOT', root), patch.object(managed_init.shared, 'PROFILE', profile), \ + patch.object(managed_init.shared, 'prepare_runtime', return_value=image_env), \ + patch.object(managed_init.os, 'fchown'), patch.object(managed_init.subprocess, 'Popen', process): + if failed: + with self.assertRaises(RuntimeError): + managed_init.initialize() + else: + managed_init.initialize() + self.assertTrue((root / 'managed-launch.json').exists()) + auth = json.loads((profile / 'auth.json').read_text()) + self.assertEqual(auth['runner_credential'], data['Credential']) + self.assertEqual(auth['runtime_id'], data['DeviceID']) + self.assertEqual((profile / 'auth.json').stat().st_mode & 0o777, 0o600) + self.assertFalse(source.exists()) + self.assertNotIn(data['Credential'], json.dumps(process.call_args.args)) + self.assertNotIn(data['Credential'], json.dumps(process.call_args.kwargs['env'])) + self.assertEqual(process.call_args.kwargs['env']['PARSAR_RUNTIME_ENVIRONMENT_ID'], data['EnvironmentID']) + self.assertEqual(process.call_args.kwargs['user'], 1000) + if failed: + self.assertFalse((root / 'managed-ready.json').exists()) + else: + receipt = json.loads((root / 'managed-ready.json').read_text()) + self.assertEqual(receipt['identity'], managed_init.identity(data)) + self.assertNotIn(data['Credential'], json.dumps(receipt)) + source.write_text(json.dumps(data)) + with self.assertRaises(RuntimeError): + managed_init.initialize() + process.assert_called_once() + + def test_managed_credentials_and_environment(self): + self.exercise() + + def test_unknown_start_preserves_claim_and_never_replays(self): + self.exercise(failed=True) + + def test_existing_self_hosted_claim_prevents_managed_start(self): + with tempfile.TemporaryDirectory() as temporary, patch.object(managed_init.shared, 'ROOT', Path(temporary)): + Path(temporary, 'launch.json').write_text('{}') + with self.assertRaises(RuntimeError): + managed_init.initialize() + + +if __name__ == '__main__': + unittest.main() diff --git a/services/agents-api/internal/api/errors.go b/services/agents-api/internal/api/errors.go index 07386cce9..6f895cc0b 100644 --- a/services/agents-api/internal/api/errors.go +++ b/services/agents-api/internal/api/errors.go @@ -90,8 +90,10 @@ func writeStoreError(w http.ResponseWriter, r *http.Request, err error, notFound writeError(w, http.StatusConflict, "project_api_key_exists", "This API key ID already exists. List its metadata and revoke it explicitly if the secret was not saved.") case errors.Is(err, store.ErrExecutorCredentialExists): writeError(w, http.StatusConflict, "executor_credential_exists", "This executor key ID already exists. Explicitly rotate it to replace the secret.") + case errors.Is(err, store.ErrSandboxCredentialUnavailable): + writeError(w, http.StatusServiceUnavailable, "sandbox_credential_unavailable", "Sandbox credentials are unavailable. Check the service credential encryption configuration.") case errors.Is(err, store.ErrSandboxDeploymentConflict): - writeError(w, http.StatusConflict, "sandbox_deployment_conflict", "The sandbox deployment is already configured or is managed by a deployment file.") + writeError(w, http.StatusConflict, "sandbox_deployment_conflict", "The sandbox deployment cannot change in its current state. Refresh the configuration, enter maintenance and finish resource cleanup before switching. File-managed deployments must use their configuration file.") case errors.Is(err, store.ErrRuntimeNodeCredential): writeError(w, http.StatusUnauthorized, "invalid_node_credential", "A valid sandbox node enrollment or node credential is required.") case errors.Is(err, store.ErrRuntimeNodeInUse): diff --git a/services/agents-api/internal/api/handler.go b/services/agents-api/internal/api/handler.go index 0eec95774..470a58f13 100644 --- a/services/agents-api/internal/api/handler.go +++ b/services/agents-api/internal/api/handler.go @@ -40,6 +40,8 @@ type Handler struct { sandboxStore *store.Store deploymentAuth *DeploymentAuthenticator sandboxSetup func(context.Context, store.SandboxDeploymentSetupRequest) (store.RuntimeDeploymentView, error) + sandboxUpdate func(context.Context, store.SandboxDeploymentUpdateRequest) (store.RuntimeDeploymentView, error) + sandboxMaintenance func(context.Context, store.SandboxMaintenanceRequest) (store.RuntimeDeploymentView, error) policy execution.Policy store ResourceStore auth *Authenticator diff --git a/services/agents-api/internal/api/sandbox_deployment_changes_test.go b/services/agents-api/internal/api/sandbox_deployment_changes_test.go new file mode 100644 index 000000000..e7076a40b --- /dev/null +++ b/services/agents-api/internal/api/sandbox_deployment_changes_test.go @@ -0,0 +1,79 @@ +package api + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +func TestSandboxDeploymentChangesAuthenticateAndDecode(t *testing.T) { + project, _ := NewAuthenticator([]APIKey{callerBinding()}) + admin, _ := NewDeploymentAuthenticator([]string{device.HashCredential("administrator")}) + updates, maintenance := 0, 0 + update := func(_ context.Context, in store.SandboxDeploymentUpdateRequest) (store.RuntimeDeploymentView, error) { + updates++ + if in.Provider != "e2b" || in.ExpectedGeneration != 2 || in.E2B == nil || in.E2B.APIKey != "synthetic-private-key" { + t.Fatal("write-only fields were lost") + } + return store.RuntimeDeploymentView{Provider: in.Provider}, nil + } + maintain := func(_ context.Context, in store.SandboxMaintenanceRequest) (store.RuntimeDeploymentView, error) { + maintenance++ + if in.ExpectedGeneration != 2 { + t.Fatal("generation was lost") + } + return store.RuntimeDeploymentView{Maintenance: in.Maintenance}, nil + } + h, err := NewHandler(&recordingStore{}, project, "codex", WithSandboxManager(&store.Store{}, admin), WithSandboxDeploymentChanges(update, maintain)) + if err != nil { + t.Fatal(err) + } + const selection = `{"provider":"e2b","core_url":"https://core.example","expected_generation":2,"e2b":{"api_key":"synthetic-private-key","template":"qualified:build"}}` + for _, tc := range []struct { + method, path, token, body string + status int + }{ + {"PUT", "/deployment", "caller", selection, 401}, + {"PATCH", "/deployment/maintenance", "caller", `{"maintenance":true,"expected_generation":2}`, 401}, + {"PUT", "/deployment", "administrator", strings.Replace(selection, `"api_key"`, `"API_KEY"`, 1), 400}, + {"PUT", "/deployment", "administrator", strings.Replace(selection, `"expected_generation":2,`, "", 1), 400}, + {"PUT", "/deployment", "administrator", selection, 200}, + {"PATCH", "/deployment/maintenance", "administrator", `{"expected_generation":2}`, 400}, + {"PATCH", "/deployment/maintenance", "administrator", `{"maintenance":null,"expected_generation":2}`, 400}, + {"PATCH", "/deployment/maintenance", "administrator", `{"maintenance":true,"expected_generation":2}`, 200}, + {"PATCH", "/deployment/maintenance", "administrator", `{"maintenance":false,"expected_generation":2}`, 200}, + } { + r := httptest.NewRequest(tc.method, "/core/v1/sandbox"+tc.path, strings.NewReader(tc.body)) + r.Header.Set("Authorization", "Bearer "+tc.token) + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + if w.Code != tc.status || strings.Contains(w.Body.String(), "synthetic-private-key") { + t.Fatalf("%s %s status=%d", tc.method, tc.path, w.Code) + } + } + if updates != 1 || maintenance != 2 { + t.Fatalf("unauthorized or invalid input reached mutation: %d %d", updates, maintenance) + } +} + +func TestSandboxDeploymentChangesUnavailableWithoutOwner(t *testing.T) { + h := &Handler{} + for _, tc := range []struct { + body string + handler http.HandlerFunc + }{ + {`{"provider":"docker","core_url":"https://core.example","expected_generation":1}`, h.updateSandboxDeployment}, + {`{"maintenance":true,"expected_generation":1}`, h.setSandboxMaintenance}, + } { + w := httptest.NewRecorder() + tc.handler(w, httptest.NewRequest("PUT", "/", strings.NewReader(tc.body))) + if w.Code != http.StatusConflict { + t.Fatal(w.Code) + } + } +} diff --git a/services/agents-api/internal/api/sandbox_deployment_setup.go b/services/agents-api/internal/api/sandbox_deployment_setup.go index 3767943f0..aa675dad2 100644 --- a/services/agents-api/internal/api/sandbox_deployment_setup.go +++ b/services/agents-api/internal/api/sandbox_deployment_setup.go @@ -7,17 +7,50 @@ import ( "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" ) +// SandboxE2BInput is write-only provider configuration. Safe responses use the +// store's separate deployment view and never serialize this request. +type SandboxE2BInput struct { + APIKey string `json:"api_key"` + Template string `json:"template"` +} + +type SandboxDeploymentInput struct { + Provider string `json:"provider"` + CoreURL string `json:"core_url"` + E2B *SandboxE2BInput `json:"e2b,omitempty"` +} + +type SandboxDeploymentChangeInput struct { + SandboxDeploymentInput + ExpectedGeneration uint64 `json:"expected_generation"` +} + +func (v SandboxDeploymentInput) request() store.SandboxDeploymentSetupRequest { + input := store.SandboxDeploymentSetupRequest{Provider: v.Provider, CoreURL: v.CoreURL} + if v.E2B != nil { + input.E2B = &store.SandboxE2BConfiguration{APIKey: v.E2B.APIKey, Template: v.E2B.Template} + } + return input +} + func WithSandboxDeploymentSetup(initialize func(context.Context, store.SandboxDeploymentSetupRequest) (store.RuntimeDeploymentView, error)) Option { return func(h *Handler) { h.sandboxSetup = initialize } } -// @Summary Initialize the deployment sandbox provider once -// @Description Selects one immutable provider and public Core origin. Exact retries return the existing selection; differing selections and file-managed deployments reject. This does not create a node or execute work. +func WithSandboxDeploymentChanges( + update func(context.Context, store.SandboxDeploymentUpdateRequest) (store.RuntimeDeploymentView, error), + maintenance func(context.Context, store.SandboxMaintenanceRequest) (store.RuntimeDeploymentView, error), +) Option { + return func(h *Handler) { h.sandboxUpdate = update; h.sandboxMaintenance = maintenance } +} + +// @Summary Initialize the deployment sandbox provider +// @Description Selects a provider and public Core origin. E2B credentials are write-only. Exact retries return the existing selection; differing selections and file-managed deployments reject. This does not create compute or execute work. // @Tags Sandbox Manager // @Produce json // @Security DeploymentAdminAuth // @Accept json -// @Param body body store.SandboxDeploymentSetupRequest true "Deployment selection" +// @Param body body api.SandboxDeploymentInput true "Deployment selection" // @Success 200 {object} store.RuntimeDeploymentView // @Failure 400,401,409,500,503 {object} v1.ErrorResponse // @Router /core/v1/sandbox/deployment [post] @@ -26,8 +59,8 @@ func (h *Handler) initializeSandboxDeployment(w http.ResponseWriter, r *http.Req if !ok { return } - var input store.SandboxDeploymentSetupRequest - if decodeInputObject(raw, &input, "provider", "core_url") != nil { + var input SandboxDeploymentInput + if decodeInputObject(raw, &input, "provider", "core_url", "e2b") != nil { writeStoreError(w, r, store.ErrInvalidInput) return } @@ -35,7 +68,75 @@ func (h *Handler) initializeSandboxDeployment(w http.ResponseWriter, r *http.Req writeStoreError(w, r, store.ErrSandboxDeploymentConflict) return } - result, err := h.sandboxSetup(r.Context(), input) + result, err := h.sandboxSetup(r.Context(), input.request()) + if err != nil { + writeStoreError(w, r, err) + return + } + writeJSON(w, http.StatusOK, result) +} + +// @Summary Change a fully drained deployment's sandbox provider +// @Description Requires maintenance, the current generation and verified cleanup of all old resources. Credentials are write-only. The public Core origin stays unchanged. Historical records are retained; old node credentials and enrollments are retired. Explicitly resume after success. Never automatically retry an uncertain write. +// @Tags Sandbox Manager +// @Produce json +// @Security DeploymentAdminAuth +// @Accept json +// @Param body body api.SandboxDeploymentChangeInput true "Replacement deployment selection" +// @Success 200 {object} store.RuntimeDeploymentView +// @Failure 400,401,409,500,503 {object} v1.ErrorResponse +// @Router /core/v1/sandbox/deployment [put] +func (h *Handler) updateSandboxDeployment(w http.ResponseWriter, r *http.Request) { + raw, ok := readJSONBody(w, r) + if !ok { + return + } + var input SandboxDeploymentChangeInput + if decodeInputObject(raw, &input, "provider", "core_url", "e2b", "expected_generation") != nil || input.ExpectedGeneration == 0 { + writeStoreError(w, r, store.ErrInvalidInput) + return + } + if h.sandboxUpdate == nil { + writeStoreError(w, r, store.ErrSandboxDeploymentConflict) + return + } + result, err := h.sandboxUpdate(r.Context(), store.SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input.request(), ExpectedGeneration: input.ExpectedGeneration}) + if err != nil { + writeStoreError(w, r, err) + return + } + writeJSON(w, http.StatusOK, result) +} + +// @Summary Pause or resume new hosted allocations +// @Description Requires the current generation. Maintenance preserves execution, cleanup and history. Resume requires successfully activated provider configuration. +// @Tags Sandbox Manager +// @Produce json +// @Security DeploymentAdminAuth +// @Accept json +// @Param body body store.SandboxMaintenanceRequest true "Maintenance state" +// @Success 200 {object} store.RuntimeDeploymentView +// @Failure 400,401,409,500,503 {object} v1.ErrorResponse +// @Router /core/v1/sandbox/deployment/maintenance [patch] +func (h *Handler) setSandboxMaintenance(w http.ResponseWriter, r *http.Request) { + raw, ok := readJSONBody(w, r) + if !ok { + return + } + // Omitted or null maintenance is not an instruction to resume. + var input struct { + Maintenance *bool `json:"maintenance"` + ExpectedGeneration uint64 `json:"expected_generation"` + } + if decodeInputObject(raw, &input, "maintenance", "expected_generation") != nil || input.Maintenance == nil || input.ExpectedGeneration == 0 { + writeStoreError(w, r, store.ErrInvalidInput) + return + } + if h.sandboxMaintenance == nil { + writeStoreError(w, r, store.ErrSandboxDeploymentConflict) + return + } + result, err := h.sandboxMaintenance(r.Context(), store.SandboxMaintenanceRequest{Maintenance: *input.Maintenance, ExpectedGeneration: input.ExpectedGeneration}) if err != nil { writeStoreError(w, r, err) return diff --git a/services/agents-api/internal/api/sandbox_manager.go b/services/agents-api/internal/api/sandbox_manager.go index de7771f54..bbd47fd70 100644 --- a/services/agents-api/internal/api/sandbox_manager.go +++ b/services/agents-api/internal/api/sandbox_manager.go @@ -41,6 +41,8 @@ func (h *Handler) registerSandboxManagerRoutes(r chi.Router) { r.Use(h.deploymentAuth.authenticate) r.Get("/deployment", h.sandboxDeployment) r.Post("/deployment", h.initializeSandboxDeployment) + r.Put("/deployment", h.updateSandboxDeployment) + r.Patch("/deployment/maintenance", h.setSandboxMaintenance) r.Get("/nodes", h.sandboxNodes) r.Patch("/nodes/{node_id}", h.updateSandboxNode) r.Delete("/nodes/{node_id}", h.removeSandboxNode) diff --git a/services/agents-api/internal/credentialcrypto/sandbox_deployment.go b/services/agents-api/internal/credentialcrypto/sandbox_deployment.go new file mode 100644 index 000000000..0346f14ff --- /dev/null +++ b/services/agents-api/internal/credentialcrypto/sandbox_deployment.go @@ -0,0 +1,30 @@ +package credentialcrypto + +import ( + "encoding/json" + "strconv" + "unicode/utf8" +) + +func sandboxDeploymentData(installation string, generation uint64) ([]byte, error) { + if installation == "" || !utf8.ValidString(installation) || generation == 0 { + return nil, errInvalidBinding + } + return json.Marshal([]string{"parsar.agents-api.sandbox-deployment.v1", installation, strconv.FormatUint(generation, 10)}) +} + +func (c *Cipher) SealSandboxDeployment(plaintext []byte, installation string, generation uint64) ([]byte, error) { + aad, err := sandboxDeploymentData(installation, generation) + if err != nil { + return nil, err + } + return c.seal(plaintext, aad) +} + +func (c *Cipher) OpenSandboxDeployment(ciphertext []byte, installation string, generation uint64) ([]byte, error) { + aad, err := sandboxDeploymentData(installation, generation) + if err != nil { + return nil, err + } + return c.open(ciphertext, aad) +} diff --git a/services/agents-api/internal/credentialcrypto/sandbox_deployment_test.go b/services/agents-api/internal/credentialcrypto/sandbox_deployment_test.go new file mode 100644 index 000000000..16c606d2e --- /dev/null +++ b/services/agents-api/internal/credentialcrypto/sandbox_deployment_test.go @@ -0,0 +1,29 @@ +package credentialcrypto + +import ( + "bytes" + "testing" +) + +func TestSandboxDeploymentCredentialBinding(t *testing.T) { + c, err := New(bytes.Repeat([]byte{9}, 32)) + if err != nil { + t.Fatal(err) + } + encrypted, err := c.SealSandboxDeployment([]byte("private-api-key"), "installation", 2) + if err != nil { + t.Fatal(err) + } + for _, binding := range []struct { + id string + generation uint64 + }{{"other", 2}, {"installation", 1}, {"installation", 3}} { + if _, err := c.OpenSandboxDeployment(encrypted, binding.id, binding.generation); err == nil { + t.Fatal("accepted different deployment binding") + } + } + plain, err := c.OpenSandboxDeployment(encrypted, "installation", 2) + if err != nil || string(plain) != "private-api-key" { + t.Fatal("credential did not roundtrip", err) + } +} diff --git a/services/agents-api/internal/db/queries/devices.sql b/services/agents-api/internal/db/queries/devices.sql index 7d9dc0af3..d7379b580 100644 --- a/services/agents-api/internal/db/queries/devices.sql +++ b/services/agents-api/internal/db/queries/devices.sql @@ -6,7 +6,7 @@ VALUES ($1, $2, $3, $4) RETURNING id; SELECT id, name FROM devices WHERE tenant_id = $1 AND id = $2 AND revoked_at IS NULL; -- name: GetDeviceCredential :one -SELECT d.id, d.name, d.credential_hash, COALESCE(a.node_id::text, '')::text AS runtime_node_id +SELECT d.id, d.name, d.credential_hash, COALESCE(a.node_id::text, '')::text AS runtime_node_id, COALESCE(a.id::text, '')::text AS runtime_allocation_id FROM runtime_device_authority d LEFT JOIN runtime_allocations a ON a.device_id = d.id WHERE d.id = $1; diff --git a/services/agents-api/internal/db/queries/initial_environment_files.sql b/services/agents-api/internal/db/queries/initial_environment_files.sql index 44d8fc807..5f16254d0 100644 --- a/services/agents-api/internal/db/queries/initial_environment_files.sql +++ b/services/agents-api/internal/db/queries/initial_environment_files.sql @@ -23,13 +23,13 @@ FROM sessions s WHERE s.tenant_id = $1 AND s.id = $2; -- name: ClaimRuntimeInitialization :one UPDATE runtime_allocations SET initialization = 'running' WHERE id = $1 AND initialization = 'pending' AND state = 'running' AND create_settled -AND (node_id IS NOT NULL OR kept_at > clock_timestamp() - interval '1 hour') +AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') RETURNING *; -- name: CompleteRuntimeInitialization :one UPDATE runtime_allocations SET initialization = 'complete' WHERE id = $1 AND initialization = 'running' AND state = 'running' AND create_settled -AND (node_id IS NOT NULL OR kept_at > clock_timestamp() - interval '1 hour') +AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') RETURNING *; -- name: LockInitialSourceFile :one diff --git a/services/agents-api/internal/db/queries/runtime_allocations.sql b/services/agents-api/internal/db/queries/runtime_allocations.sql index 496f38c9b..4e1060ff3 100644 --- a/services/agents-api/internal/db/queries/runtime_allocations.sql +++ b/services/agents-api/internal/db/queries/runtime_allocations.sql @@ -3,14 +3,14 @@ INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, no VALUES ($1, $2, $3, $4, $5, CASE WHEN EXISTS (SELECT 1 FROM initial_environment_files f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) OR EXISTS (SELECT 1 FROM environment_setups f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) THEN 'pending' ELSE 'complete' END) RETURNING *; -- name: GetRuntimeAllocation :one -SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired +SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id WHERE s.tenant_id = $1 AND a.environment_id = $2; -- name: ListRuntimeAllocations :many -SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired +SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id @@ -32,13 +32,13 @@ LIMIT $2; -- name: ObserveRuntimeRunning :one UPDATE runtime_allocations SET state = 'running', create_settled = true WHERE id = $1 AND state IN ('creating', 'running') -AND (node_id IS NOT NULL OR kept_at > clock_timestamp() - interval '1 hour') +AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') RETURNING *; -- name: KeepRuntimeAllocation :one UPDATE runtime_allocations SET kept_at = clock_timestamp() WHERE id = $1 AND state = 'running' -AND (node_id IS NOT NULL OR kept_at > clock_timestamp() - interval '1 hour') +AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') RETURNING *; -- name: RequestRuntimeCleanup :one diff --git a/services/agents-api/internal/db/queries/runtime_lifecycle_nodes.sql b/services/agents-api/internal/db/queries/runtime_lifecycle_nodes.sql index c4aa1c41d..3b6d7302f 100644 --- a/services/agents-api/internal/db/queries/runtime_lifecycle_nodes.sql +++ b/services/agents-api/internal/db/queries/runtime_lifecycle_nodes.sql @@ -1,12 +1,12 @@ -- name: ListRuntimeLifecycleNodes :many SELECT n.id FROM runtime_nodes n CROSS JOIN runtime_deployment d -WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id AND d.provider_kind<>'' +WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id AND d.mode='nodes' UNION ALL -SELECT NULL::uuid AS id FROM runtime_deployment WHERE provider_kind='' +SELECT NULL::uuid AS id FROM runtime_deployment WHERE mode IN ('','direct') ORDER BY id; -- name: ListRuntimeAllocationsForNode :many -SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired +SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id @@ -20,13 +20,14 @@ FROM environments e JOIN sessions s ON s.id=e.session_id LEFT JOIN runtime_placements p ON p.environment_id=e.id WHERE p.node_id IS NOT DISTINCT FROM sqlc.narg(node_id)::uuid AND p.released_at IS NULL + AND NOT (SELECT maintenance FROM runtime_deployment) AND e.id > sqlc.arg(after_id)::uuid AND s.deleted_at IS NULL AND e.status='pending' AND s.configuration->'environment'->>'type'='openai_hosted' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id=e.id) ORDER BY e.id LIMIT 32; -- name: GetRuntimeLifecyclePlacement :one -SELECT d.provider_kind, a.id AS allocation_id, a.node_id AS allocation_node_id, +SELECT d.provider_kind, d.mode, a.id AS allocation_id, a.node_id AS allocation_node_id, p.node_id AS placement_node_id, p.released_at FROM environments e JOIN sessions s ON s.id=e.session_id CROSS JOIN runtime_deployment d diff --git a/services/agents-api/internal/db/queries/runtime_nodes.sql b/services/agents-api/internal/db/queries/runtime_nodes.sql index b3f0f585f..caba1a174 100644 --- a/services/agents-api/internal/db/queries/runtime_nodes.sql +++ b/services/agents-api/internal/db/queries/runtime_nodes.sql @@ -1,5 +1,5 @@ -- name: SetRuntimeManagerDeployment :exec -UPDATE runtime_deployment SET provider_kind=$1, local_node_id=$2, owner_epoch=owner_epoch+1 WHERE singleton=true; +UPDATE runtime_deployment SET provider_kind=$1, local_node_id=$2, mode='nodes', generation=GREATEST(generation,1), owner_epoch=owner_epoch+1 WHERE singleton=true; -- name: GetRuntimeDeployment :one SELECT * FROM runtime_deployment WHERE singleton=true; diff --git a/services/agents-api/internal/db/queries/runtime_suspension.sql b/services/agents-api/internal/db/queries/runtime_suspension.sql index c7bfe90d3..fb5a32403 100644 --- a/services/agents-api/internal/db/queries/runtime_suspension.sql +++ b/services/agents-api/internal/db/queries/runtime_suspension.sql @@ -6,7 +6,7 @@ SET compute_phase = sqlc.arg(phase), compute_state = sqlc.arg(state)::jsonb, kept_at = CASE WHEN sqlc.arg(phase)::text = 'running' THEN clock_timestamp() ELSE kept_at END WHERE id = sqlc.arg(id) AND compute_revision = sqlc.arg(revision) AND state = 'running' AND initialization = 'complete' - AND ((compute_phase IN ('disabled','running') AND (node_id IS NOT NULL OR kept_at > clock_timestamp() - interval '1 hour')) + AND ((compute_phase IN ('disabled','running') AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour')) OR (compute_phase NOT IN ('disabled','running') AND compute_retained_until > clock_timestamp())) RETURNING *; diff --git a/services/agents-api/internal/db/queries/sandbox_deployment_setup.sql b/services/agents-api/internal/db/queries/sandbox_deployment_setup.sql index fe85ba126..a69244074 100644 --- a/services/agents-api/internal/db/queries/sandbox_deployment_setup.sql +++ b/services/agents-api/internal/db/queries/sandbox_deployment_setup.sql @@ -4,4 +4,16 @@ owner_epoch=owner_epoch+1, updated_at=clock_timestamp() WHERE singleton=true; -- name: InitializeSandboxDeployment :exec UPDATE runtime_deployment SET provider_kind=$1, core_url=$2, backend_fingerprint=$3, -idle_seconds=$4, retention_seconds=$5, updated_at=clock_timestamp() WHERE singleton=true; +idle_seconds=$4, retention_seconds=$5, generation=$6, mode=$7, e2b_template=$8, e2b_credential=$9, updated_at=clock_timestamp() WHERE singleton=true; + +-- name: SetSandboxMaintenance :exec +UPDATE runtime_deployment SET maintenance=$1,updated_at=clock_timestamp() WHERE singleton=true; + +-- name: RetireSandboxNodes :exec +UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL,provider_ready=false WHERE removed_at IS NULL; + +-- name: RetireSandboxEnrollments :exec +UPDATE runtime_node_enrollments SET expires_at=clock_timestamp() WHERE consumed_at IS NULL; + +-- name: AdvanceSandboxOwnerEpoch :exec +UPDATE runtime_deployment SET owner_epoch=owner_epoch+1 WHERE singleton=true; diff --git a/services/agents-api/internal/db/sqlc/devices.sql.go b/services/agents-api/internal/db/sqlc/devices.sql.go index 9f5c0d13b..7a2ff38a6 100644 --- a/services/agents-api/internal/db/sqlc/devices.sql.go +++ b/services/agents-api/internal/db/sqlc/devices.sql.go @@ -82,17 +82,18 @@ func (q *Queries) GetDevice(ctx context.Context, arg GetDeviceParams) (GetDevice } const getDeviceCredential = `-- name: GetDeviceCredential :one -SELECT d.id, d.name, d.credential_hash, COALESCE(a.node_id::text, '')::text AS runtime_node_id +SELECT d.id, d.name, d.credential_hash, COALESCE(a.node_id::text, '')::text AS runtime_node_id, COALESCE(a.id::text, '')::text AS runtime_allocation_id FROM runtime_device_authority d LEFT JOIN runtime_allocations a ON a.device_id = d.id WHERE d.id = $1 ` type GetDeviceCredentialRow struct { - ID pgtype.UUID `json:"id"` - Name string `json:"name"` - CredentialHash string `json:"credential_hash"` - RuntimeNodeID string `json:"runtime_node_id"` + ID pgtype.UUID `json:"id"` + Name string `json:"name"` + CredentialHash string `json:"credential_hash"` + RuntimeNodeID string `json:"runtime_node_id"` + RuntimeAllocationID string `json:"runtime_allocation_id"` } func (q *Queries) GetDeviceCredential(ctx context.Context, id pgtype.UUID) (GetDeviceCredentialRow, error) { @@ -103,6 +104,7 @@ func (q *Queries) GetDeviceCredential(ctx context.Context, id pgtype.UUID) (GetD &i.Name, &i.CredentialHash, &i.RuntimeNodeID, + &i.RuntimeAllocationID, ) return i, err } diff --git a/services/agents-api/internal/db/sqlc/initial_environment_files.sql.go b/services/agents-api/internal/db/sqlc/initial_environment_files.sql.go index fb240a05d..2f4068192 100644 --- a/services/agents-api/internal/db/sqlc/initial_environment_files.sql.go +++ b/services/agents-api/internal/db/sqlc/initial_environment_files.sql.go @@ -14,7 +14,7 @@ import ( const claimRuntimeInitialization = `-- name: ClaimRuntimeInitialization :one UPDATE runtime_allocations SET initialization = 'running' WHERE id = $1 AND initialization = 'pending' AND state = 'running' AND create_settled -AND (node_id IS NOT NULL OR kept_at > clock_timestamp() - interval '1 hour') +AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error ` @@ -47,7 +47,7 @@ func (q *Queries) ClaimRuntimeInitialization(ctx context.Context, id pgtype.UUID const completeRuntimeInitialization = `-- name: CompleteRuntimeInitialization :one UPDATE runtime_allocations SET initialization = 'complete' WHERE id = $1 AND initialization = 'running' AND state = 'running' AND create_settled -AND (node_id IS NOT NULL OR kept_at > clock_timestamp() - interval '1 hour') +AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error ` diff --git a/services/agents-api/internal/db/sqlc/models.go b/services/agents-api/internal/db/sqlc/models.go index 94afb91b0..555a40f80 100644 --- a/services/agents-api/internal/db/sqlc/models.go +++ b/services/agents-api/internal/db/sqlc/models.go @@ -198,6 +198,10 @@ type RuntimeDeployment struct { CoreUrl string `json:"core_url"` IdleSeconds int64 `json:"idle_seconds"` RetentionSeconds int64 `json:"retention_seconds"` + Generation int64 `json:"generation"` + Mode string `json:"mode"` + E2bTemplate string `json:"e2b_template"` + E2bCredential []byte `json:"e2b_credential"` } type RuntimeDeviceAuthority struct { diff --git a/services/agents-api/internal/db/sqlc/runtime_allocations.sql.go b/services/agents-api/internal/db/sqlc/runtime_allocations.sql.go index 468557487..a21fd6227 100644 --- a/services/agents-api/internal/db/sqlc/runtime_allocations.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_allocations.sql.go @@ -57,7 +57,7 @@ func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntime } const getRuntimeAllocation = `-- name: GetRuntimeAllocation :one -SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired +SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id @@ -110,7 +110,7 @@ func (q *Queries) GetRuntimeAllocation(ctx context.Context, arg GetRuntimeAlloca const keepRuntimeAllocation = `-- name: KeepRuntimeAllocation :one UPDATE runtime_allocations SET kept_at = clock_timestamp() WHERE id = $1 AND state = 'running' -AND (node_id IS NOT NULL OR kept_at > clock_timestamp() - interval '1 hour') +AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error ` @@ -141,7 +141,7 @@ func (q *Queries) KeepRuntimeAllocation(ctx context.Context, id pgtype.UUID) (Ru } const listRuntimeAllocations = `-- name: ListRuntimeAllocations :many -SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired +SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id @@ -280,7 +280,7 @@ func (q *Queries) ListUnallocatedHostedEnvironments(ctx context.Context, id pgty const observeRuntimeRunning = `-- name: ObserveRuntimeRunning :one UPDATE runtime_allocations SET state = 'running', create_settled = true WHERE id = $1 AND state IN ('creating', 'running') -AND (node_id IS NOT NULL OR kept_at > clock_timestamp() - interval '1 hour') +AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error ` diff --git a/services/agents-api/internal/db/sqlc/runtime_deployment.sql.go b/services/agents-api/internal/db/sqlc/runtime_deployment.sql.go index 4572cc053..ef4b0fabe 100644 --- a/services/agents-api/internal/db/sqlc/runtime_deployment.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_deployment.sql.go @@ -33,7 +33,7 @@ func (q *Queries) CountRuntimeDeploymentResources(ctx context.Context) (CountRun } const lockRuntimeDeployment = `-- name: LockRuntimeDeployment :one -SELECT singleton, installation_id, backend_fingerprint, maintenance, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, core_url, idle_seconds, retention_seconds FROM runtime_deployment WHERE singleton = true FOR UPDATE +SELECT singleton, installation_id, backend_fingerprint, maintenance, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, core_url, idle_seconds, retention_seconds, generation, mode, e2b_template, e2b_credential FROM runtime_deployment WHERE singleton = true FOR UPDATE ` func (q *Queries) LockRuntimeDeployment(ctx context.Context) (RuntimeDeployment, error) { @@ -52,6 +52,10 @@ func (q *Queries) LockRuntimeDeployment(ctx context.Context) (RuntimeDeployment, &i.CoreUrl, &i.IdleSeconds, &i.RetentionSeconds, + &i.Generation, + &i.Mode, + &i.E2bTemplate, + &i.E2bCredential, ) return i, err } diff --git a/services/agents-api/internal/db/sqlc/runtime_lifecycle_nodes.sql.go b/services/agents-api/internal/db/sqlc/runtime_lifecycle_nodes.sql.go index fb55ff11e..2bf9d7510 100644 --- a/services/agents-api/internal/db/sqlc/runtime_lifecycle_nodes.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_lifecycle_nodes.sql.go @@ -12,7 +12,7 @@ import ( ) const getRuntimeLifecyclePlacement = `-- name: GetRuntimeLifecyclePlacement :one -SELECT d.provider_kind, a.id AS allocation_id, a.node_id AS allocation_node_id, +SELECT d.provider_kind, d.mode, a.id AS allocation_id, a.node_id AS allocation_node_id, p.node_id AS placement_node_id, p.released_at FROM environments e JOIN sessions s ON s.id=e.session_id CROSS JOIN runtime_deployment d @@ -28,6 +28,7 @@ type GetRuntimeLifecyclePlacementParams struct { type GetRuntimeLifecyclePlacementRow struct { ProviderKind string `json:"provider_kind"` + Mode string `json:"mode"` AllocationID pgtype.UUID `json:"allocation_id"` AllocationNodeID pgtype.UUID `json:"allocation_node_id"` PlacementNodeID pgtype.UUID `json:"placement_node_id"` @@ -39,6 +40,7 @@ func (q *Queries) GetRuntimeLifecyclePlacement(ctx context.Context, arg GetRunti var i GetRuntimeLifecyclePlacementRow err := row.Scan( &i.ProviderKind, + &i.Mode, &i.AllocationID, &i.AllocationNodeID, &i.PlacementNodeID, @@ -48,7 +50,7 @@ func (q *Queries) GetRuntimeLifecyclePlacement(ctx context.Context, arg GetRunti } const listRuntimeAllocationsForNode = `-- name: ListRuntimeAllocationsForNode :many -SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired +SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id @@ -115,9 +117,9 @@ func (q *Queries) ListRuntimeAllocationsForNode(ctx context.Context, arg ListRun const listRuntimeLifecycleNodes = `-- name: ListRuntimeLifecycleNodes :many SELECT n.id FROM runtime_nodes n CROSS JOIN runtime_deployment d -WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id AND d.provider_kind<>'' +WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id AND d.mode='nodes' UNION ALL -SELECT NULL::uuid AS id FROM runtime_deployment WHERE provider_kind='' +SELECT NULL::uuid AS id FROM runtime_deployment WHERE mode IN ('','direct') ORDER BY id ` @@ -147,6 +149,7 @@ FROM environments e JOIN sessions s ON s.id=e.session_id LEFT JOIN runtime_placements p ON p.environment_id=e.id WHERE p.node_id IS NOT DISTINCT FROM $1::uuid AND p.released_at IS NULL + AND NOT (SELECT maintenance FROM runtime_deployment) AND e.id > $2::uuid AND s.deleted_at IS NULL AND e.status='pending' AND s.configuration->'environment'->>'type'='openai_hosted' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id=e.id) diff --git a/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go b/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go index bf388f0e0..9e0b2ca5e 100644 --- a/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go @@ -153,7 +153,7 @@ func (q *Queries) DisconnectRuntimeNode(ctx context.Context, arg DisconnectRunti } const getRuntimeDeployment = `-- name: GetRuntimeDeployment :one -SELECT singleton, installation_id, backend_fingerprint, maintenance, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, core_url, idle_seconds, retention_seconds FROM runtime_deployment WHERE singleton=true +SELECT singleton, installation_id, backend_fingerprint, maintenance, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, core_url, idle_seconds, retention_seconds, generation, mode, e2b_template, e2b_credential FROM runtime_deployment WHERE singleton=true ` func (q *Queries) GetRuntimeDeployment(ctx context.Context) (RuntimeDeployment, error) { @@ -172,6 +172,10 @@ func (q *Queries) GetRuntimeDeployment(ctx context.Context) (RuntimeDeployment, &i.CoreUrl, &i.IdleSeconds, &i.RetentionSeconds, + &i.Generation, + &i.Mode, + &i.E2bTemplate, + &i.E2bCredential, ) return i, err } @@ -553,7 +557,7 @@ func (q *Queries) RemoveRuntimeNode(ctx context.Context, id pgtype.UUID) error { } const setRuntimeManagerDeployment = `-- name: SetRuntimeManagerDeployment :exec -UPDATE runtime_deployment SET provider_kind=$1, local_node_id=$2, owner_epoch=owner_epoch+1 WHERE singleton=true +UPDATE runtime_deployment SET provider_kind=$1, local_node_id=$2, mode='nodes', generation=GREATEST(generation,1), owner_epoch=owner_epoch+1 WHERE singleton=true ` type SetRuntimeManagerDeploymentParams struct { diff --git a/services/agents-api/internal/db/sqlc/runtime_suspension.sql.go b/services/agents-api/internal/db/sqlc/runtime_suspension.sql.go index 8f88c646d..1b1c73b0f 100644 --- a/services/agents-api/internal/db/sqlc/runtime_suspension.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_suspension.sql.go @@ -136,7 +136,7 @@ SET compute_phase = $1, compute_state = $2::jsonb, kept_at = CASE WHEN $1::text = 'running' THEN clock_timestamp() ELSE kept_at END WHERE id = $4 AND compute_revision = $5 AND state = 'running' AND initialization = 'complete' - AND ((compute_phase IN ('disabled','running') AND (node_id IS NOT NULL OR kept_at > clock_timestamp() - interval '1 hour')) + AND ((compute_phase IN ('disabled','running') AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour')) OR (compute_phase NOT IN ('disabled','running') AND compute_retained_until > clock_timestamp())) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error ` diff --git a/services/agents-api/internal/db/sqlc/sandbox_deployment_setup.sql.go b/services/agents-api/internal/db/sqlc/sandbox_deployment_setup.sql.go index 5603b17f7..c2ccf1c5b 100644 --- a/services/agents-api/internal/db/sqlc/sandbox_deployment_setup.sql.go +++ b/services/agents-api/internal/db/sqlc/sandbox_deployment_setup.sql.go @@ -11,6 +11,15 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +const advanceSandboxOwnerEpoch = `-- name: AdvanceSandboxOwnerEpoch :exec +UPDATE runtime_deployment SET owner_epoch=owner_epoch+1 WHERE singleton=true +` + +func (q *Queries) AdvanceSandboxOwnerEpoch(ctx context.Context) error { + _, err := q.db.Exec(ctx, advanceSandboxOwnerEpoch) + return err +} + const claimWebSandboxDeployment = `-- name: ClaimWebSandboxDeployment :exec UPDATE runtime_deployment SET installation_id=$1, web_managed=true, owner_epoch=owner_epoch+1, updated_at=clock_timestamp() WHERE singleton=true @@ -23,7 +32,7 @@ func (q *Queries) ClaimWebSandboxDeployment(ctx context.Context, installationID const initializeSandboxDeployment = `-- name: InitializeSandboxDeployment :exec UPDATE runtime_deployment SET provider_kind=$1, core_url=$2, backend_fingerprint=$3, -idle_seconds=$4, retention_seconds=$5, updated_at=clock_timestamp() WHERE singleton=true +idle_seconds=$4, retention_seconds=$5, generation=$6, mode=$7, e2b_template=$8, e2b_credential=$9, updated_at=clock_timestamp() WHERE singleton=true ` type InitializeSandboxDeploymentParams struct { @@ -32,6 +41,10 @@ type InitializeSandboxDeploymentParams struct { BackendFingerprint string `json:"backend_fingerprint"` IdleSeconds int64 `json:"idle_seconds"` RetentionSeconds int64 `json:"retention_seconds"` + Generation int64 `json:"generation"` + Mode string `json:"mode"` + E2bTemplate string `json:"e2b_template"` + E2bCredential []byte `json:"e2b_credential"` } func (q *Queries) InitializeSandboxDeployment(ctx context.Context, arg InitializeSandboxDeploymentParams) error { @@ -41,6 +54,37 @@ func (q *Queries) InitializeSandboxDeployment(ctx context.Context, arg Initializ arg.BackendFingerprint, arg.IdleSeconds, arg.RetentionSeconds, + arg.Generation, + arg.Mode, + arg.E2bTemplate, + arg.E2bCredential, ) return err } + +const retireSandboxEnrollments = `-- name: RetireSandboxEnrollments :exec +UPDATE runtime_node_enrollments SET expires_at=clock_timestamp() WHERE consumed_at IS NULL +` + +func (q *Queries) RetireSandboxEnrollments(ctx context.Context) error { + _, err := q.db.Exec(ctx, retireSandboxEnrollments) + return err +} + +const retireSandboxNodes = `-- name: RetireSandboxNodes :exec +UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL,provider_ready=false WHERE removed_at IS NULL +` + +func (q *Queries) RetireSandboxNodes(ctx context.Context) error { + _, err := q.db.Exec(ctx, retireSandboxNodes) + return err +} + +const setSandboxMaintenance = `-- name: SetSandboxMaintenance :exec +UPDATE runtime_deployment SET maintenance=$1,updated_at=clock_timestamp() WHERE singleton=true +` + +func (q *Queries) SetSandboxMaintenance(ctx context.Context, maintenance bool) error { + _, err := q.db.Exec(ctx, setSandboxMaintenance, maintenance) + return err +} diff --git a/services/agents-api/internal/execution/runtime_lifecycle.go b/services/agents-api/internal/execution/runtime_lifecycle.go index a1599dcb1..e2a93f2ab 100644 --- a/services/agents-api/internal/execution/runtime_lifecycle.go +++ b/services/agents-api/internal/execution/runtime_lifecycle.go @@ -20,6 +20,8 @@ import ( // RuntimeProvider binds one deployment to one sandbox installation. // BackendFingerprint identifies its namespace independently of mutable sizing. type RuntimeProvider struct { + Generation uint64 + Mode string loadDeployment func(context.Context) (*RuntimeProvider, error) VerifyLegacyOwnership store.RuntimeOwnershipVerifier ProviderKind string @@ -67,7 +69,7 @@ func newRuntimeManager(s *store.Store, registry *gateway.Registry, config *Runti } } ctx, stop := context.WithCancel(context.Background()) - return &runtimeManager{store: s, registry: registry, config: copied, setupInstallationID: config.InstallationID, loadDeployment: config.loadDeployment, setupGate: make(chan struct{}, 1), ctx: ctx, cancel: stop, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1)}, nil + return &runtimeManager{store: s, registry: registry, config: copied, setupInstallationID: config.InstallationID, loadDeployment: config.loadDeployment, setupGate: make(chan struct{}, 1), mutationGate: make(chan struct{}, 1), ctx: ctx, cancel: stop, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1)}, nil } func validatedRuntimeProvider(config *RuntimeProvider, registry *gateway.Registry) (RuntimeProvider, error) { @@ -81,6 +83,15 @@ func validatedRuntimeProvider(config *RuntimeProvider, registry *gateway.Registr return RuntimeProvider{}, sandbox.ErrInvalid } copied := *config + if copied.Mode == "" && copied.ProviderKind != "" { + copied.Mode = "nodes" + } + if copied.Mode != "" && copied.Mode != "nodes" && copied.Mode != "direct" { + return RuntimeProvider{}, sandbox.ErrInvalid + } + if copied.Mode == "direct" && (copied.ProviderKind != "e2b" || copied.LocalNodeID != "" || copied.Suspension != nil) { + return RuntimeProvider{}, sandbox.ErrInvalid + } if config.Suspension != nil { policy := *config.Suspension if _, ok := config.Provider.(sandbox.CheckpointProvider); !ok || policy.IdleTimeout < time.Second || policy.Retention < time.Second || policy.MaxActive < 1 || policy.MaxRetained < policy.MaxActive { @@ -161,7 +172,7 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p return store.RuntimeAllocation{}, sandbox.ErrInvalid } if _, err := r.store.GetRuntimeAllocation(ctx, tenant, environment); errors.Is(err, store.ErrNotFound) { - if r.config.Maintenance { + if r.config.Maintenance && r.config.Generation == 0 { return store.RuntimeAllocation{}, ErrExecutionUnavailable } if err := r.computeFreshCapacity(ctx, providerKey); err != nil { @@ -201,6 +212,25 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p Reference: runtimeReference(owner), SessionID: owner.SessionID, DeviceID: owner.DeviceID, CoreURL: r.config.CoreURL, Credential: token, NetworkAccess: placement.NetworkAccess, AllowedDomains: placement.AllowedDomains, }) + if info.Reference == runtimeReference(owner) && info.CreateSettled && info.State == "absent" { + record, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + released, releaseErr := r.store.ReleaseAbsentRuntimeCreation(record, owner) + cancel() + if releaseErr != nil { + return owner, releaseErr + } + if err == nil { + err = sandbox.ErrComputeUnconfirmed + } + return released, err + } + if info.Reference == runtimeReference(owner) && info.CreateSettled { + settled, settleErr := r.store.SettleRuntimeCreation(ctx, owner) + if settleErr != nil { + return owner, settleErr + } + owner = settled + } if err != nil { // Explicit invalid/foreign bootstrap cannot become an authorized Runtime. // Other failures may hide a successful Create; retain recovery for those. @@ -317,7 +347,7 @@ func (r *runtimeLifecycle) observe(ctx context.Context, owner store.RuntimeAlloc if err == nil && info.Reference != runtimeReference(owner) { return sandbox.ErrOwnership } - if running && info.BootstrapComplete && !owner.CreateSettled { + if err == nil && (info.CreateSettled || running && info.BootstrapComplete) && !owner.CreateSettled { // Only the adapter can qualify completion of its bootstrap writes. owner, err = r.store.SettleRuntimeCreation(ctx, owner) if err != nil { diff --git a/services/agents-api/internal/execution/runtime_manager.go b/services/agents-api/internal/execution/runtime_manager.go index 271e03a79..0fb790a87 100644 --- a/services/agents-api/internal/execution/runtime_manager.go +++ b/services/agents-api/internal/execution/runtime_manager.go @@ -2,6 +2,8 @@ package execution import ( "context" + "errors" + "fmt" "sync" "time" @@ -11,6 +13,8 @@ import ( // runtimeManager owns node membership, not provider operations. Each registered // node has one serial lifecycle; slow provider work cannot occupy another node. +var errRuntimeTransition = fmt.Errorf("%w: sandbox configuration is changing", ErrExecutionUnavailable) + type runtimeManager struct { store *store.Store registry *gateway.Registry @@ -18,6 +22,9 @@ type runtimeManager struct { setupInstallationID string loadDeployment func(context.Context) (*RuntimeProvider, error) setupGate chan struct{} + mutationGate chan struct{} + switching bool + switchDrained chan struct{} ctx context.Context cancel context.CancelFunc mu sync.Mutex @@ -42,6 +49,10 @@ func (m *runtimeManager) enter(parent context.Context) (context.Context, func(), m.mu.Unlock() return nil, nil, ErrExecutionUnavailable } + if m.switching { + m.mu.Unlock() + return nil, nil, errRuntimeTransition + } m.active.Add(1) m.mu.Unlock() ctx, cancel := context.WithCancel(parent) @@ -54,7 +65,11 @@ func (m *runtimeManager) enter(parent context.Context) (context.Context, func(), func (m *runtimeManager) node(id string) (*runtimeNode, error) { m.mu.Lock() - if m.closed || (m.loadDeployment != nil && m.config.Provider == nil) || (id == "") != (m.config.ProviderKind == "") { + if m.switching { + m.mu.Unlock() + return nil, errRuntimeTransition + } + if m.closed || (m.loadDeployment != nil && m.config.Provider == nil) || (id == "") != (m.config.ProviderKind == "" || m.config.Mode == "direct") { m.mu.Unlock() return nil, ErrExecutionUnavailable } @@ -142,6 +157,10 @@ func (m *runtimeManager) applyInventory(previous map[string]*runtimeNode, ids [] } var retired []*runtimeNode m.mu.Lock() + if m.switching { + m.mu.Unlock() + return nil, errRuntimeTransition + } if m.closed { m.mu.Unlock() return nil, ErrExecutionUnavailable @@ -201,7 +220,7 @@ func (m *runtimeManager) run(ctx context.Context) error { } m.running = true m.mu.Unlock() - if _, err := m.syncNodes(ctx); err != nil { + if _, err := m.syncNodes(ctx); err != nil && !errors.Is(err, errRuntimeTransition) { return err } ticker := time.NewTicker(5 * time.Second) @@ -215,7 +234,7 @@ func (m *runtimeManager) run(ctx context.Context) error { case err := <-m.failed: return err case <-ticker.C: - if _, err := m.syncNodes(ctx); err != nil { + if _, err := m.syncNodes(ctx); err != nil && !errors.Is(err, errRuntimeTransition) { return err } } diff --git a/services/agents-api/internal/execution/runtime_pending.go b/services/agents-api/internal/execution/runtime_pending.go index 129ac46cd..407da7af7 100644 --- a/services/agents-api/internal/execution/runtime_pending.go +++ b/services/agents-api/internal/execution/runtime_pending.go @@ -10,7 +10,7 @@ import ( // provisionPending shares the existing lifecycle owner and serial gate. This // also recovers idle Session creation interrupted after its database commit. func (r *runtimeLifecycle) provisionPending(ctx context.Context) error { - if r.config.Maintenance { + if r.config.Maintenance && r.config.Generation == 0 { return nil } rows, err := r.store.ListUnallocatedHostedEnvironmentsForNode(ctx, r.nodeID, r.pendingCursor) diff --git a/services/agents-api/internal/execution/runtime_wake_hint.go b/services/agents-api/internal/execution/runtime_wake_hint.go index c7a694462..1aa6af62a 100644 --- a/services/agents-api/internal/execution/runtime_wake_hint.go +++ b/services/agents-api/internal/execution/runtime_wake_hint.go @@ -11,7 +11,14 @@ import ( // delivery failure leaves that input for the normal maintenance scan. func (w *Worker) hintRuntimeWake(ctx context.Context, session store.Session) { r := w.runtimes - if r == nil || r.config.Suspension == nil || r.ctx.Err() != nil { + if r == nil { + return + } + r.mu.Lock() + config := r.config + available := !r.closed && !r.switching + r.mu.Unlock() + if !available || config.Suspension == nil || r.ctx.Err() != nil { return } lookup, cancel := context.WithTimeout(ctx, time.Second) @@ -21,7 +28,7 @@ func (w *Worker) hintRuntimeWake(ctx context.Context, session store.Session) { return } owner, err := w.admission.GetRuntimeAllocation(lookup, session.TenantID, environment.ID) - if err != nil || owner.ProviderKey != r.config.InstallationID || owner.State != "running" || + if err != nil || owner.ProviderKey != config.InstallationID || owner.State != "running" || !owner.CreateSettled || owner.Initialization != "complete" || owner.SessionDeleted || owner.Expired { return } diff --git a/services/agents-api/internal/execution/sandbox_deployment_setup.go b/services/agents-api/internal/execution/sandbox_deployment_setup.go index 70fae6f1a..6c3d146ac 100644 --- a/services/agents-api/internal/execution/sandbox_deployment_setup.go +++ b/services/agents-api/internal/execution/sandbox_deployment_setup.go @@ -2,22 +2,25 @@ package execution import ( "context" + "errors" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" ) // NewDeferredRuntimeProvider enables Web setup for one fixed installation. The -// loader returns nil until selection, then an immutable remote-node provider. -// This is one-time activation, not provider replacement or configuration reload. +// loader returns nil until selection, then the committed immutable generation. +// Replacement is serialized by the deployment maintenance and drain flow. func NewDeferredRuntimeProvider(installationID string, load func(context.Context) (*RuntimeProvider, error)) *RuntimeProvider { return &RuntimeProvider{InstallationID: installationID, loadDeployment: load} } func (w *Worker) InitializeSandboxDeployment(ctx context.Context, input store.SandboxDeploymentSetupRequest) (store.RuntimeDeploymentView, error) { - if w.runtimes == nil || w.runtimes.loadDeployment == nil { - return store.RuntimeDeploymentView{}, store.ErrSandboxDeploymentConflict + unlock, err := w.runtimes.lockMutation(ctx) + if err != nil { + return store.RuntimeDeploymentView{}, err } + defer unlock() return w.dispatcher.Store.InitializeSandboxDeployment(ctx, w.runtimes.setupInstallationID, input) } @@ -45,10 +48,16 @@ func (m *runtimeManager) ensureDeployment(parent context.Context) (bool, error) return true, nil } config, err := m.loadDeployment(ctx) + // A missing local provider dependency blocks hosted execution, not the + // administrator's recovery API. The existing scan can load it after repair. + // Storage and ownership failures still stop the execution owner. + if errors.Is(err, ErrExecutionUnavailable) { + return false, nil + } if err != nil || config == nil { return false, err } - if config.InstallationID != m.setupInstallationID || config.LocalNodeID != "" || config.loadDeployment != nil || (config.ProviderKind != "docker" && config.ProviderKind != "microsandbox") { + if config.InstallationID != m.setupInstallationID || config.LocalNodeID != "" || config.loadDeployment != nil || (config.ProviderKind != "docker" && config.ProviderKind != "microsandbox" && config.ProviderKind != "e2b") { return false, sandbox.ErrInvalid } copied, err := validatedRuntimeProvider(config, m.registry) @@ -57,6 +66,9 @@ func (m *runtimeManager) ensureDeployment(parent context.Context) (bool, error) } m.mu.Lock() defer m.mu.Unlock() + if m.switching { + return false, errRuntimeTransition + } if m.closed || ctx.Err() != nil { return false, ErrExecutionUnavailable } diff --git a/services/agents-api/internal/execution/sandbox_deployment_setup_test.go b/services/agents-api/internal/execution/sandbox_deployment_setup_test.go index ac16c7de2..3b7741357 100644 --- a/services/agents-api/internal/execution/sandbox_deployment_setup_test.go +++ b/services/agents-api/internal/execution/sandbox_deployment_setup_test.go @@ -82,3 +82,36 @@ func TestDeferredSandboxDeploymentShutdownCancelsLoad(t *testing.T) { t.Fatal("shutdown did not cancel setup load", err) } } + +func TestDeferredSandboxProviderFailureKeepsRecoveryAvailable(t *testing.T) { + hub := node.NewHub(node.HubOptions{}) + defer hub.Close() + id := uuid.NewString() + available := false + loadErr := ErrExecutionUnavailable + configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker")} + m, err := newRuntimeManager(nil, gateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { + if !available { + return nil, loadErr + } + return configuration, nil + })) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { m.stop(); m.drain() }) + if nodes, err := m.syncNodes(t.Context()); err != nil || len(nodes) != 0 { + t.Fatal("unavailable provider stopped the manager", err) + } + if _, err := m.node("node"); !errors.Is(err, ErrExecutionUnavailable) { + t.Fatal("unavailable provider admitted execution", err) + } + loadErr = errors.New("storage failure") + if _, err := m.ensureDeployment(t.Context()); !errors.Is(err, loadErr) { + t.Fatal("provider recovery hid a storage failure", err) + } + available = true + if ready, err := m.ensureDeployment(t.Context()); err != nil || !ready { + t.Fatal("repaired provider did not activate", ready, err) + } +} diff --git a/services/agents-api/internal/execution/sandbox_deployment_switch.go b/services/agents-api/internal/execution/sandbox_deployment_switch.go new file mode 100644 index 000000000..c9ab529af --- /dev/null +++ b/services/agents-api/internal/execution/sandbox_deployment_switch.go @@ -0,0 +1,155 @@ +package execution + +import ( + "context" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +func (m *runtimeManager) lockMutation(ctx context.Context) (func(), error) { + if m == nil || m.loadDeployment == nil { + return nil, store.ErrSandboxDeploymentConflict + } + select { + case m.mutationGate <- struct{}{}: + return func() { <-m.mutationGate }, nil + case <-ctx.Done(): + return nil, ctx.Err() + case <-m.ctx.Done(): + return nil, ErrExecutionUnavailable + } +} + +// pauseDeployment blocks new callers before waiting for the existing accounting. +// The coordinator remains alive; no database lock spans cancellation or draining. +func (m *runtimeManager) pauseDeployment(ctx context.Context) error { + m.mu.Lock() + if m.closed { + m.mu.Unlock() + return ErrExecutionUnavailable + } + if !m.switching { + m.switching = true + m.switchDrained = make(chan struct{}) + go m.drainDeployment(m.switchDrained) + } + drained := m.switchDrained + m.mu.Unlock() + select { + case <-drained: + return nil + case <-ctx.Done(): + return ctx.Err() + case <-m.ctx.Done(): + return ErrExecutionUnavailable + } +} + +// One drain outlives the initiating HTTP request. A cancelled waiter cannot +// permit a replacement generation while an old caller is still completing. +func (m *runtimeManager) drainDeployment(done chan struct{}) { + defer close(done) + for _, gate := range []chan struct{}{m.setupGate, m.inventory} { + select { + case gate <- struct{}{}: + <-gate + case <-m.ctx.Done(): + } + } + m.mu.Lock() + for _, n := range m.nodes { + n.lifecycle.stop() + } + m.mu.Unlock() + m.active.Wait() +} + +// activateDeployment is serialized with every Web configuration mutation. The +// loader reads the committed generation and publishes the same server snapshot. +func (m *runtimeManager) activateDeployment(ctx context.Context, expected store.RuntimeDeploymentView) error { + m.mu.Lock() + if m.closed { + m.mu.Unlock() + return ErrExecutionUnavailable + } + if !m.switching && m.config.Provider != nil && m.config.Generation == expected.Generation { + m.mu.Unlock() + return nil + } + switching := m.switching + m.mu.Unlock() + if switching { + if err := m.pauseDeployment(ctx); err != nil { + return err + } + } + config, err := m.loadDeployment(ctx) + if err != nil { + return err + } + if config == nil || config.InstallationID != expected.InstallationID || config.Generation != expected.Generation || config.Mode != expected.Mode || config.ProviderKind != expected.Provider || config.loadDeployment != nil || config.LocalNodeID != "" { + return sandbox.ErrInvalid + } + copied, err := validatedRuntimeProvider(config, m.registry) + if err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + if m.closed || ctx.Err() != nil { + return ErrExecutionUnavailable + } + // Switching was only admitted with no retained or pending resources. + if m.switching { + m.nodes = make(map[string]*runtimeNode) + } + m.config = copied + m.switching = false + m.switchDrained = nil + return nil +} + +func (w *Worker) UpdateSandboxDeployment(ctx context.Context, input store.SandboxDeploymentUpdateRequest) (store.RuntimeDeploymentView, error) { + unlock, err := w.runtimes.lockMutation(ctx) + if err != nil { + return store.RuntimeDeploymentView{}, err + } + defer unlock() + m := w.runtimes + if err := m.store.CheckSandboxDeploymentSwitch(ctx, m.setupInstallationID, input); err != nil { + return store.RuntimeDeploymentView{}, err + } + if err := m.pauseDeployment(ctx); err != nil { + return store.RuntimeDeploymentView{}, err + } + result, err := m.store.UpdateSandboxDeployment(ctx, m.setupInstallationID, input) + if err != nil { + return store.RuntimeDeploymentView{}, err + } + if err := m.activateDeployment(ctx, result); err != nil { + return store.RuntimeDeploymentView{}, err + } + return result, nil +} + +func (w *Worker) SetSandboxMaintenance(ctx context.Context, input store.SandboxMaintenanceRequest) (store.RuntimeDeploymentView, error) { + unlock, err := w.runtimes.lockMutation(ctx) + if err != nil { + return store.RuntimeDeploymentView{}, err + } + defer unlock() + m := w.runtimes + if !input.Maintenance { + expected, err := m.store.GetRuntimeDeployment(ctx) + if err != nil { + return store.RuntimeDeploymentView{}, err + } + if expected.Generation != input.ExpectedGeneration { + return store.RuntimeDeploymentView{}, store.ErrSandboxDeploymentConflict + } + if err := m.activateDeployment(ctx, expected); err != nil { + return store.RuntimeDeploymentView{}, err + } + } + return m.store.SetSandboxMaintenance(ctx, m.setupInstallationID, input) +} diff --git a/services/agents-api/internal/execution/sandbox_deployment_switch_test.go b/services/agents-api/internal/execution/sandbox_deployment_switch_test.go new file mode 100644 index 000000000..ade1c822f --- /dev/null +++ b/services/agents-api/internal/execution/sandbox_deployment_switch_test.go @@ -0,0 +1,151 @@ +package execution + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/node" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" +) + +func TestSandboxManagerSwitchDrainsBeforeDirectActivation(t *testing.T) { + hub := node.NewHub(node.HubOptions{}) + defer hub.Close() + id := uuid.NewString() + config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker")} + m, err := newRuntimeManager(nil, gateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil })) + if err != nil { + t.Fatal(err) + } + defer func() { m.stop(); m.drain() }() + if _, err := m.ensureDeployment(t.Context()); err != nil { + t.Fatal(err) + } + old, err := m.node(uuid.NewString()) + if err != nil { + t.Fatal(err) + } + _, finish, err := m.enter(t.Context()) + if err != nil { + t.Fatal(err) + } + paused := make(chan error, 1) + go func() { paused <- m.pauseDeployment(t.Context()) }() + select { + case <-old.lifecycle.ctx.Done(): + case <-time.After(time.Second): + finish() + t.Fatal("old lifecycle not cancelled") + } + if _, err := m.node(uuid.NewString()); !errors.Is(err, errRuntimeTransition) { + finish() + t.Fatal("transition admitted new lifecycle", err) + } + select { + case err := <-paused: + finish() + t.Fatal("switch skipped active caller", err) + default: + } + finish() + select { + case err := <-paused: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("switch drain blocked") + } + config = &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", Generation: 2, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("b", 64), Provider: hub.Proxy(uuid.NewString(), "docker")} + if err := m.activateDeployment(t.Context(), store.RuntimeDeploymentView{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b"}); err != nil { + t.Fatal(err) + } + if _, err := m.node(uuid.NewString()); !errors.Is(err, ErrExecutionUnavailable) { + t.Fatal("cloud created physical lifecycle", err) + } + current, err := m.node("") + if err != nil || current.lifecycle.config.Generation != 2 { + t.Fatal("cloud lifecycle not activated", err) + } + if len(m.nodes) != 1 { + t.Fatal("old lifecycle retained after activation") + } +} + +func TestSandboxManagerFailedActivationStaysPaused(t *testing.T) { + id := uuid.NewString() + m, err := newRuntimeManager(nil, gateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, errors.New("provider unavailable") })) + if err != nil { + t.Fatal(err) + } + defer func() { m.stop(); m.drain() }() + if err := m.pauseDeployment(t.Context()); err != nil { + t.Fatal(err) + } + if err := m.activateDeployment(t.Context(), store.RuntimeDeploymentView{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b"}); err == nil { + t.Fatal("failed provider activated") + } + if _, _, err := m.enter(t.Context()); !errors.Is(err, errRuntimeTransition) { + t.Fatal("failed activation reopened work", err) + } +} + +func TestSandboxManagerCancelledSwitchCannotResumeBeforeDrain(t *testing.T) { + hub := node.NewHub(node.HubOptions{}) + defer hub.Close() + id := uuid.NewString() + config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker")} + m, err := newRuntimeManager(nil, gateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil })) + if err != nil { + t.Fatal(err) + } + if _, err := m.ensureDeployment(t.Context()); err != nil { + t.Fatal(err) + } + _, finish, err := m.enter(t.Context()) + if err != nil { + t.Fatal(err) + } + released := false + defer func() { + if !released { + finish() + } + m.stop() + m.drain() + }() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond) + if err := m.pauseDeployment(ctx); !errors.Is(err, context.DeadlineExceeded) { + cancel() + t.Fatal("initial pause did not wait for old caller", err) + } + cancel() + m.mu.Lock() + originalDrain := m.switchDrained + m.mu.Unlock() + config = &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", Generation: 2, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("b", 64), Provider: hub.Proxy(uuid.NewString(), "docker")} + expected := store.RuntimeDeploymentView{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b"} + ctx, cancel = context.WithTimeout(t.Context(), 10*time.Millisecond) + if err := m.activateDeployment(ctx, expected); !errors.Is(err, context.DeadlineExceeded) { + cancel() + t.Fatal("resume bypassed outstanding drain", err) + } + cancel() + m.mu.Lock() + sameDrain := m.switchDrained == originalDrain + generation := m.config.Generation + m.mu.Unlock() + if !sameDrain || generation != 1 { + t.Fatal("retry replaced drain or activated configuration") + } + finish() + released = true + if err := m.activateDeployment(t.Context(), expected); err != nil { + t.Fatal("drained generation did not resume", err) + } +} diff --git a/services/agents-api/internal/sandbox/e2b/process.go b/services/agents-api/internal/sandbox/e2b/process.go new file mode 100644 index 000000000..0a76dbfa3 --- /dev/null +++ b/services/agents-api/internal/sandbox/e2b/process.go @@ -0,0 +1,80 @@ +package e2b + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "strings" +) + +// ProcessCaller retains and drains an outstanding helper after caller timeout. +// The helper keeps its allocation lock until its bounded SDK operation settles. +var errHelperNotStarted = errors.New("helper did not start") + +type ProcessCaller struct{} + +func (*ProcessCaller) Call(ctx context.Context, q Request) (Response, error) { + data, err := json.Marshal(q) + if err != nil || len(data) > MaxRequestBytes { + return Response{}, errHelperNotStarted + } + cmd := exec.Command(q.Config.Binary) + cmd.Stdin = bytes.NewReader(data) + for _, entry := range os.Environ() { + key, _, _ := strings.Cut(entry, "=") + if key == "HOME" || key == "PATH" || key == "TMPDIR" || key == "LANG" || key == "SSL_CERT_FILE" || key == "SSL_CERT_DIR" { + cmd.Env = append(cmd.Env, entry) + } + } + cmd.Env = append(cmd.Env, "PYTHONNOUSERSITE=1", "PYTHONDONTWRITEBYTECODE=1") + stdout, stderr := &limitBuffer{limit: MaxResponseBytes}, &limitBuffer{limit: MaxOutputBytes} + cmd.Stdout, cmd.Stderr = stdout, stderr + if ctx.Err() != nil { + return Response{}, errHelperNotStarted + } + if cmd.Start() != nil { + return Response{}, errHelperNotStarted + } + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + select { + case <-ctx.Done(): + return Response{}, ctx.Err() + case err = <-done: + if err != nil || stdout.exceeded || stderr.exceeded { + return Response{}, errors.New("helper result unconfirmed") + } + } + var out Response + decoder := json.NewDecoder(bytes.NewReader(stdout.Bytes())) + decoder.DisallowUnknownFields() + if decoder.Decode(&out) != nil { + return Response{}, errors.New("invalid helper response") + } + var extra any + if decoder.Decode(&extra) != io.EOF { + return Response{}, errors.New("trailing helper response") + } + return out, nil +} + +type limitBuffer struct { + bytes.Buffer + limit int + exceeded bool +} + +func (b *limitBuffer) Write(data []byte) (int, error) { + n := len(data) + left := b.limit - b.Len() + if n > left { + b.exceeded = true + data = data[:left] + } + _, _ = b.Buffer.Write(data) + return n, nil +} diff --git a/services/agents-api/internal/sandbox/e2b/process_test.go b/services/agents-api/internal/sandbox/e2b/process_test.go new file mode 100644 index 000000000..2c46b245a --- /dev/null +++ b/services/agents-api/internal/sandbox/e2b/process_test.go @@ -0,0 +1,61 @@ +package e2b + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestProcessWireRejectsUnknownAndTrailingOutput(t *testing.T) { + for _, output := range []string{`{"Version":1,"Secret":"private"}`, `{"Version":1} {"Version":1}`} { + root := t.TempDir() + binary := filepath.Join(root, "helper") + if err := os.WriteFile(binary, []byte("#!/bin/sh\nprintf '%s' '"+output+"'\n"), 0700); err != nil { + t.Fatal(err) + } + if _, err := (&ProcessCaller{}).Call(bounded(t), Request{Config: Config{Binary: binary}}); err == nil || strings.Contains(err.Error(), "private") { + t.Fatal("invalid helper response accepted or leaked", err) + } + } +} +func TestTimeoutDoesNotTerminateOwnedHelper(t *testing.T) { + root := t.TempDir() + binary := filepath.Join(root, "helper") + marker := filepath.Join(root, "settled") + // The private test path contains no shell syntax; the real adapter never puts + // credentials or request contents in argv or inherited environment. + script := "#!/bin/sh\nsleep 0.15\ntouch '" + marker + "'\nprintf '%s' '{\"Version\":1}'\n" + if err := os.WriteFile(binary, []byte(script), 0700); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond) + defer cancel() + if _, err := (&ProcessCaller{}).Call(ctx, Request{Config: Config{Binary: binary}}); err == nil { + t.Fatal("timeout not reported") + } + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(marker); err == nil { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("helper was killed before it could settle") +} +func TestEnvironmentDropsProviderSelectorsAndCredentials(t *testing.T) { + root := t.TempDir() + binary := filepath.Join(root, "helper") + script := "#!/bin/sh\nif test -n \"${E2B_API_KEY:-}${E2B_API_URL:-}${PRIVATE_MODEL_KEY:-}\"; then exit 1; fi\nprintf '%s' '{\"Version\":1}'\n" + if err := os.WriteFile(binary, []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("E2B_API_KEY", "secret") + t.Setenv("E2B_API_URL", "https://wrong.example") + t.Setenv("PRIVATE_MODEL_KEY", "secret") + if _, err := (&ProcessCaller{}).Call(bounded(t), Request{Config: Config{Binary: binary}}); err != nil { + t.Fatal(err) + } +} diff --git a/services/agents-api/internal/sandbox/e2b/provider.go b/services/agents-api/internal/sandbox/e2b/provider.go new file mode 100644 index 000000000..4cccee312 --- /dev/null +++ b/services/agents-api/internal/sandbox/e2b/provider.go @@ -0,0 +1,195 @@ +// Package e2b adapts the official SDK helper to managed compute operations. +package e2b + +import ( + "context" + "errors" + "net/url" + "os" + "path/filepath" + "strings" + "time" + "unicode" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentnetwork" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/google/uuid" +) + +const ProtocolVersion = 1 +const SDKVersion = "2.51.0" +const MaxOutputBytes = 1024 * 1024 +const MaxRequestBytes = 72 * 1024 * 1024 +const MaxResponseBytes = 16 * 1024 * 1024 + +// Config contains trusted deployment configuration; APIKey travels only on stdin. +type Config struct { + Binary, StateDir, InstallationID, APIKey, Template string + TimeoutSeconds int +} + +type Request struct { + Version int + Operation string + Config Config + Reference sandbox.Reference + Bootstrap *sandbox.Bootstrap `json:",omitempty"` + Command *sandbox.Command `json:",omitempty"` + Deadline time.Time +} +type Response struct { + Version int + Info *sandbox.Info `json:",omitempty"` + Command *sandbox.CommandResult `json:",omitempty"` + ErrorCode string +} +type Caller interface { + Call(context.Context, Request) (Response, error) +} +type Provider struct { + config Config + caller Caller +} + +var _ sandbox.Provider = (*Provider)(nil) + +func validID(value string) bool { + id, err := uuid.Parse(value) + return err == nil && id != uuid.Nil && id.String() == value +} +func validReference(r sandbox.Reference) bool { + return validID(r.TenantID) && validID(r.EnvironmentID) && validID(r.AllocationID) +} +func (c Config) Validate() error { + template, build, ok := strings.Cut(c.Template, ":") + if !ok || template == "" || !validID(build) || !validID(c.InstallationID) || c.TimeoutSeconds < 1 || c.TimeoutSeconds > 86400 || c.APIKey == "" || len(c.APIKey) > 4096 || strings.ContainsFunc(c.APIKey, func(r rune) bool { return unicode.IsSpace(r) || r == 0 }) { + return sandbox.ErrInvalid + } + for _, r := range template { + if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_') { + return sandbox.ErrInvalid + } + } + for _, path := range []string{c.Binary, c.StateDir} { + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return sandbox.ErrInvalid + } + } + binary, err := os.Stat(c.Binary) + if err != nil || !binary.Mode().IsRegular() || binary.Mode().Perm()&0111 == 0 { + return sandbox.ErrInvalid + } + state, err := os.Lstat(c.StateDir) + if err != nil || !state.IsDir() || state.Mode().Perm()&0077 != 0 { + return sandbox.ErrInvalid + } + return nil +} +func New(c Config) (*Provider, error) { return NewWithCaller(c, &ProcessCaller{}) } +func NewWithCaller(c Config, caller Caller) (*Provider, error) { + if c.Validate() != nil || caller == nil { + return nil, sandbox.ErrInvalid + } + return &Provider{config: c, caller: caller}, nil +} +func (p *Provider) call(ctx context.Context, operation string, r sandbox.Reference, b *sandbox.Bootstrap, command *sandbox.Command) (Response, error) { + deadline, ok := ctx.Deadline() + if !ok || !validReference(r) { + return unstarted(operation, r), sandbox.ErrInvalid + } + if err := ctx.Err(); err != nil { + return unstarted(operation, r), err + } + out, err := p.caller.Call(ctx, Request{Version: ProtocolVersion, Operation: operation, Config: p.config, Reference: r, Bootstrap: b, Command: command, Deadline: deadline}) + if errors.Is(err, errHelperNotStarted) { + return unstarted(operation, r), sandbox.ErrComputeUnconfirmed + } + if err != nil || out.Version != ProtocolVersion { + if operation == "command" { + return Response{}, sandbox.ErrCommandUnconfirmed + } + return Response{}, sandbox.ErrComputeUnconfirmed + } + if out.Info != nil && (out.Info.Reference != r || len(out.Info.ProviderID) > 256 || len(out.Info.State) > 64 || out.Info.BootstrapComplete && !out.Info.CreateSettled) { + return Response{}, sandbox.ErrComputeUnconfirmed + } + switch out.ErrorCode { + case "": + return out, nil + case "invalid": + return out, sandbox.ErrInvalid + case "ownership": + return out, sandbox.ErrOwnership + case "exists": + return out, sandbox.ErrExists + case "not_found": + return out, sandbox.ErrNotFound + case "command_unconfirmed": + return out, sandbox.ErrCommandUnconfirmed + default: + return out, sandbox.ErrComputeUnconfirmed + } +} +func (p *Provider) info(ctx context.Context, operation string, r sandbox.Reference, b *sandbox.Bootstrap) (sandbox.Info, error) { + out, err := p.call(ctx, operation, r, b, nil) + if out.Info != nil { + return *out.Info, err + } + if err == nil { + err = sandbox.ErrComputeUnconfirmed + } + return sandbox.Info{Reference: r}, err +} +func (p *Provider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { + u, err := url.Parse(b.CoreURL) + policy := agentnetwork.Policy{Access: b.NetworkAccess, AllowedDomains: b.AllowedDomains} + if !validReference(b.Reference) || !validID(b.SessionID) || !validID(b.DeviceID) || policy.Validate() != nil || err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.RawQuery != "" || u.Fragment != "" || strings.TrimSpace(b.Credential) == "" { + info := sandbox.Info{Reference: b.Reference} + if validReference(b.Reference) { + info.State, info.CreateSettled = "absent", true + } + return info, sandbox.ErrInvalid + } + b.AllowedDomains = policy.Hosts() + return p.info(ctx, "create", b.Reference, &b) +} +func (p *Provider) GetInfo(ctx context.Context, r sandbox.Reference) (sandbox.Info, error) { + return p.info(ctx, "inspect", r, nil) +} +func (p *Provider) Renew(ctx context.Context, r sandbox.Reference) (sandbox.Info, error) { + return p.info(ctx, "renew", r, nil) +} +func (p *Provider) Kill(ctx context.Context, r sandbox.Reference) error { + out, err := p.call(ctx, "kill", r, nil, nil) + if err == nil && (out.Info == nil || !out.Info.CreateSettled || out.Info.State != "absent") { + return sandbox.ErrComputeUnconfirmed + } + return err +} +func (p *Provider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { + if len(c.Args) == 0 || len(c.Stdin) > sandbox.MaxCommandInputBytes || c.Directory != "" && !filepath.IsAbs(c.Directory) { + return sandbox.CommandResult{}, sandbox.ErrInvalid + } + for _, arg := range c.Args { + if strings.ContainsRune(arg, 0) { + return sandbox.CommandResult{}, sandbox.ErrInvalid + } + } + out, err := p.call(ctx, "command", r, nil, &c) + if err != nil { + return sandbox.CommandResult{}, err + } + if out.Command == nil || len(out.Command.Stdout) > MaxOutputBytes || len(out.Command.Stderr) > MaxOutputBytes { + return sandbox.CommandResult{}, sandbox.ErrCommandUnconfirmed + } + return *out.Command, nil +} + +// A fresh allocation Create rejected before process startup has no cloud effects. +// The common Provider contract forbids replaying an earlier unknown Create. +func unstarted(operation string, r sandbox.Reference) Response { + if operation == "create" && validReference(r) { + return Response{Info: &sandbox.Info{Reference: r, State: "absent", CreateSettled: true}} + } + return Response{} +} diff --git a/services/agents-api/internal/sandbox/e2b/provider_test.go b/services/agents-api/internal/sandbox/e2b/provider_test.go new file mode 100644 index 000000000..05fae786c --- /dev/null +++ b/services/agents-api/internal/sandbox/e2b/provider_test.go @@ -0,0 +1,160 @@ +package e2b + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/google/uuid" +) + +type fakeCaller struct { + response Response + err error + requests []Request +} + +func (f *fakeCaller) Call(_ context.Context, q Request) (Response, error) { + f.requests = append(f.requests, q) + return f.response, f.err +} +func fixture(t *testing.T) (*Provider, *fakeCaller, sandbox.Reference) { + t.Helper() + root := t.TempDir() + if err := os.Chmod(root, 0700); err != nil { + t.Fatal(err) + } + binary := filepath.Join(root, "helper") + if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 1\n"), 0700); err != nil { + t.Fatal(err) + } + config := Config{Binary: binary, StateDir: root, InstallationID: uuid.NewString(), APIKey: "private-test-key", Template: "test:" + uuid.NewString(), TimeoutSeconds: 300} + caller := &fakeCaller{response: Response{Version: ProtocolVersion}} + provider, err := NewWithCaller(config, caller) + if err != nil { + t.Fatal(err) + } + return provider, caller, sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()} +} +func bounded(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + t.Cleanup(cancel) + return ctx +} +func TestReadOnlyConstructionAndValidation(t *testing.T) { + p, f, _ := fixture(t) + if len(f.requests) != 0 { + t.Fatal("construction invoked helper") + } + for _, change := range []func(*Config){func(c *Config) { c.Template = "mutable" }, func(c *Config) { c.APIKey = "key\n" }, func(c *Config) { c.StateDir = "relative" }, func(c *Config) { c.TimeoutSeconds = 0 }, func(c *Config) { c.Binary = "/missing/helper" }} { + c := p.config + change(&c) + if _, err := NewWithCaller(c, f); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("invalid configuration accepted") + } + } +} +func TestReferenceBoundSettlement(t *testing.T) { + p, f, r := fixture(t) + f.response.Info = &sandbox.Info{Reference: r, State: "absent", CreateSettled: true} + info, err := p.GetInfo(bounded(t), r) + if err != nil || !info.CreateSettled || info.State != "absent" { + t.Fatalf("absence proof: %+v %v", info, err) + } + f.response.Info.AllocationID = uuid.NewString() + if _, err = p.GetInfo(bounded(t), r); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { + t.Fatal("foreign proof accepted", err) + } +} +func TestUnknownDoesNotLeakHelperDiagnosticOrProveAbsence(t *testing.T) { + p, f, r := fixture(t) + f.err = errors.New("private-account-key in SDK response") + info, err := p.GetInfo(bounded(t), r) + if !errors.Is(err, sandbox.ErrComputeUnconfirmed) || info.CreateSettled { + t.Fatal(info, err) + } + if err.Error() != "sandbox lifecycle outcome unconfirmed" { + t.Fatal("helper diagnostic leaked") + } + f.err = nil + f.response.ErrorCode = "not_found" + if _, err = p.GetInfo(bounded(t), r); !errors.Is(err, sandbox.ErrNotFound) { + t.Fatal(err) + } +} +func TestKillRequiresTerminalProof(t *testing.T) { + p, f, r := fixture(t) + for _, info := range []*sandbox.Info{nil, {Reference: r, State: "absent"}, {Reference: r, State: "running", CreateSettled: true}} { + f.response.Info = info + if !errors.Is(p.Kill(bounded(t), r), sandbox.ErrComputeUnconfirmed) { + t.Fatal("unproven cleanup accepted") + } + } + f.response.Info = &sandbox.Info{Reference: r, State: "absent", CreateSettled: true} + if err := p.Kill(bounded(t), r); err != nil { + t.Fatal(err) + } +} +func TestCreateAndCommandUseOnlyPrivateRequest(t *testing.T) { + p, f, r := fixture(t) + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "private-bootstrap", NetworkAccess: "enabled"} + f.response.Info = &sandbox.Info{Reference: r, State: "running", ProviderID: "native-id", CreateSettled: true, BootstrapComplete: true} + if _, err := p.Create(bounded(t), b); err != nil { + t.Fatal(err) + } + q := f.requests[0] + if q.Operation != "create" || q.Bootstrap == nil || q.Bootstrap.Credential != b.Credential || q.Config.APIKey != p.config.APIKey { + t.Fatal("private request lost") + } + f.response.Info = nil + f.response.Command = &sandbox.CommandResult{ExitCode: 7, Stdout: "output"} + got, err := p.RunCommand(bounded(t), r, sandbox.Command{Args: []string{"cat"}, Stdin: []byte("private input")}) + if err != nil || got.ExitCode != 7 { + t.Fatal(got, err) + } + f.err = context.DeadlineExceeded + if _, err = p.RunCommand(bounded(t), r, sandbox.Command{Args: []string{"true"}}); !errors.Is(err, sandbox.ErrCommandUnconfirmed) { + t.Fatal(err) + } + if len(f.requests) != 3 { + t.Fatal("operation was replayed") + } +} +func TestDeadlineAndCommandAdmission(t *testing.T) { + p, f, r := fixture(t) + if _, err := p.GetInfo(context.Background(), r); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal(err) + } + for _, cmd := range []sandbox.Command{{}, {Args: []string{"x\x00"}}, {Args: []string{"cat"}, Directory: "relative"}} { + if _, err := p.RunCommand(bounded(t), r, cmd); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal(err) + } + } + if len(f.requests) != 0 { + t.Fatal("invalid operation reached helper") + } +} + +func TestDefinitePreHelperCreateFailureCarriesAbsenceProof(t *testing.T) { + p, f, r := fixture(t) + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "private", NetworkAccess: "enabled"} + for _, err := range []error{errHelperNotStarted, context.DeadlineExceeded} { + f.err = err + info, gotErr := p.Create(bounded(t), b) + if gotErr == nil { + t.Fatal("failure lost") + } + if errors.Is(err, errHelperNotStarted) { + if !info.CreateSettled || info.State != "absent" { + t.Fatal("missing definite absence proof") + } + } else if info.CreateSettled { + t.Fatal("timeout proved absence") + } + } +} diff --git a/services/agents-api/internal/sandbox/provider.go b/services/agents-api/internal/sandbox/provider.go index 6fbf34f91..f5a69b770 100644 --- a/services/agents-api/internal/sandbox/provider.go +++ b/services/agents-api/internal/sandbox/provider.go @@ -35,6 +35,10 @@ type Info struct { // BootstrapComplete is provider evidence that initialization has reached its // last mutating step. It does not establish daemon or native readiness. BootstrapComplete bool + // CreateSettled proves that the original create and initialization attempt can + // no longer mutate resources. An absent observation needs this explicit proof; + // an ordinary missing resource or empty provider listing is not sufficient. + CreateSettled bool } type Command struct { Args []string diff --git a/services/agents-api/internal/store/devices.go b/services/agents-api/internal/store/devices.go index b13687c4e..ed4050d0b 100644 --- a/services/agents-api/internal/store/devices.go +++ b/services/agents-api/internal/store/devices.go @@ -74,7 +74,7 @@ func (s *Store) GetDeviceCredential(ctx context.Context, deviceID string) (devic return device.Credential{}, false, err } return device.Credential{ID: uuid.UUID(row.ID.Bytes).String(), Name: row.Name, - Type: gateway.RuntimeTypeAgentDaemon, CredentialHash: row.CredentialHash, RuntimeNodeID: row.RuntimeNodeID}, true, nil + Type: gateway.RuntimeTypeAgentDaemon, CredentialHash: row.CredentialHash, RuntimeNodeID: row.RuntimeNodeID, RuntimeAllocationID: row.RuntimeAllocationID}, true, nil } func (s *Store) RevokeDevice(ctx context.Context, tenantID, deviceID string) error { diff --git a/services/agents-api/internal/store/runtime_allocation_state.go b/services/agents-api/internal/store/runtime_allocation_state.go index 067ce66ff..50535992a 100644 --- a/services/agents-api/internal/store/runtime_allocation_state.go +++ b/services/agents-api/internal/store/runtime_allocation_state.go @@ -41,17 +41,23 @@ func (s *Store) SettleRuntimeCreation(ctx context.Context, owner RuntimeAllocati // Cancellation requests do not prove existing native work has stopped. A live, // unexpired Environment fails with the generic provisioning reason. func (s *Store) RequestRuntimeCleanup(ctx context.Context, owner RuntimeAllocation) (RuntimeAllocation, error) { - return s.requestRuntimeCleanup(ctx, owner, provisioningFailureReason) + return s.requestRuntimeCleanup(ctx, owner, provisioningFailureReason, false) } // FailRuntimeInitialization is RequestRuntimeCleanup after a confirmed failed // initialization step: the step's safe reason becomes the Session error. Deleted // Sessions, expired and already terminal Environments keep their existing outcome. func (s *Store) FailRuntimeInitialization(ctx context.Context, owner RuntimeAllocation, failure ProvisioningFailure) (RuntimeAllocation, error) { - return s.requestRuntimeCleanup(ctx, owner, failure.reason()) + return s.requestRuntimeCleanup(ctx, owner, failure.reason(), false) } -func (s *Store) requestRuntimeCleanup(ctx context.Context, owner RuntimeAllocation, reason string) (RuntimeAllocation, error) { +// ReleaseAbsentRuntimeCreation consumes provider proof that the original attempt +// is settled and owns no resources. Authority revocation and release commit together. +func (s *Store) ReleaseAbsentRuntimeCreation(ctx context.Context, owner RuntimeAllocation) (RuntimeAllocation, error) { + return s.requestRuntimeCleanup(ctx, owner, provisioningFailureReason, true) +} + +func (s *Store) requestRuntimeCleanup(ctx context.Context, owner RuntimeAllocation, reason string, absent bool) (RuntimeAllocation, error) { return s.mutateRuntimeAllocation(ctx, owner, false, func(ctx context.Context, q *sqlc.Queries, row sqlc.RuntimeAllocation) (sqlc.RuntimeAllocation, error) { if row.State == "released" { return row, nil @@ -74,7 +80,21 @@ func (s *Store) requestRuntimeCleanup(ctx context.Context, owner RuntimeAllocati return sqlc.RuntimeAllocation{}, err } - return q.RequestRuntimeCleanup(ctx, row.ID) + pending, err := q.RequestRuntimeCleanup(ctx, row.ID) + if err != nil || !absent { + return pending, err + } + if _, err := q.SettleRuntimeCreation(ctx, row.ID); err != nil { + return sqlc.RuntimeAllocation{}, err + } + released, err := q.ReleaseRuntimeAllocation(ctx, row.ID) + if err != nil { + return sqlc.RuntimeAllocation{}, err + } + if err := q.ReleaseRuntimePlacement(ctx, row.EnvironmentID); err != nil { + return sqlc.RuntimeAllocation{}, err + } + return released, nil }) } diff --git a/services/agents-api/internal/store/runtime_allocations.go b/services/agents-api/internal/store/runtime_allocations.go index 4e2949aa2..71831b36c 100644 --- a/services/agents-api/internal/store/runtime_allocations.go +++ b/services/agents-api/internal/store/runtime_allocations.go @@ -100,7 +100,7 @@ func (s *Store) ReserveRuntimeAllocation(ctx context.Context, tenant, environmen if err != nil { return err } - if deployment.ProviderKind != "" { + if deployment.Mode == "nodes" { placement, err := q.GetRuntimePlacement(ctx, lookup.ID) if err != nil { return err diff --git a/services/agents-api/internal/store/runtime_creation_settlement_test.go b/services/agents-api/internal/store/runtime_creation_settlement_test.go new file mode 100644 index 000000000..e741af22a --- /dev/null +++ b/services/agents-api/internal/store/runtime_creation_settlement_test.go @@ -0,0 +1,110 @@ +package store_test + +import ( + "context" + "errors" + "testing" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" +) + +type absentCreationProvider struct { + lifecycleProvider + cancel context.CancelFunc + foreign bool + observeSettled bool +} + +func (p *absentCreationProvider) Create(_ context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { + p.creates++ + if p.cancel != nil { + p.cancel() + } + if p.observeSettled { + return sandbox.Info{}, errors.New("fixture: uncertain response") + } + ref := b.Reference + if p.foreign { + ref.AllocationID = uuid.NewString() + } + return sandbox.Info{Reference: ref, State: "absent", CreateSettled: true}, sandbox.ErrInvalid +} +func (p *absentCreationProvider) GetInfo(_ context.Context, r sandbox.Reference) (sandbox.Info, error) { + if p.observeSettled { + return sandbox.Info{Reference: r, State: "absent", CreateSettled: true}, nil + } + return sandbox.Info{}, sandbox.ErrNotFound +} +func TestManagedRuntimeConfirmedAbsentCreateReleasesAtomically(t *testing.T) { + for _, cancelled := range []bool{false, true} { + t.Run(map[bool]string{false: "live caller", true: "cancelled caller"}[cancelled], func(t *testing.T) { + s, _ := store.NewManagedTestStore(t) + key := uuid.NewString() + p := &absentCreationProvider{} + w, _ := managedWorker(t, s, key, p) + tenant, session, environment := managedSession(t, s) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + if cancelled { + p.cancel = cancel + } + owner, err := w.ProvisionEnvironment(ctx, tenant, environment.ID, key) + if err == nil || owner.State != "released" || !owner.CreateSettled { + t.Fatal("confirmed absence not atomically released", owner, err) + } + if p.kills != 0 || p.creates != 1 { + t.Fatal("absence proof still called external cleanup", p.kills, p.creates) + } + stored, err := s.GetRuntimeAllocation(t.Context(), tenant, environment.ID) + if err != nil || stored.State != "released" || !stored.CreateSettled { + t.Fatal("release not durable", err) + } + if _, ok, err := s.GetDeviceCredential(t.Context(), owner.DeviceID); err != nil || ok { + t.Fatal("released credential retained authority", err) + } + value, err := s.GetSession(t.Context(), tenant, session.ID) + if err != nil || value.Environment.Status != "failed" { + t.Fatal("environment not terminated", err) + } + if err := w.ReconcileManagedRuntimes(t.Context()); err != nil { + t.Fatal(err) + } + if p.kills != 0 { + t.Fatal("released absence was cleaned again") + } + }) + } +} +func TestManagedRuntimeForeignAbsenceCannotReleaseCreation(t *testing.T) { + s, _ := store.NewManagedTestStore(t) + key := uuid.NewString() + p := &absentCreationProvider{foreign: true} + w, _ := managedWorker(t, s, key, p) + tenant, _, environment := managedSession(t, s) + if _, err := w.ProvisionEnvironment(t.Context(), tenant, environment.ID, key); err == nil { + t.Fatal("foreign absence accepted") + } + owner, err := s.GetRuntimeAllocation(t.Context(), tenant, environment.ID) + if err != nil || owner.CreateSettled || owner.State == "released" { + t.Fatal("foreign proof settled original attempt", owner, err) + } +} +func TestManagedRuntimeObservedSettlementAllowsOwnedCleanup(t *testing.T) { + s, _ := store.NewManagedTestStore(t) + key := uuid.NewString() + p := &absentCreationProvider{observeSettled: true} + w, _ := managedWorker(t, s, key, p) + tenant, session, environment := managedSession(t, s) + if _, err := w.ProvisionEnvironment(t.Context(), tenant, environment.ID, key); err == nil { + t.Fatal("uncertain Create succeeded") + } + if err := s.DeleteSession(t.Context(), tenant, session.ID); err != nil { + t.Fatal(err) + } + reconcileManagedState(t, w, s, tenant, environment.ID, "released") + if p.kills != 1 || p.creates != 1 { + t.Fatal("settled observation replayed Create or skipped cleanup", p.creates, p.kills) + } +} diff --git a/services/agents-api/internal/store/runtime_lifecycle_nodes.go b/services/agents-api/internal/store/runtime_lifecycle_nodes.go index 6c36e0deb..3e6e60824 100644 --- a/services/agents-api/internal/store/runtime_lifecycle_nodes.go +++ b/services/agents-api/internal/store/runtime_lifecycle_nodes.go @@ -108,7 +108,7 @@ func (s *Store) ResolveRuntimeLifecycleNode(ctx context.Context, tenant, environ if err != nil { return "", err } - if row.ProviderKind == "" { + if row.ProviderKind == "" || row.Mode == "direct" { if row.PlacementNodeID.Valid || row.AllocationNodeID.Valid { return "", ErrRuntimeNodeUnavailable } diff --git a/services/agents-api/internal/store/runtime_node_types.go b/services/agents-api/internal/store/runtime_node_types.go index 3c4ab619e..3a0aec4ae 100644 --- a/services/agents-api/internal/store/runtime_node_types.go +++ b/services/agents-api/internal/store/runtime_node_types.go @@ -66,12 +66,24 @@ type RuntimePlacement struct { State string `json:"state"` ComputePhase string `json:"compute_phase"` } +type SandboxDeploymentResources struct { + Allocations int64 `json:"allocations"` + Pending int64 `json:"pending"` +} +type SandboxE2BView struct { + Template string `json:"template"` + CredentialConfigured bool `json:"credential_configured"` +} type RuntimeDeploymentView struct { - CoreURL string `json:"core_url"` - InstallationID string `json:"installation_id"` - Provider string `json:"provider"` - Maintenance bool `json:"maintenance"` - OwnerEpoch uint64 `json:"owner_epoch"` + Generation uint64 `json:"generation"` + Mode string `json:"mode"` + Resources SandboxDeploymentResources `json:"resources"` + E2B *SandboxE2BView `json:"e2b,omitempty"` + CoreURL string `json:"core_url"` + InstallationID string `json:"installation_id"` + Provider string `json:"provider"` + Maintenance bool `json:"maintenance"` + OwnerEpoch uint64 `json:"owner_epoch"` } type RuntimeNodeAllocation struct { Diagnostic string `json:"diagnostic"` diff --git a/services/agents-api/internal/store/runtime_nodes.go b/services/agents-api/internal/store/runtime_nodes.go index 5bf1b22c5..4bf528417 100644 --- a/services/agents-api/internal/store/runtime_nodes.go +++ b/services/agents-api/internal/store/runtime_nodes.go @@ -44,11 +44,20 @@ func (s *Store) runtimeManagerTransaction(ctx context.Context, apply func(*sqlc. }) } func (s *Store) GetRuntimeDeployment(ctx context.Context) (RuntimeDeploymentView, error) { - d, err := s.queries.GetRuntimeDeployment(ctx) + return getRuntimeDeploymentView(ctx, s.queries) +} +func getRuntimeDeploymentView(ctx context.Context, q *sqlc.Queries) (RuntimeDeploymentView, error) { + d, err := q.GetRuntimeDeployment(ctx) + if err != nil { + return RuntimeDeploymentView{}, err + } + resources, err := q.CountRuntimeDeploymentResources(ctx) if err != nil { return RuntimeDeploymentView{}, err } - return runtimeDeploymentView(d), nil + result := runtimeDeploymentView(d) + result.Resources = SandboxDeploymentResources{Allocations: resources.Allocations, Pending: resources.Pending} + return result, nil } func (s *Store) RuntimeOwnerEpoch(ctx context.Context) (uint64, error) { d, err := s.queries.GetRuntimeDeployment(ctx) @@ -89,6 +98,9 @@ func (s *Store) CreateRuntimeEnrollment(ctx context.Context) (string, time.Time, token := hex.EncodeToString(bytes[:]) var expires time.Time err := s.runtimeManagerTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { + if d.Mode != "nodes" || d.Maintenance { + return ErrSandboxDeploymentConflict + } if err := q.CreateRuntimeEnrollment(ctx, sqlc.CreateRuntimeEnrollmentParams{TokenSha256: runtimeTokenDigest(token), InstallationID: d.InstallationID}); err != nil { return err } @@ -108,9 +120,6 @@ func (s *Store) EnrollRuntimeNode(ctx context.Context, token string, input Runti } var result RuntimeNodeIdentity err = s.runtimeManagerTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { - if input.Provider != d.ProviderKind { - return ErrInvalidInput - } receipt, err := q.GetRuntimeEnrollment(ctx, runtimeTokenDigest(token)) if errors.Is(err, pgx.ErrNoRows) { return ErrRuntimeNodeCredential @@ -121,6 +130,9 @@ func (s *Store) EnrollRuntimeNode(ctx context.Context, token string, input Runti if receipt.ConsumedAt.Valid || !receipt.ExpiresAt.Time.After(time.Now()) || receipt.InstallationID != d.InstallationID { return ErrRuntimeNodeCredential } + if d.Mode != "nodes" || d.Maintenance || input.Provider != d.ProviderKind { + return ErrInvalidInput + } if _, err := q.GetRuntimeNode(ctx, id); err == nil { return ErrIdempotencyConflict } else if !errors.Is(err, pgx.ErrNoRows) { @@ -164,7 +176,7 @@ func (s *Store) AuthenticateRuntimeNode(ctx context.Context, nodeID, credential if err != nil { return RuntimeNodeIdentity{}, ErrRuntimeNodeUnavailable } - if n.InstallationID != d.InstallationID || d.ProviderKind == "" { + if n.InstallationID != d.InstallationID || d.Mode != "nodes" { return RuntimeNodeIdentity{}, ErrRuntimeNodeCredential } return nodeIdentity(n, d.ProviderKind), nil diff --git a/services/agents-api/internal/store/runtime_placements.go b/services/agents-api/internal/store/runtime_placements.go index d380701ac..e5f3fcff9 100644 --- a/services/agents-api/internal/store/runtime_placements.go +++ b/services/agents-api/internal/store/runtime_placements.go @@ -16,6 +16,15 @@ func reserveRuntimePlacement(ctx context.Context, q *sqlc.Queries, session pgtyp if err != nil { return err } + if d.Mode == "direct" { + if selected != "" { + return ErrInvalidInput + } + if d.Maintenance { + return ErrRuntimeNodeUnavailable + } + return nil + } if d.ProviderKind == "" { if d.WebManaged || selected != "" { return ErrRuntimeNodeUnavailable diff --git a/services/agents-api/internal/store/sandbox_deployment_mutations.go b/services/agents-api/internal/store/sandbox_deployment_mutations.go new file mode 100644 index 000000000..37ea04555 --- /dev/null +++ b/services/agents-api/internal/store/sandbox_deployment_mutations.go @@ -0,0 +1,246 @@ +package store + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "math" + "strings" + "unicode" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +var ErrSandboxCredentialUnavailable = errors.New("sandbox credential encryption is unavailable") + +// APIKey is internal configuration. HTTP requests use a write-only DTO. +type SandboxE2BConfiguration struct { + APIKey string `json:"-"` + Template string `json:"template"` +} +type SandboxDeploymentUpdateRequest struct { + SandboxDeploymentSetupRequest + ExpectedGeneration uint64 `json:"expected_generation"` +} +type SandboxMaintenanceRequest struct { + Maintenance bool `json:"maintenance"` + ExpectedGeneration uint64 `json:"expected_generation"` +} + +func validateSandboxSelection(input SandboxDeploymentSetupRequest) error { + if ValidateSandboxCoreURL(input.CoreURL) != nil { + return ErrInvalidInput + } + switch input.Provider { + case "docker", "microsandbox": + if input.E2B != nil { + return ErrInvalidInput + } + case "e2b": + if input.E2B == nil || input.E2B.APIKey == "" || len(input.E2B.APIKey) > 4096 || strings.IndexFunc(input.E2B.APIKey, func(r rune) bool { return unicode.IsSpace(r) || r == 0 }) >= 0 { + return ErrInvalidInput + } + template, build, ok := strings.Cut(input.E2B.Template, ":") + id, err := uuid.Parse(build) + if !ok || template == "" || len(template) > 128 || err != nil || id == uuid.Nil || id.String() != build { + return ErrInvalidInput + } + for _, c := range template { + if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_' || c == '-') { + return ErrInvalidInput + } + } + default: + return ErrInvalidInput + } + return nil +} + +func (s *Store) sandboxSelectionEqual(d sqlc.RuntimeDeployment, input SandboxDeploymentSetupRequest) (bool, error) { + if d.ProviderKind != input.Provider || d.CoreUrl != input.CoreURL { + return false, nil + } + if input.E2B == nil { + return d.E2bTemplate == "", nil + } + if d.E2bTemplate != input.E2B.Template { + return false, nil + } + key, err := s.credentialCipher.OpenSandboxDeployment(d.E2bCredential, runtimeUUID(d.InstallationID), uint64(d.Generation)) + if err != nil { + return false, ErrSandboxCredentialUnavailable + } + return string(key) == input.E2B.APIKey, nil +} + +func (s *Store) saveSandboxSelection(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment, input SandboxDeploymentSetupRequest) error { + if d.Generation == math.MaxInt64 { + return ErrSandboxDeploymentConflict + } + generation := d.Generation + 1 + mode := "nodes" + namespace := "nodes:" + runtimeUUID(d.InstallationID) + if input.Provider == "e2b" { + mode = "direct" + namespace = "e2b:" + runtimeUUID(d.InstallationID) + } + digest := sha256.Sum256([]byte(input.Provider + "\x00" + namespace)) + params := sqlc.InitializeSandboxDeploymentParams{ProviderKind: input.Provider, CoreUrl: input.CoreURL, BackendFingerprint: hex.EncodeToString(digest[:]), Generation: generation, Mode: mode} + if input.Provider == "microsandbox" { + params.IdleSeconds, params.RetentionSeconds = 300, 86400 + } + if input.E2B != nil { + encrypted, err := s.credentialCipher.SealSandboxDeployment([]byte(input.E2B.APIKey), runtimeUUID(d.InstallationID), uint64(generation)) + if err != nil { + return ErrSandboxCredentialUnavailable + } + params.E2bCredential, params.E2bTemplate = encrypted, input.E2B.Template + } + return q.InitializeSandboxDeployment(ctx, params) +} + +func (s *Store) InitializeSandboxDeployment(ctx context.Context, installationID string, input SandboxDeploymentSetupRequest) (RuntimeDeploymentView, error) { + if s.executionLease == nil { + return RuntimeDeploymentView{}, ErrInvalidInput + } + if err := validateSandboxSelection(input); err != nil { + return RuntimeDeploymentView{}, err + } + id, err := parseConnectionGeneration(installationID) + if err != nil { + return RuntimeDeploymentView{}, err + } + ctx, cancel := context.WithTimeout(ctx, executionTransactionTimeout) + defer cancel() + var result RuntimeDeploymentView + err = s.executionLease.transaction(ctx, func(tx pgx.Tx) error { + q := s.queries.WithTx(tx) + d, err := q.LockRuntimeDeployment(ctx) + if err != nil { + return err + } + if !d.WebManaged || d.InstallationID != id { + return ErrSandboxDeploymentConflict + } + if d.ProviderKind != "" { + equal, err := s.sandboxSelectionEqual(d, input) + if err != nil { + return err + } + if !equal { + return ErrSandboxDeploymentConflict + } + } else if err := s.saveSandboxSelection(ctx, q, d, input); err != nil { + return err + } + result, err = getRuntimeDeploymentView(ctx, q) + return err + }) + return result, err +} + +// CheckSandboxDeploymentSwitch is a preliminary check only. The mutation repeats +// it after execution has drained; no database lock spans provider work. +func (s *Store) CheckSandboxDeploymentSwitch(ctx context.Context, installation string, input SandboxDeploymentUpdateRequest) error { + if s.executionLease == nil { + return ErrInvalidInput + } + if err := validateSandboxSelection(input.SandboxDeploymentSetupRequest); err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, executionTransactionTimeout) + defer cancel() + return s.executionLease.transaction(ctx, func(tx pgx.Tx) error { + q := s.queries.WithTx(tx) + d, err := q.LockRuntimeDeployment(ctx) + if err != nil { + return err + } + return checkSandboxSwitch(ctx, q, d, installation, input) + }) +} +func checkSandboxSwitch(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment, installation string, input SandboxDeploymentUpdateRequest) error { + if !d.WebManaged || runtimeUUID(d.InstallationID) != installation || d.ProviderKind == "" || !d.Maintenance || uint64(d.Generation) != input.ExpectedGeneration || d.CoreUrl != input.CoreURL { + return ErrSandboxDeploymentConflict + } + resources, err := q.CountRuntimeDeploymentResources(ctx) + if err != nil { + return err + } + if resources.Allocations != 0 || resources.Pending != 0 { + return ErrSandboxDeploymentConflict + } + return nil +} + +func (s *Store) UpdateSandboxDeployment(ctx context.Context, installation string, input SandboxDeploymentUpdateRequest) (RuntimeDeploymentView, error) { + if s.executionLease == nil { + return RuntimeDeploymentView{}, ErrInvalidInput + } + if err := validateSandboxSelection(input.SandboxDeploymentSetupRequest); err != nil { + return RuntimeDeploymentView{}, err + } + ctx, cancel := context.WithTimeout(ctx, executionTransactionTimeout) + defer cancel() + var result RuntimeDeploymentView + err := s.executionLease.transaction(ctx, func(tx pgx.Tx) error { + q := s.queries.WithTx(tx) + d, err := q.LockRuntimeDeployment(ctx) + if err != nil { + return err + } + if err := checkSandboxSwitch(ctx, q, d, installation, input); err != nil { + return err + } + equal, err := s.sandboxSelectionEqual(d, input.SandboxDeploymentSetupRequest) + if err != nil { + return err + } + if !equal { + if err := s.saveSandboxSelection(ctx, q, d, input.SandboxDeploymentSetupRequest); err != nil { + return err + } + if err := q.RetireSandboxNodes(ctx); err != nil { + return err + } + if err := q.RetireSandboxEnrollments(ctx); err != nil { + return err + } + if err := q.AdvanceSandboxOwnerEpoch(ctx); err != nil { + return err + } + } + result, err = getRuntimeDeploymentView(ctx, q) + return err + }) + return result, err +} + +// The Worker verifies activation before calling this with maintenance=false. +func (s *Store) SetSandboxMaintenance(ctx context.Context, installation string, input SandboxMaintenanceRequest) (RuntimeDeploymentView, error) { + if s.executionLease == nil { + return RuntimeDeploymentView{}, ErrInvalidInput + } + ctx, cancel := context.WithTimeout(ctx, executionTransactionTimeout) + defer cancel() + var result RuntimeDeploymentView + err := s.executionLease.transaction(ctx, func(tx pgx.Tx) error { + q := s.queries.WithTx(tx) + d, err := q.LockRuntimeDeployment(ctx) + if err != nil { + return err + } + if !d.WebManaged || runtimeUUID(d.InstallationID) != installation || d.ProviderKind == "" || uint64(d.Generation) != input.ExpectedGeneration { + return ErrSandboxDeploymentConflict + } + if err := q.SetSandboxMaintenance(ctx, input.Maintenance); err != nil { + return err + } + result, err = getRuntimeDeploymentView(ctx, q) + return err + }) + return result, err +} diff --git a/services/agents-api/internal/store/sandbox_deployment_resources_test.go b/services/agents-api/internal/store/sandbox_deployment_resources_test.go new file mode 100644 index 000000000..38f54d37f --- /dev/null +++ b/services/agents-api/internal/store/sandbox_deployment_resources_test.go @@ -0,0 +1,58 @@ +package store + +import ( + "bytes" + "testing" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/credentialcrypto" + "github.com/google/uuid" +) + +func TestSandboxDeploymentMutationViewsIncludeActualResources(t *testing.T) { + _, pool := newManagedTestStore(t) + cipher, err := credentialcrypto.New(bytes.Repeat([]byte{9}, 32)) + if err != nil { + t.Fatal(err) + } + s := NewWithCredentialCipher(pool, cipher) + w := executionLease(t, s).Store() + installation := uuid.NewString() + if err := w.ClaimWebSandboxDeployment(t.Context(), installation); err != nil { + t.Fatal(err) + } + selection := e2bSelection() + if _, err := w.InitializeSandboxDeployment(t.Context(), installation, selection); err != nil { + t.Fatal(err) + } + tenant := uuid.NewString() + session, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString(), "")) + if err != nil { + t.Fatal(err) + } + if _, err := w.ReserveRuntimeAllocation(t.Context(), tenant, session.Environment.ID, installation, device.HashCredential(uuid.NewString())); err != nil { + t.Fatal(err) + } + if _, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString(), "")); err != nil { + t.Fatal(err) + } + want := SandboxDeploymentResources{Allocations: 1, Pending: 1} + // Replaying setup must report the current resources rather than initial zeros. + replay, err := w.InitializeSandboxDeployment(t.Context(), installation, selection) + if err != nil || replay.Resources != want { + t.Fatalf("setup replay resources = %+v, error = %v", replay.Resources, err) + } + for _, maintenance := range []bool{true, false} { + response, err := w.SetSandboxMaintenance(t.Context(), installation, SandboxMaintenanceRequest{Maintenance: maintenance, ExpectedGeneration: 1}) + if err != nil { + t.Fatal(err) + } + current, err := s.GetRuntimeDeployment(t.Context()) + if err != nil { + t.Fatal(err) + } + if response.Maintenance != maintenance || response.Resources != want || response.Resources != current.Resources { + t.Fatalf("maintenance %v response = %+v, current resources = %+v", maintenance, response, current.Resources) + } + } +} diff --git a/services/agents-api/internal/store/sandbox_deployment_setup.go b/services/agents-api/internal/store/sandbox_deployment_setup.go index ee739ab91..8e3e2b57d 100644 --- a/services/agents-api/internal/store/sandbox_deployment_setup.go +++ b/services/agents-api/internal/store/sandbox_deployment_setup.go @@ -2,8 +2,6 @@ package store import ( "context" - "crypto/sha256" - "encoding/hex" "errors" "net" "net/url" @@ -17,14 +15,19 @@ import ( var ErrSandboxDeploymentConflict = errors.New("sandbox deployment is already configured differently") type SandboxDeploymentSetupRequest struct { - Provider string `json:"provider"` - CoreURL string `json:"core_url"` + Provider string `json:"provider"` + CoreURL string `json:"core_url"` + E2B *SandboxE2BConfiguration `json:"e2b,omitempty"` } // SandboxSetup is the immutable configuration selected by the deployment admin. // An empty Provider means that Web setup has not yet selected an adapter. type SandboxSetup struct { InstallationID, Provider, CoreURL, BackendFingerprint string + Generation uint64 + Mode string + Maintenance bool + E2B *SandboxE2BConfiguration IdleSeconds, RetentionSeconds int64 } @@ -38,7 +41,15 @@ func (s *Store) GetSandboxSetup(ctx context.Context) (SandboxSetup, error) { if !d.WebManaged { return SandboxSetup{}, ErrSandboxDeploymentConflict } - return SandboxSetup{InstallationID: runtimeUUID(d.InstallationID), Provider: d.ProviderKind, CoreURL: d.CoreUrl, BackendFingerprint: d.BackendFingerprint, IdleSeconds: d.IdleSeconds, RetentionSeconds: d.RetentionSeconds}, nil + result := SandboxSetup{InstallationID: runtimeUUID(d.InstallationID), Provider: d.ProviderKind, CoreURL: d.CoreUrl, BackendFingerprint: d.BackendFingerprint, IdleSeconds: d.IdleSeconds, RetentionSeconds: d.RetentionSeconds, Generation: uint64(d.Generation), Mode: d.Mode, Maintenance: d.Maintenance} + if d.ProviderKind == "e2b" { + credential, err := s.credentialCipher.OpenSandboxDeployment(d.E2bCredential, result.InstallationID, result.Generation) + if err != nil { + return SandboxSetup{}, ErrSandboxCredentialUnavailable + } + result.E2B = &SandboxE2BConfiguration{APIKey: string(credential), Template: d.E2bTemplate} + } + return result, nil } // ClaimWebSandboxDeployment runs exactly once per execution-owner startup. It @@ -113,51 +124,10 @@ func ValidateSandboxCoreURL(value string) error { return nil } -func (s *Store) InitializeSandboxDeployment(ctx context.Context, installationID string, input SandboxDeploymentSetupRequest) (RuntimeDeploymentView, error) { - if s.executionLease == nil || (input.Provider != "docker" && input.Provider != "microsandbox") || ValidateSandboxCoreURL(input.CoreURL) != nil { - return RuntimeDeploymentView{}, ErrInvalidInput - } - id, err := parseConnectionGeneration(installationID) - if err != nil { - return RuntimeDeploymentView{}, err - } - ctx, cancel := context.WithTimeout(ctx, executionTransactionTimeout) - defer cancel() - var result RuntimeDeploymentView - err = s.executionLease.transaction(ctx, func(tx pgx.Tx) error { - q := s.queries.WithTx(tx) - d, err := q.LockRuntimeDeployment(ctx) - if err != nil { - return err - } - if !d.WebManaged || d.InstallationID != id { - return ErrSandboxDeploymentConflict - } - if d.ProviderKind != "" { - if d.ProviderKind != input.Provider || d.CoreUrl != input.CoreURL { - return ErrSandboxDeploymentConflict - } - result = runtimeDeploymentView(d) - return nil - } - digest := sha256.Sum256([]byte(input.Provider + "\x00nodes:" + installationID)) - params := sqlc.InitializeSandboxDeploymentParams{ProviderKind: input.Provider, CoreUrl: input.CoreURL, BackendFingerprint: hex.EncodeToString(digest[:])} - if input.Provider == "microsandbox" { - params.IdleSeconds, params.RetentionSeconds = 300, 86400 - } - if err := q.InitializeSandboxDeployment(ctx, params); err != nil { - return err - } - d, err = q.GetRuntimeDeployment(ctx) - if err != nil { - return err - } - result = runtimeDeploymentView(d) - return nil - }) - return result, err -} - func runtimeDeploymentView(d sqlc.RuntimeDeployment) RuntimeDeploymentView { - return RuntimeDeploymentView{InstallationID: runtimeUUID(d.InstallationID), Provider: d.ProviderKind, CoreURL: d.CoreUrl, Maintenance: d.Maintenance, OwnerEpoch: uint64(d.OwnerEpoch)} + result := RuntimeDeploymentView{InstallationID: runtimeUUID(d.InstallationID), Provider: d.ProviderKind, CoreURL: d.CoreUrl, Maintenance: d.Maintenance, OwnerEpoch: uint64(d.OwnerEpoch), Generation: uint64(d.Generation), Mode: d.Mode} + if d.ProviderKind == "e2b" { + result.E2B = &SandboxE2BView{Template: d.E2bTemplate, CredentialConfigured: len(d.E2bCredential) > 0} + } + return result } diff --git a/services/agents-api/internal/store/sandbox_deployment_setup_test.go b/services/agents-api/internal/store/sandbox_deployment_setup_test.go index 8c60c8818..3285ab881 100644 --- a/services/agents-api/internal/store/sandbox_deployment_setup_test.go +++ b/services/agents-api/internal/store/sandbox_deployment_setup_test.go @@ -3,6 +3,7 @@ package store import ( "context" "errors" + "fmt" "sync" "testing" @@ -145,3 +146,18 @@ func TestSandboxDeploymentSetupRejectsFileManagedAndUnleasedWrites(t *testing.T) t.Fatal("file-managed deployment adopted", err) } } + +func TestSandboxSelectionRejectsWhitespaceInE2BCredential(t *testing.T) { + for _, separator := range []string{" ", "\t", "\r", "\n", "\x00", "\u00a0", "\u2003", "\u3000"} { + t.Run(fmt.Sprintf("U+%04X", []rune(separator)[0]), func(t *testing.T) { + selection := e2bSelection() + selection.E2B.APIKey = "prefix" + separator + "suffix" + if err := validateSandboxSelection(selection); !errors.Is(err, ErrInvalidInput) { + t.Fatalf("credential containing whitespace or NUL accepted: %v", err) + } + }) + } + if err := validateSandboxSelection(e2bSelection()); err != nil { + t.Fatal(err) + } +} diff --git a/services/agents-api/internal/store/sandbox_deployment_switch_test.go b/services/agents-api/internal/store/sandbox_deployment_switch_test.go new file mode 100644 index 000000000..39d4c586b --- /dev/null +++ b/services/agents-api/internal/store/sandbox_deployment_switch_test.go @@ -0,0 +1,304 @@ +package store + +import ( + "bytes" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/credentialcrypto" + "github.com/google/uuid" +) + +func e2bSelection() SandboxDeploymentSetupRequest { + return SandboxDeploymentSetupRequest{Provider: "e2b", CoreURL: "https://core.example", E2B: &SandboxE2BConfiguration{APIKey: "fixture-private-api-key", Template: "runtime:" + uuid.NewString()}} +} +func TestSandboxDirectDeploymentOwnershipAndCleanSwitch(t *testing.T) { + _, pool := newManagedTestStore(t) + cipher, err := credentialcrypto.New(bytes.Repeat([]byte{4}, 32)) + if err != nil { + t.Fatal(err) + } + s := NewWithCredentialCipher(pool, cipher) + w := executionLease(t, s).Store() + id := uuid.NewString() + if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + t.Fatal(err) + } + input := e2bSelection() + view, err := w.InitializeSandboxDeployment(t.Context(), id, input) + if err != nil || view.Generation != 1 || view.Mode != "direct" || view.E2B == nil || !view.E2B.CredentialConfigured { + t.Fatal("direct setup", view, err) + } + raw, _ := json.Marshal(view) + if bytes.Contains(raw, []byte(input.E2B.APIKey)) { + t.Fatal("credential in public view") + } + var ciphertext []byte + if err := pool.QueryRow(t.Context(), "SELECT e2b_credential FROM runtime_deployment").Scan(&ciphertext); err != nil || bytes.Contains(ciphertext, []byte(input.E2B.APIKey)) { + t.Fatal("credential not encrypted", err) + } + setup, err := s.GetSandboxSetup(t.Context()) + if err != nil || setup.E2B.APIKey != input.E2B.APIKey { + t.Fatal("internal credential unavailable", err) + } + if _, _, err := s.CreateRuntimeEnrollment(t.Context()); !errors.Is(err, ErrSandboxDeploymentConflict) { + t.Fatal("cloud enrolled a machine", err) + } + tenant := uuid.NewString() + if _, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString(), uuid.NewString())); !errors.Is(err, ErrInvalidInput) { + t.Fatal("direct accepted node selection", err) + } + session, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString(), "")) + if err != nil { + t.Fatal(err) + } + environment := session.Environment.ID + nodes, err := w.ListRuntimeLifecycleNodes(t.Context()) + if err != nil || len(nodes) != 1 || nodes[0] != "" { + t.Fatal("cloud lifecycle requires node", nodes, err) + } + owner, err := w.ReserveRuntimeAllocation(t.Context(), tenant, environment, id, device.HashCredential(uuid.NewString())) + if err != nil || owner.NodeID != "" { + t.Fatal(owner, err) + } + credential, ok, err := s.GetDeviceCredential(t.Context(), owner.DeviceID) + if err != nil || !ok || credential.RuntimeAllocationID != owner.ID || credential.RuntimeNodeID != "" { + t.Fatal("direct bootstrap lost managed identity", err) + } + if _, err := pool.Exec(t.Context(), "UPDATE runtime_allocations SET kept_at=clock_timestamp()-interval '2 hours' WHERE id=$1", owner.ID); err != nil { + t.Fatal(err) + } + observed, err := w.GetRuntimeAllocation(t.Context(), tenant, environment) + if err != nil || observed.Expired { + t.Fatal("cloud inherited legacy node-less expiry", err) + } + if _, err := w.SetSandboxMaintenance(t.Context(), id, SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 1}); err != nil { + t.Fatal(err) + } + update := SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: SandboxDeploymentSetupRequest{Provider: "docker", CoreURL: input.CoreURL}, ExpectedGeneration: 1} + if _, err := w.UpdateSandboxDeployment(t.Context(), id, update); !errors.Is(err, ErrSandboxDeploymentConflict) { + t.Fatal("unknown allocation allowed switch", err) + } + if _, err := w.RequestRuntimeCleanup(t.Context(), owner); err != nil { + t.Fatal(err) + } + if _, err := w.ReleaseRuntimeAllocation(t.Context(), owner); !errors.Is(err, ErrTurnConflict) { + t.Fatal("unsettled create released", err) + } + if _, err := w.SettleRuntimeCreation(t.Context(), owner); err != nil { + t.Fatal(err) + } + if _, err := w.ReleaseRuntimeAllocation(t.Context(), owner); err != nil { + t.Fatal(err) + } + changed, err := w.UpdateSandboxDeployment(t.Context(), id, update) + if err != nil || changed.Generation != 2 || changed.Mode != "nodes" || !changed.Maintenance || changed.E2B != nil || changed.Resources != (SandboxDeploymentResources{}) { + t.Fatal("clean switch", changed, err) + } + if _, err := w.SetSandboxMaintenance(t.Context(), id, SandboxMaintenanceRequest{ExpectedGeneration: 1}); !errors.Is(err, ErrSandboxDeploymentConflict) { + t.Fatal("stale resume accepted", err) + } + if _, err := s.GetSession(t.Context(), tenant, session.ID); err != nil { + t.Fatal("historical Session lost", err) + } +} + +func TestSandboxSwitchRetiresNodesAndEnrollment(t *testing.T) { + _, pool := newManagedTestStore(t) + cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{5}, 32)) + s := NewWithCredentialCipher(pool, cipher) + w := executionLease(t, s).Store() + id := uuid.NewString() + if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + t.Fatal(err) + } + if _, err := w.InitializeSandboxDeployment(t.Context(), id, SandboxDeploymentSetupRequest{Provider: "docker", CoreURL: "https://core.example"}); err != nil { + t.Fatal(err) + } + token, _, err := s.CreateRuntimeEnrollment(t.Context()) + if err != nil { + t.Fatal(err) + } + node := RuntimeNodeEnrollment{NodeID: uuid.NewString(), Name: "Machine", Provider: "docker", Credential: strings.Repeat("c", 64), BackendFingerprint: strings.Repeat("b", 64), MaxActive: 2, MaxRetained: 4} + if _, err := s.EnrollRuntimeNode(t.Context(), token, node); err != nil { + t.Fatal(err) + } + unused, _, err := s.CreateRuntimeEnrollment(t.Context()) + if err != nil { + t.Fatal(err) + } + if _, err := w.SetSandboxMaintenance(t.Context(), id, SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 1}); err != nil { + t.Fatal(err) + } + // Maintenance rejects a valid enrollment without consuming it. Authentication + // still precedes deployment details for invalid or retired credentials. + spareNode := node + spareNode.NodeID = uuid.NewString() + if _, err := s.EnrollRuntimeNode(t.Context(), unused, spareNode); !errors.Is(err, ErrInvalidInput) { + t.Fatal("maintenance accepted enrollment", err) + } + var consumed bool + if err := pool.QueryRow(t.Context(), "SELECT consumed_at IS NOT NULL FROM runtime_node_enrollments WHERE token_sha256=$1", runtimeTokenDigest(unused)).Scan(&consumed); err != nil || consumed { + t.Fatal("maintenance consumed enrollment", err) + } + spareNode.Provider = "microsandbox" + if _, err := s.EnrollRuntimeNode(t.Context(), strings.Repeat("invalid", 8), spareNode); !errors.Is(err, ErrRuntimeNodeCredential) { + t.Fatal("invalid token disclosed deployment validation", err) + } + spareNode.Provider = "docker" + input := e2bSelection() + if _, err := w.UpdateSandboxDeployment(t.Context(), id, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}); err != nil { + t.Fatal(err) + } + + if _, err := s.EnrollRuntimeNode(t.Context(), unused, spareNode); !errors.Is(err, ErrRuntimeNodeCredential) { + t.Fatal("retired token did not reject before cloud deployment validation", err) + } + if _, err := s.AuthenticateRuntimeNode(t.Context(), node.NodeID, node.Credential); !errors.Is(err, ErrRuntimeNodeCredential) { + t.Fatal("old node credential survived", err) + } + if _, err := w.SetSandboxMaintenance(t.Context(), id, SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 2}); err != nil { + t.Fatal(err) + } + if _, err := w.UpdateSandboxDeployment(t.Context(), id, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: SandboxDeploymentSetupRequest{Provider: "docker", CoreURL: input.CoreURL}, ExpectedGeneration: 2}); err != nil { + t.Fatal(err) + } + if _, err := w.SetSandboxMaintenance(t.Context(), id, SandboxMaintenanceRequest{ExpectedGeneration: 3}); err != nil { + t.Fatal(err) + } + node.NodeID = uuid.NewString() + if _, err := s.EnrollRuntimeNode(t.Context(), unused, node); !errors.Is(err, ErrRuntimeNodeCredential) { + t.Fatal("old enrollment survived roundtrip", err) + } + nodes, err := s.ListRuntimeNodes(t.Context()) + if err != nil || len(nodes) != 0 { + t.Fatal("retired nodes reappeared", err) + } +} + +func TestSandboxMaintenanceSerializesFreshDirectSessions(t *testing.T) { + _, pool := newManagedTestStore(t) + cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{6}, 32)) + s := NewWithCredentialCipher(pool, cipher) + w := executionLease(t, s).Store() + id := uuid.NewString() + if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + t.Fatal(err) + } + input := e2bSelection() + if _, err := w.InitializeSandboxDeployment(t.Context(), id, input); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for range 12 { + wg.Add(1) + go func() { + defer wg.Done() + _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString(), "")) + if err != nil && !errors.Is(err, ErrEnvironmentUnavailable) && !errors.Is(err, ErrRuntimeNodeUnavailable) { + t.Error(err) + } + }() + } + if _, err := w.SetSandboxMaintenance(t.Context(), id, SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 1}); err != nil { + t.Fatal(err) + } + wg.Wait() + for range 3 { + if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString(), "")); !errors.Is(err, ErrEnvironmentUnavailable) { + t.Fatal("fresh creation bypassed maintenance", err) + } + } + view, err := s.GetRuntimeDeployment(t.Context()) + if err != nil { + t.Fatal(err) + } + _, err = w.UpdateSandboxDeployment(t.Context(), id, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: SandboxDeploymentSetupRequest{Provider: "docker", CoreURL: input.CoreURL}, ExpectedGeneration: 1}) + if view.Resources.Pending > 0 && !errors.Is(err, ErrSandboxDeploymentConflict) { + t.Fatal("committed pending Session bypassed switch guard", err) + } +} + +func TestSandboxSwitchPreservesReleasedAllocationAndItemHistory(t *testing.T) { + _, pool := newManagedTestStore(t) + cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{8}, 32)) + s := NewWithCredentialCipher(pool, cipher) + w := executionLease(t, s).Store() + installation := uuid.NewString() + if err := w.ClaimWebSandboxDeployment(t.Context(), installation); err != nil { + t.Fatal(err) + } + selection := e2bSelection() + if _, err := w.InitializeSandboxDeployment(t.Context(), installation, selection); err != nil { + t.Fatal(err) + } + tenant := uuid.NewString() + session, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString(), "")) + if err != nil { + t.Fatal(err) + } + owner, err := w.ReserveRuntimeAllocation(t.Context(), tenant, session.Environment.ID, installation, device.HashCredential(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + if _, err := w.ReleaseAbsentRuntimeCreation(t.Context(), owner); err != nil { + t.Fatal(err) + } + // A separate completed Session supplies public Items without calling a model. + history, err := s.CreateSession(t.Context(), tenant, CreateSessionInput{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString()}) + if err != nil { + t.Fatal(err) + } + input, err := s.SubmitMessage(t.Context(), tenant, history.ID, "history", json.RawMessage(`{"text":"retained request"}`)) + if err != nil { + t.Fatal(err) + } + if _, err := w.TransitionTurn(t.Context(), tenant, history.ID, input.TurnID, TurnTransition{ExpectedStatus: TurnQueued, Status: TurnInProgress}); err != nil { + t.Fatal(err) + } + if err := w.AppendTurnEvents(t.Context(), tenant, history.ID, input.TurnID, 1, []ExecutionEvent{{Kind: "output_message", Payload: json.RawMessage(`{"id":"answer","status":"completed","text":"retained answer"}`)}}); err != nil { + t.Fatal(err) + } + if _, err := w.TransitionTurn(t.Context(), tenant, history.ID, input.TurnID, TurnTransition{ExpectedStatus: TurnInProgress, Status: TurnCompleted}); err != nil { + t.Fatal(err) + } + items, err := s.ListItems(t.Context(), tenant, history.ID, "", 100, true) + if err != nil || len(items.Items) != 2 { + t.Fatal("history fixture", err) + } + before, _ := json.Marshal(items) + var allocationBefore []byte + if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(a) FROM runtime_allocations a WHERE id=$1", owner.ID).Scan(&allocationBefore); err != nil { + t.Fatal(err) + } + if _, err := w.SetSandboxMaintenance(t.Context(), installation, SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 1}); err != nil { + t.Fatal(err) + } + if _, err := w.UpdateSandboxDeployment(t.Context(), installation, SandboxDeploymentUpdateRequest{ExpectedGeneration: 1, SandboxDeploymentSetupRequest: SandboxDeploymentSetupRequest{Provider: "docker", CoreURL: selection.CoreURL}}); err != nil { + t.Fatal(err) + } + items, err = s.ListItems(t.Context(), tenant, history.ID, "", 100, true) + if err != nil { + t.Fatal(err) + } + after, _ := json.Marshal(items) + if !bytes.Equal(before, after) { + t.Fatal("switch rewrote public Item history") + } + var allocationAfter []byte + if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(a) FROM runtime_allocations a WHERE id=$1", owner.ID).Scan(&allocationAfter); err != nil { + t.Fatal(err) + } + if !bytes.Equal(allocationBefore, allocationAfter) { + t.Fatal("switch rewrote released allocation ownership") + } + for _, id := range []string{session.ID, history.ID} { + if _, err := s.GetSession(t.Context(), tenant, id); err != nil { + t.Fatal("switch lost undeleted Session", err) + } + } +} diff --git a/services/agents-api/internal/store/sandbox_deployment_switch_worker_test.go b/services/agents-api/internal/store/sandbox_deployment_switch_worker_test.go new file mode 100644 index 000000000..4de0e8f4e --- /dev/null +++ b/services/agents-api/internal/store/sandbox_deployment_switch_worker_test.go @@ -0,0 +1,100 @@ +package store_test + +import ( + "bytes" + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/credentialcrypto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" +) + +func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { + _, pool := store.NewManagedTestStore(t) + cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{7}, 32)) + s := store.NewWithCredentialCipher(pool, cipher) + id := uuid.NewString() + p := &lifecycleProvider{resources: map[string]sandbox.Info{}} + var fail atomic.Bool + configuration := execution.NewDeferredRuntimeProvider(id, func(ctx context.Context) (*execution.RuntimeProvider, error) { + if fail.Load() { + return nil, errors.New("fixture provider unavailable") + } + setup, err := s.GetSandboxSetup(ctx) + if err != nil || setup.Provider == "" { + return nil, err + } + return &execution.RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, Maintenance: setup.Maintenance, CoreURL: setup.CoreURL + "/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}, nil + }) + w, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: s, Registry: gateway.NewRegistry(), ManagedRuntimes: configuration}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- w.Run(ctx) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("worker shutdown blocked") + } + }) + if _, err := w.InitializeSandboxDeployment(t.Context(), store.SandboxDeploymentSetupRequest{Provider: "docker", CoreURL: "https://core.example"}); err != nil { + t.Fatal(err) + } + if _, err := w.SetSandboxMaintenance(t.Context(), store.SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 1}); err != nil { + t.Fatal(err) + } + input := store.SandboxDeploymentUpdateRequest{ExpectedGeneration: 1, SandboxDeploymentSetupRequest: store.SandboxDeploymentSetupRequest{Provider: "e2b", CoreURL: "https://core.example", E2B: &store.SandboxE2BConfiguration{APIKey: "fixture-api-key", Template: "runtime:" + uuid.NewString()}}} + fail.Store(true) + if _, err := w.UpdateSandboxDeployment(t.Context(), input); err == nil { + t.Fatal("failed activation reported success") + } + view, err := s.GetRuntimeDeployment(t.Context()) + if err != nil || view.Generation != 2 || !view.Maintenance || view.Provider != "e2b" { + t.Fatal("failed activation lost persisted maintenance", view, err) + } + if _, err := w.SetSandboxMaintenance(t.Context(), store.SandboxMaintenanceRequest{ExpectedGeneration: 2}); err == nil { + t.Fatal("failed activation resumed") + } + fail.Store(false) + if _, err := w.SetSandboxMaintenance(t.Context(), store.SandboxMaintenanceRequest{ExpectedGeneration: 2}); err != nil { + t.Fatal(err) + } + tenant, session, environment := managedSession(t, s) + allocation, err := w.ProvisionEnvironment(t.Context(), tenant, environment.ID, id) + if err != nil || allocation.NodeID != "" || allocation.State != "running" { + t.Fatal("direct provider not available after resume", allocation, err) + } + if _, err := w.SetSandboxMaintenance(t.Context(), store.SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 2}); err != nil { + t.Fatal(err) + } + next := store.SandboxDeploymentUpdateRequest{ExpectedGeneration: 2, SandboxDeploymentSetupRequest: store.SandboxDeploymentSetupRequest{Provider: "docker", CoreURL: "https://core.example"}} + if _, err := w.UpdateSandboxDeployment(t.Context(), next); !errors.Is(err, store.ErrSandboxDeploymentConflict) { + t.Fatal("dirty switch accepted", err) + } + if err := s.DeleteSession(t.Context(), tenant, session.ID); err != nil { + t.Fatal(err) + } + reconcileManagedState(t, w, s, tenant, environment.ID, "released") + if _, err := w.UpdateSandboxDeployment(t.Context(), next); err != nil { + t.Fatal("clean switch failed", err) + } + if _, err := w.SetSandboxMaintenance(t.Context(), store.SandboxMaintenanceRequest{ExpectedGeneration: 3}); err != nil { + t.Fatal(err) + } + select { + case err := <-done: + t.Fatalf("provider switch stopped Worker: %v", err) + default: + } +} diff --git a/services/agents-api/migrations/000063_hosted_provider_selection.sql b/services/agents-api/migrations/000063_hosted_provider_selection.sql new file mode 100644 index 000000000..c7fe69d42 --- /dev/null +++ b/services/agents-api/migrations/000063_hosted_provider_selection.sql @@ -0,0 +1,46 @@ +-- +goose Up +ALTER TABLE runtime_deployment DROP CONSTRAINT runtime_deployment_provider_kind_check; +ALTER TABLE runtime_deployment ADD CONSTRAINT runtime_deployment_provider_kind_check CHECK (provider_kind IN ('','docker','microsandbox','e2b')); +ALTER TABLE runtime_deployment ADD COLUMN generation bigint NOT NULL DEFAULT 0 CHECK (generation >= 0); +ALTER TABLE runtime_deployment ADD COLUMN mode text NOT NULL DEFAULT '' CHECK (mode IN ('','nodes','direct')); +ALTER TABLE runtime_deployment ADD COLUMN e2b_template text NOT NULL DEFAULT ''; +ALTER TABLE runtime_deployment ADD COLUMN e2b_credential bytea; +UPDATE runtime_deployment SET generation=1,mode='nodes' WHERE provider_kind<>''; +ALTER TABLE runtime_deployment DROP CONSTRAINT runtime_deployment_setup_check; +ALTER TABLE runtime_deployment ADD CONSTRAINT runtime_deployment_setup_check CHECK ( + NOT web_managed OR (local_node_id IS NULL AND ( + (provider_kind='' AND core_url='' AND mode='' AND generation=0 AND idle_seconds=0 AND retention_seconds=0) OR + (provider_kind='docker' AND core_url<>'' AND mode='nodes' AND generation>0 AND idle_seconds=0 AND retention_seconds=0) OR + (provider_kind='microsandbox' AND core_url<>'' AND mode='nodes' AND generation>0 AND idle_seconds>0 AND retention_seconds>0) OR + (provider_kind='e2b' AND core_url<>'' AND mode='direct' AND generation>0 AND idle_seconds=0 AND retention_seconds=0) + )) +); +ALTER TABLE runtime_deployment ADD CONSTRAINT runtime_deployment_e2b_check CHECK ( + (provider_kind='e2b' AND e2b_template<>'' AND e2b_credential IS NOT NULL) OR + (provider_kind<>'e2b' AND e2b_template='' AND e2b_credential IS NULL) +); + +-- +goose Down +-- +goose StatementBegin +DO $$ BEGIN + IF EXISTS (SELECT 1 FROM runtime_deployment WHERE provider_kind='e2b' OR generation>1) THEN + RAISE EXCEPTION 'Cannot discard hosted provider configuration generations'; + END IF; +END $$; +-- +goose StatementEnd + +ALTER TABLE runtime_deployment DROP CONSTRAINT runtime_deployment_e2b_check; +ALTER TABLE runtime_deployment DROP CONSTRAINT runtime_deployment_setup_check; +ALTER TABLE runtime_deployment DROP COLUMN e2b_credential; +ALTER TABLE runtime_deployment DROP COLUMN e2b_template; +ALTER TABLE runtime_deployment DROP COLUMN mode; +ALTER TABLE runtime_deployment DROP COLUMN generation; +ALTER TABLE runtime_deployment DROP CONSTRAINT runtime_deployment_provider_kind_check; +ALTER TABLE runtime_deployment ADD CONSTRAINT runtime_deployment_provider_kind_check CHECK (provider_kind IN ('','docker','microsandbox')); +ALTER TABLE runtime_deployment ADD CONSTRAINT runtime_deployment_setup_check CHECK ( + NOT web_managed OR (local_node_id IS NULL AND ( + (provider_kind='' AND core_url='' AND idle_seconds=0 AND retention_seconds=0) OR + (provider_kind='docker' AND core_url<>'' AND idle_seconds=0 AND retention_seconds=0) OR + (provider_kind='microsandbox' AND core_url<>'' AND idle_seconds>0 AND retention_seconds>0) + )) +); diff --git a/services/agents-api/tools/e2b-provider/Build.Dockerfile b/services/agents-api/tools/e2b-provider/Build.Dockerfile new file mode 100644 index 000000000..f0c98b4e4 --- /dev/null +++ b/services/agents-api/tools/e2b-provider/Build.Dockerfile @@ -0,0 +1,5 @@ +# Fixed CPython and glibc baseline shared by native Core and the Debian Core image. +FROM python:3.12.12-slim-bookworm@sha256:2986c55feb36e6cae00fa1fefb454283e4b33f35e75ff8bdd123b134130be301 +RUN apt-get update && apt-get install -y --no-install-recommends binutils \ + && rm -rf /var/lib/apt/lists/* +ENTRYPOINT ["python3", "/source/services/agents-api/tools/e2b-provider/build.py"] diff --git a/services/agents-api/tools/e2b-provider/README.md b/services/agents-api/tools/e2b-provider/README.md new file mode 100644 index 000000000..e2ba15eb5 --- /dev/null +++ b/services/agents-api/tools/e2b-provider/README.md @@ -0,0 +1,76 @@ +# Managed E2B SDK helper + +Core's Go adapter invokes this one-shot helper for Create, GetInfo, Renew, Kill +and initialization RunCommand. Daily execution and Files remain on the existing +Runtime connection. The helper uses the official E2B Python SDK 2.51.0; it does +not implement provider HTTP, envd RPC, a scheduler or a network service. + +The private JSON boundary has version 1. Requests and credentials enter stdin; +stdout contains one bounded response with sanitized error codes. API keys never +enter arguments, inherited environment or receipts. The process retains its +allocation lock when the Core caller times out, until the bounded SDK operation +returns. Core must serialize lifecycle requests and never replay Create. + +`StateDir` must already exist, be owned by the service user and have mode 0700. +Keep it on durable private storage through Core upgrades/restarts. Its receipts +contain provider connection credentials, ownership identities and one-shot +creation claims, not Core execution state. Losing this directory cannot authorize +recreation or successful cleanup. Do not delete receipts after an uncertain call. + +GetInfo uses SDK metadata/ID reads. SDK `connect` is never used because it can +resume paused compute. The SDK's version-pinned constructor restores clients +from private connection material; this deprecated constructor is intentionally +contained in `sdk.py` and covered by a no-connect/no-create test. An unknown +Create without connection material can be discovered and reclaimed, but cannot +resume bootstrap. Empty lookup cannot settle an unknown Create. Cleanup retains +every matching candidate and confirms exact-ID absence before writing a tombstone. + +`CreateSettled` proves the original Create/bootstrap can no longer mutate. It is +independent of `BootstrapComplete`, which acknowledges the protected initializer's +last step, not enrollment or native readiness. Explicitly settled absence returns +successful Info with `State=absent`; ordinary missing compute has no such proof. +An explicitly rejected Create with a settled receipt and no provider IDs proves +absence without another cloud request. GetInfo and Kill retain that rejection +receipt, so repeated recovery remains possible even when the API key is invalid. +Other receipts still require cloud discovery and ownership checks. +Unconfirmed initialization commands require reclaiming the whole allocation. + +## Build + +From the repository root: + +```sh +E2B_PROVIDER_BUILD_DIR="$HOME/.parsar/build/e2b-provider" scripts/build-e2b-provider.sh +``` + +Docker builds Linux amd64 output with the pinned CPython 3.12.12/Debian 12 image. +The full Python dependency closure, including PyInstaller, has hashes in +`requirements.lock`. Native `pyqwest` and `protobuf-py-ext` wheels are included. +No account credential is needed for builds or `--check`. When building from an +archived source tree, supply `E2B_SOURCE_REVISION` with its actual commit. + +The output is `agents-api-e2b-provider-linux-amd64.tar.gz` and its `.sha256` file. +Extraction yields `agents-api-e2b-provider/agents-api-e2b-provider`, `_internal/`, +`licenses/`, `requirements.lock` and `manifest.json`. The artifact contains only +regular files/directories, with executable permissions preserved. Core's image +and native installer use the same tree; the target needs compatible Linux/glibc +and CA certificates, but no separately installed Python. The installation owns +the durable receipt path independently of this immutable helper payload. + +`deploy/e2b/build-template.py` packages `init.py` and `managed_init.py` with the +qualified Runtime image. Existing templates without these files must be rebuilt. +The shared protected image preparation is used by both managed and self-hosted +startup; their credential formats and one-shot receipts remain separate. + +## Verification + +```sh +python -m unittest discover -s services/agents-api/tools/e2b-provider -p '*_test.py' -v +python -m unittest discover -s services/agents-api/deploy/e2b -p '*_test.py' -v +``` + +The build runs the first suite and validates the relocated helper's `--check` +report. These checks do not establish cloud authentication, native isolation or +real execution. Live acceptance must use owned E2B compute and the same Runtime, +with actual lease renewal, restart/unknown outcome reconciliation and confirmed +cleanup. Initializer and three-harness qualification remain deployment checks. diff --git a/services/agents-api/tools/e2b-provider/build.py b/services/agents-api/tools/e2b-provider/build.py new file mode 100644 index 000000000..8eaef686e --- /dev/null +++ b/services/agents-api/tools/e2b-provider/build.py @@ -0,0 +1,77 @@ +"""Build and verify the fixed Linux SDK helper inside its pinned build image.""" +import hashlib +import importlib.metadata +import json +import os +from pathlib import Path +import platform +import shutil +import subprocess +import sys +import tarfile +import tempfile + +SOURCE = Path('/source/services/agents-api/tools/e2b-provider') +OUTPUT = Path('/output') +NAME = 'agents-api-e2b-provider' +BASE = 'python:3.12.12-slim-bookworm@sha256:2986c55feb36e6cae00fa1fefb454283e4b33f35e75ff8bdd123b134130be301' + + +def checked(args, **options): + subprocess.run(args, check=True, **options) + + +def main(): + if sys.version_info[:3] != (3, 12, 12) or platform.system() != 'Linux' or platform.machine() != 'x86_64': + raise RuntimeError('Use the pinned Linux amd64 build image') + os.umask(0o022) + with tempfile.TemporaryDirectory(prefix='e2b-build-') as temporary: + root = Path(temporary) + source = root / 'source' + shutil.copytree(SOURCE, source, ignore=shutil.ignore_patterns('__pycache__')) + venv = root / 'venv' + checked([sys.executable, '-m', 'venv', str(venv)]) + python = str(venv / 'bin/python') + checked([python, '-m', 'pip', 'install', '--disable-pip-version-check', '--require-hashes', + '--only-binary=:all:', '-r', str(source / 'requirements.lock')]) + checked([python, '-m', 'unittest', 'discover', '-s', str(source), '-p', '*_test.py', '-v']) + checked([python, '-m', 'PyInstaller', '--noconfirm', '--clean', '--onedir', '--noupx', + '--name', NAME, '--distpath', str(root / 'dist'), '--workpath', str(root / 'work'), + '--specpath', str(root), '--recursive-copy-metadata', 'e2b', + # The SDK's native pyqwest transport imports tracing modules + # dynamically; Python bytecode analysis cannot discover them. + '--collect-submodules', 'opentelemetry', str(source / 'main.py')]) + exported = root / 'export' / NAME + # PyInstaller uses library aliases. Native payloads deliberately contain + # only regular files/directories so the installer can reject all links. + shutil.copytree(root / 'dist' / NAME, exported, symlinks=False) + licenses = exported / 'licenses' + licenses.mkdir() + shutil.copy2(source / 'requirements.lock', exported / 'requirements.lock') + checked([python, str(source / 'licenses.py'), str(licenses)]) + shutil.copy2('/source/LICENSE', licenses / 'Parsar-Core-LICENSE') + report = json.loads(subprocess.check_output([str(exported / NAME), '--check'], text=True)) + if report != {'Version': 1, 'SDKVersion': '2.51.0'}: + raise RuntimeError('Unexpected helper readiness report') + manifest = {'format_version': 1, 'source_revision': os.environ['E2B_SOURCE_REVISION'], + 'sdk_version': report['SDKVersion'], 'python_version': platform.python_version(), + 'platform': 'linux-amd64', 'libc': platform.libc_ver(), 'build_image': BASE, + 'entrypoint': NAME, + 'source_sha256': {file.name: hashlib.sha256(file.read_bytes()).hexdigest() + for file in sorted(source.glob('*.py'))}, + 'requirements_sha256': hashlib.sha256((exported / 'requirements.lock').read_bytes()).hexdigest()} + (exported / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n') + for entry in exported.rglob('*'): + if entry.is_symlink() or not (entry.is_file() or entry.is_dir()): + raise RuntimeError('Unsupported artifact entry') + entry.chmod(0o755 if entry.is_dir() or entry.stat().st_mode & 0o111 else 0o644) + destination = OUTPUT / (NAME + '-linux-amd64.tar.gz') + with tarfile.open(destination, 'w:gz', dereference=True) as archive: + archive.add(exported, arcname=NAME) + checksum = hashlib.sha256(destination.read_bytes()).hexdigest() + destination.with_suffix(destination.suffix + '.sha256').write_text(checksum + ' ' + destination.name + '\n') + print(json.dumps({'archive': str(destination), 'sha256': checksum, 'bytes': destination.stat().st_size})) + + +if __name__ == '__main__': + main() diff --git a/services/agents-api/tools/e2b-provider/licenses.py b/services/agents-api/tools/e2b-provider/licenses.py new file mode 100644 index 000000000..6c4e56992 --- /dev/null +++ b/services/agents-api/tools/e2b-provider/licenses.py @@ -0,0 +1,31 @@ +"""Retain installed wheel license files/metadata and bundled system-library notices.""" +import importlib.metadata +import json +from pathlib import Path +import shutil +import sys +import sysconfig + +output = Path(sys.argv[1]) +packages = [] +for distribution in importlib.metadata.distributions(): + name, version = distribution.metadata['Name'], distribution.version + directory = output / (name + '-' + version) + directory.mkdir(exist_ok=True) + directory.joinpath('METADATA').write_text(distribution.read_text('METADATA') or '') + for entry in distribution.files or []: + if any(word in entry.name.lower() for word in ['license', 'copying', 'notice']) or '/licenses/' in str(entry): + source = Path(distribution.locate_file(entry)) + if source.is_file(): + target = directory / str(entry).replace('/', '_').replace('..', '_') + shutil.copyfile(source, target) + packages.append({'name': name, 'version': version}) +(output / 'packages.json').write_text(json.dumps(sorted(packages, key=lambda p: p['name']), indent=2) + '\n') +python_license = Path(sysconfig.get_path('stdlib')) / 'LICENSE.txt' +if not python_license.is_file(): + raise RuntimeError('CPython license is missing') +shutil.copyfile(python_license, output / 'CPython-LICENSE.txt') +system = output / 'system' +system.mkdir() +for copyright_file in Path('/usr/share/doc').glob('*/copyright'): + shutil.copyfile(copyright_file, system / copyright_file.parent.name) diff --git a/services/agents-api/tools/e2b-provider/main.py b/services/agents-api/tools/e2b-provider/main.py new file mode 100644 index 000000000..a908c6c93 --- /dev/null +++ b/services/agents-api/tools/e2b-provider/main.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""One request over stdin; no service, model calls or provider credentials in logs.""" +import importlib.metadata +import json +import logging +import os +import sys + +from provider import Provider +from sdk import SDK_VERSION +from state import Failure + +MAX_REQUEST = 72 * 1024 * 1024 + + +def main(): + logging.disable(logging.CRITICAL) + os.umask(0o077) + # Private SDK routing selectors may otherwise override the official cloud. + for key in list(os.environ): + if key.startswith(('E2B_', 'PYTHON')): + os.environ.pop(key) + try: + if importlib.metadata.version('e2b') != SDK_VERSION: + raise Failure('invalid') + if sys.argv[1:] == ['--check']: + # Construct the SDK's native transport without sending any request. + # This also verifies its dynamically imported dependencies survived + # freezing; importing e2b alone does not exercise that boundary. + from pyqwest import SyncHTTPTransport + SyncHTTPTransport(tls_include_system_certs=True) + print(json.dumps({'Version': 1, 'SDKVersion': SDK_VERSION})) + return + if len(sys.argv) != 1: + raise Failure('invalid') + data = sys.stdin.buffer.read(MAX_REQUEST + 1) + if len(data) > MAX_REQUEST: + raise Failure('invalid') + result = Provider(json.loads(data)).execute() + except Failure as error: + result = {'Version': 1, 'ErrorCode': error.code} + except BaseException: + result = {'Version': 1, 'ErrorCode': 'unconfirmed'} + sys.stdout.write(json.dumps(result) + '\n') + + +if __name__ == '__main__': + main() diff --git a/services/agents-api/tools/e2b-provider/provider.py b/services/agents-api/tools/e2b-provider/provider.py new file mode 100644 index 000000000..48b6770b5 --- /dev/null +++ b/services/agents-api/tools/e2b-provider/provider.py @@ -0,0 +1,200 @@ +"""Five bounded SDK operations for an already authorized Core allocation.""" +import json +import time +from datetime import datetime, timezone +from uuid import UUID + +from e2b import Sandbox, SandboxQuery, SandboxState +from e2b.exceptions import FileNotFoundException, SandboxNotFoundException + +from sdk import connection_material, definitely_rejected, restore, run +from state import Failure, Receipt + +PREFIX = 'parsar_' +FIELDS = ('InstallationID', 'TenantID', 'EnvironmentID', 'AllocationID') + + +def valid_id(value): + try: + return isinstance(value, str) and str(UUID(value)) == value and UUID(value).int != 0 + except (ValueError, AttributeError): + return False + + +class Provider: + def __init__(self, request): + self.q = request + self.config = request['Config'] + self.reference = request['Reference'] + if (request['Version'] != 1 or request['Operation'] not in + ('create', 'inspect', 'renew', 'kill', 'command') or + set(self.reference) != set(FIELDS[1:]) or + any(not valid_id(v) for v in self.reference.values()) or + not valid_id(self.config['InstallationID'])): + raise Failure('invalid') + deadline = datetime.fromisoformat(request['Deadline'].replace('Z', '+00:00')) + self.deadline = time.monotonic() + (deadline - datetime.now(timezone.utc)).total_seconds() + self.metadata = {PREFIX + field.lower(): value for field, value in + dict(self.reference, InstallationID=self.config['InstallationID']).items()} + self.receipt = None + + def remaining(self): + remaining = self.deadline - time.monotonic() + if remaining <= 0: + raise Failure('unconfirmed') + return remaining + + def options(self): + return {'api_key': self.config['APIKey'], 'retries': 0, 'debug': False, + 'request_timeout': self.remaining()} + + def info(self, cloud=None, absent=False): + record = self.receipt.data or {} + ids = record.get('ids', []) + absent = absent or self.rejected_absence() + value = dict(self.reference, ProviderID=ids[0] if len(ids) == 1 else '', State='absent' if absent else 'unknown', + BootstrapComplete=False, CreateSettled=record.get('settled', False)) + if cloud is not None: + value.update(ProviderID=cloud.sandbox_id, State=cloud.state, + BootstrapComplete=record.get('bootstrap_complete', False)) + return value + + def rejected_absence(self): + record = self.receipt.data or {} + return (record.get('status') == 'rejected' and record.get('settled') is True + and record.get('ids') == []) + + def owns(self, cloud): + if any(cloud.metadata.get(key) != value for key, value in self.metadata.items()): + raise Failure('ownership') + return cloud + + def discover(self): + record = self.receipt.data or {} + ids = record.get('ids', []) + found = [] + if ids: + for sandbox_id in ids: + try: + found.append(self.owns(Sandbox.get_info(sandbox_id, **self.options()))) + except SandboxNotFoundException: + pass + return found + paginator = Sandbox.list(query=SandboxQuery(metadata=self.metadata, + state=[SandboxState.RUNNING, SandboxState.PAUSED]), **self.options()) + while paginator.has_next: + self.remaining() + for cloud in paginator.next_items(**self.options()): + found.append(self.owns(cloud)) + # Retain every candidate; never pick one of several for initialization. + if found: + self.receipt.save(ids=sorted({cloud.sandbox_id for cloud in found})) + return found + + def client(self, cloud): + material = (self.receipt.data or {}).get('connection') + if not material or material['sandbox_id'] != cloud.sandbox_id: + raise Failure('unconfirmed') + return restore(material, self.options()) + + def inspect(self): + if self.rejected_absence(): + return None + found = self.discover() + if not found: + if (self.receipt.data or {}).get('settled'): + return None + raise Failure('not_found') + if len(found) != 1: + raise Failure('unconfirmed') + cloud = found[0] + record = self.receipt.data or {} + if (cloud.state == 'running' and not record.get('bootstrap_complete') and record.get('connection') + and record.get('status') not in ('bootstrap_failed', 'killed')): + try: + receipt = json.loads(self.client(cloud).files.read('/root/.parsar/e2b/managed-ready.json', + user='root', request_timeout=self.remaining())) + except FileNotFoundException: + receipt = None + if receipt is not None: + expected = record.get('bootstrap_identity') + if (receipt.get('identity') != expected or receipt.get('status') != 'daemon_started' or + type(receipt.get('daemon_pid')) is not int or receipt['daemon_pid'] <= 0): + raise Failure('ownership') + self.receipt.save(settled=True, bootstrap_complete=True) + return cloud + + def create(self): + if self.receipt.data is not None: + raise Failure('exists') + bootstrap = self.q['Bootstrap'] + if any(bootstrap.get(field) != value for field, value in self.reference.items()): + raise Failure('invalid') + identity = dict(self.reference, InstallationID=self.config['InstallationID'], + SessionID=bootstrap['SessionID'], DeviceID=bootstrap['DeviceID']) + self.receipt.save(status='create_pending', bootstrap_identity=identity) + try: + cloud = Sandbox.create(template=self.config['Template'], timeout=self.config['TimeoutSeconds'], + metadata=self.metadata, lifecycle={'on_timeout': 'kill', 'auto_resume': False}, + **self.options()) + except Exception as error: + if definitely_rejected(error): + self.receipt.save(status='rejected', settled=True) + raise Failure('unconfirmed') from None + self.receipt.save(status='created', ids=[cloud.sandbox_id], connection=connection_material(cloud)) + payload = dict(bootstrap, InstallationID=self.config['InstallationID']) + cloud.files.write('/root/.parsar/e2b/managed-bootstrap.json', json.dumps(payload), + user='root', request_timeout=self.remaining()) + self.receipt.save(status='bootstrap_pending') + result = run(cloud, {'Args': ['/usr/bin/python3', '-I', '/opt/parsar-e2b/managed_init.py']}, + self.remaining, user='root') + if result['ExitCode'] != 0: + self.receipt.save(status='bootstrap_failed', settled=True) + raise Failure('unconfirmed') + self.receipt.save(status='bootstrap_exited', settled=True) + return self.inspect() + + def renew(self): + cloud = self.inspect() + if cloud is None or cloud.state != 'running': + raise Failure('unconfirmed') + Sandbox.set_timeout(cloud.sandbox_id, self.config['TimeoutSeconds'], **self.options()) + return self.owns(Sandbox.get_info(cloud.sandbox_id, **self.options())) + + def kill(self): + if self.rejected_absence(): + return + found = self.discover() + # The allocation flock excludes any still-running local Create helper. + # A matching actual VM proves the original request reached allocation. + known = bool((self.receipt.data or {}).get('ids')) + if not known and not (self.receipt.data or {}).get('settled'): + raise Failure('unconfirmed') + for cloud in found: + Sandbox.kill(cloud.sandbox_id, **self.options()) + if self.discover(): + raise Failure('unconfirmed') + self.receipt.save(status='killed', settled=True, bootstrap_complete=False, connection=None) + + def execute(self): + with Receipt(self.q, self.remaining) as self.receipt: + try: + operation = self.q['Operation'] + if operation == 'kill': + self.kill() + return {'Version': 1, 'Info': self.info(absent=True), 'ErrorCode': ''} + if operation == 'command': + cloud = self.inspect() + if cloud.state != 'running' or not self.receipt.data.get('bootstrap_complete'): + raise Failure('unconfirmed') + result = run(self.client(cloud), self.q['Command'], self.remaining) + return {'Version': 1, 'Command': result, 'ErrorCode': ''} + cloud = {'create': self.create, 'inspect': self.inspect, 'renew': self.renew}[operation]() + return {'Version': 1, 'Info': self.info(cloud, absent=cloud is None), 'ErrorCode': ''} + except Failure as error: + info = self.info() + if error.code == 'not_found': + info = dict(self.reference, ProviderID='', State='absent', BootstrapComplete=False, CreateSettled=False) + return {'Version': 1, 'Info': info, 'ErrorCode': error.code} + except Exception: + return {'Version': 1, 'Info': self.info(), 'ErrorCode': 'unconfirmed'} diff --git a/services/agents-api/tools/e2b-provider/provider_test.py b/services/agents-api/tools/e2b-provider/provider_test.py new file mode 100644 index 000000000..bced46e8f --- /dev/null +++ b/services/agents-api/tools/e2b-provider/provider_test.py @@ -0,0 +1,246 @@ +"""Controlled SDK boundary failures; live qualification remains separate.""" +import copy +from datetime import datetime, timedelta, timezone +import json +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch +from uuid import uuid4 + +from e2b import SandboxState +from e2b.exceptions import AuthenticationException, SandboxNotFoundException + +from provider import Provider +from sdk import restore, run +from state import Failure, Receipt + + +class ProviderTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.reference = {key: str(uuid4()) for key in ['TenantID', 'EnvironmentID', 'AllocationID']} + self.config = {'StateDir': self.temporary.name, 'InstallationID': str(uuid4()), + 'APIKey': 'private-account-secret', 'Template': 'test:' + str(uuid4()), 'TimeoutSeconds': 120} + self.request = {'Version': 1, 'Operation': 'create', 'Config': self.config, + 'Reference': self.reference, 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=30)).isoformat(), + 'Bootstrap': dict(self.reference, SessionID=str(uuid4()), DeviceID=str(uuid4()), + CoreURL='https://core.example/api/v1', Credential='private-runtime-secret', + NetworkAccess='enabled', AllowedDomains=[])} + self.cloud = Mock(sandbox_id='owned-id', sandbox_domain='e2b.app', _envd_version='0.5.0', + _envd_access_token='private-envd-secret', traffic_access_token=None, state='running') + self.cloud.metadata = Provider(self.request).metadata + self.identity = dict(self.reference, InstallationID=self.config['InstallationID'], + SessionID=self.request['Bootstrap']['SessionID'], DeviceID=self.request['Bootstrap']['DeviceID']) + self.ready = json.dumps({'identity': self.identity, 'status': 'daemon_started', 'daemon_pid': 123}) + self.cloud.files.read.return_value = self.ready + self.api = Mock() + self.api.create.return_value = self.cloud + self.api.get_info.return_value = self.cloud + self.api.list.return_value = SimpleNamespace(has_next=False) + self.patch = patch('provider.Sandbox', self.api) + self.patch.start() + self.addCleanup(self.patch.stop) + self.runtime = patch('provider.restore', return_value=self.cloud) + self.runtime.start() + self.addCleanup(self.runtime.stop) + self.command = patch('provider.run', return_value={'Stdout': '', 'Stderr': '', 'ExitCode': 0}) + self.command.start() + self.addCleanup(self.command.stop) + + def call(self, operation): + return Provider(dict(self.request, Operation=operation)).execute() + + def record(self): + return json.loads(next(Path(self.temporary.name).glob('*.json')).read_text()) + + def test_create_recover_and_never_replay(self): + result = self.call('create') + self.assertEqual(result['ErrorCode'], '') + self.assertTrue(result['Info']['BootstrapComplete']) + self.assertTrue(result['Info']['CreateSettled']) + self.assertEqual(self.call('create')['ErrorCode'], 'exists') + self.assertTrue(self.call('inspect')['Info']['BootstrapComplete']) + self.api.create.assert_called_once() + kwargs = self.api.create.call_args.kwargs + self.assertEqual(kwargs['retries'], 0) + self.assertEqual(kwargs['lifecycle'], {'on_timeout': 'kill', 'auto_resume': False}) + self.assertEqual(kwargs['metadata'], self.cloud.metadata) + serialized = json.dumps(self.record()) + self.assertNotIn(self.config['APIKey'], serialized) + self.assertNotIn(self.request['Bootstrap']['Credential'], serialized) + self.assertEqual(self.record()['connection']['envd_access_token'], 'private-envd-secret') + self.api.connect.assert_not_called() + + def test_unknown_create_empty_lookup_never_proves_cleanup(self): + self.api.create.side_effect = TimeoutError('confidential SDK diagnostic') + self.assertEqual(self.call('create')['ErrorCode'], 'unconfirmed') + missing = self.call('inspect') + self.assertEqual(missing['ErrorCode'], 'not_found') + self.assertFalse(missing['Info']['CreateSettled']) + self.assertEqual(self.api.list.call_args.kwargs['query'].state, + [SandboxState.RUNNING, SandboxState.PAUSED]) + self.assertEqual([value.value for value in self.api.list.call_args.kwargs['query'].state], ['running', 'paused']) + self.assertEqual(self.call('kill')['ErrorCode'], 'unconfirmed') + self.assertEqual(self.call('create')['ErrorCode'], 'exists') + self.api.create.assert_called_once() + + def test_explicit_rejected_create_has_successful_absence_proof(self): + self.api.create.side_effect = AuthenticationException('secret diagnostic') + self.assertTrue(self.call('create')['Info']['CreateSettled']) + rejection = self.record() + self.api.reset_mock() + self.api.list.side_effect = AuthenticationException('revoked key') + self.api.get_info.side_effect = AuthenticationException('revoked key') + for operation in ('inspect', 'kill', 'inspect', 'kill'): + with self.subTest(operation=operation): + missing = self.call(operation) + self.assertEqual(missing['ErrorCode'], '') + self.assertEqual(missing['Info']['State'], 'absent') + self.assertEqual(missing['Info']['ProviderID'], '') + self.assertTrue(missing['Info']['CreateSettled']) + self.assertEqual(self.record(), rejection) + self.assertEqual(self.api.mock_calls, []) + + def test_other_empty_receipts_still_require_cloud_discovery(self): + for status, settled in [('create_pending', False), ('rejected', False), ('killed', True)]: + with self.subTest(status=status, settled=settled): + with Receipt(self.request, lambda: 30) as receipt: + receipt.save(status=status, settled=settled, ids=[]) + self.api.reset_mock() + self.api.list.side_effect = AuthenticationException('revoked key') + for operation in ('inspect', 'kill'): + self.assertEqual(self.call(operation)['ErrorCode'], 'unconfirmed') + self.assertEqual(self.api.list.call_count, 2) + self.api.kill.assert_not_called() + + def test_rejected_receipt_with_ids_still_verifies_ownership(self): + with Receipt(self.request, lambda: 30) as receipt: + receipt.save(status='rejected', settled=True, ids=[self.cloud.sandbox_id]) + self.cloud.metadata = {} + for operation in ('inspect', 'kill'): + self.assertEqual(self.call(operation)['ErrorCode'], 'ownership') + self.assertEqual(self.api.get_info.call_count, 2) + self.api.kill.assert_not_called() + + def test_killed_known_vm_still_requires_cloud_absence_check(self): + self.call('create') + self.api.get_info.side_effect = SandboxNotFoundException() + self.assertEqual(self.call('kill')['ErrorCode'], '') + self.api.reset_mock() + self.api.get_info.side_effect = AuthenticationException('revoked key') + for operation in ('inspect', 'kill'): + self.assertEqual(self.call(operation)['ErrorCode'], 'unconfirmed') + self.assertEqual(self.api.get_info.call_count, 2) + self.api.kill.assert_not_called() + + def test_unknown_bootstrap_only_settles_after_exact_kill(self): + self.cloud.files.write.side_effect = TimeoutError() + self.assertFalse(self.call('create')['Info']['CreateSettled']) + self.api.kill.side_effect = lambda *args, **kwargs: setattr(self.api.get_info, 'side_effect', SandboxNotFoundException()) + self.assertEqual(self.call('kill')['ErrorCode'], '') + self.assertTrue(self.call('inspect')['Info']['CreateSettled']) + self.assertIsNone(self.record()['connection']) + self.api.kill.assert_called_once_with('owned-id', **self.api.kill.call_args.kwargs) + + def test_bootstrap_failure_is_settled_but_not_ready(self): + with patch('provider.run', return_value={'ExitCode': 1}): + result = self.call('create') + self.assertTrue(result['Info']['CreateSettled']) + self.assertFalse(result['Info']['BootstrapComplete']) + self.assertEqual(self.call('inspect')['Info']['State'], 'running') + + def test_mismatched_receipt_cannot_prove_bootstrap_complete(self): + self.cloud.files.read.return_value = json.dumps({'identity': {}, 'status': 'daemon_started', 'daemon_pid': 2}) + self.assertEqual(self.call('create')['ErrorCode'], 'ownership') + self.assertTrue(self.record()['settled']) + self.assertFalse(self.record()['bootstrap_complete']) + + def test_foreign_owner_prevents_renew_and_delete(self): + self.call('create') + self.cloud.metadata = {} + self.assertEqual(self.call('renew')['ErrorCode'], 'ownership') + self.assertEqual(self.call('kill')['ErrorCode'], 'ownership') + self.api.set_timeout.assert_not_called() + self.api.kill.assert_not_called() + + def test_expired_unsettled_known_vm_is_cleanable(self): + self.cloud.files.write.side_effect = TimeoutError() + self.call('create') + self.api.get_info.side_effect = SandboxNotFoundException() + self.assertEqual(self.call('inspect')['ErrorCode'], 'not_found') + self.assertEqual(self.call('kill')['ErrorCode'], '') + self.assertTrue(self.call('inspect')['Info']['CreateSettled']) + + def test_paused_vm_never_resumes(self): + self.call('create') + self.cloud.state = 'paused' + self.assertEqual(self.call('inspect')['Info']['State'], 'paused') + self.assertEqual(self.call('renew')['ErrorCode'], 'unconfirmed') + self.api.set_timeout.assert_not_called() + self.api.connect.assert_not_called() + + def test_lost_create_discovers_every_page_and_cleans_all_candidates(self): + self.api.create.side_effect = TimeoutError() + self.call('create') + other = SimpleNamespace(sandbox_id='other-owned-id', metadata=self.cloud.metadata, state='running') + pages = iter([[self.cloud], [other]]) + paginator = SimpleNamespace(has_next=True) + def next_items(**options): + self.assertEqual(options["retries"], 0) + page = next(pages) + if page == [other]: + paginator.has_next = False + return page + paginator.next_items = next_items + self.api.list.return_value = paginator + self.assertEqual(self.call('inspect')['ErrorCode'], 'unconfirmed') + self.assertEqual(self.record()['ids'], ['other-owned-id', 'owned-id']) + deleted = set() + self.api.get_info.side_effect = lambda sandbox_id, **kwargs: self.maybe_missing(sandbox_id, deleted, other) + self.api.kill.side_effect = lambda sandbox_id, **kwargs: deleted.add(sandbox_id) + self.assertEqual(self.call('kill')['ErrorCode'], '') + self.assertEqual(deleted, {'other-owned-id', 'owned-id'}) + + def maybe_missing(self, sandbox_id, deleted, other): + if sandbox_id in deleted: + raise SandboxNotFoundException() + return self.cloud if sandbox_id == self.cloud.sandbox_id else other + + +class SDKTest(unittest.TestCase): + def test_constructor_restores_without_a_control_plane_operation(self): + material = {'sandbox_id': 'fixture', 'sandbox_domain': 'e2b.app', 'envd_version': '0.5.0', + 'envd_access_token': 'synthetic-envd-secret', 'traffic_access_token': None} + with patch('e2b.sandbox_sync.sandbox_api.SandboxApi._cls_connect', side_effect=AssertionError('connect forbidden')), \ + patch('e2b.sandbox_sync.sandbox_api.SandboxApi._create_sandbox', side_effect=AssertionError('create forbidden')): + client = restore(material, {'api_key': 'synthetic-key', 'retries': 0, 'debug': False}) + self.assertEqual(client.sandbox_id, material['sandbox_id']) + self.assertEqual(client.connection_config.sandbox_headers['X-Access-Token'], material['envd_access_token']) + self.assertEqual(client.connection_config.retries, 0) + + def test_stdin_is_separate_from_quoted_command_and_receives_eof(self): + import base64 + client = Mock() + client.commands.run.return_value.pid = 23 + client.commands.run.return_value.wait.return_value = SimpleNamespace(stdout='ok', stderr='', exit_code=7) + result = run(client, {'Args': ['cat', 'a;touch /unwanted'], 'Directory': '/workspace', + 'Stdin': base64.b64encode(b'private-input').decode()}, lambda: 3) + self.assertEqual(result['ExitCode'], 7) + self.assertEqual(client.commands.run.call_args.args, ("cat 'a;touch /unwanted'",)) + client.commands.send_stdin.assert_called_once_with(23, b'private-input', request_timeout=3) + client.commands.close_stdin.assert_called_once_with(23, request_timeout=3) + + def test_unknown_command_does_not_replay(self): + client = Mock() + client.commands.run.side_effect = TimeoutError('private command') + with self.assertRaises(Failure) as result: + run(client, {'Args': ['true']}, lambda: 3) + self.assertEqual(result.exception.code, 'command_unconfirmed') + client.commands.run.assert_called_once() + + +if __name__ == '__main__': + unittest.main() diff --git a/services/agents-api/tools/e2b-provider/requirements.in b/services/agents-api/tools/e2b-provider/requirements.in new file mode 100644 index 000000000..b47bbe30e --- /dev/null +++ b/services/agents-api/tools/e2b-provider/requirements.in @@ -0,0 +1,3 @@ +# The helper and its artifact builder are locked together for Linux amd64/Python 3.12. +e2b==2.51.0 +pyinstaller==6.22.3 diff --git a/services/agents-api/tools/e2b-provider/requirements.lock b/services/agents-api/tools/e2b-provider/requirements.lock new file mode 100644 index 000000000..f58fcfc85 --- /dev/null +++ b/services/agents-api/tools/e2b-provider/requirements.lock @@ -0,0 +1,217 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --generate-hashes --python-platform x86_64-manylinux_2_28 --python-version 3.12 services/agents-api/tools/e2b-provider/requirements.in -o services/agents-api/tools/e2b-provider/requirements.lock +altgraph==0.17.5 \ + --hash=sha256:c87b395dd12fabde9c99573a9749d67da8d29ef9de0125c7f536699b4a9bc9e7 \ + --hash=sha256:f3a22400bce1b0c701683820ac4f3b159cd301acab067c51c653e06961600597 + # via pyinstaller +anyio==4.15.1 \ + --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \ + --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94 + # via httpx +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ + --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 + # via e2b +bracex==3.0.1 \ + --hash=sha256:4e38e32392e4a4780fe15d644bfc7c8514057cfc3861e060b11814ce829c25e4 \ + --hash=sha256:6523ad83aeb5098a4ee597cff0f964442ff74e460bd3fafaffab6a013ff2288c + # via wcmatch +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via + # httpcore + # httpx +connectrpc==0.11.1 \ + --hash=sha256:18277f7838847b4271ca38d40c7d2387b5a2ea6a29f240689c19e1ec84aaff66 \ + --hash=sha256:8a52e2e92a485fa9681c1101a79a5ebeb31807e3ea3d5aabd41f484a6398bc7b + # via e2b +dockerfile-parse==2.0.1 \ + --hash=sha256:3184ccdc513221983e503ac00e1aa504a2aa8f84e5de673c46b0b6eee99ec7bc \ + --hash=sha256:bdffd126d2eb26acf1066acb54cb2e336682e1d72b974a40894fac76a4df17f6 + # via e2b +e2b==2.51.0 \ + --hash=sha256:902ee861b30c8ec3e46e7f84db8c993f30695872c7b4490efaad1627ec6e08ba \ + --hash=sha256:951d0fd069f017e760b338a435bdb83589b4bfb9574fc5203a7f777e9cb994c2 + # via -r services/agents-api/tools/e2b-provider/requirements.in +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via httpcore +h2==4.4.1 \ + --hash=sha256:0e25f1462b23c9cb82d9eb02e28bc706dac2a68cb457c6a0d74d63c8a2a5d0e6 \ + --hash=sha256:4e866ffb1a869ae14dd9b5e6beb5c24a13da0495ad72b65925ded182521c1516 + # via e2b +hpack==4.2.0 \ + --hash=sha256:0895cfa3b5531fc65fe439c05eb65144f123bf7a394fcaa56aa423548d8e45c0 \ + --hash=sha256:858ac0b02280fa582b5080d68db0899c62a80375e0e5413a74970c5e518b6986 + # via h2 +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via e2b +hyperframe==6.1.0 \ + --hash=sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5 \ + --hash=sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08 + # via h2 +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via + # anyio + # httpx +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +opentelemetry-api==1.44.0 \ + --hash=sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a \ + --hash=sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef + # via pyqwest +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # e2b + # pyinstaller + # pyinstaller-hooks-contrib +protobuf-py==0.1.1 \ + --hash=sha256:6bd08ac4d8f1661965bbe2685429d79043704cdd1ee720a7a89617331742240b \ + --hash=sha256:efc4f50f275ed6dae10a1f30bb81ad1a75368557b3ff22a532b7a472050368f1 + # via + # connectrpc + # e2b +protobuf-py-ext==0.1.1 \ + --hash=sha256:10992141a8282a71ac3e3530d1a489efb27618d84000b9a47918cf70e5816d9b \ + --hash=sha256:1610865622e2e27568277ca63d8d2d23dcc55eaa767398865c21539ddf4ce24d \ + --hash=sha256:182798e4861aba72d05855bd06febe4926aa7265e6f444a1b8af5252beee4f7e \ + --hash=sha256:21572764f625d829604fc4d83635f533f35775f11c6da142345b3f3c8d64bd09 \ + --hash=sha256:2a4cc478eef7a2acc1daaebcde479a3ac2396d47e6bdc7e776ca4c4147ba8b4c \ + --hash=sha256:310039e03cb15181781a0b78017419f6d4ee302e988c3c70b87f1facdf05532d \ + --hash=sha256:359dccdc1c3eafed2a913c570bb082b8df848a1d27d548ce8385f246a7d68be2 \ + --hash=sha256:4411ffe0e06a774b83c5c71c546ce097640a25f596c45f95f53d3e3148e3f22d \ + --hash=sha256:53a6b3590f6aa7f97b8ed60f62fef9b096babfeae82f7283fd3a4c405827d4f8 \ + --hash=sha256:55a17d80ea419501ff221a6627523f38a431bb33e6aa3de81ae3a7f271c49c75 \ + --hash=sha256:6b0c615c48e95acc53cf33e9310eeaff8b30d2d7555bf93e7bca8fb4f40e9a5c \ + --hash=sha256:72956cd0af5dee24b41c6f5ba5e42622d17e6d555002b5efc1634e27a1446de2 \ + --hash=sha256:79eee3bcbb289d6ea114eb8fe3a1469c5b59bf53e207238469f567d9c53ba56f \ + --hash=sha256:8995efba9476e1ea18ef9873306871dba697308994bc2766558fec3387acc3de \ + --hash=sha256:89ec8348d1ba045f79b2fedd14e40cca36f2a41b52f2c4fdf55a60c58add2353 \ + --hash=sha256:8a4adc65ab6a5e4885c67fc808bcacb83d755d05b566d312a0e10c2a873f2ad4 \ + --hash=sha256:91c38b4a10a306366443273ee03ca554537a0965bf05b7ada8e7dbdee04cc93e \ + --hash=sha256:9a9c2f026096ca4c595c89a297067ef371e241d3ff8e1f6d7c779aa8419cdca7 \ + --hash=sha256:9dbb518b5638403ceaa08ac4fc7dac626f45ed9b856b3517de3906cf3de4d632 \ + --hash=sha256:9eb25c3a329c0551cc86b209a5e5d8ecb8d834b9924a3aa019377853a703b6d3 \ + --hash=sha256:aaecbde82bef10c7c40578cbb61b7a19896bf7fa450972050a3bb302acb7d5d6 \ + --hash=sha256:ae4373845cbb85bdade3ef5368cc2f4b5f80bf173383afc1ab063f95644e5599 \ + --hash=sha256:ba61d49dace8f874361583030a7c48139b42eb37c9ffbb1e7e8a227a51576f44 \ + --hash=sha256:c6f0cd58620f415a3534d195358338f4999a774e12510f91c592b38d13d37388 \ + --hash=sha256:cf78707a040b9294e5e1ec4a1875f0046acfe52e92150cea27de4e9fc9db39bb \ + --hash=sha256:e7f14f00c2678bfbe7ad46f057b9f9938c1677bcf39e405e163e272c6f9814b8 \ + --hash=sha256:e85bfdfdb3ed50634db8ccc7429dd9286520109489c735463971a418707b4fef \ + --hash=sha256:e97f45c49676efacfb8e95bfcb1a002bc337e618a6780be1e55df2ba5ebd2f1e + # via protobuf-py +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via rich +pyinstaller==6.22.3 \ + --hash=sha256:020d4fe0627f5c73dfbcfad1a6578ef354d5fe4bdccd9d8e5a615528f5a45d2b \ + --hash=sha256:052f4a1cd4f81092ccb7a18fe8ebc9ecf832a5917317e39fb3775af4e959b253 \ + --hash=sha256:05eb2f5615503e72939a7224d68b4aff572c6b0438ee4a17d0a4b481f399362d \ + --hash=sha256:312da84b4b31ab9750066a823734c11b7c68757fdd67c9f038bf19c330a954e4 \ + --hash=sha256:31df3cc4261e804a53d358b7897c293ec8b88b167a7cca06c78018085c855f97 \ + --hash=sha256:451a4ae14b719365bf1a2f0a99dae7b3463060061c3a394c70d5264cfb439528 \ + --hash=sha256:46a3e30a118341a1ec20667a5a95aa7e0aa8c4fa129893c0151b3eb3f801a722 \ + --hash=sha256:500bd58c7bf7e584a8435adccbd763a0b918d5c12b08d74ff50fd79b2915458b \ + --hash=sha256:9e4f7bd64b2103f14b4a6240fb9155e9fe7bc61391b4043bbddb7d3ebc8ab0fa \ + --hash=sha256:a2b6fc26601e8c2596c3d54981566934a3382e85d05d0fe91b55b6a8dcc6b282 \ + --hash=sha256:ea240b5dfa831ff673c97505eee0ec24670bccf71396c83109440348944c83d0 \ + --hash=sha256:f82d0fb89f21c3ed6f482ee16f341bc59e8a31dbabb9bae9ef04e53ade4dc0d9 + # via -r services/agents-api/tools/e2b-provider/requirements.in +pyinstaller-hooks-contrib==2026.7 \ + --hash=sha256:24257a04c7a5a7a034cf28e39dcee20fbeeb9f043076729480f2e1b69904408a \ + --hash=sha256:5fbcaacb22c4f4aac869a127dce283f67a4b4cfcc37d496f2446603e6d68aefa + # via pyinstaller +pyqwest==0.10.0 \ + --hash=sha256:07a0eb595f4096232c2d22549b6e4612c1ecada7934e46462c2c37ce14a89cfb \ + --hash=sha256:0879a3f0b37876372aa328b9fee956165db174c1ab72464146fe515f49399ddb \ + --hash=sha256:09364115761579eabfc79d1e954cdb3ded508dac1903fac7285d4c6f058c683f \ + --hash=sha256:0f9163d6dd991bf1bf27308ba38ba021af660b15fffa47ebca98e41cf6f00309 \ + --hash=sha256:14b875d2273212d7fa8e4b755d8d736ffd226b1c707a9c0017dfdc8393a96eca \ + --hash=sha256:1aaccd8a9db9430b2aedb5bad8ead80742cbc056b85c229516c70dc80539f906 \ + --hash=sha256:234b3f71e3f314d997c203d8cf829b7117edd041153f9c277d0060ab90134148 \ + --hash=sha256:23a28beb55fa6d975949bffae4adfb69378f3229bb5cbd71231e95bf66f5b26c \ + --hash=sha256:26401baf7dafc71c8d12d2e8389519d141e6f7c14094d0dd4cf9ec1d3b5555bd \ + --hash=sha256:317a74d633abe3bc5bccabf479e069c515dab9e6a755274b0ccb1d8a5bbfede3 \ + --hash=sha256:35b472877e73dd63fed089c2bc8fa198407f005c8c19e0a93f025ebefde01a81 \ + --hash=sha256:3978e794b9cfd8eaa500fb5d7aee63bc6172c605efa0abc1f62d85485bc049e1 \ + --hash=sha256:399802647ea646c6ac9b5460e541b7c209b7a13563c667b2690ded2060185f2e \ + --hash=sha256:559674a98a8b1217e1830ecd41c9905bf2b60983c6b8017063dfac199f00727c \ + --hash=sha256:5637256a0dac0ef57e0eaa02b032014965e4a4c995e1deca1b1b97e6d1765f78 \ + --hash=sha256:564ec360b7848b35e009038ffbca00466305a9708ab21829477f64aa8cad4c64 \ + --hash=sha256:59f3f16628e518c674102e7b5fcff2101bba6abb4f6737ec5fade9b9278e6a53 \ + --hash=sha256:6bb511c434f79c641efb5573e5795e56dc972252f4b96e52a9636d4ece5231a4 \ + --hash=sha256:6c1a693be17d57d2c2eca4085e32c2809c53090c16719a907c90ebcf1f40dc01 \ + --hash=sha256:715991fd4f04862cd7a9d7452daabcdbd74dff4dff55eb20c22d60382dc2a4ed \ + --hash=sha256:73d9eb438ab4a957a1ce0619d3af8c1c1126bfb9181033b123d792fcf4224531 \ + --hash=sha256:78662158093f9d5c742368f4dd9956aa595f44c6ac0860777c98853ddd5e1610 \ + --hash=sha256:7ea761937acf3a00d1a7e70e982949d18946e5471d1419266ab3a78bbfa19759 \ + --hash=sha256:83578e24cccd5e0dc04d60a0af7bfb43325b5f22d03ff74ff79ed0ecf553b50d \ + --hash=sha256:850b8de6ade09a60bdb2f969871a177c2c304b594b2034ba3f5962c7bea75551 \ + --hash=sha256:a21f1f15252a8303623b4f17b9c6de595ace11b3ade07f2adb6d07121e8191aa \ + --hash=sha256:a5c757cfac5f53c8671dcb4850d5fc4c4339ea3e90636331c9318f8e3ddabc06 \ + --hash=sha256:aa492d5777dd145a60795ed95d9d4707a3cd1091fdcdfc93a82ac7fdc43ebacd \ + --hash=sha256:b5071491e416ea54e3b95bf9ffbed0bd065b093cb96e10a75c3d8f2cbe3c9823 \ + --hash=sha256:b5c80e88a5967c1cadb3237c450f91a84a3683f8838c8dca96f09fee3612e762 \ + --hash=sha256:b68b5e68d513a4c63a072f8f40e38015160cf90bfbf7e8ef7c3935ca87e9e022 \ + --hash=sha256:c04798bed79c1dfa0e5b0e30fb137124311083490d44d6dfbe068d3dd254349e \ + --hash=sha256:c26f3de1feb5d066d7a66802a47407a93ba043696064ad80beda4a0a4bf10056 \ + --hash=sha256:c48910d27820b9c46fcd001b0fe514a3cf47d4784f59512dcdb8c91c395f82e4 \ + --hash=sha256:cfcc7ba0229baa17831582befb046ace167b368140dae022d0b89b8d586ba12c \ + --hash=sha256:d03ba2cd17948b623a6210981d342eb122546d8a8e910ec77511aff4b1acdd00 \ + --hash=sha256:d6e7db305a8318b1f3218053e87501f8f245ca8bd63e948e0282d04bf0883470 \ + --hash=sha256:dc3d80b402fb59dbe015e25993ac8147456fb231a4c949f92a89f31315ad50f9 \ + --hash=sha256:e4415ae40b8eedb1713dab14d7f9fecc3f79d26f3206c561087b88b99d5ce24b \ + --hash=sha256:e5e3c436e041d8873ce5bb0fdcf9f9e86f5604e8f0ef9e03149efebd8cb474f6 \ + --hash=sha256:eb472c6e5d6833ebfec79db310e426eb17b01ac64c0e2c251bd9192c0d2ee0c5 \ + --hash=sha256:eff9ccf427604d34c635954def07b6113d4754f968073eef2df40bdb80b05bf5 \ + --hash=sha256:f399a696392fff3db3eef0a18ef65b8a3b8396d129193487d966b8eb11006376 + # via + # connectrpc + # e2b +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via e2b +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via e2b +setuptools==84.0.0 \ + --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 \ + --hash=sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73 + # via pyinstaller +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # anyio + # e2b + # opentelemetry-api +wcmatch==11.0.1 \ + --hash=sha256:1ea2b4fa678b8ca268253798d5963935df39132d47c3e241c0a0732224005e7d \ + --hash=sha256:fd149ecddb9f0a88ea780017d6dde17c994e494e7f7303d4e3c9d6251f978f4b + # via e2b diff --git a/services/agents-api/tools/e2b-provider/sdk.py b/services/agents-api/tools/e2b-provider/sdk.py new file mode 100644 index 000000000..da32d2adb --- /dev/null +++ b/services/agents-api/tools/e2b-provider/sdk.py @@ -0,0 +1,79 @@ +"""The version-pinned SDK boundary. No handwritten provider HTTP or envd RPC.""" +import base64 +import shlex + +from e2b import Sandbox +from e2b.connection_config import ConnectionConfig +from e2b.exceptions import AuthenticationException, SandboxException, ServiceBusyException +from e2b.sandbox.commands.command_handle import CommandExitException +from packaging.version import Version + +from state import Failure + +SDK_VERSION = '2.51.0' +MAX_OUTPUT = 1024 * 1024 + + +def connection_material(sandbox): + # The pinned SDK has no non-mutating attach method; connect can resume a VM. + # Keep this deprecated constructor boundary isolated and covered by SDK tests. + return {'sandbox_id': sandbox.sandbox_id, 'sandbox_domain': sandbox.sandbox_domain, + 'envd_version': str(sandbox._envd_version), + 'envd_access_token': sandbox._envd_access_token, + 'traffic_access_token': sandbox.traffic_access_token} + + +def restore(material, options): + headers = {'E2b-Sandbox-Id': material['sandbox_id'], + 'E2b-Sandbox-Port': str(ConnectionConfig.envd_port)} + if material['envd_access_token']: + headers['X-Access-Token'] = material['envd_access_token'] + return Sandbox(**dict(material, envd_version=Version(material['envd_version']), + connection_config=ConnectionConfig(extra_sandbox_headers=headers, **options))) + + +def definitely_rejected(error): + # These SDK classes/statuses represent an explicit refused Create. Transport + # failures and arbitrary 5xx responses can conceal a committed allocation. + return (isinstance(error, (AuthenticationException, ServiceBusyException)) or + isinstance(error, SandboxException) and error.status_code in (400, 401, 403, 404, 422, 429)) + + +def run(sandbox, command, remaining, user='runtime'): + raw = command.get('Stdin') + data = base64.b64decode(raw, validate=True) if raw is not None else None + if data is not None and len(data) > 50 * 1024 * 1024 + 32: + raise Failure('invalid') + args = command.get('Args') + if not isinstance(args, list) or not args or any(not isinstance(arg, str) or '\0' in arg for arg in args): + raise Failure('invalid') + directory = command.get('Directory') or None + if directory is not None and not directory.startswith('/'): + raise Failure('invalid') + counts = [0, 0] + + def bounded(index, text): + counts[index] += len(text.encode()) + if counts[index] > MAX_OUTPUT: + raise Failure('command_unconfirmed') + + try: + process = sandbox.commands.run(shlex.join(args), user=user, cwd=directory, + background=True, stdin=data is not None, + timeout=remaining(), request_timeout=remaining()) + if data is not None: + # Avoid an oversized unary SDK message; every chunk is submitted once. + for offset in range(0, len(data), 64 * 1024): + sandbox.commands.send_stdin(process.pid, data[offset:offset + 64 * 1024], + request_timeout=remaining()) + sandbox.commands.close_stdin(process.pid, request_timeout=remaining()) + try: + result = process.wait(on_stdout=lambda text: bounded(0, text), + on_stderr=lambda text: bounded(1, text)) + except CommandExitException as error: + result = error + return {'Stdout': result.stdout, 'Stderr': result.stderr, 'ExitCode': result.exit_code} + except Failure: + raise + except Exception: + raise Failure('command_unconfirmed') from None diff --git a/services/agents-api/tools/e2b-provider/state.py b/services/agents-api/tools/e2b-provider/state.py new file mode 100644 index 000000000..c56c71a7a --- /dev/null +++ b/services/agents-api/tools/e2b-provider/state.py @@ -0,0 +1,82 @@ +"""Private SDK connection receipts; Core alone owns execution and allocation truth.""" +import fcntl +import hashlib +import json +import os +from pathlib import Path +import stat +import time + + +class Failure(Exception): + def __init__(self, code): + self.code = code + + +def fsync_directory(path): + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + os.fsync(fd) + finally: + os.close(fd) + + +class Receipt: + def __init__(self, request, remaining): + self.identity = dict(request['Reference'], InstallationID=request['Config']['InstallationID']) + self.root = Path(request['Config']['StateDir']) + value = self.root.lstat() + if not stat.S_ISDIR(value.st_mode) or value.st_mode & 0o077 or value.st_uid != os.getuid(): + raise Failure('invalid') + digest = hashlib.sha256(json.dumps(self.identity, sort_keys=True).encode()).hexdigest() + self.path = self.root / (digest + '.json') + self.lock = os.open(self.root / (digest + '.lock'), os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + self.remaining = remaining + self.data = None + + def __enter__(self): + try: + while True: + self.remaining() + try: + fcntl.flock(self.lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + time.sleep(min(.05, self.remaining())) + try: + fd = os.open(self.path, os.O_RDONLY | os.O_NOFOLLOW) + except FileNotFoundError: + return self + with os.fdopen(fd) as source: + value = os.fstat(source.fileno()) + if not stat.S_ISREG(value.st_mode) or value.st_mode & 0o077 or value.st_uid != os.getuid() or value.st_size > 65536: + raise Failure('ownership') + self.data = json.load(source) + if self.data.get('identity') != self.identity or self.data.get('version') != 1: + raise Failure('ownership') + return self + except BaseException: + os.close(self.lock) + raise + + def __exit__(self, *unused): + os.close(self.lock) + + def save(self, **values): + data = dict(self.data or {'version': 1, 'identity': self.identity, + 'ids': [], 'settled': False, 'bootstrap_complete': False}) + data.update(values) + temporary = self.path.with_suffix('.tmp') + # A crashed writer's private temporary file has no committed meaning. + try: + temporary.unlink() + except FileNotFoundError: + pass + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, 'w') as target: + json.dump(data, target) + target.flush() + os.fsync(target.fileno()) + os.replace(temporary, self.path) + fsync_directory(self.root) + self.data = data diff --git a/services/agents-api/tools/e2b-provider/state_test.py b/services/agents-api/tools/e2b-provider/state_test.py new file mode 100644 index 000000000..9b01877aa --- /dev/null +++ b/services/agents-api/tools/e2b-provider/state_test.py @@ -0,0 +1,75 @@ +"""Private receipt durability and lifecycle exclusion.""" +from datetime import datetime, timedelta, timezone +import json +import os +from pathlib import Path +import tempfile +import threading +import time +import unittest +from uuid import uuid4 +from unittest.mock import patch + +from provider import Provider +from state import Failure, Receipt + + +class StateTest(unittest.TestCase): + def setUp(self): + self.root = tempfile.TemporaryDirectory() + self.addCleanup(self.root.cleanup) + self.request = {'Version': 1, 'Operation': 'kill', + 'Config': {'StateDir': self.root.name, 'InstallationID': str(uuid4()), 'APIKey': 'test'}, + 'Reference': {key: str(uuid4()) for key in ['TenantID', 'EnvironmentID', 'AllocationID']}, + 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=2)).isoformat()} + + def test_existing_helper_holds_lock_beyond_callers_timeout(self): + finished = [] + with Receipt(self.request, lambda: 1) as first: + first.save(status='create_pending') + request = dict(self.request, Deadline=(datetime.now(timezone.utc) + timedelta(milliseconds=50)).isoformat()) + def call(): + try: + Provider(request).execute() + except Failure as error: + finished.append(error.code) + with patch('provider.Sandbox') as sdk: + worker = threading.Thread(target=call) + worker.start() + worker.join(1) + self.assertFalse(worker.is_alive()) + self.assertEqual(finished, ['unconfirmed']) + sdk.kill.assert_not_called() + with Receipt(self.request, lambda: 1) as recovered: + self.assertEqual(recovered.data['status'], 'create_pending') + self.assertFalse(recovered.data['settled']) + + def test_receipt_symlink_is_never_followed(self): + with Receipt(self.request, lambda: 1) as receipt: + path = receipt.path + witness = Path(self.root.name, 'witness') + witness.write_text('unchanged') + path.symlink_to(witness) + with self.assertRaises(OSError): + with Receipt(self.request, lambda: 1): + pass + self.assertEqual(witness.read_text(), 'unchanged') + + def test_foreign_identity_and_world_readable_receipt_rejected(self): + with Receipt(self.request, lambda: 1) as receipt: + receipt.save(status='create_pending') + path = receipt.path + data = json.loads(path.read_text()) + data['identity']['TenantID'] = str(uuid4()) + path.write_text(json.dumps(data)) + with self.assertRaises(Failure): + with Receipt(self.request, lambda: 1): + pass + path.chmod(0o644) + with self.assertRaises(Failure): + with Receipt(self.request, lambda: 1): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/services/core-console/node_installation_test.go b/services/core-console/node_installation_test.go index 14cb6c9c7..a7f2ac3b6 100644 --- a/services/core-console/node_installation_test.go +++ b/services/core-console/node_installation_test.go @@ -50,6 +50,10 @@ func TestPairedConsoleKeepsAdminAndNodeCredentialsSeparated(t *testing.T) { }{ {"POST", "/core/v1/sandbox/deployment", "basic", 200}, {"POST", "/core/v1/sandbox/deployment", "none", 401}, + {"PUT", "/core/v1/sandbox/deployment", "basic", 200}, + {"PUT", "/core/v1/sandbox/deployment", "none", 401}, + {"PATCH", "/core/v1/sandbox/deployment/maintenance", "basic", 200}, + {"PATCH", "/core/v1/sandbox/deployment/maintenance", "none", 401}, {"GET", "/core/v1/sandbox/nodes", "node", 401}, {"GET", "/v1/agents", "basic", 200}, {"POST", "/core/v1/sandbox/enroll", "node", 200}, @@ -82,7 +86,7 @@ func TestPairedConsoleKeepsAdminAndNodeCredentialsSeparated(t *testing.T) { t.Fatal("paired mode missing") } } - if calls.Load() != 4 { + if calls.Load() != 6 { t.Fatalf("unexpected upstream requests: %d", calls.Load()) } r := consoleRequest(t, server, "POST", "/core/v1/sandbox/deployment") diff --git a/services/core-console/sandbox_admin.go b/services/core-console/sandbox_admin.go index ed93803d0..cfcdd48fc 100644 --- a/services/core-console/sandbox_admin.go +++ b/services/core-console/sandbox_admin.go @@ -20,13 +20,16 @@ func sandboxAdminRequest(r *http.Request) bool { if len(parts) == 1 { switch parts[0] { case "deployment": - return r.Method == http.MethodGet || r.Method == http.MethodPost + return r.Method == http.MethodGet || r.Method == http.MethodPost || r.Method == http.MethodPut case "nodes": return r.Method == http.MethodGet case "enrollment-tokens": return r.Method == http.MethodPost } } + if len(parts) == 2 && parts[0] == "deployment" && parts[1] == "maintenance" { + return r.Method == http.MethodPatch + } if len(parts) >= 2 && parts[0] == "nodes" && parts[1] != "" { if len(parts) == 2 { return r.Method == http.MethodPatch || r.Method == http.MethodDelete