diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b32f49e9c..c3e0da432 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -788,40 +788,76 @@ addresses from the validated installation public URL, never inbound Host headers current selection from the live deployment API; there is no startup configuration read. -Provider, resources and Runtime changes require maintenance, the current generation -and verified zero retained or pending resources. They never change the Core -address, which comes from the installation public URL. Maintenance prevents fresh hosted Sessions and allocations while -retaining admitted work, known receipts, queries and explicit cleanup. Unreleased -allocations include stopped compute, snapshots, uncertain operations and pending -cleanup. Pending hosted Environments without allocations also block changes. -Failed checks never authorize resource deletion. Explicitly resume after a successful -change. Preserve historical allocation ownership and Session placement; never migrate -an existing Session to another provider or recreate a released allocation. - -Explicit administrator Session archive requires a Web-managed deployment in -maintenance and its current generation. Keep Project scope and managed-hosted -eligibility checks in the existing administrative Store transaction, alongside -Environment expiry (preserving failure), cancellation requests, Runtime authority -revocation and audit. The ordinary Worker and provider lifecycle own compute and -snapshot release; archive must not call a provider in that transaction or delete -the Session. Retain history and persisted Files/Artifacts. The archive GET reports -actual resource disposition, not archive provenance or Turn finalization. Keep -unknown cleanup blocking, never automatically retry the mutation, and never -resume an archived Session or restore its unpersisted workspace. +All sandbox writes require the observed generation, including initial POST at +zero; reject stale state before provider preparation, reset or no-op checks, and +repeat it under the committing row lock. Same-provider PUT currently requires no +reset and zero held resources. A different backend or unspecified old selection +requires reset. No online rollout is implemented at this stage; omit that projection. +Preserve historical allocation ownership and placement, never migrate a Session. + +Reset is durable execution state, advanced by the existing manager outside its +counted work. Serialize start, escalation, cancel, setup, update and finalization +through the mutation gate. Auto waits only for in-progress/waiting root/subagent +Turns and pending file writes; queued work, idle and suspended Sessions can archive. +Recheck idleness with Session then deployment locks; never invert that order during +finalization. Keyset-page bounded work, bind candidates to the reset request time +as well as generation, and persist the original absolute deadline and validated +audit provenance. Cancellation cannot revive archived work. Auto's deadline +escalates to force durably; self-hosted Sessions are excluded. + +Use one snapshot and timestamp for the held-resource partition: unreleased +allocations plus pending hosted Environments without any allocation row. Cleanup +has precedence, then busy/idle. Count deleted/expired receipts until release. +Offline ownership comes from allocation or active placement node identity, with +the existing 45-second connection/owner-epoch predicate; provider readiness is +independent. Project every offline node and make its sum equal on_offline_nodes. +No cleanup failure, offline state or empty read authorizes a synthetic release. + +At zero held resources, drain outside database transactions, recheck under the +deployment lock, and atomically clear all provider policy, E2B credential/build +metadata and specification, retire nodes/tokens, advance generation/owner epoch +and audit completion. Publish a generation-bearing nil provider tombstone without +fallible work after commit, preventing delayed loads from reviving the old provider. +On a failed final write or interrupted drain, restore the committed provider using +a bounded owner context before releasing the mutation gate; recovery failure keeps +admission fenced and stops the owner. Ordinary migration resumes old Web-managed +maintenance with reset null. Refuse downgrade during reset or for an unconfigured +completed reset with generation above zero; never discard the generation. + +Explicit administrator Session archive requires a Web-managed deployment and its +current generation, without a reset precondition. Keep Project scope and hosted +eligibility checks in its Session-first transaction, alongside Environment expiry, +cancellation, Runtime authority revocation and audit. Background reset reconstructs +the actual Project scope from trusted records while retaining requester provenance. +The ordinary provider lifecycle owns compute/snapshot release. Retain history and +persisted Files/Artifacts; archived Sessions cannot resume unpersisted workspace. +The archive GET reports resource disposition, not provenance or Turn finalization. + +Archive cancellation must not destroy a healthy receipt path before its terminal +commit. Only the archive that first revokes a device may persist its exact +`archive_cancel_turn_id`; ordinary revocation clears it, and repeated cleanup +preserves rather than recreates it. The existing authenticated delivery can drain +its cancellation for at most 20 seconds from the Turn's original +`cancel_requested_at`. Track that delivery through Done, cancellation ACK and +terminal commit, independently of subscription removal. Heartbeats and both normal +and checkpoint cleanup use the same identity/deadline check; no new connection, +input, file/MCP authority or lease renewal is granted. Do not hold a transaction or +lifecycle gate waiting for the receipt. Lost peers, expiry and restart retain the +ordinary failure/cleanup fallback, never a fabricated cancelled outcome. Preserve +unknown Create ownership and reject schema downgrade with unsettled markers. Core rejects `AGENTS_API_MANAGED_RUNTIMES_FILE`; there is no file-managed startup path or embedded local node. An older file-managed database is not automatically adopted after its environment variable is removed. Settle and drain that deployment with its previous release and original backend, preserving business data, private -receipts, identities and storage. This change provides no old-database conversion, -force reset or automatic deletion. The supported current path is a database-managed +receipts, identities and storage. The legacy file-managed path has no automatic adoption or force conversion. The supported current path is a database-managed deployment. Harness selection and public/self-hosted contracts remain unchanged. The paired console serves only matched, non-secret distribution artifacts for node installation. Never serve private installation files or arbitrary paths. Installation reads `GET /api/v1/sandbox-node/configuration` using an unconsumed enrollment token, or a retained node credential with `X-OAC-Node-ID`. Reads never consume enrollment; -registered nodes can read their matching configuration during maintenance. Validate +registered nodes can read their matching configuration during reset. Validate installation, generation, specification digest and release before writing node files, registering or reconnecting. Reject drift rather than overwriting retained identity or using local resource defaults. Registration consumes a token only after these @@ -908,7 +944,19 @@ existing allocation must agree with the placement. Never choose another node. E2B direct allocations use one serial lifecycle without a node identity. The coordinator stops accepting work and cancels and drains all node workers and direct callers before releasing the sole execution lease. Lease loss is global; -ordinary provider failures stay within their node. Session locks, deployment +ordinary provider failures stay within their node. A planned deployment drain or +inventory retirement cancels lifecycle contexts synchronously between leased +operations, using the existing lease gate with its five-second bound. Include +an active manual reconcile's cancel function: an asynchronous `AfterFunc` alone +can cancel a later leased query after the gate reopens. Never cancel an in-flight +leased query merely to change deployment configuration. A failed cancellation +fence synchronously closes manager admission and reports owner failure, even +before its coordinator starts. Failed inventory retirement retains the original +lifecycle identity and gate through owner shutdown; gate availability is not +proof that cancellation succeeded. The drain barrier stays closed and cannot +activate a replacement. Release the lease gate before waiting for provider +settlement or lifecycle accounting. Ordinary caller deadlines and owner shutdown retain their +existing cancellation and fail-closed lease-loss behavior. Session locks, deployment capacity transactions and revision/one-shot receipts remain authoritative, with no external operation holding a database lock. @@ -925,7 +973,7 @@ Do not add a startup adoption path to bypass the database-managed selection. Do not add node-level drain controls. Refuse node removal with pending allocations, instances, snapshots, unknown results or cleanup resources. Offline ownership is retained. Removing a node does not delete compute. Local and remote nodes share -the same resource guard. Keep the deployment-wide maintenance and configuration +the same resource guard. Keep the deployment-wide reset and configuration change guard. This boundary does not add cross-node Session migration, Core multi-active, autoscaling, Kubernetes or harness residency. @@ -1740,18 +1788,22 @@ project; config.json's independent schema format stays 1. The operator command i `oac` (`oac_cli.py`, packaged as `oac.pyz`), with default installation directory `~/.oac/core`, private `~/.oac`, generated `x-oac` annotations and `.oac.lock`. Only `install.sh --convert` reads pre-rename state. Its `rename.py` coordinator first -validates the source, target, bundle and generated files. A stopped old Core may be -started from its own files for authenticated drain checks; refusal restores only -services the probe started and reports any failed restoration. Require maintenance, -zero unreleased allocations and pending work, and no registered nodes. One explicit -confirmation covers legacy config-layout conversion and the rename. Never delete a -target volume or copy container without matching the recorded conversion ownership; -copy the stopped PostgreSQL volume, retain the old volume, and carry the atomic -journal across the default-directory move. Preserve ports, secrets, installation -identity and stored history. Resume only with the recorded bundle. Docker and -microsandbox replace the Runtime through the existing guarded PUT and resume -admission; E2B stays in maintenance for its rebuilt template. Node and self-hosted -installations are outside this converter. +validates the source, target, bundle and generated files. The previous Core must +already be running and readable: do not start an unknown provider to inspect it. +Keep old-release maintenance, zero-resource and no-registered-node preflight. +Refuse E2B conversion before any conversion mutation, also on journal resume after +the original source-bundle guard. Its safe Core-owned upgrade preparation remains +a release blocker; never patch execution tables or change ordinary migration's +resume decision to work around it. One explicit confirmation covers legacy layout +and rename. Never delete a target volume/container without recorded ownership; +copy the stopped database, retain the old volume and journal across the atomic move. +Preserve ports, secrets, installation identity and history. Resume only with the +recorded bundle. The new-Core node-provider leg requires matching identity/provider, +reset null and zero resources; send one expected-generation PUT of exact bundle +Runtime and prior resources. Read back exact generation/specification. Reconcile an +uncertain write through reads, never blind replay or the retired maintenance PATCH. +Keep native stop proof and interrupted-copy recovery. Node/self-hosted installations +are outside this converter. Core process settings use `OAC_*`, Web settings use `OAC_WEB_*`, and shared Go logging uses `OAC_LOG_*`. Retired settings fail startup even when empty or when the new name is also set; report every matching name without values. Only the @@ -2918,8 +2970,12 @@ replaced; do not carry obsolete compatibility code forward to satisfy this secti every Session transaction: binding, claim/reconciliation, journal/Items/Usage, function callbacks/application receipts and terminal/native continuity. Serialize these short transactions and lease pings; execution transactions have a five-second - deadline including gate and Session-lock waits. Never hold a transaction across - daemon/model work, reconnect the writer or fall back to the pool after lease loss. + deadline including gate and Session-lock waits. Sandbox reset snapshots bound the + deployment relation explicitly to its legal singleton row before resource joins. + Keep this cardinality visible even on fresh databases without statistics: inflated + join estimates can trigger expensive JIT compilation inside the lease deadline. + Never hold a transaction across daemon/model work, reconnect the writer or fall + back to the pool after lease loss. The original Store handles public admission and device/auth maintenance on pooled connections. Execution reads may also use the pool; a read grants no write authority. At startup, reconcile previously claimed work as failed, preserve queued inputs diff --git a/Makefile b/Makefile index b1e3112bf..90c1b28c2 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,9 @@ build-agents-api-release: ./scripts/build-agents-api-release.sh check-agents-api: build-agents-api - go test ./services/agents-api/... ./packages/agents-client/... -count=1 + # Persistence integration tests include bounded lifecycle waits that together exceed Go's 10m default. + go test ./services/agents-api/... ./packages/agents-client/... -count=1 -timeout=20m + PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s services/agents-api/tests -p 'official_diagnostics_test.py' PYTHONDONTWRITEBYTECODE=1 python3 services/agents-api/deploy/runtime/initialize_receipt_test.py PYTHONDONTWRITEBYTECODE=1 python3 services/agents-api/deploy/e2b/managed_init_test.py diff --git a/README.md b/README.md index fb6bd0525..e3d6d192a 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ native differences. | [Configuration reference](docs/configuration.md) | Every setting in `config.json` and in Web | | [Call the API](docs/getting-started/quickstart.md) | The application developer's quickstart | | [API reference](docs/api/README.md) | The `/v1`, `/core/v1` and `/api/v1` namespaces | -| [Nodes and sandbox backends: operator reference](services/agents-api/HOSTED-SANDBOX-MANAGER.md) | Node protocol, manual registration, placement, maintenance | +| [Nodes and sandbox backends: operator reference](services/agents-api/HOSTED-SANDBOX-MANAGER.md) | Node protocol, manual registration, placement, reset | | [Protocol coverage](contracts/agents-api/README.md) and [harness selection](contracts/agents-api/harness-selection.md) | Supported operations and native differences | | [Landing page source](site/index.html) | The public site | diff --git a/apps/docs/FOLLOW-UP.md b/apps/docs/FOLLOW-UP.md index 232f913c1..e74b7e1a4 100644 --- a/apps/docs/FOLLOW-UP.md +++ b/apps/docs/FOLLOW-UP.md @@ -1,14 +1,13 @@ # Feature integration follow-up -This candidate documents the actual maintenance and drain model at source -`75bf4484ba957344ae374354858bbdfe104f5c4f`. It is not final documentation for design 50. -The Core error-envelope foundation is included; the later configuration validators and -diagnostic endpoints are not. Do not replace current procedures with the planned -behavior before the features land. +This candidate documents durable reset and the interim zero-resource same-provider +update rule at source `f36ac3592bfe9f77a2172680aa52b14c6f525062`. +The Core error-envelope foundation is included; online generations, later configuration +validators and diagnostic endpoints are not. Do not replace current procedures with +planned behavior before those features land. | Feature dependency | Site pages to reconcile | Source and verification work | | --- | --- | --- | -| Reset replaces maintenance (design 50 PR-R) | configure, hosted-providers, troubleshooting, admin-api, observability, API reference/core | Regenerate the canonical guides and management contract; describe reset admission, cleanup, failures, archive preconditions and offline-node limits from merged code. Remove retired maintenance instructions only after their replacement exists. | | Online generations and E2B changes (design 50 PR-G) | configure, hosted-providers, troubleshooting, execution-model, API reference/core | Document actual generation/rollout states, the supported change matrix and existing-sandbox behavior. Verify PUT examples against the final contract. | | Node update protocol (design 50 PR-N) | hosted-providers, troubleshooting, API reference/machine | Explain the implemented update command, old-node handling and ownership preservation. Do not claim an update command now. | | Later configuration/error changes (design 40 and remaining installer work) | install, configure, troubleshooting, quickstart | Regenerate current configuration fields and revise diagnostics against merged code; keep secrets out of examples. | diff --git a/apps/docs/content/docs/admin-api.mdx b/apps/docs/content/docs/admin-api.mdx index 1e9b830cc..d0b79bf04 100644 --- a/apps/docs/content/docs/admin-api.mdx +++ b/apps/docs/content/docs/admin-api.mdx @@ -8,7 +8,7 @@ resource inspection, Project and key management, credential issuance, audit, usa and sandbox operations. It never calls `/v1` or `/api/v1`, and has no Agent execution, copy or arbitrary asset editing operation. -Core request failures use the [Core administration error envelope](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/core-errors.md), +Core request failures use the [Core administration error envelope](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/core-errors.md), including optional typed safe details. Console sign-in endpoints retain their separate error shape described below. @@ -49,9 +49,9 @@ label, but could set any value. Never use it for authorization or as proof of origin. Web calls Core only through the typed Core clients in `packages/agents-client`: -[AdminClient](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/packages/agents-client/src/admin-client.ts) (`/core/v1`), +[AdminClient](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/packages/agents-client/src/admin-client.ts) (`/core/v1`), `SandboxAdminClient` (`/core/v1/sandbox`) and `CoreMetricsClient` -(`/core/v1/metrics`). See [the complete administrator reference](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/admin-api.md) +(`/core/v1/metrics`). See [the complete administrator reference](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md) for methods, fields, filters, pagination and response shapes. Routes below are relative to `/core/v1`: @@ -62,8 +62,8 @@ Routes below are relative to `/core/v1`: | Resource lists/details/deletion | `/projects/{id}/agents`, `/sessions`, `/environment-templates`, `/skills`, `/files`, `/vaults`, including the documented nested reads | | Asset ownership | `GET /projects/{id}/resource-owners` with batched resource IDs | | Key operation history | `GET /projects/{id}/write-operations` with key/resource/time filters | -| Executor credentials | `GET/POST /projects/{id}/environments/{environment_id}/executor-credentials`, `DELETE …/executor-credentials/{key_id}` ([contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/environment-executor-credentials.md)) | -| Deployment model providers | `GET /harnesses`, `GET/PUT/DELETE /harnesses/{harness}/model-provider`; the key is write-only ([contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/model-execution.md#deployment-defaults)) | +| Executor credentials | `GET/POST /projects/{id}/environments/{environment_id}/executor-credentials`, `DELETE …/executor-credentials/{key_id}` ([contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/environment-executor-credentials.md)) | +| Deployment model providers | `GET /harnesses`, `GET/PUT/DELETE /harnesses/{harness}/model-provider`; the key is write-only ([contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/model-execution.md#deployment-defaults)) | | Usage and health | `GET /summary`, `/sandbox/runtime-observations`, `/metrics` | | Administrator audit | `GET /audit-log`; deployment-wide entries have `project_id: null` | @@ -76,21 +76,21 @@ it is operational attribution, not per-key billing. ## Sandbox administration -The [deployment configuration contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/sandbox-deployment.md), -[nodes and sandbox backends reference](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/HOSTED-SANDBOX-MANAGER.md) -and [generated OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/core.openapi.yaml) +The [deployment configuration contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/sandbox-deployment.md), +[nodes and sandbox backends reference](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/HOSTED-SANDBOX-MANAGER.md) +and [generated OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/core.openapi.yaml) define deployment and node operations: - `GET/POST/PUT /core/v1/sandbox/deployment` and - `PATCH /core/v1/sandbox/deployment/maintenance`. + `POST/DELETE /core/v1/sandbox/deployment/reset`. - `GET /core/v1/sandbox/nodes`, `PATCH/DELETE /core/v1/sandbox/nodes/{node_id}`, and `GET /core/v1/sandbox/nodes/{node_id}/allocations`. - `POST /core/v1/sandbox/enrollment-tokens` for a one-time node installation command. Its non-secret `enrollment_id` reappears on the node that command registers. PostgreSQL owns one provider, per-sandbox resource specification and immutable -Runtime selection. POST initializes it; PUT replaces the complete selection using -`expected_generation`. Requests carry `resources` and, for Docker/microsandbox, +Runtime selection. POST initializes it and PUT updates the same provider; both +require the observed `expected_generation`, including zero at first setup. Requests carry `resources` and, for Docker/microsandbox, `runtime`; safe responses return `specification` and `specification_digest`. Response `resources.allocations` and `resources.pending` are cleanup counts, not CPU, memory or disk settings. E2B accepts a write-only key and exact template build @@ -99,23 +99,26 @@ may omit `resources` to adopt the validated build's CPU and memory; responses sh the build as read at selection time in `e2b.template_build`. Microsandbox responses return its idle `suspension` policy; other providers return null. -Provider, resource and Runtime changes all require global maintenance and verified -cleanup of retained/pending resources. Core validates the candidate before commit; -a rejection preserves the previous configuration. A changed commit advances the -generation and retires old nodes and enrollment tokens atomically. Explicitly resume -after success. Neither switching nor editing configuration deletes resources or -migrates existing Sessions. During maintenance, administrators can explicitly -[archive each retained hosted Session](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/admin-api.md#administrative-session-archive) -through the Core administrator API at the current generation, then read its -resource disposition and recheck deployment counts. Archive preserves history -and persisted Files/Artifacts; unpersisted workspace contents are lost and the -original Session cannot resume. This API does not add a console archive control. +Same-provider updates currently require zero retained/pending resources and no +reset. Changing backend requires explicit durable reset before a new POST. +Auto archives idle/queued/suspended hosted Sessions, waits for started work and +file writes, and escalates at its persisted deadline; force requests cancellation +and verified cleanup. The deployment response supplies the authoritative +`reset.remaining` partition and offline-node subset; Web must not derive either +from independently loaded lists. No rollout projection is provided yet. + +Administrators may [archive an individual hosted Session](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md#administrative-session-archive) +at the current generation without reset. History and persisted Files/Artifacts +survive; unpersisted workspace is lost and the original Session cannot resume. +Cancel reset stops further archives, not cleanup already requested. After zero +resources Core clears the selection and advances generation; configure again using +that new generation. Never automatically replay an uncertain write. Public Environment Templates, the `/v1` contract and caller-owned `self_hosted` provisioning remain unchanged. `GET /api/v1/sandbox-node/configuration` uses an enrollment Bearer token, or a retained node Bearer credential with `X-OAC-Node-ID`. This read does not consume -enrollment. Retained matching nodes can read their configuration during maintenance. +enrollment. Retained matching nodes can read their configuration during reset. Installers must verify the returned generation, specification digest and Runtime before registration; local files cannot override the saved limits. A mismatch returns `sandbox_specification_mismatch` without replacing node state. @@ -145,7 +148,7 @@ generated schema; do not interpret every empty or failed read as an absent resou `GET /core/v1/metrics?range=1h|6h|24h|7d` returns Core process, execution queue/slots, PostgreSQL and background-job measurements. It requires the Core key, rejects arbitrary query filters and never grants -Agent execution access. See the [exact measurement contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/core-metrics.md) +Agent execution access. See the [exact measurement contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/core-metrics.md) for complete buckets, null values, units and process-local retention. Frontend implementation is maintained separately; this backend change does not modify Agent metrics or the public Agent API. @@ -154,7 +157,7 @@ Agent metrics or the public Agent API. The Core key reads a node and its host history through `GET /core/v1/sandbox/nodes/{node_id}?range=1h|6h|24h`. See the -[node host history contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/node-host-history.md) +[node host history contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/node-host-history.md) for nullable observations, freshness and aggregation. The node list is unchanged. -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/api/web-management.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/api/web-management.md) diff --git a/apps/docs/content/docs/agents-and-tools.mdx b/apps/docs/content/docs/agents-and-tools.mdx index ed238409b..1f5432c42 100644 --- a/apps/docs/content/docs/agents-and-tools.mdx +++ b/apps/docs/content/docs/agents-and-tools.mdx @@ -4,7 +4,7 @@ description: "Native harness tools, shared declarations and the limits of execut --- Assessed against Core `206474a4c10901442b1faa094281bfb5559082b5` on 2026-09-22. -The contract remains [OpenAI Python 3.13.0 at `d7c41ef`](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/upstream.json), Beta +The contract remains [OpenAI Python 3.13.0 at `d7c41ef`](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/upstream.json), Beta `agents=v1`. This matrix records the bounded execution/tools milestone. It does not establish complete protocol compatibility; the remaining limits below stay open. @@ -22,20 +22,20 @@ and Environment Plugin MCP have separate inventories and qualification. | Operation | Implemented behavior and real qualification | Unsupported or unverified boundary | | --- | --- | --- | -| Initial message input, `sessions.create` | String input and ordered user-message arrays share atomic admission. Codex/Claude text and inline image execution: M1/M2; ordinary MiniMax text: M1/P1. [Input contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/message-input.md), [initial parser](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/internal/api/session_initial_input.go). | Whitespace-only text is admitted verbatim for Codex and rejected at admission for Claude SDK and MiniMax Code ([text content](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/message-input.md#text-content)). Empty parts beside text (SES-08) and local size-limit parity need upstream evidence. Images are only qualified for inline PNG/JPEG on `none` and Core-managed Docker `openai_hosted`; MiniMax, self-hosted images and remote URLs remain gaps. | -| Prepared and active messages, `sessions.events.create` | Ordered `input_text`/`input_image` arrays retain original content and distinct public user Items. HTTP 202 confirms persistence; native receipts establish application. Initial/prepared/active Docker PNG/JPEG: M2; active PNG on `none`: M1. [Shared admission](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/internal/api/inputs.go). | Codex flattens native messages with blank-line separators. Claude can fold or queue native turns; it does not promise Codex's same-native-turn behavior. Public durability does not prove native consumption. Claude SDK and MiniMax Code reject messages without an image or non-whitespace text at admission (`unsupported_or_invalid_configuration`); Codex delivers them unchanged. | -| Structured output, `agent.text.format` | Save/inherit/freeze `{type:json_schema,schema:...}`. Claude SDK object-root, single Agent, medium verbosity, ordinary functions returning text: S1 (`none`) and S2 (Docker, including prepared/active input and Files/Artifacts). Native final text is retained unchanged; its stream follows the official message sequence with the whole text in one `output_text.delta` ([item serialization](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/history-events-usage.md#item-serialization-2026-09-23)). [Contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/structured-output.md), [profile](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/internal/engine/claude.go). | An explicit non-object root type is a protocol error for every harness ([validation](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/official-semantics-alignment.md#agent-configuration-validation--september-23)). Codex/MiniMax, schemas without an object root, schema numbers changed by binary64, self-hosted, Skills/Plugins, MCP, Subagent and discovery combinations reject execution. No output repair, coercion or extra model loop. Arbitrary schema dialects are unverified. | -| Function configuration, saved/inline Agents | Required name/description/schema; `defer_loading` defaults false. Protocol errors, repeated names and explicit non-object root types reject with the official fields ([validation](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/official-semantics-alignment.md#agent-configuration-validation--september-23)). Saved references resolve into an immutable Session snapshot. Codex/Claude real calls: F1/F2/M2/S1/S2. [Parser](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/internal/api/function_configuration.go), [saved tools](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/internal/api/saved_tools.go). | MiniMax public functions reject. Claude requires an explicit object root. Local nonblank/512-byte name and 64-definition Session bounds are compatibility gaps. Saving configuration alone does not qualify execution. | -| Function-result admission, `events.create` | Required `turn_id`, `call_id`, `success`; optional nullable `error` and `output`. Output is string or ordered text/image content. Scoped atomic batches retain field presence, original content and retry identity in storage; public result Items and events always carry `output` and `error`, null when not submitted ([item serialization](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/history-events-usage.md#item-serialization-2026-09-23)). Same result retries are accepted; changed results and results after cancellation return 409 `conflict_error`, including after terminal state. In the caller's Session, an unknown call or a call of another Turn returns 400 `invalid_request_error` without changing the pending action; missing and foreign Sessions stay 404 ([error rows](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/official-semantics-alignment.md#session-input-conflicts-and-result-targets--september-23)). F1/F2/M2 plus [controlled SDK/raw checks](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/tests/official_function_inputs.py). [Parser](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/internal/api/function_inputs.go), [Store](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/internal/store/function_inputs.go). | Admission is separate from application and public Item publication. Invalid or unqualified content cannot consume a pending call. Core messages omit the call and executor IDs that official messages name; hosted defaults and publication timing remain unverified. | -| Function text results and native application | Codex waits for a matching live root dynamic-tool completion; Claude waits for a matching live root native tool result. Text/error results, retry/conflict, cancellation and cold continuation: F1/F2. [Receipt contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/function-result-images.md). | Transport writes alone do not confirm application. Confirmation is not provider consumption, crash recovery or exactly-once external effects. No automatic result replay. | +| Initial message input, `sessions.create` | String input and ordered user-message arrays share atomic admission. Codex/Claude text and inline image execution: M1/M2; ordinary MiniMax text: M1/P1. [Input contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/message-input.md), [initial parser](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/internal/api/session_initial_input.go). | Whitespace-only text is admitted verbatim for Codex and rejected at admission for Claude SDK and MiniMax Code ([text content](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/message-input.md#text-content)). Empty parts beside text (SES-08) and local size-limit parity need upstream evidence. Images are only qualified for inline PNG/JPEG on `none` and Core-managed Docker `openai_hosted`; MiniMax, self-hosted images and remote URLs remain gaps. | +| Prepared and active messages, `sessions.events.create` | Ordered `input_text`/`input_image` arrays retain original content and distinct public user Items. HTTP 202 confirms persistence; native receipts establish application. Initial/prepared/active Docker PNG/JPEG: M2; active PNG on `none`: M1. [Shared admission](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/internal/api/inputs.go). | Codex flattens native messages with blank-line separators. Claude can fold or queue native turns; it does not promise Codex's same-native-turn behavior. Public durability does not prove native consumption. Claude SDK and MiniMax Code reject messages without an image or non-whitespace text at admission (`unsupported_or_invalid_configuration`); Codex delivers them unchanged. | +| Structured output, `agent.text.format` | Save/inherit/freeze `{type:json_schema,schema:...}`. Claude SDK object-root, single Agent, medium verbosity, ordinary functions returning text: S1 (`none`) and S2 (Docker, including prepared/active input and Files/Artifacts). Native final text is retained unchanged; its stream follows the official message sequence with the whole text in one `output_text.delta` ([item serialization](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/history-events-usage.md#item-serialization-2026-09-23)). [Contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/structured-output.md), [profile](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/internal/engine/claude.go). | An explicit non-object root type is a protocol error for every harness ([validation](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/official-semantics-alignment.md#agent-configuration-validation--september-23)). Codex/MiniMax, schemas without an object root, schema numbers changed by binary64, self-hosted, Skills/Plugins, MCP, Subagent and discovery combinations reject execution. No output repair, coercion or extra model loop. Arbitrary schema dialects are unverified. | +| Function configuration, saved/inline Agents | Required name/description/schema; `defer_loading` defaults false. Protocol errors, repeated names and explicit non-object root types reject with the official fields ([validation](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/official-semantics-alignment.md#agent-configuration-validation--september-23)). Saved references resolve into an immutable Session snapshot. Codex/Claude real calls: F1/F2/M2/S1/S2. [Parser](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/internal/api/function_configuration.go), [saved tools](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/internal/api/saved_tools.go). | MiniMax public functions reject. Claude requires an explicit object root. Local nonblank/512-byte name and 64-definition Session bounds are compatibility gaps. Saving configuration alone does not qualify execution. | +| Function-result admission, `events.create` | Required `turn_id`, `call_id`, `success`; optional nullable `error` and `output`. Output is string or ordered text/image content. Scoped atomic batches retain field presence, original content and retry identity in storage; public result Items and events always carry `output` and `error`, null when not submitted ([item serialization](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/history-events-usage.md#item-serialization-2026-09-23)). Same result retries are accepted; changed results and results after cancellation return 409 `conflict_error`, including after terminal state. In the caller's Session, an unknown call or a call of another Turn returns 400 `invalid_request_error` without changing the pending action; missing and foreign Sessions stay 404 ([error rows](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/official-semantics-alignment.md#session-input-conflicts-and-result-targets--september-23)). F1/F2/M2 plus [controlled SDK/raw checks](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/tests/official_function_inputs.py). [Parser](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/internal/api/function_inputs.go), [Store](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/internal/store/function_inputs.go). | Admission is separate from application and public Item publication. Invalid or unqualified content cannot consume a pending call. Core messages omit the call and executor IDs that official messages name; hosted defaults and publication timing remain unverified. | +| Function text results and native application | Codex waits for a matching live root dynamic-tool completion; Claude waits for a matching live root native tool result. Text/error results, retry/conflict, cancellation and cold continuation: F1/F2. [Receipt contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/function-result-images.md). | Transport writes alone do not confirm application. Confirmation is not provider consumption, crash recovery or exactly-once external effects. No automatic result replay. | | Function image results | Successful ordered inline PNG/JPEG with text, large PNG and image-only JPEG: Codex/Claude `none` F1/F2; Docker M2. Public Items retain submitted bytes. Codex receipt regression: F2. | Claude rejects failed images and remote references before persistence; native resizing may change its image bytes. MiniMax functions and self-hosted image workflows are unqualified. Other Codex image/error/reference combinations cannot be inferred from successful-inline evidence. | -| Deferred function discovery | Type-only `tool_search` plus mixed eager/deferred functions: Claude SDK 0.3.269/native 2.1.269, Kimi K3, single Agent, medium, `none`, text results; text and PNG input D1. Native provider observations establish lazy schema loading. [Contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/tool-search.md). | A repeated `tool_search` is a protocol error. Codex/MiniMax discovery, search-only/missing-search, workspace, MCP, structured-output and Subagent combinations remain gaps. Opaque native policy changes lack a reliable pre-input deferral signal. Saved tools include `tool_search`; the pinned Session response union excludes it. Exact hosted projection is unverified. | -| Explicit disabled search/PTC | Saved and inline `web_search.mode=disabled` and `programmatic_tool_calling.enabled=false`; shared native controls on initial and cold execution. All three harnesses on `none`: P1, including native inventory/control evidence. [Contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/tool-policy.md), [parser](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/internal/api/disabled_tools.go). | Unsupported explicit enablement rejects at Session admission; saved Agents keep every pinned search mode as resource data (TV-05), and Sessions from such Agents reject unless they replace the tools. A repeated `web_search` is a protocol error. Omission retains approved native behavior, which does not establish official default-on PTC parity. Enabled search and default/error parity remain gaps; unrelated native utilities are not implicitly removed. | -| Agent service-origin MCP | Implemented Codex/Claude HTTP `none` profiles, anonymous/static bearer, scoped Vault selection and native `mcp_call` Items. [Configuration and qualification limits](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/README.md#http-mcp-execution), [profiles](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/internal/engine/profile.go). | This closure batch does not requalify MCP/provider combinations. MiniMax, self-hosted/hosted service-origin MCP, OAuth, nonempty inline headers/metadata and other transports/origins reject. Claude requires a static connected inventory; original MCP-envelope fidelity and continuing server health are unverified. | -| Environment Plugin MCP/native tools | Separate [Docker Plugin transport matrix](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/environment-templates.md#environment-origin-mcp-plugins) and [V1 deployment evidence](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/user-managed-runtime-v1.md). Native tools stay within the existing colocated Runtime. | Plugin MCP is not Agent service-origin MCP or a public function action. No new optional cross-product is qualified here. Native utility inventories need not be identical. | -| Required action: `function_call` | Session GET/list and Session SSE expose persisted `{arguments,call_id,name,turn_id,type}`. Actions remain pending until native application or cancellation/terminal settlement. [Projection](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/internal/store/function_state.go), [confirmation](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/internal/store/function_results.go); real handling F1/F2/M2/S1/S2/D1; explicit client disconnect/query/reconnect: F3. | Function-call history is not pending-state authority. Client reconnect does not replay events or redo external application effects. | -| Required action: `environment_connection` | Offline waiting input exposes `{environment_id,type}` before Turn creation. Connect the exact Session Environment through our scoped daemon enrollment. Three-harness Docker/E2B execution: E1; explicit initial pending-input/query/connection/native completion: E2; expiry: [controlled initial-input test](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/tests/official_self_hosted_initial.py). | Idle offline Sessions without pending input request nothing. Registration alone is not connectivity or native readiness. Stock `exec-server`/Noise transport is outside the approved V1 route. Exact upstream registration/action-removal timing remains unverified. | -| Stream disconnect and execution loss | SSE is live-only. Reconnect, buffer events, retrieve Session/Turn/Items and deduplicate Item IDs. Worker recovery fails previously claimed work without replay; queued work can remain. [Recovery contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/README.md#public-execution-admission), [connection reconciliation](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/internal/store/environment_connection_recovery.go). | Client disconnect and daemon/Core loss are different cases. F3 qualifies client-stream loss and continued pending function handling; F2 qualifies native process loss with an unapplied saved result. Neither promises replay or recovery of unknown external effects. | +| Deferred function discovery | Type-only `tool_search` plus mixed eager/deferred functions: Claude SDK 0.3.269/native 2.1.269, Kimi K3, single Agent, medium, `none`, text results; text and PNG input D1. Native provider observations establish lazy schema loading. [Contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/tool-search.md). | A repeated `tool_search` is a protocol error. Codex/MiniMax discovery, search-only/missing-search, workspace, MCP, structured-output and Subagent combinations remain gaps. Opaque native policy changes lack a reliable pre-input deferral signal. Saved tools include `tool_search`; the pinned Session response union excludes it. Exact hosted projection is unverified. | +| Explicit disabled search/PTC | Saved and inline `web_search.mode=disabled` and `programmatic_tool_calling.enabled=false`; shared native controls on initial and cold execution. All three harnesses on `none`: P1, including native inventory/control evidence. [Contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/tool-policy.md), [parser](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/internal/api/disabled_tools.go). | Unsupported explicit enablement rejects at Session admission; saved Agents keep every pinned search mode as resource data (TV-05), and Sessions from such Agents reject unless they replace the tools. A repeated `web_search` is a protocol error. Omission retains approved native behavior, which does not establish official default-on PTC parity. Enabled search and default/error parity remain gaps; unrelated native utilities are not implicitly removed. | +| Agent service-origin MCP | Implemented Codex/Claude HTTP `none` profiles, anonymous/static bearer, scoped Vault selection and native `mcp_call` Items. [Configuration and qualification limits](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/README.md#http-mcp-execution), [profiles](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/internal/engine/profile.go). | This closure batch does not requalify MCP/provider combinations. MiniMax, self-hosted/hosted service-origin MCP, OAuth, nonempty inline headers/metadata and other transports/origins reject. Claude requires a static connected inventory; original MCP-envelope fidelity and continuing server health are unverified. | +| Environment Plugin MCP/native tools | Separate [Docker Plugin transport matrix](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/environment-templates.md#environment-origin-mcp-plugins) and [V1 deployment evidence](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/user-managed-runtime-v1.md). Native tools stay within the existing colocated Runtime. | Plugin MCP is not Agent service-origin MCP or a public function action. No new optional cross-product is qualified here. Native utility inventories need not be identical. | +| Required action: `function_call` | Session GET/list and Session SSE expose persisted `{arguments,call_id,name,turn_id,type}`. Actions remain pending until native application or cancellation/terminal settlement. [Projection](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/internal/store/function_state.go), [confirmation](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/internal/store/function_results.go); real handling F1/F2/M2/S1/S2/D1; explicit client disconnect/query/reconnect: F3. | Function-call history is not pending-state authority. Client reconnect does not replay events or redo external application effects. | +| Required action: `environment_connection` | Offline waiting input exposes `{environment_id,type}` before Turn creation. Connect the exact Session Environment through our scoped daemon enrollment. Three-harness Docker/E2B execution: E1; explicit initial pending-input/query/connection/native completion: E2; expiry: [controlled initial-input test](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/tests/official_self_hosted_initial.py). | Idle offline Sessions without pending input request nothing. Registration alone is not connectivity or native readiness. Stock `exec-server`/Noise transport is outside the approved V1 route. Exact upstream registration/action-removal timing remains unverified. | +| Stream disconnect and execution loss | SSE is live-only. Reconnect, buffer events, retrieve Session/Turn/Items and deduplicate Item IDs. Worker recovery fails previously claimed work without replay; queued work can remain. [Recovery contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/README.md#public-execution-admission), [connection reconciliation](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/internal/store/environment_connection_recovery.go). | Client disconnect and daemon/Core loss are different cases. F3 qualifies client-stream loss and continued pending function handling; F2 qualifies native process loss with an unapplied saved result. Neither promises replay or recovery of unknown external effects. | ## Required-action recovery contract @@ -52,7 +52,7 @@ application already performed the function, keep and submit that saved result; do not run an external effect again merely because an action remains visible. Core's acknowledgement boundary is native application. For an environment action, connect its exact Environment using the existing enrollment authorization. -[Connection observations](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/internal/store/environment_connections.go) +[Connection observations](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/internal/store/environment_connections.go) fence stale generations; transport connection can clear pre-Turn activity before native preparation. Neither registration nor an action disappearing proves that a model ran successfully. Query the matching Turn and Items for its outcome. @@ -61,8 +61,8 @@ Two timing questions remain open. Repeated `requires_action` notification order and acknowledgement timing are local implementation choices, not proven upstream semantics. Also, an admitted result that is cancelled before native observation can remain internally saved without a public output Item or `item.added`. -[Item publication](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/internal/store/item_projection.go) and -[the existing coverage record](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/README.md#public-function-configuration) preserve +[Item publication](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/internal/store/item_projection.go) and +[the existing coverage record](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/README.md#public-function-configuration) preserve this gap. Successful retry cannot manufacture the missing observation or establish that the model consumed the result. @@ -82,12 +82,12 @@ not a claim that every historical workflow was rerun at the current baseline. | M2 | `20260922/workspace-images/validation-summary.json`; Codex `public-run-_lr5uiy_`, Claude `public-run-sb65p9e9`; candidate `2c295116d66ed9aafc808ec49a8cb6dcb5c674c2`, merge `1adb2489bc3415039440224e9a7905c53e68a557` ([#17](https://github.com/MiniMax-AI/parsar-core/pull/17)). Kimi K3, Codex 0.153.4, Claude SDK 0.3.269/native 2.1.269; Docker seven-Turn image/workspace workflow. Codex's receipt conclusion is superseded by F2. | | F1 | `20260922/function-result-images/validation-summary.json`, `validated/function-image-public-2567456666/public.json`; candidate `d3cdb225bc7628b968b76c7e1b400101894ce8cd`, merge `fd23f169e1ede1b2f1c39a1d9dcce71886e3c006` ([#16](https://github.com/MiniMax-AI/parsar-core/pull/16)). Claude SDK 0.3.269/native 2.1.269 with Kimi K3 on `none`; success PNG/JPEG, failed text, retries, cancellation, history continuation. Earlier Codex transport-only evidence does not qualify current receipts. | | F2 | `20260922/function-result-receipts/validation-summary.json`, `candidate-public-owner-sdk.log`; hosted `public-run-4fn70foy` (104.13s), `none` `function-image-public-945075091` (83.918s). Candidate `6dceb98e1c56469fcc70cd82025ce77fc870ed3d`, merge `206474a4c10901442b1faa094281bfb5559082b5` ([#20](https://github.com/MiniMax-AI/parsar-core/pull/20)); real Kimi, Codex 0.153.4. Qualifies native receipts, not completed provider consumption. | -| F3 | `20260922/execution-tools-closure/pending-actions/validation-summary.json`; Codex `codex/public-run-8gexf5a8` (65.58s), Claude `claude_sdk/public-run-5f2tr7e_` (74.33s), production source `206474a`. The same [public verifier](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/tests/official_pending_actions_native.py) checks disconnected-client pending queries, success/error/cancel, original results, target isolation, retry/conflict and no implicit call replay with real Kimi on Docker. No Core restart or native-loss injection is claimed by these runs. | +| F3 | `20260922/execution-tools-closure/pending-actions/validation-summary.json`; Codex `codex/public-run-8gexf5a8` (65.58s), Claude `claude_sdk/public-run-5f2tr7e_` (74.33s), production source `206474a`. The same [public verifier](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/tests/official_pending_actions_native.py) checks disconnected-client pending queries, success/error/cancel, original results, target isolation, retry/conflict and no implicit call replay with real Kimi on Docker. No Core restart or native-loss injection is claimed by these runs. | | S1 | `20260922/structured-output/acceptance-summary.json`, `structured-public-1905281777/public.json`; real candidate `7f533d46c472f3cdf7c16ce9471c225a2ba7eded`, merge `8716e699dbc34904499c94b6b0b03857bbebfaad` ([#11](https://github.com/MiniMax-AI/parsar-core/pull/11)). Claude/Kimi `none` schema/function execution and cold continuation. Schema-specific active steering was not separately qualified here. | | S2 | `20260922/hosted-structured-output/validation-summary.json`, `validated/public-inline.log`, `public-run-qvq3csf1` (186.09s); candidate `f6d2c3f2a1dc2ff5f177213bd2b20d78496011d0`, merge `aa85d8ecc60e0a8b7f2494b73caa81216ee197e2` ([#18](https://github.com/MiniMax-AI/parsar-core/pull/18)). Claude SDK 0.3.269/native 2.1.269, real Kimi; Docker saved/inline schema, prepared/active input, native files and four completed structured answers. | | D1 | `20260922/deferred-tools/tool-search-public-2039155387/public.json`, `public-image-isolated-tests.log` (86.71s); merge `178507ae7a63d4068e1e82bef9cd256ba398ae00` ([#13](https://github.com/MiniMax-AI/parsar-core/pull/13)). Claude SDK 0.3.269/native 2.1.269/Kimi K3 `none`; text results with initial/active PNGs, cancellation and cold continuation. `claude-native-1790052750` separately records provider schema inventories and native feasibility. | | P1 | `20260922/tool-policy/acceptance.json`, `server-evidence/tool-policy-{codex-2495445746,claude_sdk-2539346823,mcode-3747575401}/public.json`; candidate `eeb432c7495265ae3a9309e32f5b4323611c3245`, merge `4b8754a6eda6696d9b18eb8d583953ac16c6800f` ([#14](https://github.com/MiniMax-AI/parsar-core/pull/14)). Codex/Claude Kimi K3, MiniMax M2.7; four `none` Sessions per harness, native disable controls and cold continuation. | -| E1 | `20260921/self-hosted-onboarding/{source-candidate.json,source-verified.json,live,live-e2b}`; candidate `8f0cd2530d7b58cb7fb3ea124a1fc43dacecca36`. [Exact Docker/E2B run paths and limits](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/user-managed-runtime-v1.md#evidence-and-verification-boundaries): Codex Kimi K3 Responses, Claude Kimi K3 Anthropic-compatible API, MiniMax M2.7, immutable Linux amd64 Runtime builds. Native execution, restart/history, cancellation, Files/Artifacts and isolation; shared credential lifecycle evidence is not a separate live rotation run for every E2B profile. | +| E1 | `20260921/self-hosted-onboarding/{source-candidate.json,source-verified.json,live,live-e2b}`; candidate `8f0cd2530d7b58cb7fb3ea124a1fc43dacecca36`. [Exact Docker/E2B run paths and limits](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/user-managed-runtime-v1.md#evidence-and-verification-boundaries): Codex Kimi K3 Responses, Claude Kimi K3 Anthropic-compatible API, MiniMax M2.7, immutable Linux amd64 Runtime builds. Native execution, restart/history, cancellation, Files/Artifacts and isolation; shared credential lifecycle evidence is not a separate live rotation run for every E2B profile. | | E2 | `20260922/execution-tools-closure/environment-actions/result.json` contains Codex/Claude Kimi passes (13 checks each) and the retained MiniMax direct-network failure; `mcode-relay-retry/result.json` separately passes MiniMax M2.7 (13 checks) after correcting provider routing. Source `206474a`; same shared SDK/raw workflow, isolated Core/PostgreSQL and user-managed Docker. Covers initial creation SSE, client disconnect, exact pending Environment/no Turn or Items, scoped enrollment, one actual completed model Turn and creation retries preserving history. This is connection-action qualification, not another full lifecycle or E2B regression. | Controlled tests establish parser, projection, atomicity and race behavior; native @@ -121,4 +121,4 @@ remain separate limitations. They do not authorize silently relabeling an unprov principal workflow as deferred. OAuth, full Usage, release publication, new tool kinds and new execution loops are outside this closure batch. -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/execution-tools.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/execution-tools.md) diff --git a/apps/docs/content/docs/api-reference/core/core-administration.mdx b/apps/docs/content/docs/api-reference/core/core-administration.mdx index af6d73f49..0ed580f68 100644 --- a/apps/docs/content/docs/api-reference/core/core-administration.mdx +++ b/apps/docs/content/docs/api-reference/core/core-administration.mdx @@ -30,12 +30,12 @@ _openapi: assert active Turn settlement. Read this after an uncertain archive response; never infer released from a missing sandbox alone. - content: >- - Core key only. Requires maintenance and the current deployment - generation. Permanently closes execution, requests cancellation and - releases sandbox/snapshots through existing cleanup. Session history - and persisted files/artifacts remain; unpersisted workspace contents - are lost. A cleanup_pending response is not proof of resource release. - Does not affect caller-managed Runtime. + Core key only. Requires the current deployment generation; no + maintenance mode is required. Permanently closes execution, requests + cancellation and releases sandbox/snapshots through existing cleanup. + Session history and persisted files/artifacts remain; unpersisted + workspace contents are lost. A cleanup_pending response is not proof + of resource release. Does not affect caller-managed Runtime. - content: >- Core key only. Each observation is labelled with its owning Project ID. Uses the existing read-only Runtime sampler, with bounded diff --git a/apps/docs/content/docs/api-reference/core/sandbox-manager.mdx b/apps/docs/content/docs/api-reference/core/sandbox-manager.mdx index f05ee6363..c0f819bcf 100644 --- a/apps/docs/content/docs/api-reference/core/sandbox-manager.mdx +++ b/apps/docs/content/docs/api-reference/core/sandbox-manager.mdx @@ -21,21 +21,29 @@ _openapi: sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in - specification.resources. Exact retries return the existing selection; - differing selections and file-managed deployments reject. This does - not create compute or execute work. - - content: >- - Requires maintenance, the current generation and verified cleanup of - all old resources. Credentials are write-only. E2B may omit resources - to adopt the validated template build's CPU and memory. A core_url - member is rejected with 400; the address comes from the installation - public URL. Historical records are retained; old node credentials and - enrollments are retired. Explicitly resume after success. Never - automatically retry an uncertain write. - - content: >- - Requires the current generation. Maintenance preserves execution, - cleanup and history. Resume requires successfully activated provider - configuration. + specification.resources. Requires explicit expected_generation, + including zero at first setup. Stale retries reject before provider + validation. An identical selection at the current generation is a + no-op; differing selections and file-managed deployments reject. This + does not create compute or execute work. + - content: >- + Requires the current generation, the same backend type, no active + reset and verified cleanup of all old resources. A backend type change + requires reset. Credentials are write-only. E2B may omit resources to + adopt the validated template build's CPU and memory. A core_url member + is rejected with 400; the address comes from the installation public + URL. Historical records are retained; old node credentials and + enrollments are retired. Never automatically retry an uncertain write. + - content: >- + Archives hosted Sessions and waits for confirmed provider cleanup, + preserving history and Files/Artifacts. Auto waits for started or + waiting Turns and file writes until the durable deadline; force + cancels them. The same clear is idempotent; force escalates auto. + Requires the current generation. Self-hosted Sessions are unchanged. + - content: >- + Restores admission but never restores Sessions already archived. With + no reset running this is an idempotent read, provided the generation + still matches. - content: >- Core key only. Does not grant project resource access. Responses contain only explicit safe fields. @@ -58,4 +66,4 @@ _openapi: {/* This file was generated by Fumadocs. Do not edit this file directly. Any changes should be made by running the generation command again. */} - \ No newline at end of file + \ No newline at end of file diff --git a/apps/docs/content/docs/bootstrap-projects-keys.mdx b/apps/docs/content/docs/bootstrap-projects-keys.mdx index 31aef04ce..7b44799a4 100644 --- a/apps/docs/content/docs/bootstrap-projects-keys.mdx +++ b/apps/docs/content/docs/bootstrap-projects-keys.mdx @@ -69,7 +69,7 @@ and display only. Keep deployment, application, node and provider credentials ou Installation creates no Project or application key. An administrator creates a Project and issues named keys on the console's **Projects and keys** page, which -uses the [administrator API](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/admin-api.md), or with that +uses the [administrator API](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md), or with that API directly. The Project owns one tenant and one principal; its keys share assets and permissions. Writes retain each key's provenance. All Projects and application keys live in @@ -98,4 +98,4 @@ management request points to the console's Core key or Core connection. A resour deletion conflict must remain visible; it does not authorize an execution call. Node and daemon `/api/v1` routes and native Runtime interfaces retain their own authentication. -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/web/core-connection.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/web/core-connection.md) diff --git a/apps/docs/content/docs/concepts.mdx b/apps/docs/content/docs/concepts.mdx index cd2b5bd00..9f8505337 100644 --- a/apps/docs/content/docs/concepts.mdx +++ b/apps/docs/content/docs/concepts.mdx @@ -83,6 +83,6 @@ ownership. No request bodies, secrets or file contents enter audit records. Do not add product users, RBAC, cross-Project shared assets, administrator execution, or old private-protocol compatibility to this management model. Existing public -execution and Runtime ownership rules remain in [CONTRIBUTING.md](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/CONTRIBUTING.md). +execution and Runtime ownership rules remain in [CONTRIBUTING.md](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/CONTRIBUTING.md). -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/design-principles.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/design-principles.md) diff --git a/apps/docs/content/docs/configure.mdx b/apps/docs/content/docs/configure.mdx index 11653093f..33a9b1aaf 100644 --- a/apps/docs/content/docs/configure.mdx +++ b/apps/docs/content/docs/configure.mdx @@ -116,7 +116,7 @@ output or in Core's settings snapshot. Model providers are not process settings; The schema is -[`deploy/install/config.schema.json`](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/deploy/install/config.schema.json); each +[`deploy/install/config.schema.json`](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/deploy/install/config.schema.json); each installation keeps a copy in `generated/config.schema.json` for editors. Core serves the non-secret settings snapshot, with the path of `config.json` and the apply command, at `GET /core/v1/installation`. @@ -128,7 +128,7 @@ Core API with the Core key. | Setting | Where in Web | Core API | Notes | | --- | --- | --- | --- | -| Sandbox backend: Docker, microsandbox or E2B | **Nodes** (**Sandbox backend** with E2B): the setup wizard, ending with **Save configuration** | `/core/v1/sandbox/deployment` | One backend per deployment. `install.sh --sandbox` saves the first choice. To change it, see the Nodes page; that flow will change in a coming release | +| Sandbox backend: Docker, microsandbox or E2B | **Nodes** (**Sandbox backend** with E2B): the setup wizard, ending with **Save configuration** | `/core/v1/sandbox/deployment` | One backend per deployment. `install.sh --sandbox` saves the first choice. Changing backend requires explicit reset and a new setup at the resulting generation | | Sandbox size and Runtime release | **Nodes**: the setup wizard | `/core/v1/sandbox/deployment` | Every sandbox gets the same size. See [Sandbox deployment](#sandbox-deployment) | | E2B API key and template build | **Nodes**: the setup wizard's **E2B cloud** (the page is then called **Sandbox backend**) | `/core/v1/sandbox/deployment` | The key is write-only and encrypted | | Nodes and their capacity | **Nodes**: **Add node**, **Edit node**, **Remove node** on a node's page (**Remove** in its list row) | `/core/v1/sandbox/enrollment-tokens`, `/core/v1/sandbox/nodes` | See [Node capacity](#node-capacity) and the [nodes guide](/hosted-providers) | @@ -145,7 +145,7 @@ Which harnesses are enabled, and the default one, are process settings Exactly one provider serves the deployment: Docker, microsandbox or E2B. The deployment specification holds the per-sandbox resources and, for Docker and microsandbox, the Runtime release. Web's setup proposes the Standard size from -[`standard-sizes.json`](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/apps/web/src/features/sandbox/standard-sizes.json) and +[`standard-sizes.json`](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/apps/web/src/features/sandbox/standard-sizes.json) and derives Small and Large from it; the installer applies the bundle's copy of the same file. The database owns the saved values. @@ -165,8 +165,8 @@ Changing the provider, the size or the Runtime applies to the whole deployment; **Nodes** page, and note that this flow will change in a coming release. A successful change retires the old nodes and enrollment commands; history stays, and existing Sessions never move between providers. See the -[operator reference](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-maintenance) -and the [deployment contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/sandbox-deployment.md). +[operator reference](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-reset) +and the [deployment contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/sandbox-deployment.md). ### Node capacity @@ -193,7 +193,7 @@ Session freezes the provider it starts with, so a later change affects only new Sessions. Self-hosted Sessions never use it. A hosted or self-hosted Session with no provider is refused with 400 `model_provider_required`. See [model providers](/quickstart#model-providers) and the -[model execution contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/model-execution.md#deployment-defaults). +[model execution contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/model-execution.md#deployment-defaults). The operator file `AGENTS_API_EXECUTION_OPTIONS_FILE` is retired; see [Upgrade notes](/troubleshooting#upgrade-notes). @@ -213,7 +213,7 @@ A Web-only installation has only `secrets/core.key`, a copy of its Core's key. Core reads only its environment. The installer renders `generated/core.env` from `config.json`; if you run Core yourself (see -[Maintainers and advanced deployments](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/maintainers.md)), set these variables. Compose +[Maintainers and advanced deployments](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/maintainers.md)), set these variables. Compose loads the file with `env_file` and systemd with `EnvironmentFile`, so Compose must be 2.26.0 or newer. @@ -228,7 +228,7 @@ loads the file with `env_file` and systemd with `EnvironmentFile`, so Compose mu | `OAC_INSTALLATION_ID` | `state.json`; pinned to the database | | `OAC_SETTINGS_FILE` | `generated/settings.json`, the snapshot Core serves at `GET /core/v1/installation`; Core does not act on it | | `OAC_EXECUTION_CONCURRENCY`, `OAC_DEFAULT_HARNESS`, `OAC_HARNESSES`, `OAC_WRITE_AUDIT_RETENTION`, `OAC_OAUTH_TRUSTED_ORIGINS` | The matching `core.*` settings | -| `OAC_HISTORY_SETTINGS_FILE` | `generated/runtime-history.json` when `core.runtime_history` is set; see the [history contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/runtime-history-api.md) | +| `OAC_HISTORY_SETTINGS_FILE` | `generated/runtime-history.json` when `core.runtime_history` is set; see the [history contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/runtime-history-api.md) | | `OAC_LOG_LEVEL`, `OAC_LOG_FORMAT`, `OAC_LOG_ADD_SOURCE` | `log.*`; Web reads the same three | | `OAC_E2B_STATE_DIR`, `OAC_E2B_PROVIDER_BIN` | The E2B receipt directory and, for native Core, the bundled helper. The E2B key and template live in the database | @@ -250,4 +250,4 @@ paths must be canonical absolute paths without control characters, quotes, backs or wildcards. Keep the installation ID and the database together; Core refuses a missing installation ID when its database already has a deployment. -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/configuration.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/configuration.md) diff --git a/apps/docs/content/docs/console.mdx b/apps/docs/content/docs/console.mdx index de30947cf..6af130720 100644 --- a/apps/docs/content/docs/console.mdx +++ b/apps/docs/content/docs/console.mdx @@ -24,10 +24,10 @@ management client, and it sends nothing to `/v1`. | Monitor | Sandbox metrics | Node capacity and hosted Runtime CPU and memory across Projects | | Monitor | Session log | Every Session, opening one Session's read-only conversation, trace and Turns; a self-hosted Session's page also manages its executor credentials and gives the command that connects a host | | Resources | Agents, Environment templates, Skills, Files, Vaults | Inspection and permitted deletion | -| Platform | Projects and keys, Nodes, System | Project and key lifecycle; sandbox deployment and nodes; System: the installation's public address, API base URL, ID and source commit (read-only), each harness's default model provider (write-only key) beside its read-only startup state, the sandbox configuration every Project shares: provider, sandbox size, Runtime or E2B template build, idle suspension and maintenance, and Core's config.json startup settings with where to change them | +| Platform | Projects and keys, Nodes, System | Project and key lifecycle; sandbox deployment and nodes; System: the installation's public address, API base URL, ID and source commit (read-only), each harness's default model provider (write-only key) beside its read-only startup state, the sandbox configuration every Project shares: provider, sandbox size, Runtime or E2B template build, idle suspension and reset, and Core's config.json startup settings with where to change them | Missing data is shown as missing (—), never as zero. How each figure is read and -bounded is recorded in [management interface coverage](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/web/protocol-coverage.md). +bounded is recorded in [management interface coverage](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/web/protocol-coverage.md). ## Management scope @@ -60,15 +60,15 @@ only the console server sends it to Core. - [Connection and authentication](/bootstrap-projects-keys) - [Architecture and ownership](/execution-model) -- [Management interface coverage](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/web/protocol-coverage.md) -- [Frontend handoff and acceptance](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/web/roadmap.md) -- [React application](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/apps/web/README.md) +- [Management interface coverage](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/web/protocol-coverage.md) +- [Frontend handoff and acceptance](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/web/roadmap.md) +- [React application](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/apps/web/README.md) -The [administrator API contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/admin-api.md) defines -management routes and resource behavior. The [public API contracts](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/README.md) +The [administrator API contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md) defines +management routes and resource behavior. The [public API contracts](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/README.md) define the separate application interface. See the [design principles](/concepts) -for ownership and [contributor guide](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/CONTRIBUTING.md) for required checks. +for ownership and [contributor guide](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/CONTRIBUTING.md) for required checks. -OpenAgentCore Web is available under the [MIT License](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/LICENSE). +OpenAgentCore Web is available under the [MIT License](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/LICENSE). -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/web/README.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/web/README.md) diff --git a/apps/docs/content/docs/environments-and-files.mdx b/apps/docs/content/docs/environments-and-files.mdx index ab045960d..566f1d3c9 100644 --- a/apps/docs/content/docs/environments-and-files.mdx +++ b/apps/docs/content/docs/environments-and-files.mdx @@ -3,16 +3,16 @@ title: "Environments and files" description: "Execution placement, Environment resources and workspace ownership." --- -This assessment covers the fixed [Python SDK contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/upstream.json), with partial +This assessment covers the fixed [Python SDK contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/upstream.json), with partial coverage. Core-managed Docker runs the three qualified native harnesses. V1 user-managed `self_hosted` enrollment uses the same colocated Runtime for Codex, -Claude SDK and MiniMax at `/workspace`; see its [real deployment qualification](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/user-managed-runtime-v1.md). +Claude SDK and MiniMax at `/workspace`; see its [real deployment qualification](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/user-managed-runtime-v1.md). For hosted compute, the deployment selects E2B, Docker or microsandbox through -[Hosted Sandbox Manager](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/HOSTED-SANDBOX-MANAGER.md). +[Hosted Sandbox Manager](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/HOSTED-SANDBOX-MANAGER.md). For the separate caller-managed E2B path, the user owns allocation, renewal and -cleanup through the official SDK and [Runtime packaging](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/deploy/e2b/README.md). -[Templates](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/environment-templates.md) remain a hosted-only resource path; -[Files](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/environment-files.md) reuse the exact authorized local workspace. +cleanup through the official SDK and [Runtime packaging](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/deploy/e2b/README.md). +[Templates](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/environment-templates.md) remain a hosted-only resource path; +[Files](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/environment-files.md) reuse the exact authorized local workspace. The internal Store now owns a durable Environment association for newly created `self_hosted` and `openai_hosted` snapshots, atomically with Session creation. @@ -30,15 +30,15 @@ readiness. Rotation/revocation, Session deletion and ownership loss deny further access without promising immediate cessation of native effects. Native history remains local to the bound Runtime and cannot be replaced on retry. There is no registry/Noise relay or transient service-side harness credential. -See the [enrollment guide](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/README.md#user-managed-runtime-enrollment). +See the [enrollment guide](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/README.md#user-managed-runtime-enrollment). ## Basic public Docker-hosted profile An explicitly configured default managed Docker provider enables `type=openai_hosted` for the qualified Codex, Claude Code and MiniMax Code profiles. Each uses the same Runtime lifecycle and workspace interfaces with its own native adapter/isolation. -See the [engine profile guides](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/README.md#public-engine-profiles) for setup and limits. -The standalone [operator configuration](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/deploy/codex/README.md#standalone-operator-configuration) +See the [engine profile guides](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/README.md#public-engine-profiles) for setup and limits. +The standalone [operator configuration](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/deploy/codex/README.md#standalone-operator-configuration) selects the qualified immutable Runtime image; advertised capabilities alone do not enable admission. An idle or initial-text creation commits Session, Environment and retry identity before the existing leased Worker provisions its allocation. @@ -50,7 +50,7 @@ policies use the same qualified image with adapter-selected immutable native pol Templates and inline configuration share initial files, env, packages, ordered setup and inline or tenant-owned referenced Skills through the hosted initializer. Unsupported hostname forms and installation combinations reject explicitly; see -the [Template coverage and limits](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/environment-templates.md). Empty/null installation +the [Template coverage and limits](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/environment-templates.md). Empty/null installation defaults produce safe empty metadata, not a live workspace inventory. Service-origin hosted MCP remains unsupported; Environment Plugin MCP has its own qualified transport matrix. @@ -70,11 +70,11 @@ semantics; this profile does not establish complete Environment compatibility. The application creates, renews and destroys its E2B sandbox through the official SDK. It deploys the shared Runtime, then enrolls that Runtime into a `self_hosted` Session. Core neither keeps an E2B allocation nor issues Provider renew/kill calls. -The [E2B guide](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/deploy/e2b/README.md) owns packaging and +The [E2B guide](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/deploy/e2b/README.md) owns packaging and user-side lifecycle instructions. Expiry or lost workspace/history must not trigger -transparent replacement or replay. The [new enrollment qualification](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/user-managed-runtime-v1.md) +transparent replacement or replay. The [new enrollment qualification](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/user-managed-runtime-v1.md) records its own real deployment evidence. -The [prior E2B qualification](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/README.md#e2b-v1-qualification) records a historical +The [prior E2B qualification](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/README.md#e2b-v1-qualification) records a historical Core-managed route; it does not establish this caller-managed enrollment result. Current deployment-managed E2B is a separate hosted configuration in the manager guide. @@ -111,7 +111,7 @@ connection alone is not readiness. Retries preserve identity and the five-minute database deadline. HTTP disconnect retains the reservation. Local expired/cancelled outcomes return 409, lost ownership 503, and deletion 404; exact hosted error status/body and pending-input crash recovery are unverified. See the -[canonical wait rules](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/CONTRIBUTING.md#environment-ownership-and-placement). +[canonical wait rules](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/CONTRIBUTING.md#environment-ownership-and-placement). Cancellation-only batches use the existing locked admission and native delivery. An idle cancellation creates no Turn, and retry identity preserves the original @@ -191,7 +191,7 @@ from Session and Turn status. ## Ownership and placement decision -The canonical [architecture rules](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/CONTRIBUTING.md#environment-ownership-and-placement) +The canonical [architecture rules](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/CONTRIBUTING.md#environment-ownership-and-placement) keep Environment lifecycle common while leaving process placement and native transport to adapters. Logical ownership does not require a machine per object. @@ -215,4 +215,4 @@ native history independently of disposable compute, or prove native restoration; never treat an Environment ID as a filesystem or history backup. Do not silently move an existing Session away from its bound device. -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/environments.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/environments.md) diff --git a/apps/docs/content/docs/execution-model.mdx b/apps/docs/content/docs/execution-model.mdx index 07b150217..1c57f5eee 100644 --- a/apps/docs/content/docs/execution-model.mdx +++ b/apps/docs/content/docs/execution-model.mdx @@ -10,7 +10,7 @@ Agents API independently with their own Project keys. The management backend, `AdminClient` and the React console built on them are implemented. The [design principles](/concepts) define identity and authority. -The [administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/admin-api.md) defines exact +The [administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md) defines exact routes, payloads, pagination and audit records. ## Request boundaries @@ -79,7 +79,7 @@ route. Deployment sandbox management selects one provider at a time: E2B, Docker or microsandbox. E2B uses the deployment's provider integration; Docker and microsandbox -use operator-managed machines. Provider setup, maintenance and node administration +use operator-managed machines. Provider setup, reset and node administration belong to the existing sandbox management surface. An application's `self_hosted` Runtime, including one it provisions in its own E2B @@ -106,6 +106,6 @@ comes from the deployment's allocations; the hosted Runtime total counts hosted observation records across projects and reported lifecycle states. These sources have different coverage and refresh independently, so the console does not infer resource retention or cleanup from their difference. -Native execution ownership remains governed by [CONTRIBUTING.md](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/CONTRIBUTING.md). +Native execution ownership remains governed by [CONTRIBUTING.md](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/CONTRIBUTING.md). -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/web/architecture.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/web/architecture.md) diff --git a/apps/docs/content/docs/hosted-providers.mdx b/apps/docs/content/docs/hosted-providers.mdx index 7b8638a06..bad34441a 100644 --- a/apps/docs/content/docs/hosted-providers.mdx +++ b/apps/docs/content/docs/hosted-providers.mdx @@ -10,7 +10,7 @@ With E2B you need no nodes. Self-hosted executors, which applications run for th own Sessions, are a different thing; see [Self-hosted executors](/self-hosted-execution). You add a node by generating a command in Web and pasting it on the host. The -[operator reference](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/HOSTED-SANDBOX-MANAGER.md) covers the +[operator reference](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/HOSTED-SANDBOX-MANAGER.md) covers the node protocol, manual registration, placement and failure handling. ## Before you add a node @@ -187,7 +187,7 @@ The installation ID is in the command (`--installation-id`) and on the **System* 1. In Web, open **Nodes** and choose **Remove node** on the node's page, or **Remove** in its list row, then **Confirm removal**. Core refuses while the node still holds sandboxes, snapshots or pending cleanup; let them finish, or - [archive their Sessions](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-maintenance) + [archive their Sessions](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-reset) through the Core API. Removal is permanent: the host can come back only as a new node. 2. Web then shows **Clean up the host** with the uninstall command. Run it on the host: @@ -271,7 +271,7 @@ this flow will change in a coming release. A change applies to the whole deploym and retires every node. Retired nodes drop out of the list, so there is nothing to remove in Web: on each host, run the [uninstall command](#remove-a-node) (Core no longer accepts the node, so it needs no `--force`), then add the host again with a new command. -The [operator reference](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-maintenance) +The [operator reference](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-reset) describes today's procedure and API. @@ -306,4 +306,4 @@ installation's resources: Then use the new Core's Add node command. The new account, service and state use `oac-node` and `~/.oac/nodes`; historical Core Session records remain in Core. -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/getting-started/nodes.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/getting-started/nodes.md) diff --git a/apps/docs/content/docs/index.mdx b/apps/docs/content/docs/index.mdx index 1be8be55b..9311b1bfb 100644 --- a/apps/docs/content/docs/index.mdx +++ b/apps/docs/content/docs/index.mdx @@ -19,10 +19,10 @@ applications call Core with. | [API reference](/public-api) | Developers | The `/v1`, `/core/v1` and `/api/v1` namespaces and their contracts | Deeper references: the -[nodes and sandbox backends](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/HOSTED-SANDBOX-MANAGER.md) +[nodes and sandbox backends](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/HOSTED-SANDBOX-MANAGER.md) reference for operators, the -[protocol coverage and native differences](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/README.md), and -[maintainers and advanced deployments](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/maintainers.md) for building distributions +[protocol coverage and native differences](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/README.md), and +[maintainers and advanced deployments](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/maintainers.md) for building distributions and running Core without the installer. -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/getting-started/README.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/getting-started/README.md) diff --git a/apps/docs/content/docs/install.mdx b/apps/docs/content/docs/install.mdx index 4778a096b..a4934a55a 100644 --- a/apps/docs/content/docs/install.mdx +++ b/apps/docs/content/docs/install.mdx @@ -139,7 +139,7 @@ A deployment runs its sandboxes on exactly one backend: | `none` | Nothing yet | Choose the backend on the **Nodes** page in Web | The Standard sizes come from the bundle's copy of Web's -[`standard-sizes.json`](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/apps/web/src/features/sandbox/standard-sizes.json). The +[`standard-sizes.json`](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/apps/web/src/features/sandbox/standard-sizes.json). The installer saves the choice through the same administrator API that Web uses once the services are healthy. If Core refuses it, for example because E2B rejects the key, the installer prints Core's message and exits with an error; the services keep @@ -163,7 +163,7 @@ it installs anything: ``` Prepare the template build with the -[E2B guide](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/deploy/e2b/README.md). To change the backend or +[E2B guide](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/deploy/e2b/README.md). To change the backend or the sandbox size later, use the Nodes page in Web; see [Nodes](/hosted-providers#change-the-sandbox-backend-or-size). @@ -530,4 +530,4 @@ missing images, restores a missing `oac` command, copies node files newly placed `--install-dir`. It refuses a bundle from another release; see [Upgrades](/troubleshooting#upgrade-an-installation). -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/getting-started/install.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/getting-started/install.md) diff --git a/apps/docs/content/docs/observability.mdx b/apps/docs/content/docs/observability.mdx index 91d7f2d57..4ca748148 100644 --- a/apps/docs/content/docs/observability.mdx +++ b/apps/docs/content/docs/observability.mdx @@ -7,7 +7,7 @@ Status: Phase 2 and initial Core Web consumption implemented. The current-snapsh `packages/agents-client` projection, and generated `core.openapi.yaml` contract are implemented and consumed by the Dashboard through complete Session/observation identity joins. Durable history uses the separate optional -[Runtime history API](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/runtime-history-api.md); lifecycle controls remain outside +[Runtime history API](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/runtime-history-api.md); lifecycle controls remain outside this phase. These are administrator reads under `/core/v1`, authenticated by the Core key, @@ -92,7 +92,7 @@ Authorization: Bearer ... ``` This returns the same object shape as a list item's `observation`, without the -administrator list's `disk` (see the [administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/admin-api.md)). It never starts a Turn, creates +administrator list's `disk` (see the [administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md)). It never starts a Turn, creates an Environment, provisions compute, renews a lease, or changes lifecycle state. A valid `environment:none` Session returns `200` with status `unsupported`; the @@ -193,7 +193,7 @@ Use the existing Agents API error envelope. | 400 | `invalid_request_error` / `invalid_request_error` | List: a repeated supported query key, or an empty or invalid limit or order, with the shared Beta list messages. Unknown list query keys are ignored. | | 400 | `invalid_request_error` / `unsupported_parameter` | Single-Session retrieval with any query parameter. | | 401 | `invalid_request_error` / `invalid_admin_key` | Missing or invalid Core key. | -| 404 | `not_found_error` / `not_found_error` | Missing, malformed or foreign Session/cursor, indistinguishably, as for the [Session list cursor](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/list-query-semantics.md#list-cursor-errors--september-23-2026). | +| 404 | `not_found_error` / `not_found_error` | Missing, malformed or foreign Session/cursor, indistinguishably, as for the [Session list cursor](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/list-query-semantics.md#list-cursor-errors--september-23-2026). | | 500 | `server_error` / `internal_error` | Integrity, ownership, or invalid provider evidence. | | 503 | `server_error` / `execution_unavailable` | Required Runtime observation service is not configured, or list collection exceeded its request budget. | @@ -273,10 +273,10 @@ deletion makes the candidate incomplete and prevents publication. - Token usage: use existing Session/Turn Usage. - Billing and cost: product/backend concern. - Historical series in these routes: the optional capability uses the separate - [Runtime history API](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/runtime-history-api.md). + [Runtime history API](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/runtime-history-api.md). - Container logs and command output. - Provider credentials or native configuration. - Start, stop, pause, resume, restart, renew, or delete operations. - Idle classification and automatic shutdown. -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/runtime-observability-api.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/runtime-observability-api.md) diff --git a/apps/docs/content/docs/public-api.mdx b/apps/docs/content/docs/public-api.mdx index 13cec4d9c..5c55bf834 100644 --- a/apps/docs/content/docs/public-api.mdx +++ b/apps/docs/content/docs/public-api.mdx @@ -9,8 +9,8 @@ no credential works in another namespace. | Namespace | Caller | Credential | Contents | Reference | | --- | --- | --- | --- | --- | | `/v1` | Applications (business systems, SDKs) | Project API key | Exactly the pinned official Agents API routes. Core-only fields live only in `x_agents_core` (`harness`, `model_provider`) | [Public API](/sessions) | -| `/core/v1` | Core Web's server and operator scripts | [Core key](/troubleshooting#core-key) | Installation facts, Projects and keys, resource reads and deletion, Session archive, credential issuance, metrics, audit, sandbox deployment and nodes, deployment model providers | [Core API](#core-api), [Web API](/admin-api), [Core OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/core.openapi.yaml) | -| `/api/v1` | Nodes, Runtime daemons, self-hosted executors | Machine credentials: node enrollment tokens and executor credentials issued through `/core/v1`, node credentials registered with an enrollment token, and daemon credentials Core writes into hosted sandboxes | Machine connections only: `/api/v1/sandbox-node/*` and `/api/v1/agent-daemon/*`, including WebSockets; each credential works only on its own routes | [Node operations](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/HOSTED-SANDBOX-MANAGER.md#register-a-host), [executor credentials](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/environment-executor-credentials.md), [machine OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/runtime.openapi.yaml) | +| `/core/v1` | Core Web's server and operator scripts | [Core key](/troubleshooting#core-key) | Installation facts, Projects and keys, resource reads and deletion, Session archive, credential issuance, metrics, audit, sandbox deployment and nodes, deployment model providers | [Core API](#core-api), [Web API](/admin-api), [Core OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/core.openapi.yaml) | +| `/api/v1` | Nodes, Runtime daemons, self-hosted executors | Machine credentials: node enrollment tokens and executor credentials issued through `/core/v1`, node credentials registered with an enrollment token, and daemon credentials Core writes into hosted sandboxes | Machine connections only: `/api/v1/sandbox-node/*` and `/api/v1/agent-daemon/*`, including WebSockets; each credential works only on its own routes | [Node operations](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/HOSTED-SANDBOX-MANAGER.md#register-a-host), [executor credentials](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/environment-executor-credentials.md), [machine OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/runtime.openapi.yaml) | A Project API key gets 401 on `/core/v1` and `/api/v1`; the Core key gets 401 on `/v1` and `/api/v1`. Projects own assets. Multiple equally privileged keys share @@ -34,7 +34,7 @@ with the Core key on Core's loopback port, not through the public entry. Applications call every `/v1` route with a Project API key. The routes are exactly the 58 method and path pairs in -[upstream-routes.json](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/upstream-routes.json); the +[upstream-routes.json](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/upstream-routes.json); the [Public API guide](/sessions) groups them by resource. `/v1` has no Core-only route. @@ -45,17 +45,17 @@ with the Core key. | Routes | Contents | Contract | | --- | --- | --- | -| `projects`, `projects/{project_id}[/archive]`, `projects/{project_id}/keys[/{key_id}]` | Projects and their API keys | [Administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/admin-api.md) | -| `projects/{project_id}/{agents,environment-templates,skills,files,vaults,sessions}/**` | Resource reads and deletion, Session history, artifacts and archive | [Administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/admin-api.md) | -| `projects/{project_id}/sessions/{session_id}/execution-configuration` | Committed harness and model selection | [Execution configuration](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/execution-configuration.md) | +| `projects`, `projects/{project_id}[/archive]`, `projects/{project_id}/keys[/{key_id}]` | Projects and their API keys | [Administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md) | +| `projects/{project_id}/{agents,environment-templates,skills,files,vaults,sessions}/**` | Resource reads and deletion, Session history, artifacts and archive | [Administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md) | +| `projects/{project_id}/sessions/{session_id}/execution-configuration` | Committed harness and model selection | [Execution configuration](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/execution-configuration.md) | | `projects/{project_id}/sessions/{session_id}/runtime-observation`, `sandbox/runtime-observations` | Current Runtime observations | [Runtime observations](/observability) | -| `projects/{project_id}/sessions/{session_id}/runtime-history` | Stored Runtime history | [Runtime history](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/runtime-history-api.md) | -| `projects/{project_id}/{resource-owners,write-operations}`, `audit-log`, `summary` | Provenance, write history, administrator audit and usage summary | [Write audit](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/write-audit.md), [administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/admin-api.md) | -| `projects/{project_id}/environments/{environment_id}/executor-credentials[/{key_id}]` | Executor credentials for a self-hosted Environment | [Executor credentials](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/environment-executor-credentials.md) | -| `installation` | Public URL, API base URL, source commit, the installer's process settings and what is bound to the public URL; available before any deployment | [Installation](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/installation.md) | -| `metrics` | Core's own process metrics | [Core metrics](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/core-metrics.md) | -| `sandbox/deployment[/maintenance]`, `sandbox/enrollment-tokens`, `sandbox/nodes[/{node_id}[/allocations]]` | Sandbox deployment, node enrollment tokens and nodes | [Sandbox deployment](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/sandbox-deployment.md), [node operations](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/HOSTED-SANDBOX-MANAGER.md), [node host history](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/node-host-history.md) | -| `harnesses`, `harnesses/{harness}/model-provider` | Supported harnesses and each harness's deployment default model provider (write-only key) | [Model execution](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/model-execution.md#deployment-defaults) | +| `projects/{project_id}/sessions/{session_id}/runtime-history` | Stored Runtime history | [Runtime history](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/runtime-history-api.md) | +| `projects/{project_id}/{resource-owners,write-operations}`, `audit-log`, `summary` | Provenance, write history, administrator audit and usage summary | [Write audit](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/write-audit.md), [administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md) | +| `projects/{project_id}/environments/{environment_id}/executor-credentials[/{key_id}]` | Executor credentials for a self-hosted Environment | [Executor credentials](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/environment-executor-credentials.md) | +| `installation` | Public URL, API base URL, source commit, the installer's process settings and what is bound to the public URL; available before any deployment | [Installation](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/installation.md) | +| `metrics` | Core's own process metrics | [Core metrics](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/core-metrics.md) | +| `sandbox/deployment[/reset]`, `sandbox/enrollment-tokens`, `sandbox/nodes[/{node_id}[/allocations]]` | Sandbox deployment, node enrollment tokens and nodes | [Sandbox deployment](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/sandbox-deployment.md), [node operations](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/HOSTED-SANDBOX-MANAGER.md), [node host history](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/node-host-history.md) | +| `harnesses`, `harnesses/{harness}/model-provider` | Supported harnesses and each harness's deployment default model provider (write-only key) | [Model execution](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/model-execution.md#deployment-defaults) | ## Machine connection API @@ -64,42 +64,42 @@ the Core key or a Project API key. | Routes | Caller | Credential | Contract | | --- | --- | --- | --- | -| `POST sandbox-node/enroll` | Node installer | One-use enrollment token from `POST /core/v1/sandbox/enrollment-tokens` | [Node operations](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/HOSTED-SANDBOX-MANAGER.md#register-a-host), [machine OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/runtime.openapi.yaml) | -| `GET sandbox-node/configuration` | Node installer and node | Enrollment token, or node credential with `X-OAC-Node-ID` | [Sandbox deployment](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/sandbox-deployment.md), [machine OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/runtime.openapi.yaml) | -| `GET sandbox-node/identity`, WebSocket `GET sandbox-node/connect` | Node | Node credential registered at enrollment | [Node operations](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/HOSTED-SANDBOX-MANAGER.md#register-a-host) | -| `POST agent-daemon/enroll`, `GET agent-daemon/connection` | Self-hosted executor and its installer | Executor credential from `/core/v1/projects/{project_id}/environments/{environment_id}/executor-credentials` | [Executor credentials](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/environment-executor-credentials.md) | -| WebSocket `GET agent-daemon/ws`, `POST agent-daemon/bootstrap`, `GET agent-daemon/device-status` | Runtime daemons | Daemon credential: Core writes one into each hosted sandbox it prepares; a self-hosted executor uses its executor credential | [Runtime enrollment](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/README.md#user-managed-runtime-enrollment) | +| `POST sandbox-node/enroll` | Node installer | One-use enrollment token from `POST /core/v1/sandbox/enrollment-tokens` | [Node operations](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/HOSTED-SANDBOX-MANAGER.md#register-a-host), [machine OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/runtime.openapi.yaml) | +| `GET sandbox-node/configuration` | Node installer and node | Enrollment token, or node credential with `X-OAC-Node-ID` | [Sandbox deployment](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/sandbox-deployment.md), [machine OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/runtime.openapi.yaml) | +| `GET sandbox-node/identity`, WebSocket `GET sandbox-node/connect` | Node | Node credential registered at enrollment | [Node operations](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/HOSTED-SANDBOX-MANAGER.md#register-a-host) | +| `POST agent-daemon/enroll`, `GET agent-daemon/connection` | Self-hosted executor and its installer | Executor credential from `/core/v1/projects/{project_id}/environments/{environment_id}/executor-credentials` | [Executor credentials](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/environment-executor-credentials.md) | +| WebSocket `GET agent-daemon/ws`, `POST agent-daemon/bootstrap`, `GET agent-daemon/device-status` | Runtime daemons | Daemon credential: Core writes one into each hosted sandbox it prepares; a self-hosted executor uses its executor credential | [Runtime enrollment](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/README.md#user-managed-runtime-enrollment) | ## Contract sources -- [Pinned upstream baseline](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/upstream.json): OpenAI +- [Pinned upstream baseline](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/upstream.json): OpenAI Python SDK 3.13.0, exact upstream commit and `agents=v1`. -- [Pinned routes](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/upstream-routes.json) and - [fields](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/upstream-fields.json): extracted from the +- [Pinned routes](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/upstream-routes.json) and + [fields](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/upstream-fields.json): extracted from the pinned SDK by `scripts/extract-agents-api-upstream.py`. Contract tests require the public OpenAPI to have exactly these routes and to keep every other field inside `x_agents_core`. -- [Public OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/openapi.yaml): public schema snapshot; - combine it with the fixed SDK and [operation evidence](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/operation-evidence.md). -- [Core OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/core.openapi.yaml): generated `/core/v1` +- [Public OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/openapi.yaml): public schema snapshot; + combine it with the fixed SDK and [operation evidence](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/operation-evidence.md). +- [Core OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/core.openapi.yaml): generated `/core/v1` routes, all authenticated by the Core key. -- [Machine connection OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/runtime.openapi.yaml): +- [Machine connection OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/runtime.openapi.yaml): generated `/api/v1` sandbox node routes. The node WebSocket and the private daemon transport are described in the node and Runtime credential guides. -- [Administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/admin-api.md): Project/key +- [Administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md): Project/key lifecycle, resources, explicit hosted Session archive, summary, errors/deletion preconditions, audit and historical copy provenance. - [Web integration](/admin-api): browser, console and Core boundaries and frontend handoff. -- [Design rules](/concepts) and [contributor guide](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/CONTRIBUTING.md): +- [Design rules](/concepts) and [contributor guide](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/CONTRIBUTING.md): ownership, security and change requirements. Generated schemas do not establish complete compatibility or real execution -support. The [coverage record](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/README.md) identifies +support. The [coverage record](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/README.md) identifies qualified workflows, native differences and unresolved behavior. Update the relevant contract and this index when adding or moving an API surface. -Core administration failures use the [Core error envelope](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/core-errors.md), +Core administration failures use the [Core error envelope](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/core-errors.md), including typed optional safe details and distinct console proxy rejection codes. The public and machine error contracts remain unchanged. -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/api/README.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/api/README.md) diff --git a/apps/docs/content/docs/quickstart.mdx b/apps/docs/content/docs/quickstart.mdx index b6ebfacd8..bd3558799 100644 --- a/apps/docs/content/docs/quickstart.mdx +++ b/apps/docs/content/docs/quickstart.mdx @@ -101,8 +101,8 @@ The provider's protocol must match the harness: | Claude Code (`claude_sdk`) | `anthropic` | The provider's exact model ID | | MiniMax Code (`mcode`) | `anthropic` | `context_window` and `max_output_tokens` | -See [harness selection](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/harness-selection.md) and -[model execution](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/model-execution.md) for the full contract. +See [harness selection](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/harness-selection.md) and +[model execution](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/model-execution.md) for the full contract. Selecting a harness doesn't make an unsupported model or operation work. ## Model providers @@ -142,8 +142,8 @@ not replay history. Don't submit the same work as a new Session when a response lost. Files, Artifacts, cancellation and their current limits are in the -[coverage ledger](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/README.md). MCP credentials use the +[coverage ledger](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/README.md). MCP credentials use the separate Vault API; they are neither Project API keys nor model provider keys. The [API index](/public-api) lists every route. -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/getting-started/quickstart.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/getting-started/quickstart.md) diff --git a/apps/docs/content/docs/self-hosted-execution.mdx b/apps/docs/content/docs/self-hosted-execution.mdx index 3de987fef..3e81ea259 100644 --- a/apps/docs/content/docs/self-hosted-execution.mdx +++ b/apps/docs/content/docs/self-hosted-execution.mdx @@ -143,7 +143,7 @@ key_id=$(python3 -c 'import uuid; print(uuid.uuid4())'); echo "credential ID: $k Rotate with `{"key_id":"…","rotate":true}` on the same route; revoke with `DELETE …/executor-credentials/`. After an uncertain response, list the credentials with `GET` before trying again. The -[credential contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/environment-executor-credentials.md) +[credential contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/environment-executor-credentials.md) has every rule and error. @@ -166,4 +166,4 @@ Credential replacement for executors created by the new installer still updates and restarts the same owned container. The legacy refusal does not authorize adopting an old container to replace its credential. -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/getting-started/self-hosted.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/getting-started/self-hosted.md) diff --git a/apps/docs/content/docs/sessions.mdx b/apps/docs/content/docs/sessions.mdx index c70cbe237..36944b8b4 100644 --- a/apps/docs/content/docs/sessions.mdx +++ b/apps/docs/content/docs/sessions.mdx @@ -34,8 +34,8 @@ leaves the others valid; archiving the Project revokes all its keys. The optiona ## Public resources Paths are relative to `/v1`. Detailed fields, union types and response shapes are -in the [public OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/openapi.yaml), fixed SDK source -and linked semantic records. The [resource inventory](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/README.md#upstream-resource-inventory) +in the [public OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/openapi.yaml), fixed SDK source +and linked semantic records. The [resource inventory](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/README.md#upstream-resource-inventory) lists each operation and its coverage. | Resource | Route family | Semantics | @@ -44,11 +44,11 @@ lists each operation and its coverage. | Sessions and Turns | `/agents/sessions`, nested `/turns` | Admission, metadata, state and recovery queries | | Items and events | Session `/items`, `/events` | Persisted history and live SSE | | Artifacts | Session `/artifacts` | Captured output metadata, immutable content and deletion | -| Subagents | Session `/subagents`, nested `/turns` and `/items` | Scoped child execution reads; [native limits](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/subagents.md) | -| Environments and workspace Files | `/agents/environments`, nested `/files` | [Environment state](/environments-and-files) and [workspace access](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/environment-files.md) | -| Environment Templates | `/agents/environments/templates` | [Reusable initialization](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/environment-templates.md) | -| Source Files | `/files` | [Uploaded content resources](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/source-files.md), distinct from live workspace files | -| Skills and versions | `/skills`, nested `/versions` | [Bundles and versions](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/file-resource-semantics.md) | +| Subagents | Session `/subagents`, nested `/turns` and `/items` | Scoped child execution reads; [native limits](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/subagents.md) | +| Environments and workspace Files | `/agents/environments`, nested `/files` | [Environment state](/environments-and-files) and [workspace access](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/environment-files.md) | +| Environment Templates | `/agents/environments/templates` | [Reusable initialization](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/environment-templates.md) | +| Source Files | `/files` | [Uploaded content resources](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/source-files.md), distinct from live workspace files | +| Skills and versions | `/skills`, nested `/versions` | [Bundles and versions](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/file-resource-semantics.md) | | Vaults and Credentials | `/vaults`, nested `/credentials` | [Scoped write-only credential storage](/environments-and-files) | SSE is live, not a historical replay service. After a disconnect, query the Session, @@ -58,14 +58,14 @@ native execution. Do not automatically replay uncertain tool or file effects. ## Core extension fields The `/v1` route set is exactly the pinned SDK's operations, listed in -[upstream-routes.json](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/upstream-routes.json); Core adds +[upstream-routes.json](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/upstream-routes.json); Core adds no route. It adds fields only inside `x_agents_core`, because it runs several harnesses and accepts custom model access: | Field | Where | Reference | | --- | --- | --- | -| `x_agents_core.harness` | Saved Agent create, update and read; the inline Session `agent`; the Session's effective `agent` | [Harness selection](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/harness-selection.md) | -| `x_agents_core.model_provider` | Saved Agent create, update and read; Session creation. It takes `protocol`, `base_url`, optional `context_window` and `max_output_tokens`, and a write-only `api_key`; reads return `api_key_configured` instead of the key | [Model execution](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/model-execution.md) | +| `x_agents_core.harness` | Saved Agent create, update and read; the inline Session `agent`; the Session's effective `agent` | [Harness selection](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/harness-selection.md) | +| `x_agents_core.model_provider` | Saved Agent create, update and read; Session creation. It takes `protocol`, `base_url`, optional `context_window` and `max_output_tokens`, and a write-only `api_key`; reads return `api_key_configured` instead of the key | [Model execution](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/model-execution.md) | Any other member of `x_agents_core` is rejected with 400. Deployment, placement, credential issuance and operational reads are not part of `/v1`; they belong to @@ -78,7 +78,7 @@ never to `self_hosted`; `none` Sessions accept only the deployment default. `openai_hosted` and `self_hosted` Sessions must end up with one: otherwise creation returns 400 with code `model_provider_required` and param `x_agents_core.model_provider`, and nothing is written. See -[model execution](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/model-execution.md). +[model execution](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/model-execution.md). An application reads a Session's model from `agent.model` and an explicitly selected harness from `agent.x_agents_core.harness`; a Session on the deployment @@ -90,8 +90,8 @@ The deployment chooses E2B or its own Docker/microsandbox nodes. A public `self_hosted` Environment uses caller-owned compute and the same colocated Runtime. For a `self_hosted` Session, the deployment operator issues the Environment's executor credential with the Core key; see -[executor credentials](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/environment-executor-credentials.md). +[executor credentials](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/environment-executor-credentials.md). Our daemon transport and the supported native harness differences are explicit; they do not imply interoperability with OpenAI's stock executor transport. -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/api/public-agent-api.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/api/public-agent-api.md) diff --git a/apps/docs/content/docs/troubleshooting.mdx b/apps/docs/content/docs/troubleshooting.mdx index 3c9cd97f3..8369b2b1e 100644 --- a/apps/docs/content/docs/troubleshooting.mdx +++ b/apps/docs/content/docs/troubleshooting.mdx @@ -123,7 +123,7 @@ refuses. `rotate-core-key` refuses to run on a Web-only installation. ## Projects and API keys Create Projects and issue their keys in Web, on **Projects and keys**, or with the -[Core API](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/admin-api.md) under `/core/v1/projects`. A key's +[Core API](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md) under `/core/v1/projects`. A key's plaintext is shown once; Core stores only its digest. All keys of a Project share its assets and execution principal, and each write records the key that made it. @@ -195,10 +195,12 @@ The latter rename their directory, project and payload without copying a databas Preflight reports generated-file edits, invalid configuration, destination conflicts, insufficient database-copy space, and every available drain blocker. It reads the -old Core with its Core key. When that Core is stopped and static checks pass, it -announces a temporary start using only the existing files. Refusal restores the -services this probe started; a restoration failure is reported explicitly. No -installation conversion happens before confirmation. +old Core with its Core key. Start the previous release yourself before conversion: +a stopped or unreadable Core is refused without starting services. This is required +to identify the provider before any conversion writes. E2B conversion is currently +refused, including interrupted journal resumes; keep the previous release until +the safe Core-owned E2B upgrade transition is available. No installation conversion +happens before confirmation. The confirmation prints the backup command for the old project and the directory and secret moves. For a config.json installation, take the backup before confirming: @@ -223,9 +225,11 @@ The new state format is 2; config.json's schema format remains 1. Ports, public secrets, Core key, installation ID and database contents are preserved. The old `parsar` path becomes a stub naming the new `oac` command. New services use `OAC_*` settings and `io.oac.inputs` labels. Docker and microsandbox deployments retain -their resources, replace their Runtime with the bundle's release through the normal -maintenance-only API, then resume admission. E2B remains in maintenance: rebuild its -template with this release's `build-template.py`, replace it in Web, then resume. +their resources and replace their Runtime with the bundle's exact release through +an expected-generation same-provider PUT. New Core must report reset null and zero +held resources; conversion reads back the exact specification and generation. +Ordinary migration retires old maintenance and resumes admission, which is why E2B +conversion remains blocked until its safe upgrade preparation is implemented. Rerun `./install.sh --convert` with the **same bundle** after any interruption, including after the directory move. For a custom directory, repeat `--install-dir`. @@ -252,7 +256,7 @@ replacement: | `AGENTS_API_EXECUTION_OPTIONS_FILE` | Default models, set per harness in Web (**System**, **Default model**) or with `PUT /core/v1/harnesses/{harness}/model-provider` | `config/core.env`. `--convert` does not carry it over; it leaves the file, which may hold model keys, and reports it. Delete the file once the defaults are set | | `AGENTS_API_DAEMON_WS_URL` (a `wss://…/api/v1/agent-daemon/ws` URL) | `OAC_PUBLIC_URL` (the origin, such as `https://core.example`), or `public_url` in `config.json` | `config/core.env` | | `AGENTS_API_CONFIG_FILE` | None; delete the line | `config/core.env` | -| `AGENTS_API_MANAGED_RUNTIMES_FILE` | The sandbox deployment in the database, set in Web | `config/core.env`; see [older file-managed installations](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/HOSTED-SANDBOX-MANAGER.md#older-file-managed-installations) | +| `AGENTS_API_MANAGED_RUNTIMES_FILE` | The sandbox deployment in the database, set in Web | `config/core.env`; see [older file-managed installations](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/HOSTED-SANDBOX-MANAGER.md#older-file-managed-installations) | | `admin/sandbox-admin.key`, `admin/digests.json` | `admin/core.key`, `admin/core-key-digests.json` (conversion then moves them to `secrets/` and `generated/`) | Files and their `compose.json` mounts | | `AGENTS_API_SANDBOX_ADMIN_DIGESTS_FILE` | `OAC_CORE_KEY_DIGESTS_FILE` | `config/core.env` | | `CORE_CONSOLE_ADMIN_TOKEN_FILE`, `install.sh --admin-token-file` | `OAC_WEB_CORE_KEY_FILE`, `--core-key-file` | Web environment; installer flag | @@ -329,10 +333,11 @@ Nodes from releases that used the removed `/core/v1/sandbox/enroll` and microsandbox deployment: 1. Drain with the previous release while its nodes are connected: enter maintenance, - archive retained hosted Sessions and wait until nothing is retained (the - [maintenance procedure](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-maintenance)). + archive retained hosted Sessions and wait until nothing is retained using that + previous release's maintenance procedure. 2. Upgrade Core and Web, and change the routing as above. -3. Save the new release's Runtime, resume, and add the nodes again. On each node host, +3. Reset a drained selection that lacks a specification, then configure the new + release's Runtime and add nodes again. On each node host, stop the old node service and move its state directory aside as a backup first. A Web-selected Docker or microsandbox deployment saved before deployment specifications @@ -399,9 +404,9 @@ Core records which key made each public resource write, for the console. Set `core.write_audit_retention` in `config.json` (Go duration, at least `1h`, default `2160h`) to control how long non-creation history is kept; creation ownership is kept for good. Removing keys or resources does not delete these records. See the -[query contract](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/write-audit.md); administrator mutations +[query contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/write-audit.md); administrator mutations have a separate -[audit log](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/contracts/agents-api/admin-api.md#monitoring-and-audit). Neither +[audit log](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md#monitoring-and-audit). Neither logs bodies or secrets. -[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/75bf4484ba957344ae374354858bbdfe104f5c4f/docs/getting-started/operations.md) +[Repository source](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/docs/getting-started/operations.md) diff --git a/apps/docs/content/guide-sources.json b/apps/docs/content/guide-sources.json index e768b0c2b..614a1706f 100644 --- a/apps/docs/content/guide-sources.json +++ b/apps/docs/content/guide-sources.json @@ -1,61 +1,61 @@ { - "revision": "75bf4484ba957344ae374354858bbdfe104f5c4f", + "revision": "f36ac3592bfe9f77a2172680aa52b14c6f525062", "sources": { "docs/getting-started/README.md": "e4ca3bcbfdcbf3832c1d9e0525f98083ca11a4e42148f9f45090bbe1bc0f2b5c", "docs/design-principles.md": "5e1bff3cdcc77cd02ad692f67977cb4e0a5654989982d722bac0e3c79dbf82cb", - "docs/web/architecture.md": "23e5758d72687c2c821975f326f24a789545ce0b0f566802cbe901edf2ebe3f1", + "docs/web/architecture.md": "80bdb8053c06c2b2030b193aae17d5434545f9c72af50e9d3df3ade522fe25ed", "docs/getting-started/install.md": "15cfe81d4005a7337ac82bf6c35a8b441e9f9b0f2a2299021ad6c539f3f336a3", - "docs/configuration.md": "7fe061b93a09c0f7baaa70f33f0863d8b29d308bbbc9b0e42b35cbef0ef0c648", + "docs/configuration.md": "401bf72003cd2eb03ae217d49591273ea0a5c42d68190d1a7ad2f12169c287ea", "docs/web/core-connection.md": "46f86520e70bf41079708e2c398655aa9087ad9488c8e661d9a73035d11098c4", "docs/getting-started/quickstart.md": "5effeffcfb343f7939910f0c1758e1492d49e7bd66c409b1819cefe67af592d7", - "docs/api/README.md": "f8c3872b012af7c490a06578c7d56560e0ec5998865ef043523bc66cc10ae4be", + "docs/api/README.md": "b2b4c7fdf9fbde62d2169014180698a77c671f85c1c68c2e5de0e73bbdf3b13c", "contracts/agents-api/execution-tools.md": "e0b61f6c0c236c362186c5f6d0ad69a16dd8a8d9afabe1329a1fe22e5f4a71e9", "docs/api/public-agent-api.md": "18829b366bd2e49fe5beb76dc8a709bc1aa8e35d4fbfd2484a0c52b46cf75417", "contracts/agents-api/environments.md": "89ffc903ee44d00ee87c16950202a3ca4c8345160780dcdd4ed2a71b25b1c2dc", - "docs/getting-started/nodes.md": "00d98f03d0ac967575bfa35ee960ff24b971bff1531a264517b384376986ac6f", + "docs/getting-started/nodes.md": "a79e0007c1712c457b40b7573423fd1fdb95d52e283567b4c3d732393fac5109", "docs/getting-started/self-hosted.md": "3f1b402cf28bb320ba414eaf36110e9df4d792b81da0456a3c42d469357d8d99", "apps/web/public/onboarding/monitor-en.webp": "29dc220cb1250c7016b7c4bf7f30e9510b07c07b814a48c3aa9303b2d76f20b2", - "docs/web/README.md": "3b96e673591da67a6f666dddafd49a565c066c347bdfa2eaba0f92ab03daf514", - "docs/api/web-management.md": "5480bfcf9dd24a8ffc8d442e4d8582e007dadd22dba6a686d71b4bede0616506", + "docs/web/README.md": "9fce295f6f6ff04acefe338679c1e54723e123dd0027a778202adfa9ab95b6c4", + "docs/api/web-management.md": "ce248fbeb1ded0f3fe4c6021c800c268ebaf4bd37eb1292aecc3d919d138a391", "contracts/agents-api/runtime-observability-api.md": "cd46e777fe716a7e76374f4655c96dfbad9d8be1bfca6203ed6118ab87874c2b", - "docs/getting-started/operations.md": "abc965e6ba19d2a2c148c8b5afde7203ca39891383840d85379d0d433b39df1d", + "docs/getting-started/operations.md": "b930724d4518621bd6bd8457c70e6aac8c041677ad532a2b63894ef0d20ba892", "apps/docs/scripts/guides.json": "9d1ba424f8fadd65ffa6175eaa9c7bc4878d3e1437971099812c984e1e593080" }, "outputs": { - "content/docs/index.mdx": "2cd2e1d7adb6b38bfb1808bc5f9d9b2f699d02b02eb233c0ee72438884dffe9a", + "content/docs/index.mdx": "75339c35292bb1f3c6b4313d717b82686bee8caa265bb575fcecbd0ba33bb780", "content/docs/index.zh.mdx": "5bdd41893946f1d229eb4cf1599a97f2e38cb9e09722d2a69f62053376ac88a9", - "content/docs/concepts.mdx": "32120464dd98c1b80d5d4c57d7f2c7e4365f623532024a632b9fccbda730384a", + "content/docs/concepts.mdx": "304e18737cdf807562501981e5b8312466a7c7ce29db3bf4823075850dec10c4", "content/docs/concepts.zh.mdx": "c60644f395dbac5ee607cdb053249e05ec8352d28de97ef1c38d6c1e4443bbb1", - "content/docs/execution-model.mdx": "b3c928e353e173aa7eea5c4fbad06b6896fd1e97f69c87701de8af39fae054fe", + "content/docs/execution-model.mdx": "8dd05250255fdb39a4aee6ffa51a16379fbd19d2761f510e1ad9cc3a193f750c", "content/docs/execution-model.zh.mdx": "9976a9203c6a83d49c2c753e3a8528a369078e87aeb88850654da948ac8c17e1", - "content/docs/install.mdx": "439809c11f5d805e6c6f964c779ec01468dcdad88bd11a56bb180a6e6a13b98a", + "content/docs/install.mdx": "cba5c14e55c6f474d810dc8e545fadbb4f35bcad50eeb6ceef0cbd967b3fb1ab", "content/docs/install.zh.mdx": "7ed0b726cf721b18e85b89145ef8ebd2f5d382afd0489633a7b63962aa2fc4ed", - "content/docs/configure.mdx": "4ab159c54f7b9c6e1a2d0c3873de5e3641b48c4e1badf47f63311bef94f8778f", + "content/docs/configure.mdx": "ac8a8788881707287805811474aaabfc0d0f273aa2423b010b167443996d2530", "content/docs/configure.zh.mdx": "a0c3a1df3e9f75e672e1d0a8c68c82c24ac36a8479d10a951de441960d07927f", - "content/docs/bootstrap-projects-keys.mdx": "4a39e95986851828e3fbcc0550398146de9bb37cfe22f10df566c51a911ca9ed", + "content/docs/bootstrap-projects-keys.mdx": "fc2da0aa255278281d8a57c664d7d30e5a458add4e58402386e0374f23973da4", "content/docs/bootstrap-projects-keys.zh.mdx": "509b4dcea4c865d6834bcaef78baa681180d48274931230f958a39e46011c88f", - "content/docs/quickstart.mdx": "92f62b5df213c363bd7148de0462b2ca66f2054f4458d790196e75daa678bb24", + "content/docs/quickstart.mdx": "4ebc8ac16df66042e816ff51cf6672d8a85f44e119211efba584025666dce845", "content/docs/quickstart.zh.mdx": "c5f119b100b102918a45763834d0ed7fd436479fb411e16a62a408411dbcb3e1", - "content/docs/public-api.mdx": "a3d39d873054614bdf410d8c2a1e5c33320f1f3bffe155657b64579ea28fdc48", + "content/docs/public-api.mdx": "ee1be51bf72bc48809efc0af8a460c26ae1cbf962771e132ec521447b89d5479", "content/docs/public-api.zh.mdx": "fb0efb9be2fba3f4a3dc71bf053d2c7eddffbfe010f59fa5a1ae60a3f068545b", - "content/docs/agents-and-tools.mdx": "78789f7e6a1d327ff514d872b9f8b55a88754ffbfc34b7b10e5f7c115f242e3c", + "content/docs/agents-and-tools.mdx": "b8cc50792e579cbf66bcc597685b745d8721266970b75e928d8edee2ce11b08b", "content/docs/agents-and-tools.zh.mdx": "cc8d4e0ef0ea894f068f3bbd143d5ee70c6346bf98f425b1d5fe07ec3931f128", - "content/docs/sessions.mdx": "eaef17849bce13c9f3c22966910e081fcc4851c59a7a8b972e917a45bdebfb57", + "content/docs/sessions.mdx": "8dba8c2a94ef5e95fde6f53e7af3cc481e4d3c5db71815372f01ddd1bf729316", "content/docs/sessions.zh.mdx": "b1b1a7f8aacd9a902320f9692173ede24036124f85761cf02f23ef3ef2bcc4c1", - "content/docs/environments-and-files.mdx": "411328cb87962ee9849021ed14de144ab9d4b31ecb8a42954079c9142d479e98", + "content/docs/environments-and-files.mdx": "0fce7c8806d532d64eff6319bba7d0a8e44da46f888d224644b3ab5c23428c41", "content/docs/environments-and-files.zh.mdx": "ee64577220a798f023b330c65302c14438f25a34a78c4e36144bfdb4304a3ae5", - "content/docs/hosted-providers.mdx": "7d1825a5ef7e81d71a0c64bd00ce8fcc3c00d632a318eaa4063dacf876fdfee5", + "content/docs/hosted-providers.mdx": "e73657db13cbde92c1291534d910eba76172781a58645916ec7bfa770c42a5cf", "content/docs/hosted-providers.zh.mdx": "f54d80d6e65596dca6b20262abaa3dcb5af245227ffad146fcaed4bd69454dee", - "content/docs/self-hosted-execution.mdx": "67a134939247a5a3d16b4a935895259c37d8264377de53179e6598bdb82957d5", + "content/docs/self-hosted-execution.mdx": "34877624962b24c22e8371ebd2e2f83bbf5a0d004c74d5d9b5db57f5133b692c", "content/docs/self-hosted-execution.zh.mdx": "f11d34ce30d48039376256474bc9509a508a2f6b9906f49b5cf6e396707ae5c0", "public/images/source/apps/web/public/onboarding/monitor-en.webp": "29dc220cb1250c7016b7c4bf7f30e9510b07c07b814a48c3aa9303b2d76f20b2", - "content/docs/console.mdx": "a708f26941840375299f14b26dc69a278c1f44a7aedca90d50bb806ba05685c0", + "content/docs/console.mdx": "cf0e6a1f9dd690ffe74c5d83111bb18df7eebe287848497c01257692ab3edc6a", "content/docs/console.zh.mdx": "72f953ba46dcc95933773ac0c4dffdc0d0a499be9d9a1cb9467db2659f1fa10d", - "content/docs/admin-api.mdx": "23b0ebdbb3ab93cee6f93e083026318465fae581546de5af9953fbf779444452", + "content/docs/admin-api.mdx": "1f4917510d3335ec1e9fe0dd7527d0e625b4b9257df39c1691d2564261603959", "content/docs/admin-api.zh.mdx": "07006de30df3080e54c3cd79ed0ea5337dce557aa2f11afea84edb7f92884d38", - "content/docs/observability.mdx": "4894808be45568b3d6e1039183350a5315dbbde1144db94cc592eb8bd8c433e4", + "content/docs/observability.mdx": "c5e2963a4dd7abddc60869a3ecd3b6b32a97c2ba1ae28b88b7453136656c1201", "content/docs/observability.zh.mdx": "9574b3412a5754f41ab2d3a16062ac1cad06766be481a8e6bff1b84acbde7e7e", - "content/docs/troubleshooting.mdx": "2008f1bcdc5e8cfb55602e2d849093140a61682c68795d9c9f707d62e30b0c0a", + "content/docs/troubleshooting.mdx": "7ecf60de8a60e8765e7fe6cd3eba681c9ee6b76e3e374dcf1731bfc93a5f1a94", "content/docs/troubleshooting.zh.mdx": "a47714ea9890385e811b163bbb2dc1412ddd9c12747792cf53630fc06062e5e2" } } diff --git a/apps/docs/openapi/core-api.yaml b/apps/docs/openapi/core-api.yaml index 472e1f59e..87bd862f1 100644 --- a/apps/docs/openapi/core-api.yaml +++ b/apps/docs/openapi/core-api.yaml @@ -1696,7 +1696,7 @@ paths: tags: - Core Administration post: - description: Core key only. Requires maintenance and the current deployment generation. Permanently closes execution, requests cancellation and releases sandbox/snapshots through existing cleanup. Session history and persisted files/artifacts remain; unpersisted workspace contents are lost. A cleanup_pending response is not proof of resource release. Does not affect caller-managed Runtime. + description: Core key only. Requires the current deployment generation; no maintenance mode is required. Permanently closes execution, requests cancellation and releases sandbox/snapshots through existing cleanup. Session history and persisted files/artifacts remain; unpersisted workspace contents are lost. A cleanup_pending response is not proof of resource release. Does not affect caller-managed Runtime. parameters: - schema: type: string @@ -3257,7 +3257,7 @@ paths: tags: - Sandbox Manager post: - description: Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Exact retries return the existing selection; differing selections and file-managed deployments reject. This does not create compute or execute work. + description: Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Requires explicit expected_generation, including zero at first setup. Stale retries reject before provider validation. An identical selection at the current generation is a no-op; differing selections and file-managed deployments reject. This does not create compute or execute work. responses: '200': description: OK @@ -3308,7 +3308,7 @@ paths: description: Deployment selection required: true put: - description: Requires maintenance, the current generation and verified cleanup of all old resources. Credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory. A core_url member is rejected with 400; the address comes from the installation public URL. Historical records are retained; old node credentials and enrollments are retired. Explicitly resume after success. Never automatically retry an uncertain write. + description: Requires the current generation, the same backend type, no active reset and verified cleanup of all old resources. A backend type change requires reset. Credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory. A core_url member is rejected with 400; the address comes from the installation public URL. Historical records are retained; old node credentials and enrollments are retired. Never automatically retry an uncertain write. responses: '200': description: OK @@ -3358,9 +3358,16 @@ paths: $ref: '#/components/schemas/api.SandboxDeploymentChangeInput' description: Replacement deployment selection required: true - /core/v1/sandbox/deployment/maintenance: - patch: - description: Requires the current generation. Maintenance preserves execution, cleanup and history. Resume requires successfully activated provider configuration. + /core/v1/sandbox/deployment/reset: + delete: + description: Restores admission but never restores Sessions already archived. With no reset running this is an idempotent read, provided the generation still matches. + parameters: + - schema: + type: integer + description: Current deployment generation + in: query + name: expected_generation + required: true responses: '200': description: OK @@ -3400,15 +3407,59 @@ paths: $ref: '#/components/schemas/api.CoreErrorResponse' security: - DeploymentAdminAuth: [] - summary: Pause or resume new hosted allocations + summary: Cancel a sandbox deployment reset + tags: + - Sandbox Manager + post: + description: Archives hosted Sessions and waits for confirmed provider cleanup, preserving history and Files/Artifacts. Auto waits for started or waiting Turns and file writes until the durable deadline; force cancels them. The same clear is idempotent; force escalates auto. Requires the current generation. Self-hosted Sessions are unchanged. + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/store.RuntimeDeploymentView' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/api.CoreErrorResponse' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/api.CoreErrorResponse' + '409': + description: Conflict + content: + application/json: + schema: + $ref: '#/components/schemas/api.CoreErrorResponse' + '500': + description: Internal Server Error + content: + application/json: + schema: + $ref: '#/components/schemas/api.CoreErrorResponse' + '503': + description: Service Unavailable + content: + application/json: + schema: + $ref: '#/components/schemas/api.CoreErrorResponse' + security: + - DeploymentAdminAuth: [] + summary: Start or escalate a durable sandbox deployment reset tags: - Sandbox Manager requestBody: content: application/json: schema: - $ref: '#/components/schemas/store.SandboxMaintenanceRequest' - description: Maintenance state + $ref: '#/components/schemas/store.SandboxResetRequest' + description: Reset mode and current deployment generation required: true /core/v1/sandbox/enrollment-tokens: post: @@ -4368,11 +4419,15 @@ components: them; Core then uses the validated template build's cpus and memory_mib. runtime: $ref: '#/components/schemas/sandbox.RuntimeRelease' + required: + - expected_generation type: object api.SandboxDeploymentInput: properties: e2b: $ref: '#/components/schemas/api.SandboxE2BInput' + expected_generation: + type: integer provider: type: string resources: @@ -4383,6 +4438,8 @@ components: them; Core then uses the validated template build's cpus and memory_mib. runtime: $ref: '#/components/schemas/sandbox.RuntimeRelease' + required: + - expected_generation type: object api.SandboxE2BInput: properties: @@ -4902,14 +4959,17 @@ components: type: integer installation_id: type: string - maintenance: - type: boolean mode: type: string owner_epoch: type: integer provider: type: string + reset: + anyOf: + - allOf: + - $ref: '#/components/schemas/store.SandboxResetView' + - type: 'null' resources: $ref: '#/components/schemas/store.SandboxDeploymentResources' specification: @@ -5140,12 +5200,64 @@ components: - $ref: '#/components/schemas/store.SandboxE2BTemplateBuildView' description: The fixed template build as Core read it when this selection was saved. type: object - store.SandboxMaintenanceRequest: + store.SandboxResetOfflineNode: properties: + name: + type: string + node_id: + type: string + resources: + type: integer + type: object + store.SandboxResetRemaining: + properties: + busy: + type: integer + cleanup: + type: integer + idle: + type: integer + offline_nodes: + items: + $ref: '#/components/schemas/store.SandboxResetOfflineNode' + type: array + on_offline_nodes: + type: integer + type: object + store.SandboxResetRequest: + properties: + clear: + enum: + - auto + - force + type: string + deadline_seconds: + maximum: 86400 + minimum: 300 + type: integer expected_generation: + minimum: 0 type: integer - maintenance: - type: boolean + required: + - clear + - expected_generation + type: object + store.SandboxResetView: + properties: + clear: + type: string + deadline_at: + anyOf: + - type: string + - type: 'null' + forced_at: + anyOf: + - type: string + - type: 'null' + remaining: + $ref: '#/components/schemas/store.SandboxResetRemaining' + requested_at: + type: string type: object store.SandboxSuspensionView: properties: diff --git a/apps/docs/openapi/sources.json b/apps/docs/openapi/sources.json index ddcaef8ac..3633bab31 100644 --- a/apps/docs/openapi/sources.json +++ b/apps/docs/openapi/sources.json @@ -1,7 +1,7 @@ { "sources": { "contracts/agents-api/openapi.yaml": "a8896e8b67d78ed0608c7cc5636077ec33f4b4895a04bd6cfb8b038afe941e01", - "contracts/agents-api/core.openapi.yaml": "488a50fa308194224ee5ee879b7f6e38166d1b08dedc2bbc5bb39f38a844c87f", + "contracts/agents-api/core.openapi.yaml": "26500cd07c72e72bfda32dcc83594e8d182aeb08debf4b67ac3938b5a9a078fe", "contracts/agents-api/runtime.openapi.yaml": "b3fb025b20a0bcf4635d60339479b2b07f6dbed80f51e72f40a63b31e42b5f4a" }, "outputs": { @@ -21,7 +21,7 @@ "openapi/public-api.yaml": "84ae8a33717c817950d757990018edd10a4d31d5a343c8172baa099d101c5a1f", "content/docs/api-reference/meta.json": "56bf1dc145ccc8adde38c5f956f1a06f18ef18b646122250d01a294c516f8f83", "content/docs/api-reference/index.mdx": "9fd6b1c54149874d2999235067c101326f9e8a2e707a70b6672c81eb3ebfe63f", - "content/docs/api-reference/core/core-administration.mdx": "a1261c7998017de3761ca90ca073a0b89a6c528ff37a634f3d8b986ee9540fc5", + "content/docs/api-reference/core/core-administration.mdx": "5fc3ada51d2190a141f13738151e4d513390cc3bd66ce6f5013994bda68569b0", "content/docs/api-reference/core/deployment-model-providers.mdx": "b2795fadcc05a1e7195b6d34c3f234cb29cc5e3f580d1521e31965453e654347", "content/docs/api-reference/core/administrator-projects.mdx": "8fd687bb2a7119fa16e938d7c71710c2816afb9cf959eb6849ef5e6300c5419b", "content/docs/api-reference/core/agents.mdx": "b6c089fbea4b07bc4249e7c7ca5699d47f1512a4fc2d8ee01cb26910ae541a84", @@ -39,8 +39,8 @@ "content/docs/api-reference/core/skills.mdx": "ed899ade7b032b1732b3097d81419a8fdbb9ead0594fca9f2107da644b97031f", "content/docs/api-reference/core/vaults.mdx": "12d0aac96cd4a3472ef4eaa287f7945f78862017956e79bfc3ed0ef8e772e78a", "content/docs/api-reference/core/credentials.mdx": "27d586b9a8fd738b9947802f32c80458195370252886bc26abedccf2125ee981", - "content/docs/api-reference/core/sandbox-manager.mdx": "36f2895b9969d27be843d5cebf935bf14ca9400ad076e56074a25905e3df5d1b", - "openapi/core-api.yaml": "9872f80cec2661c8539b6652a294c16c668f581399afd20f76b198279faae9ed", + "content/docs/api-reference/core/sandbox-manager.mdx": "4e0d90b158d3cdb9632db53dfacaae8c2749596373422c554f81178c8a96e7e2", + "openapi/core-api.yaml": "38614bb103da6b89dceaedc8c82874bc92590a49da005f07a5e8a0147b291676", "content/docs/api-reference/core/meta.json": "588b163ed2de2100b0f8376688274f80698a6bf6ac037eef56d17c93fa8a5062", "content/docs/api-reference/core/index.mdx": "ddf0a8d87a6c21761beac703a32f2a40486a1c74abb341bd726f40c44a866405", "content/docs/api-reference/machine/sandbox-node.mdx": "5b367a08d43cea82686345d2ba1c75cc90627a7b2970fe42a0df9d77fb3e1e80", diff --git a/apps/docs/scripts/check-python.test.mjs b/apps/docs/scripts/check-python.test.mjs index e6bce3365..05d5b6697 100644 --- a/apps/docs/scripts/check-python.test.mjs +++ b/apps/docs/scripts/check-python.test.mjs @@ -21,7 +21,7 @@ test('route gate and Python regressions honor the selected interpreter without s cwd: appRoot, env: { ...process.env, OAC_TEST_OFFICIAL_SDK_PYTHON: selected }, encoding: 'utf8', }) assert.equal(result.status, 0, result.stdout + result.stderr) - if (mode === 'routes') assert.match(result.stdout, /Contract operations:\s+130/) + if (mode === 'routes') assert.match(result.stdout, /Contract operations:\s+[1-9]\d*\b/) else assert.match(result.stderr, /Ran 2 tests/) } const invocations = fs.readFileSync(log, 'utf8').trim().split('\n').map(line => JSON.parse(line)) diff --git a/apps/docs/scripts/generate-guides.mjs b/apps/docs/scripts/generate-guides.mjs index a40b69592..29cbb97b0 100644 --- a/apps/docs/scripts/generate-guides.mjs +++ b/apps/docs/scripts/generate-guides.mjs @@ -8,7 +8,7 @@ const app = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') const repo = path.resolve(app, '../..') const guides = JSON.parse(fs.readFileSync(path.join(app, 'scripts/guides.json'), 'utf8')) const routes = new Map(guides.map(g => [g.source, g.slug === 'index' ? '/' : '/' + g.slug])) -const sourceRevision = '75bf4484ba957344ae374354858bbdfe104f5c4f' +const sourceRevision = 'f36ac3592bfe9f77a2172680aa52b14c6f525062' const sourceURL = relative => `https://github.com/MiniMax-AI/parsar-core/blob/${sourceRevision}/${relative}` const digest = text => crypto.createHash('sha256').update(text).digest('hex') function mdx(text) { diff --git a/apps/web/DESIGN.md b/apps/web/DESIGN.md index 07de356dd..73d80fc76 100644 --- a/apps/web/DESIGN.md +++ b/apps/web/DESIGN.md @@ -736,8 +736,7 @@ while the deployment cannot be read; Show Getting started, a quiet row above the sidebar's account controls, opens it again at any time. ### Sandbox setup -Setting up hosted sandboxes, and changing the provider or resources in maintenance, -is a set of pages inside the Nodes page, one decision each: where sandboxes run (own +Setting up hosted sandboxes is a set of pages inside the Nodes page, one decision each: where sandboxes run (own machines or E2B), then the backend or the E2B account, then the size of each sandbox (three presets; E2B skips it, since each sandbox takes the template build's size), then a review. Choices are large cards that advance on a click; short indigo dashes @@ -756,8 +755,41 @@ then says to enter it again, with a link to that step. Advanced settings, one link away, hold the complete form: resources (not for E2B), the Runtime release and the E2B template. A change keeps the saved size and Runtime while the backend stays the same (a saved size outside the presets is -offered as Current); another backend starts from its standard size and this -console's Runtime, and E2B always needs its key again. Rules sit behind help tips. +offered as Current). Same-backend editing starts at size or E2B credentials with +the provider fixed, and is available only with zero held resources; +changing the backend requires reset and then a new setup. E2B always needs its key +again. Optional explanations sit behind help tips; errors and safety consequences +remain visible. + +### Sandbox reset +The deployment section offers explicit reset rather than maintenance/resume. Reuse +its existing panels and confirmation dialogs. Auto clear is selected first, with a +one-hour deadline editable from 5 minutes to 24 hours; Force clear and escalation +require destructive confirmation. State directly that hosted work is archived, +remaining active work may be cancelled, archived Sessions cannot resume and +unpersisted workspace contents may be lost. Histories and persisted Files/Artifacts +remain, and self-hosted execution is unaffected. Cancelling an active reset stops +further clearing but cannot undo completed archives. + +A persistent progress panel uses Core's busy, idle and cleanup counts, deadline and +named offline-node blockers. Bring blocked nodes online for confirmed cleanup; +never offer a browser-side force-release shortcut. Poll the deployment every five +seconds only while its reset is non-null. A passed deadline does not establish force +or completion; only a Core response does. Completion opens the existing setup flow, +with a new explicit save using the generation read from Core, including zero on a +fresh install. No automatic configuration replay or online-rollout progress exists. + +Read deployment progress independently of node details. Partial failures retain +successful facts with a visible stale/unavailable notice. An uncertain write opens +the existing recovery dialog and requires a new authoritative read before another +mutation; refresh reads state and never resubmits the write. The connection's +QueryClient owns both the authoritative deployment and pending or uncertain writes +across route transitions. Leaving Nodes cannot cancel or forget a submitted reset, +and a cached node snapshot cannot replace a newer reset or completion learned on +Overview or System. Returning to Nodes reads the shared deployment immediately and +refreshes node evidence separately. Only a successful authoritative read begun after +the write settles can release the mutation block; an earlier or still-pending read +cannot. Logout clears this connection-scoped state. ### System page Four sections, each saying where it changes. Installation: the public address, API diff --git a/apps/web/PRODUCT.md b/apps/web/PRODUCT.md index 7455ee3dd..881c7d8f0 100644 --- a/apps/web/PRODUCT.md +++ b/apps/web/PRODUCT.md @@ -107,6 +107,18 @@ workbench. - A node enrolled with an earlier Core address gets no new sandboxes, so on the Nodes list and its page its status is Old address, with "Remove and add again", never Available. +- **Sandbox reset** is an explicit administrator operation on Nodes. Auto clear is + the default, with a one-hour deadline (5 minutes–24 hours); Force clear requires + destructive confirmation. Reset stops new hosted Session admission, clears idle, + suspended and pending hosted work, and waits for busy Turns and file writes until + Core forces the remaining work. It does not affect self-hosted execution. + Histories and persisted Files/Artifacts remain; archived Sessions cannot resume, + and unpersisted workspace contents may be lost. Cancel stops further clearing + without undoing archives. Core alone reports progress and completion, including + resources blocked on named offline nodes; force does not bypass their cleanup. + Completion clears the backend configuration and retires old nodes/enrollment + credentials. A new configuration is then a separate deliberate save. Same-backend + editing currently requires zero held resources; online rollout is not available. - **E2B deployments** have no machines: the Nodes entry becomes Sandbox backend, and Overview and Sandbox metrics show the sandboxes Core holds in E2B's cloud (running, starting, size, template build) instead of node capacity, with no node column diff --git a/apps/web/e2e/console.ts b/apps/web/e2e/console.ts index 91e00a8d5..82aba5d5b 100644 --- a/apps/web/e2e/console.ts +++ b/apps/web/e2e/console.ts @@ -1,3 +1,4 @@ +import type { SandboxDeployment } from "@agents-core-web/agents-client"; import { expect, type APIRequestContext, type Page } from "@playwright/test"; const fixture = `http://127.0.0.1:${process.env.AGENTS_FIXTURE_PORT ?? 18092}`; @@ -42,6 +43,13 @@ export async function setNode(request: APIRequestContext, node: { id: string; na await request.post(`${fixture}/__fixture/node`, { data: node }); } +/** Supplies a later Core observation; only an explicit completion retires nodes and advances generation. */ +export async function setDeployment(request: APIRequestContext, deployment: Partial | { complete_reset: true }): Promise { + const response = await request.post(`${fixture}/__fixture/deployment`, { data: deployment }); + expect(response.ok()).toBe(true); + return response.json() as Promise; +} + /** Archives a project behind the console's back, as another administrator would. */ export async function archiveProject(request: APIRequestContext, projectId: string) { await request.post(`${fixture}/core/v1/projects/${projectId}/archive`, { headers: { cookie: "core_console=fixture-session" } }); diff --git a/apps/web/e2e/fixture-console.mjs b/apps/web/e2e/fixture-console.mjs index e2b25125b..459334ed7 100644 --- a/apps/web/e2e/fixture-console.mjs +++ b/apps/web/e2e/fixture-console.mjs @@ -71,8 +71,12 @@ function installation() { }; } +function unconfiguredDeployment(generation = 0, ownerEpoch = 3) { + return { installation_id: INSTALLATION_ID, provider: "", core_url: publicUrl(), reset: null, owner_epoch: ownerEpoch, generation, mode: "", resources: { allocations: 0, pending: 0 }, suspension: null }; +} + function configuredDeployment() { - return { installation_id: INSTALLATION_ID, provider: "docker", core_url: publicUrl(), maintenance: false, owner_epoch: 3, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, specification: { resources: { cpus: 2, memory_mib: 4096 }, runtime: release }, specification_digest: "fixture", suspension: null }; + return { installation_id: INSTALLATION_ID, provider: "docker", core_url: publicUrl(), reset: null, owner_epoch: 3, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, specification: { resources: { cpus: 2, memory_mib: 4096 }, runtime: release }, specification_digest: "fixture", suspension: null }; } /** The E2B template build as Core read it when the selection was saved. */ const templateBuild = { status: "ready", resources: { cpus: 2, memory_mib: 2048, root_disk_mib: 10240 } }; @@ -117,7 +121,7 @@ function reset(mode = "login", fresh = false, sandbox = "configured", nodes = "d // The providers whose node files the console serves (/console/config node_artifacts). nodeArtifacts: artifacts.split(",").filter(Boolean), }; - state.deployment = sandbox === "none" ? null : sandbox === "e2b" ? e2bDeployment() : configuredDeployment(); + state.deployment = sandbox === "none" ? unconfiguredDeployment() : sandbox === "e2b" ? e2bDeployment() : configuredDeployment(); // Each node reports the address it enrolled with; in "stale" mode the first one enrolled before public_url changed. // The seeded nodes enrolled before Core recorded enrollment IDs. state.nodes.forEach((node, index) => { node.core_url = address === "stale" && index === 0 ? OLD_URL : publicUrl(); node.enrollment_id = null; }); @@ -302,7 +306,34 @@ function nodeDetail(node) { }; } -async function sandboxRoute(request, response, path) { +async function sandboxRoute(request, response, path, url) { + // Retired even for authenticated callers; never reinterpret maintenance as reset. + if (path === "/deployment/maintenance") return error(response, 404, "Not found."); + if (path === "/deployment/reset" && (request.method === "POST" || request.method === "DELETE")) { + const input = request.method === "POST" ? await body(request) : { expected_generation: Number(url.searchParams.get("expected_generation")) }; + if (!Number.isInteger(input.expected_generation) || input.expected_generation < 0 || (request.method === "DELETE" && !url.searchParams.has("expected_generation"))) return error(response, 400, "expected_generation is required.", "invalid_request_error"); + if (input.expected_generation !== state.deployment.generation) return error(response, 409, "The sandbox configuration changed. Refresh before submitting again.", "sandbox_generation_stale"); + if (request.method === "DELETE") { + state.deployment.reset = null; + return send(response, 200, state.deployment); + } + if (input.clear !== "auto" && input.clear !== "force") return error(response, 400, "clear must be auto or force.", "invalid_request_error"); + const seconds = input.deadline_seconds ?? 3600; + if (!Number.isInteger(seconds) || seconds < 300 || seconds > 86400 || (input.clear === "force" && input.deadline_seconds !== undefined)) return error(response, 400, "Invalid reset deadline.", "invalid_request_error"); + if (state.deployment.reset?.clear === "force" && input.clear === "auto") return error(response, 409, "A force reset cannot return to auto clear.", "sandbox_reset_in_progress"); + if (!state.deployment.provider || state.deployment.reset?.clear === input.clear) return send(response, 200, state.deployment); + const now = new Date().toISOString(); + const previous = state.deployment.reset; + state.deployment.reset = { + clear: input.clear, requested_at: previous?.requested_at ?? now, + deadline_at: previous?.deadline_at ?? (input.clear === "auto" ? new Date(Date.now() + seconds * 1000).toISOString() : null), + forced_at: input.clear === "force" ? now : null, + remaining: previous?.remaining ?? { busy: state.deployment.resources.allocations, idle: state.deployment.resources.pending, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] }, + }; + // The fixture has no cleanup worker or deadline timer. Tests explicitly supply + // subsequent Core projections; a browser clock never completes a reset. + return send(response, 200, state.deployment); + } if (path === "/runtime-observations" && request.method === "GET") { // Only E2B reports a sandbox's disk. const e2b = state.deployment?.provider === "e2b"; @@ -314,8 +345,13 @@ async function sandboxRoute(request, response, path) { const initialize = request.method === "POST"; // As Core, before any state check: the address is config.json's public_url and read-only. if ("core_url" in input) return error(response, 400, "core_url is derived from the installation public URL (public_url in config.json, OAC_PUBLIC_URL for Core) and cannot be set here. Remove it.", "invalid_request_error", "core_url"); - if (initialize && state.deployment) return error(response, 409, "The sandbox deployment is already configured.", "sandbox_deployment_conflict"); - if (!initialize && !state.deployment) return error(response, 409, "The sandbox deployment is not configured.", "sandbox_deployment_conflict"); + if (!Number.isInteger(input.expected_generation) || input.expected_generation < 0) return error(response, 400, "expected_generation is required.", "invalid_request_error"); + if (input.expected_generation !== state.deployment.generation) return error(response, 409, "The sandbox configuration changed. Refresh before submitting again.", "sandbox_generation_stale"); + if (state.deployment.reset) return error(response, 409, "A sandbox reset is in progress.", "sandbox_reset_in_progress"); + if (!initialize && input.provider !== state.deployment.provider) return error(response, 409, "Reset before changing the sandbox backend.", "sandbox_reset_required"); + if (!initialize && state.deployment.resources.allocations + state.deployment.resources.pending > 0) return error(response, 409, "Sandbox resources remain.", "sandbox_in_use"); + if (initialize && state.deployment.provider) return error(response, 409, "The sandbox deployment is already configured.", "sandbox_deployment_conflict"); + if (!initialize && !state.deployment.provider) return error(response, 409, "The sandbox deployment is not configured.", "sandbox_deployment_conflict"); const e2b = input.provider === "e2b"; if (!e2b && (!input.resources || !input.runtime)) return error(response, 400, "resources and runtime are required.", "invalid_sandbox_configuration"); // As Core (ErrSandboxPublicURLUnreachable): E2B sandboxes reach Core over the internet, which a loopback public_url cannot serve. @@ -324,7 +360,7 @@ async function sandboxRoute(request, response, path) { const resources = input.resources ?? { cpus: templateBuild.resources.cpus, memory_mib: templateBuild.resources.memory_mib }; state.deployment = { ...configuredDeployment(), provider: input.provider, mode: e2b ? "direct" : "nodes", - ...(initialize ? {} : { maintenance: true, generation: state.deployment.generation + 1 }), + generation: state.deployment.generation + 1, owner_epoch: state.deployment.owner_epoch, specification: { resources, ...(input.runtime ? { runtime: input.runtime } : {}) }, ...(e2b ? { e2b: { template: input.e2b?.template ?? "", credential_configured: true, template_build: templateBuild } } : {}), suspension: input.provider === "microsandbox" ? { idle_seconds: 300, retention_seconds: 86400 } : null, @@ -332,10 +368,11 @@ async function sandboxRoute(request, response, path) { return send(response, 200, state.deployment); } if (path === "/deployment") { - return send(response, 200, state.deployment ?? { installation_id: INSTALLATION_ID, provider: "", core_url: publicUrl(), maintenance: false, owner_epoch: 3, generation: 0, mode: "", resources: { allocations: 0, pending: 0 }, suspension: null }); + return send(response, 200, state.deployment); } if (path === "/nodes") return send(response, 200, { data: state.nodes }); if (path === "/enrollment-tokens" && request.method === "POST") { + if (state.deployment.reset) return error(response, 409, "A sandbox reset is in progress.", "sandbox_reset_in_progress"); // As Core: a one-time token valid for ten minutes, whose limits and enrollment ID the node it enrolls takes; // only microsandbox keeps a retained limit above the active one. const input = await body(request); @@ -498,6 +535,17 @@ async function fixtureRoute(request, response, url) { reset(url.searchParams.get("auth") ?? "login", url.searchParams.get("projects") === "none", url.searchParams.get("sandbox") ?? "configured", url.searchParams.get("nodes") ?? "demo", url.searchParams.get("installation") ?? "public", url.searchParams.get("credentials") ?? "configured", url.searchParams.get("installers") !== "none", url.searchParams.get("artifacts") ?? undefined); return send(response, 200, { ok: true }); } + if (url.pathname === "/__fixture/deployment" && request.method === "POST") { + // Explicit backend observations, never a simulation driven by browser time. + const input = await body(request); + if (input.complete_reset) { + state.deployment = unconfiguredDeployment(state.deployment.generation + 1, state.deployment.owner_epoch + 1); + state.nodes = []; + state.allocations = []; + state.enrollment = null; + } else Object.assign(state.deployment, input); + return send(response, 200, state.deployment); + } if (url.pathname === "/__fixture/fail-next" && request.method === "POST") { state.failNext = await body(request); // { method, path, status, code?, message? } return send(response, 200, { ok: true }); @@ -537,7 +585,7 @@ http.createServer(async (request, response) => { state.failNext = null; return error(response, fail.status, fail.message ?? "Injected failure.", fail.code ?? null); } - if (url.pathname.startsWith("/core/v1/sandbox/")) return await sandboxRoute(request, response, url.pathname.slice("/core/v1/sandbox".length)); + if (url.pathname.startsWith("/core/v1/sandbox/")) return await sandboxRoute(request, response, url.pathname.slice("/core/v1/sandbox".length), url); if (url.pathname.startsWith("/core/v1/")) { const path = url.pathname.slice("/core/v1".length); if (path === "/harnesses" || path.startsWith("/harnesses/")) return await harnessRoute(request, response, path); diff --git a/apps/web/e2e/nodes.spec.ts b/apps/web/e2e/nodes.spec.ts index 031c20562..9b56415b0 100644 --- a/apps/web/e2e/nodes.spec.ts +++ b/apps/web/e2e/nodes.spec.ts @@ -257,71 +257,30 @@ test("saves E2B without opening Add node, as it has no machines", async ({ page, await expect(page.getByRole("dialog")).toHaveCount(0); }); -test("keeps the saved size and Runtime for the same backend, and starts another from its defaults", async ({ page, request }) => { +test("edits only the saved backend, preserving a custom size and Runtime", async ({ page, request }) => { const runtime = { source_commit: "0".repeat(40), image_id: `sha256:${"a".repeat(64)}`, image_manifest_digest: `sha256:${"b".repeat(64)}`, microsandbox_ref: `oac-runtime@sha256:${"b".repeat(64)}`, runtime_sha256: "c".repeat(64), firmware_sha256: "d".repeat(64) }; const current = { resources: { cpus: 7, memory_mib: 8192 }, runtime }; - const deployment = { installation_id: "94be54a1-138c-4f30-bc87-b13686272dbe", provider: "docker", core_url: "https://core.example", maintenance: true, + let deployment = { installation_id: "94be54a1-138c-4f30-bc87-b13686272dbe", provider: "docker", core_url: "https://core.example", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, specification: current, specification_digest: "e".repeat(64), suspension: null }; let submitted: Record | null = null; await page.route("**/core/v1/sandbox/deployment", async (route) => { - if (route.request().method() === "PUT") submitted = route.request().postDataJSON() as Record; + if (route.request().method() === "PUT") { + submitted = route.request().postDataJSON() as Record; + deployment = { ...deployment, generation: 2 }; + } await route.fulfill({ json: deployment }); }); await page.route("**/core/v1/sandbox/nodes", (route) => route.fulfill({ json: { data: [] } })); await openConsole(page, request, "nodes"); - const save = page.getByRole("button", { name: "Save and stay in maintenance" }); - const back = page.getByRole("button", { name: "Back" }); - - // The same backend keeps its saved size and Runtime. - await page.getByRole("button", { name: "Change provider or resources" }).click(); - await page.getByRole("button", { name: "Own machines" }).click(); - await page.getByRole("button", { name: "Docker" }).click(); + await page.getByRole("button", { name: "Change resources" }).click(); + await expect(page.getByRole("button", { name: "Own machines" })).toHaveCount(0); + await expect(page.getByRole("button", { name: "E2B cloud" })).toHaveCount(0); await page.getByRole("button", { name: /^Current/ }).click(); - await save.click(); - await expect.poll(() => submitted?.resources).toEqual(current.resources); - expect(submitted?.runtime).toEqual(runtime); - // Core's address is config.json's public_url: a change never sends it. + await page.getByRole("button", { name: "Save configuration" }).click(); + await expect.poll(() => submitted).toMatchObject({ provider: "docker", expected_generation: 1, resources: current.resources, runtime }); expect(submitted).not.toHaveProperty("core_url"); - - // Another backend starts from its own size, with disks, and this console's Runtime. - await back.click(); - await back.click(); - await page.getByRole("button", { name: "microsandbox" }).click(); - await page.getByRole("button", { name: /^Standard/ }).click(); - await save.click(); - await expect.poll(() => submitted?.provider).toBe("microsandbox"); - expect(submitted?.resources).toEqual({ cpus: 2, memory_mib: 4096, root_disk_mib: 8192, environment_disk_mib: 8192 }); - expect(submitted?.runtime).toMatchObject({ source_commit: "c0ffee".padEnd(40, "0") }); - - // E2B needs its key again and takes its size from the template build: no size, Runtime or disks. - await back.click(); - await back.click(); - await back.click(); - await page.getByRole("button", { name: "E2B cloud" }).click(); - await page.getByLabel("E2B API key").fill("fixture-private-key"); - await page.getByLabel("Template build").fill("template:94be54a1-138c-4f30-bc87-b13686272dbe"); - await page.getByRole("button", { name: "Next" }).click(); - await save.click(); - await expect.poll(() => submitted?.provider).toBe("e2b"); - expect(submitted?.resources).toBeUndefined(); - expect(submitted?.runtime).toBeUndefined(); -}); - -test("reports a failed sandbox change in a dialog, then reads the state again", async ({ page, request }) => { - await openConsole(page, request, "nodes"); - // Core's answer is lost, so the change may have been saved. - await page.route("**/core/v1/sandbox/deployment/maintenance", (route) => route.fulfill({ - status: 503, - contentType: "application/json", - body: JSON.stringify({ error: { message: "Unavailable.", type: "server_error", code: null, param: null } }), - })); - await page.getByRole("button", { name: "Enter maintenance to change provider" }).click(); - const failed = page.getByRole("dialog", { name: "Couldn't confirm the sandbox change" }); - await failed.getByRole("button", { name: "Refresh sandbox state" }).click(); - await expect(failed).toBeHidden(); - await expect(page.getByRole("button", { name: "Enter maintenance to change provider" })).toBeEnabled(); }); test("keeps the page usable when Core refuses a sandbox change, and shows Core's reason", async ({ page, request }) => { diff --git a/apps/web/e2e/public-url.spec.ts b/apps/web/e2e/public-url.spec.ts index 156e4f4df..b406bd4c7 100644 --- a/apps/web/e2e/public-url.spec.ts +++ b/apps/web/e2e/public-url.spec.ts @@ -19,6 +19,7 @@ test("sets up sandboxes with Core's address read-only, never sending it", async await expect(page.getByRole("dialog", { name: "Add node" })).toBeVisible(); expect(await writes(request)).toEqual(["POST /core/v1/sandbox/deployment"]); expect(bodies.some((entry) => entry.includes("core_url"))).toBe(false); + expect(bodies.filter(Boolean).map((entry) => JSON.parse(entry))).toEqual([expect.objectContaining({ expected_generation: 0 })]); }); test("explains an E2B rejection in the wizard, with the file to edit and the command to apply it", async ({ page, request }) => { diff --git a/apps/web/e2e/sandbox-reset.spec.ts b/apps/web/e2e/sandbox-reset.spec.ts new file mode 100644 index 000000000..c7a302f32 --- /dev/null +++ b/apps/web/e2e/sandbox-reset.spec.ts @@ -0,0 +1,345 @@ +import { expect, test, type Page, type TestInfo } from "@playwright/test"; +import type { SandboxReset } from "@agents-core-web/agents-client"; + +import { expectManagementBoundary, FIXTURE_CORE_KEY, openConsole, setDeployment, writes } from "./console"; + +const deploymentURL = "**/core/v1/sandbox/deployment"; +const resetURL = "**/core/v1/sandbox/deployment/reset"; +const resetPath = "/core/v1/sandbox/deployment/reset"; +const unavailable = { status: 503, json: { error: { type: "server_error", code: null, param: null, message: "Deployment read unavailable." } } }; +const observedReset = (clear: "auto" | "force" = "auto"): SandboxReset => ({ + clear, requested_at: "2026-09-26T10:00:00Z", deadline_at: "2026-09-26T11:00:00Z", + forced_at: clear === "force" ? "2026-09-26T11:00:01Z" : null, + remaining: { busy: 2, idle: 1, cleanup: 1, on_offline_nodes: 2, offline_nodes: [{ node_id: "offline-owned", name: "offline-build-worker", resources: 2 }] }, +}); +const progress = (page: Page) => page.getByRole("region", { name: "Reset in progress" }); +async function openReset(page: Page) { + await page.getByRole("button", { name: "Reset deployment", exact: true }).click(); + return page.getByRole("dialog", { name: "Reset sandbox deployment?", exact: true }); +} +async function capture(page: Page, info: TestInfo, name: string) { + const path = info.outputPath(`${name}.png`); + await page.screenshot({ path, fullPage: true }); + await info.attach(name, { path, contentType: "image/png" }); +} + +test.afterEach(async ({ request }) => { + await expectManagementBoundary(request); + expect((await writes(request)).filter((entry) => entry.includes("/maintenance"))).toEqual([]); +}); + +test("auto reset requires a bounded deadline, then preserves Core progress past that deadline", async ({ page, request }, info) => { + await page.setViewportSize({ width: 1280, height: 1000 }); + await page.emulateMedia({ colorScheme: "light", reducedMotion: "reduce" }); + await openConsole(page, request, "nodes"); + await setDeployment(request, { resources: { allocations: 3, pending: 1 } }); + await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); + const dialog = await openReset(page); + await expect(dialog).toBeVisible(); + await page.keyboard.press("Escape"); + await expect(dialog).toBeHidden(); + const entry = page.getByRole("button", { name: "Reset deployment", exact: true }); + await expect(entry).toBeFocused(); + expect(await writes(request)).toEqual([]); + await entry.press("Enter"); + await expect(dialog).toBeVisible(); + const deadline = dialog.getByLabel(/Force remaining work after \(seconds\)/); + await expect(deadline).toHaveValue("3600"); + await deadline.fill("299"); + await expect(dialog.getByRole("button", { name: "Reset deployment", exact: true })).toBeDisabled(); + await deadline.fill("86401"); + await expect(dialog.getByRole("button", { name: "Reset deployment", exact: true })).toBeDisabled(); + await deadline.fill("300"); + await expect(dialog).toContainText("Archived Sessions cannot be resumed"); + await expect(dialog).toContainText("self-hosted"); + await expect(page.locator("html")).toHaveAttribute("data-theme", "light"); + await capture(page, info, "reset-confirm-en-light-1280"); + const submitted = page.waitForRequest((sent) => sent.method() === "POST" && sent.url().endsWith(resetPath)); + await dialog.getByRole("button", { name: "Reset deployment", exact: true }).click(); + expect((await submitted).postDataJSON()).toEqual({ expected_generation: 1, clear: "auto", deadline_seconds: 300 }); + await expect(progress(page)).toBeVisible(); + // These timestamps are in the past: the browser still cannot invent force or completion. + await setDeployment(request, { resources: { allocations: 3, pending: 1 }, reset: observedReset() }); + await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); + await expect(progress(page)).toContainText("offline-build-worker"); + await expect(progress(page)).toContainText("Busy work can finish until the deadline."); + await expect(page.getByRole("button", { name: "Add node", exact: true })).toBeDisabled(); + await expect(page.getByRole("heading", { name: "Where should sandboxes run?" })).toHaveCount(0); + await capture(page, info, "reset-progress-en-light-1280"); + expect(await writes(request)).toEqual([`POST ${resetPath}`]); +}); + +test("escalation and cancellation require explicit confirmation and never undo previous clearing", async ({ page, request }) => { + await openConsole(page, request, "nodes"); + await setDeployment(request, { resources: { allocations: 3, pending: 1 }, reset: observedReset() }); + await page.reload(); + await progress(page).getByRole("button", { name: "Force reset now" }).click(); + let confirm = page.getByRole("dialog", { name: "Force reset now?" }); + await confirm.getByRole("button", { name: "Back", exact: true }).click(); + expect(await writes(request)).toEqual([]); + await progress(page).getByRole("button", { name: "Force reset now" }).click(); + confirm = page.getByRole("dialog", { name: "Force reset now?" }); + const force = page.waitForRequest((sent) => sent.method() === "POST" && sent.url().endsWith(resetPath)); + await confirm.getByRole("button", { name: "Force reset now", exact: true }).click(); + expect((await force).postDataJSON()).toEqual({ expected_generation: 1, clear: "force" }); + await expect(progress(page)).toContainText("Remaining hosted Sessions are being archived"); + await expect(progress(page)).toContainText("offline-build-worker"); + await expect(progress(page).getByRole("button", { name: "Force reset now" })).toHaveCount(0); + await progress(page).getByRole("button", { name: "Cancel reset" }).click(); + confirm = page.getByRole("dialog", { name: "Cancel reset?" }); + await expect(confirm).toContainText("Sessions already archived stay archived"); + const cancel = page.waitForRequest((sent) => sent.method() === "DELETE" && sent.url().includes(resetPath)); + await confirm.getByRole("button", { name: "Cancel reset", exact: true }).click(); + expect(new URL((await cancel).url()).searchParams.get("expected_generation")).toBe("1"); + await expect(progress(page)).toHaveCount(0); + await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); + expect(await writes(request)).toEqual([`POST ${resetPath}`, `DELETE ${resetPath}`]); +}); + +test("force reset requires confirmation and reconfiguration uses the completed generation once", async ({ page, request }) => { + await openConsole(page, request, "nodes"); + await setDeployment(request, { resources: { allocations: 1, pending: 0 } }); + await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); + const dialog = await openReset(page); + await dialog.getByLabel("Force — cancel remaining work now", { exact: true }).check(); + await expect(dialog).toContainText("Force immediately cancels remaining work"); + expect(await writes(request)).toEqual([]); + await dialog.getByRole("button", { name: "Reset deployment", exact: true }).click(); + await expect(progress(page)).toContainText("Remaining hosted Sessions are being archived"); + // Simulate the cleanup worker's completed projection, not elapsed browser time. + await setDeployment(request, { complete_reset: true }); + await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); + await page.getByRole("button", { name: "Own machines" }).click(); + await page.getByRole("button", { name: "microsandbox Recommended" }).click(); + await page.getByRole("button", { name: /^Standard/ }).click(); + const configured = page.waitForRequest((sent) => sent.method() === "POST" && sent.url().endsWith("/sandbox/deployment")); + await page.getByRole("button", { name: "Save configuration" }).click(); + expect((await configured).postDataJSON()).toMatchObject({ expected_generation: 2, provider: "microsandbox" }); + await expect(page.getByRole("dialog", { name: "Add node" })).toBeVisible(); + expect(await writes(request)).toEqual([`POST ${resetPath}`, "POST /core/v1/sandbox/deployment"]); +}); + +test("an applied reset with a lost response stays blocked through failed reads and is never replayed", async ({ page, request }) => { + await openConsole(page, request, "nodes"); + await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); + await setDeployment(request, { resources: { allocations: 1, pending: 0 } }); + await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); + await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); + let failReads = true; + await page.route(resetURL, async (route) => { + await route.fetch(); // Core accepted the mutation, but its response never reached this browser. + await route.abort("failed"); + }); + await page.route(deploymentURL, (route) => failReads ? route.fulfill(unavailable) : route.continue()); + const dialog = await openReset(page); + await dialog.getByRole("button", { name: "Reset deployment", exact: true }).click(); + const uncertain = page.getByRole("dialog", { name: "Couldn't confirm the sandbox change" }); + await expect(uncertain).toBeVisible(); + await uncertain.getByRole("button", { name: "Refresh sandbox state" }).click(); + await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeDisabled(); + expect(await writes(request)).toEqual([`POST ${resetPath}`]); + failReads = false; + await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); + await expect(progress(page)).toContainText("Busy work can finish until the deadline."); + await expect(progress(page).getByRole("button", { name: "Cancel reset" })).toBeEnabled(); + expect(await writes(request)).toEqual([`POST ${resetPath}`]); +}); + +test("a stale reset is refused until fresh state is reviewed, with no automatic resubmission", async ({ page, request }) => { + await openConsole(page, request, "nodes"); + await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); + await setDeployment(request, { generation: 2, resources: { allocations: 3, pending: 1 } }); + const dialog = await openReset(page); + const staleSubmission = page.waitForRequest((sent) => sent.method() === "POST" && sent.url().endsWith(resetPath)); + const freshRead = page.waitForResponse(async (received) => received.request().method() === "GET" + && received.url().endsWith("/sandbox/deployment") && received.ok() && (await received.json()).generation === 2); + await dialog.getByRole("button", { name: "Reset deployment", exact: true }).click(); + expect((await staleSubmission).postDataJSON()).toMatchObject({ expected_generation: 1 }); + await expect(page.getByText("Core has a newer sandbox configuration. Refresh and review it before submitting again.")).toBeVisible(); + expect(await (await freshRead).json()).toMatchObject({ generation: 2, reset: null }); + // A new generation discards the old confirmation. The read can reconcile the + // page, but submitting its old draft again would be an unauthorized replay. + await expect(dialog).toBeHidden(); + await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); + expect(await writes(request)).toEqual([`POST ${resetPath}`]); + + // The administrator reviews a new confirmation before the next explicit POST. + const reviewed = await openReset(page); + await expect(reviewed).toContainText("Archived Sessions cannot be resumed"); + expect(await writes(request)).toEqual([`POST ${resetPath}`]); + const resubmitted = page.waitForRequest((sent) => sent.method() === "POST" && sent.url().endsWith(resetPath)); + await reviewed.getByRole("button", { name: "Reset deployment", exact: true }).click(); + expect((await resubmitted).postDataJSON()).toEqual({ expected_generation: 2, clear: "auto", deadline_seconds: 3600 }); + await expect(progress(page)).toContainText("Busy work can finish until the deadline."); + expect(await writes(request)).toEqual([`POST ${resetPath}`, `POST ${resetPath}`]); +}); + +test("reset progress survives failed node and deployment reads with visible qualifications in Chinese dark mode", async ({ page, request }, info) => { + await page.setViewportSize({ width: 1280, height: 1000 }); + await page.emulateMedia({ colorScheme: "dark" }); + await openConsole(page, request, "nodes"); + await setDeployment(request, { resources: { allocations: 3, pending: 1 }, reset: observedReset() }); + await page.route("**/core/v1/sandbox/nodes", (route) => route.fulfill(unavailable)); + await page.reload(); + await expect(progress(page)).toContainText("offline-build-worker"); + // Node detail failure cannot block a control needing only fresh deployment state. + await expect(progress(page).getByRole("button", { name: "Cancel reset" })).toBeEnabled(); + await page.getByRole("button", { name: "Language and appearance" }).click(); + await page.getByRole("menuitemradio", { name: "简体中文" }).click(); + const localized = page.getByRole("region", { name: "正在重置", exact: true }); + await expect(localized).toContainText("offline-build-worker"); + await expect(localized).toContainText("正在归档空闲沙箱"); + await expect(page.locator("html")).toHaveAttribute("data-theme", "dark"); + await capture(page, info, "reset-progress-zh-dark-1280"); + await page.route(deploymentURL, (route) => route.fulfill(unavailable)); + await page.getByRole("button", { name: "刷新沙箱状态", exact: true }).click(); + await expect(localized).toContainText("offline-build-worker"); + await expect(localized.getByRole("button", { name: "取消重置" })).toBeDisabled(); + await expect(localized).toContainText("无法刷新重置进度。当前显示的是上次确认的数量。"); + expect(await writes(request)).toEqual([]); +}); + +test("a pending reset survives navigation and a lost response cannot reopen stale write controls", async ({ page, request }) => { + await openConsole(page, request, "nodes"); + await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); + await setDeployment(request, { resources: { allocations: 1, pending: 0 } }); + await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); + const dialog = await openReset(page); + let accepted!: () => void; + const coreAccepted = new Promise((resolve) => { accepted = resolve; }); + let release!: () => void; + const responseHeld = new Promise((resolve) => { release = resolve; }); + let failReads = true; + await page.route(deploymentURL, (route) => failReads ? route.fulfill(unavailable) : route.continue()); + await page.route(resetURL, async (route) => { + await route.fetch(); + accepted(); + await responseHeld; + // Navigation may already have aborted the original browser request. + await route.abort("failed").catch(() => {}); + }); + const responseLost = page.waitForEvent("requestfailed", { + predicate: (sent) => sent.method() === "POST" && sent.url().endsWith(resetPath), + }); + try { + await dialog.getByRole("button", { name: "Reset deployment", exact: true }).click(); + await coreAccepted; + const cachedAt = Date.now(); + // A hash navigation keeps the application/query cache alive, unlike a reload. + await page.evaluate(() => { window.location.hash = "overview"; }); + await expect(page.getByRole("heading", { name: "Overview", level: 1 })).toBeVisible(); + await page.getByRole("button", { name: "Nodes", exact: true }).click(); + const resetButton = page.getByRole("button", { name: "Reset deployment", exact: true }); + await expect(resetButton).toBeDisabled(); + await expect(page.getByRole("button", { name: "Add node", exact: true })).toBeDisabled(); + expect(Date.now() - cachedAt).toBeLessThan(30_000); + expect(await writes(request)).toEqual([`POST ${resetPath}`]); + + release(); + await responseLost; + // Ownership survives routing, but a departed page's local error dialog must + // not be resurrected on the new Nodes instance when that request settles. + await expect(page.getByRole("dialog", { name: "Couldn't confirm the sandbox change" })).toHaveCount(0); + await page.getByRole("button", { name: "Overview", exact: true }).click(); + await expect(page.getByRole("heading", { name: "Overview", level: 1 })).toBeVisible(); + await page.getByRole("button", { name: "Nodes", exact: true }).click(); + await expect(resetButton).toBeDisabled(); + // An explicit refresh also fails; neither navigation nor failure replays POST. + await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); + await expect(resetButton).toBeDisabled(); + expect(await writes(request)).toEqual([`POST ${resetPath}`]); + + failReads = false; + await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); + await expect(progress(page)).toContainText("Busy work can finish until the deadline."); + await expect(progress(page).getByRole("button", { name: "Cancel reset" })).toBeEnabled(); + expect(await writes(request)).toEqual([`POST ${resetPath}`]); + } finally { release(); } +}); + +test("Nodes adopts the reset and completion learned on Overview before its cached node data expires", async ({ page, request }) => { + await openConsole(page, request, "nodes"); + await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); + await expect(page.getByRole("button", { name: "Add node", exact: true })).toBeEnabled(); + const cachedAt = Date.now(); + await page.getByRole("button", { name: "Overview", exact: true }).click(); + await setDeployment(request, { resources: { allocations: 3, pending: 1 }, reset: observedReset() }); + await page.getByRole("button", { name: "Refresh", exact: true }).click(); + await expect(page.getByRole("heading", { name: "Sandbox reset in progress" })).toBeVisible(); + await page.getByRole("button", { name: "Nodes", exact: true }).click(); + await expect(progress(page)).toContainText("offline-build-worker"); + await expect(page.getByRole("button", { name: "Add node", exact: true })).toBeDisabled(); + await expect(page.getByRole("button", { name: "Change resources", exact: true })).toHaveCount(0); + expect(Date.now() - cachedAt).toBeLessThan(30_000); + + // No refresh click: the shared active reset starts bounded read-only polling on Nodes. + const forced = observedReset("force"); + forced.remaining = { busy: 0, idle: 0, cleanup: 1, on_offline_nodes: 1, offline_nodes: [{ node_id: "offline-owned", name: "offline-build-worker", resources: 1 }] }; + await setDeployment(request, { resources: { allocations: 1, pending: 0 }, reset: forced }); + await expect(progress(page)).toContainText("Remaining hosted Sessions are being archived", { timeout: 12_000 }); + await expect(progress(page).getByRole("button", { name: "Force reset now" })).toHaveCount(0); + + const completedAt = Date.now(); + await page.getByRole("button", { name: "Overview", exact: true }).click(); + await setDeployment(request, { complete_reset: true }); + await page.getByRole("button", { name: "Refresh", exact: true }).click(); + await expect(page.getByRole("heading", { name: "Sandbox reset in progress" })).toHaveCount(0); + await page.getByRole("button", { name: "Nodes", exact: true }).click(); + await expect(page.getByRole("heading", { name: "Where should sandboxes run?" })).toBeVisible(); + await expect(progress(page)).toHaveCount(0); + expect(Date.now() - completedAt).toBeLessThan(30_000); + expect(await writes(request)).toEqual([]); +}); + + +test("signing out clears pending reset ownership and a late response cannot overwrite the new login", async ({ page, request }) => { + await openConsole(page, request, "nodes"); + await setDeployment(request, { resources: { allocations: 1, pending: 0 } }); + await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); + const dialog = await openReset(page); + let accepted!: () => void; + const coreAccepted = new Promise((resolve) => { accepted = resolve; }); + let release!: () => void; + const responseHeld = new Promise((resolve) => { release = resolve; }); + await page.route(resetURL, async (route) => { + const response = await route.fetch(); + accepted(); + await responseHeld; + await route.fulfill({ response }); + }); + try { + await dialog.getByRole("button", { name: "Reset deployment", exact: true }).click(); + await coreAccepted; + await page.evaluate(() => { window.location.hash = "overview"; }); + await expect(page.getByRole("heading", { name: "Overview", level: 1 })).toBeVisible(); + await page.getByRole("button", { name: "Sign out", exact: true }).click(); + await expect(page.getByRole("heading", { name: "Sign in to OpenAgentCore" })).toBeVisible(); + // Core completes cleanup while signed out. The new login must read generation 2. + await setDeployment(request, { complete_reset: true }); + await page.getByLabel("Core key", { exact: true }).fill(FIXTURE_CORE_KEY); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await expect(page.getByRole("heading", { name: "Overview", level: 1 })).toBeVisible(); + await page.getByRole("button", { name: "Nodes", exact: true }).click(); + await page.getByRole("button", { name: "Own machines" }).click(); + await page.getByRole("button", { name: "microsandbox Recommended" }).click(); + await page.getByRole("button", { name: /^Standard/ }).click(); + const configured = page.waitForRequest((sent) => sent.method() === "POST" && sent.url().endsWith("/sandbox/deployment")); + await page.getByRole("button", { name: "Save configuration", exact: true }).click(); + expect((await configured).postDataJSON()).toMatchObject({ expected_generation: 2, provider: "microsandbox" }); + const add = page.getByRole("dialog", { name: "Add node" }); + await expect(add).toBeVisible(); + await add.getByRole("button", { name: "Close dialog" }).click(); + + const lateResponse = page.waitForResponse((received) => received.request().method() === "POST" && received.url().endsWith(resetPath)); + release(); + await (await lateResponse).finished(); + // The old generation-1 reset response cannot restore reset progress, block + // this login, or show the old route's result/error dialog. + const newDialog = await openReset(page); + await expect(newDialog).toBeVisible(); + await newDialog.getByRole("button", { name: "Back", exact: true }).click(); + await expect(progress(page)).toHaveCount(0); + await expect(page.getByRole("button", { name: "Add node", exact: true })).toBeEnabled(); + expect(await writes(request)).toEqual([`POST ${resetPath}`, "POST /core/v1/sandbox/deployment"]); + } finally { release(); } +}); diff --git a/apps/web/src/features/first-run/ConsoleAccess.tsx b/apps/web/src/features/first-run/ConsoleAccess.tsx index daa583713..f70521b1d 100644 --- a/apps/web/src/features/first-run/ConsoleAccess.tsx +++ b/apps/web/src/features/first-run/ConsoleAccess.tsx @@ -6,6 +6,7 @@ import { HelpTip } from "../../components/console-ui"; import { ThemeMenu } from "../../components/ThemeMenu"; import { useToast } from "../../components/Toast"; import { setLanguage } from "../../i18n"; +import { queryClient } from "../../lib/queries"; import { OnboardingLayout } from "../onboarding/OnboardingLayout"; import { withTransition } from "../onboarding/view-transition"; import { changeConsoleAuth, ConsoleAuthError, readConsoleAuth, type ConsoleAuth } from "./auth"; @@ -49,16 +50,24 @@ export function ConsoleAccess({ children }: { children: ReactNode }) { const [failed, setFailed] = useState(false); const [revision, setRevision] = useState(0); const generation = useRef(0); + const authMode = useRef(null); + const acceptStatus = useCallback((next: ConsoleAuth, newSession = false) => { + // clear() cancels query owners synchronously. Late mutations must retain + // their own ownership check before publishing into the new session. + if (newSession || (authMode.current === "authenticated" && next.mode === "login")) queryClient.clear(); + authMode.current = next.mode; + setStatus(next); + }, []); const refresh = useCallback(() => setRevision((current) => current + 1), []); useEffect(() => { const controller = new AbortController(); const current = ++generation.current; setFailed(false); void readConsoleAuth(controller.signal).then((value) => { - if (generation.current === current) setStatus(value); + if (generation.current === current) acceptStatus(value); }).catch(() => { if (!controller.signal.aborted && generation.current === current) setFailed(true); }); return () => { controller.abort(); generation.current++; }; - }, [revision]); + }, [revision, acceptStatus]); useEffect(() => { if (status?.mode !== "authenticated") return; const check = () => { if (document.visibilityState === "visible") refresh(); }; @@ -69,15 +78,16 @@ export function ConsoleAccess({ children }: { children: ReactNode }) { if (status?.mode === "authenticated") return { const next = await changeConsoleAuth({ action: "logout" }); generation.current++; - setStatus(next); + acceptStatus(next, true); } }}>{children}; return }> {status && !failed ? { + queryClient.clear(); // The console opens on the Overview, revealed from the pressed button. withTransition("enter", () => { generation.current++; - setStatus(next); + acceptStatus(next); }, from); }} /> :

{t(failed ? "Could not connect to your console." : "Connecting to your console…")}

diff --git a/apps/web/src/features/fleet/SandboxResetNotice.test.tsx b/apps/web/src/features/fleet/SandboxResetNotice.test.tsx new file mode 100644 index 000000000..eeefdd1df --- /dev/null +++ b/apps/web/src/features/fleet/SandboxResetNotice.test.tsx @@ -0,0 +1,33 @@ +import type { SandboxDeployment, SandboxReset } from "@agents-core-web/agents-client"; +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it } from "vitest"; + +import { SandboxResetNotice } from "./SandboxResetNotice"; + +const reset: SandboxReset = { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, + remaining: { busy: 1, idle: 0, cleanup: 2, on_offline_nodes: 1, offline_nodes: [{ node_id: "n1", name: "Node 1", resources: 1 }] } }; +const deployment = (reset: SandboxReset | null): SandboxDeployment => ({ installation_id: "i", provider: "docker", core_url: "http://core", reset, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 3, pending: 0 }, suspension: null }); +const render = (value: SandboxDeployment | undefined, failed = false) => renderToStaticMarkup( {}} />); + +describe("reset notices on read-only surfaces", () => { + it("shows active reset consequences and its existing management destination", () => { + const html = render(deployment(reset)); + expect(html).toContain("Sandbox reset in progress"); + expect(html).toContain("Waiting for Sessions and cleanup"); + expect(html).toContain("Self-hosted Sessions are unaffected"); + expect(html).toContain("View reset"); + }); + + it("shows force once Core reports the mode transition", () => { + expect(render(deployment({ ...reset, clear: "force", forced_at: "2026-09-27T11:00:00Z" }))).toContain("Forced reset"); + }); + + it("retains and qualifies the last reset on failure, then hides only after Core returns null", () => { + const html = render(deployment(reset), true); + expect(html).toContain("Sandbox reset in progress"); + expect(html).toContain("Retry"); + expect(render(deployment(null))).toBe(""); + expect(render(undefined)).toBe(""); + expect(render(undefined, true)).toContain("Retry"); + }); +}); diff --git a/apps/web/src/features/fleet/SandboxResetNotice.tsx b/apps/web/src/features/fleet/SandboxResetNotice.tsx new file mode 100644 index 000000000..75e760da7 --- /dev/null +++ b/apps/web/src/features/fleet/SandboxResetNotice.tsx @@ -0,0 +1,23 @@ +import type { SandboxDeployment } from "@agents-core-web/agents-client"; +import { useTranslation } from "react-i18next"; + +import { Section, StatusDot } from "../../components/console-ui"; +import { ReadFailure } from "../../components/ReadFailure"; +import { useConsoleNavigation } from "../../lib/console-navigation"; + +/** A reset is a deployment operation, separate from Core health and node reachability. */ +export function SandboxResetNotice({ deployment, failed, onRetry }: { + deployment: SandboxDeployment | undefined; + failed: boolean; + onRetry: () => void; +}) { + const { t } = useTranslation("overview"); + const { navigate } = useConsoleNavigation(); + return <> + {failed ? : null} + {deployment?.reset ?
navigate("nodes")}>{t("reset.view")}}> +
+

{t("reset.body")}

+
: null} + ; +} diff --git a/apps/web/src/features/fleet/fleet-model.test.ts b/apps/web/src/features/fleet/fleet-model.test.ts index dcf634fa3..97b4fe71f 100644 --- a/apps/web/src/features/fleet/fleet-model.test.ts +++ b/apps/web/src/features/fleet/fleet-model.test.ts @@ -19,10 +19,9 @@ describe("fleet model", () => { expect(nodeHealth(node("a", { online: false }))).toBe("offline"); }); - it("derives Core's own status from the Web API and maintenance", () => { - expect(coreStatus({ webApiReachable: null, maintenance: null })).toBe("checking"); - expect(coreStatus({ webApiReachable: false, maintenance: false })).toBe("unreachable"); - expect(coreStatus({ webApiReachable: true, maintenance: null })).toBe("running"); - expect(coreStatus({ webApiReachable: true, maintenance: true })).toBe("maintenance"); + it("reports Core reachability independently of sandbox operations", () => { + expect(coreStatus({ webApiReachable: null })).toBe("checking"); + expect(coreStatus({ webApiReachable: false })).toBe("unreachable"); + expect(coreStatus({ webApiReachable: true })).toBe("running"); }); }); diff --git a/apps/web/src/features/fleet/fleet-model.ts b/apps/web/src/features/fleet/fleet-model.ts index 375589410..c7007ff35 100644 --- a/apps/web/src/features/fleet/fleet-model.ts +++ b/apps/web/src/features/fleet/fleet-model.ts @@ -55,14 +55,13 @@ export function capacitySummary(nodes: readonly SandboxNode[]): CapacitySummary } /** - * Core itself, as far as the console can tell: whether the Web API answers - * and whether the sandbox deployment is in maintenance. Core reports no CPU or - * memory figures of its own yet. + * Core reachability is independent of a sandbox reset. Reset progress comes + * from the deployment read and does not imply a Core health failure. */ -export type CoreStatus = "checking" | "running" | "maintenance" | "unreachable"; +export type CoreStatus = "checking" | "running" | "unreachable"; -export function coreStatus(input: { webApiReachable: boolean | null; maintenance: boolean | null }): CoreStatus { +export function coreStatus(input: { webApiReachable: boolean | null }): CoreStatus { if (input.webApiReachable === false) return "unreachable"; if (input.webApiReachable === null) return "checking"; - return input.maintenance ? "maintenance" : "running"; + return "running"; } diff --git a/apps/web/src/features/fleet/fleet-queries.test.ts b/apps/web/src/features/fleet/fleet-queries.test.ts new file mode 100644 index 000000000..caa9f236d --- /dev/null +++ b/apps/web/src/features/fleet/fleet-queries.test.ts @@ -0,0 +1,50 @@ +import type { SandboxDeployment } from "@agents-core-web/agents-client"; +import { QueryClient } from "@tanstack/react-query"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { sandboxAdmin, sandboxDeploymentQuery, sandboxScope } from "../sandbox/sandbox-queries"; +import { fleetQuery } from "./fleet-queries"; + +const deployment: SandboxDeployment = { installation_id: "i", provider: "docker", core_url: "http://core", owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null, + reset: { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } }; + +afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); }); + +describe("fleet deployment evidence", () => { + it("retains a successful reset read in the shared cache when node reads fail", async () => { + const cache = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + vi.spyOn(sandboxAdmin, "retrieveDeployment").mockResolvedValue(deployment); + vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("node read failed"))); + await expect(cache.fetchQuery(fleetQuery(false))).rejects.toThrow(); + expect(cache.getQueryData(sandboxDeploymentQuery.queryKey)).toEqual(deployment); + cache.clear(); + }); + + it("does not clear the previous reset when its next deployment read fails", async () => { + const cache = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + cache.setQueryData(sandboxDeploymentQuery.queryKey, deployment); + vi.spyOn(sandboxAdmin, "retrieveDeployment").mockRejectedValue(new Error("deployment read failed")); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(JSON.stringify({ object: "list", data: [] }), { status: 200 }))); + await expect(cache.fetchQuery(fleetQuery(false))).rejects.toThrow(); + expect(cache.getQueryData(sandboxDeploymentQuery.queryKey)).toEqual(deployment); + expect(cache.getQueryState(sandboxDeploymentQuery.queryKey)?.status).toBe("error"); + cache.clear(); + }); + + it("cannot publish an older reset read after a sandbox write cancels the shared query", async () => { + const cache = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + let resolveRead!: (value: SandboxDeployment) => void; + vi.spyOn(sandboxAdmin, "retrieveDeployment").mockImplementation(() => new Promise((resolve) => { resolveRead = resolve; })); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(JSON.stringify({ object: "list", data: [] }), { status: 200 }))); + const fleetRead = cache.fetchQuery(fleetQuery(false)); + const rejected = expect(fleetRead).rejects.toThrow(); + await cache.cancelQueries({ queryKey: sandboxScope }); + const confirmed = { ...deployment, reset: null }; + cache.setQueryData(sandboxDeploymentQuery.queryKey, confirmed); + resolveRead(deployment); + await rejected; + expect(cache.getQueryData(sandboxDeploymentQuery.queryKey)).toEqual(confirmed); + cache.clear(); + }); + +}); diff --git a/apps/web/src/features/fleet/fleet-queries.ts b/apps/web/src/features/fleet/fleet-queries.ts index f0caaa4d0..275039af0 100644 --- a/apps/web/src/features/fleet/fleet-queries.ts +++ b/apps/web/src/features/fleet/fleet-queries.ts @@ -1,6 +1,7 @@ -import { queryOptions } from "@tanstack/react-query"; +import { queryOptions, type QueryClient } from "@tanstack/react-query"; import { SandboxAdminClient, type SandboxAllocation, type SandboxDeployment, type SandboxNode, type SandboxNodeHistoryRange } from "@agents-core-web/agents-client"; +import { sandboxDeploymentQuery } from "../sandbox/sandbox-queries"; import { sandboxConsoleConfig } from "../sandbox/console-config"; export interface FleetSnapshot { @@ -23,10 +24,11 @@ export const consoleConfigQuery = queryOptions({ queryFn: ({ signal }) => sandboxConsoleConfig(signal), }); -async function loadFleet(readAllocations: boolean, signal: AbortSignal): Promise { +async function loadFleet(readAllocations: boolean, signal: AbortSignal, cache: QueryClient): Promise { const sandbox = client(); const [deployment, nodes] = await Promise.all([ - sandbox.retrieveDeployment({ signal }), + // Keep reset truth in the shared deployment cache, even if a node read fails. + cache.fetchQuery({ ...sandboxDeploymentQuery, staleTime: 0 }), sandbox.listNodes({ signal }), ]); const allocations = readAllocations @@ -44,7 +46,7 @@ async function loadFleet(readAllocations: boolean, signal: AbortSignal): Promise export function fleetQuery(allocations: boolean) { return queryOptions({ queryKey: ["sandbox-fleet", allocations ? "with-allocations" : "nodes"], - queryFn: ({ signal }) => loadFleet(allocations, signal), + queryFn: ({ signal, client }) => loadFleet(allocations, signal, client), }); } diff --git a/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx b/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx new file mode 100644 index 000000000..c15a6dff3 --- /dev/null +++ b/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx @@ -0,0 +1,53 @@ +import type { SandboxDeployment } from "@agents-core-web/agents-client"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it } from "vitest"; + +import { gettingStartedSteps } from "../overview/getting-started"; +import { node } from "../overview/test-fixtures"; +import type { SandboxConsoleConfig } from "../sandbox/console-config"; +import { sandboxDeploymentQuery } from "../sandbox/sandbox-queries"; +import { consoleConfigQuery, fleetQuery, type FleetSnapshot } from "./fleet-queries"; +import { fleetSnapshot, useSandboxFleet } from "./use-sandbox-fleet"; + +const configured: SandboxDeployment = { installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null }; + +function Probe() { + const { state, deployment } = useSandboxFleet(); + const snapshot = fleetSnapshot(state); + const sandboxes = gettingStartedSteps({ fleet: state, sandboxReset: deployment.data ? deployment.data.reset !== null : undefined, localOnly: false, projects: [], sessions: 0, harnesses: [] }).sandboxes; + return
{snapshot ? `${snapshot.nodes.length} nodes / ${snapshot.deployment.resources.allocations} allocations` : "No current fleet evidence"}
; +} + +function render(latest: SandboxDeployment, previous = configured) { + const cache = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + const snapshot: FleetSnapshot = { deployment: previous, nodes: [node("n1")], allocations: [], loadedAt: 1 }; + const config: SandboxConsoleConfig = { sandbox_admin: true, node_installer: false, node_installer_sha256: "", self_hosted_installer: false, self_hosted_installer_sha256: "" }; + cache.setQueryData(consoleConfigQuery.queryKey, () => config); + cache.setQueryData(fleetQuery(false).queryKey, snapshot); + cache.setQueryData(sandboxDeploymentQuery.queryKey, latest); + const html = renderToStaticMarkup(); + cache.clear(); + return html; +} + +describe("fleet evidence after deployment changes", () => { + it("cannot complete onboarding or show old resource counts when reset leaves an unconfigured deployment", () => { + const resetting = { ...configured, reset: { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } } satisfies SandboxDeployment; + const completed = { ...configured, provider: "", mode: "", generation: 2, resources: { allocations: 0, pending: 0 } } satisfies SandboxDeployment; + const html = render(completed, resetting); + expect(html).toContain('data-fleet="loading"'); + expect(html).toContain('data-sandbox="checking"'); + expect(html).not.toContain("1 nodes"); + expect(html).not.toContain("1 allocations"); + }); + + it("also rejects prior installation, owner and provider evidence", () => { + for (const change of [{ installation_id: "next" }, { owner_epoch: 2 }, { provider: "e2b" as const }, { generation: 2 }]) { + const html = render({ ...configured, ...change }); + expect(html).toContain('data-fleet="loading"'); + expect(html).not.toContain('data-sandbox="done"'); + } + expect(render(configured)).toContain('data-sandbox="done"'); + }); +}); diff --git a/apps/web/src/features/fleet/use-sandbox-fleet.ts b/apps/web/src/features/fleet/use-sandbox-fleet.ts index 719257970..f28706d95 100644 --- a/apps/web/src/features/fleet/use-sandbox-fleet.ts +++ b/apps/web/src/features/fleet/use-sandbox-fleet.ts @@ -1,6 +1,8 @@ import { useQuery } from "@tanstack/react-query"; -import { useCallback } from "react"; +import { useCallback, useEffect } from "react"; +import type { SandboxDeployment } from "@agents-core-web/agents-client"; +import { sandboxDeploymentQuery } from "../sandbox/sandbox-queries"; import { consoleConfigQuery, fleetQuery, type FleetSnapshot } from "./fleet-queries"; export type { FleetSnapshot }; @@ -26,6 +28,7 @@ export function useSandboxFleet({ poll = true, allocations = false }: { poll?: b // A failed configuration read is a failure, not "no sandbox administration". const configFailed = config.isError && config.data === undefined; const adminAvailable = config.isPending || configFailed ? null : config.data?.sandbox_admin === true; + const deployment = useQuery({ ...sandboxDeploymentQuery, enabled: adminAvailable === true }); const fleet = useQuery({ ...fleetQuery(allocations), enabled: adminAvailable === true, @@ -33,23 +36,38 @@ export function useSandboxFleet({ poll = true, allocations = false }: { poll?: b refetchIntervalInBackground: false, }); + const differentDeployment = fleet.data !== undefined && deployment.data !== undefined && !fleetMatchesDeployment(fleet.data, deployment.data); + const { refetch: refetchFleet } = fleet; + // A reset/configuration change can arrive through the faster deployment poll. + // Never attach the earlier generation's nodes or counts to that new truth. + useEffect(() => { + if (adminAvailable === true && differentDeployment) void refetchFleet(); + }, [adminAvailable, differentDeployment, deployment.data?.installation_id, deployment.data?.owner_epoch, deployment.data?.generation, deployment.data?.provider, refetchFleet]); + let state: FleetState; if (configFailed) state = config.isFetching ? { status: "checking" } : { status: "failed", error: config.error }; else if (adminAvailable === null) state = { status: "checking" }; else if (!adminAvailable) state = { status: "unconfigured" }; + else if (differentDeployment) state = fleet.isError && !fleet.isFetching ? { status: "failed", error: fleet.error } : { status: "loading" }; else if (fleet.data) state = { status: "ready", snapshot: fleet.data, refreshing: fleet.isFetching, error: fleet.isError ? fleet.error : null }; else if (fleet.isError && !fleet.isFetching) state = { status: "failed", error: fleet.error }; else state = { status: "loading" }; - const { refetch: refetchFleet } = fleet; const { refetch: refetchConfig } = config; // Without sandbox administration a refresh asks the console again whether it has it. const refresh = useCallback(() => { void (adminAvailable === true ? refetchFleet() : refetchConfig()); }, [adminAvailable, refetchConfig, refetchFleet]); - return { state, refresh }; + return { state, refresh, deployment }; } export function fleetSnapshot(state: FleetState): FleetSnapshot | null { return state.status === "ready" ? state.snapshot : null; } + +/** Nodes and resource counts only describe the deployment generation they were read with. */ +function fleetMatchesDeployment(snapshot: FleetSnapshot, deployment: SandboxDeployment): boolean { + const previous = snapshot.deployment; + return previous.installation_id === deployment.installation_id && previous.owner_epoch === deployment.owner_epoch && + previous.generation === deployment.generation && previous.provider === deployment.provider; +} diff --git a/apps/web/src/features/metrics/CoreMetricsPage.tsx b/apps/web/src/features/metrics/CoreMetricsPage.tsx index 277d73001..2ec728614 100644 --- a/apps/web/src/features/metrics/CoreMetricsPage.tsx +++ b/apps/web/src/features/metrics/CoreMetricsPage.tsx @@ -19,7 +19,7 @@ const REFRESH_MS = 30_000; const GIB = 2 ** 30; const jobTone: Record = { ok: "ok", failing: "danger", stopped: "warning", unknown: "neutral" }; -const statusTone: Record = { running: "ok", maintenance: "warning", degraded: "warning", unknown: "neutral" }; +const statusTone: Record = { running: "ok", degraded: "warning", unknown: "neutral" }; function seconds(value: string | null): number | null { if (!value) return null; diff --git a/apps/web/src/features/metrics/SandboxMetricsPage.tsx b/apps/web/src/features/metrics/SandboxMetricsPage.tsx index 6655fa589..4d48ac2f8 100644 --- a/apps/web/src/features/metrics/SandboxMetricsPage.tsx +++ b/apps/web/src/features/metrics/SandboxMetricsPage.tsx @@ -32,6 +32,7 @@ import { RUNTIME_SNAPSHOT_REFRESH_MS } from "../dashboard/runtime-snapshot"; import { DiagnosticTip } from "../fleet/DiagnosticTip"; import { capacitySummary, nodeHealth, suspendedSandboxes, type NodeHealth } from "../fleet/fleet-model"; import { nodeDetailQuery } from "../fleet/fleet-queries"; +import { SandboxResetNotice } from "../fleet/SandboxResetNotice"; import { fleetSnapshot, useSandboxFleet, type FleetState } from "../fleet/use-sandbox-fleet"; import { sandboxSize, templateBuildStatus } from "../sandbox/deployment-specification"; import { formatShare, NodeHostCharts } from "./NodeHostCharts"; @@ -93,7 +94,7 @@ export function SandboxMetricsPage() { const { t, i18n } = useTranslation("metrics"); const locale = i18n.resolvedLanguage; const { navigate } = useConsoleNavigation(); - const { state: fleetState, refresh: refreshFleet } = useSandboxFleet({ allocations: true }); + const { state: fleetState, refresh: refreshFleet, deployment } = useSandboxFleet({ allocations: true }); const { state: runtimeState, refresh: refreshRuntime, stale: runtimeStale } = useHostedRuntimes(); const fleet = fleetSnapshot(fleetState); const capacity = fleet ? capacitySummary(fleet.nodes) : null; @@ -128,6 +129,7 @@ export function SandboxMetricsPage() { } /> + void deployment.refetch()} /> {cloud && fleet ? :
void }) { +export function GettingStarted({ fleet, sessions, localOnly, sandboxReset, onRetryInstallation }: { sandboxReset: boolean | "failed" | undefined; fleet: FleetState; sessions: number | "failed" | null; localOnly: boolean | "failed" | undefined; onRetryInstallation: () => void }) { const { t } = useTranslation("overview"); const { navigate } = useConsoleNavigation(); const openTour = useConsoleTour(); const projects = useQuery(projectsQuery); const harnesses = useQuery(harnessesQuery); const steps = gettingStartedSteps({ - fleet, sessions, localOnly, + fleet, sessions, localOnly, sandboxReset, projects: projects.data ?? (projects.isError ? "failed" : undefined), harnesses: harnesses.data?.data ?? (harnesses.isError ? "failed" : undefined), }); @@ -113,7 +113,9 @@ export function GettingStarted({ fleet, sessions, localOnly, onRetryInstallation const done = states.filter((state) => state === "done").length; const sandbox = steps.sandboxes; - const sandboxAction = localOnly === "failed" + const sandboxAction = sandboxReset === true + ? { label: t("reset.view"), run: () => navigate("nodes") } + : localOnly === "failed" ? { label: t("actions.retry", { ns: "common" }), run: onRetryInstallation } : sandbox.action === "setup" ? { label: t("gettingStarted.sandboxes.setup"), run: () => navigate("nodes") } @@ -145,7 +147,7 @@ export function GettingStarted({ fleet, sessions, localOnly, onRetryInstallation
    - + navigate("system", {}, "default-model") }} /> diff --git a/apps/web/src/features/overview/OverviewPage.tsx b/apps/web/src/features/overview/OverviewPage.tsx index 060ee2eff..8d0a514be 100644 --- a/apps/web/src/features/overview/OverviewPage.tsx +++ b/apps/web/src/features/overview/OverviewPage.tsx @@ -26,6 +26,7 @@ import { useConsoleNavigation } from "../../lib/console-navigation"; import { formatClock, formatCompact, formatInteger, formatPercent, formatRelative, MISSING } from "../../lib/format"; import { ProjectName, useProjects } from "../../lib/projects"; import { capacitySummary, coreStatus, type CoreStatus } from "../fleet/fleet-model"; +import { SandboxResetNotice } from "../fleet/SandboxResetNotice"; import { fleetSnapshot, useSandboxFleet, type FleetSnapshot, type FleetState } from "../fleet/use-sandbox-fleet"; import { type InProject } from "../metrics/project-sessions"; import { FleetTopology, TOPOLOGY_LIMIT, type CloudHost } from "./FleetTopology"; @@ -53,7 +54,7 @@ export const OVERVIEW_REFRESH_MS = 30_000; const ATTENTION_LIMIT = 8; const serviceTone: Record = { healthy: "ok", degraded: "warning", down: "danger", unknown: "pending" }; -const coreTone: Record = { checking: "pending", running: "ok", maintenance: "warning", unreachable: "danger" }; +const coreTone: Record = { checking: "pending", running: "ok", unreachable: "danger" }; type LoadState = | { status: "loading"; data: OverviewData | null } @@ -99,7 +100,7 @@ export function OverviewPage() { const projectsFailed = projectsState.state.status === "failed" || projectsState.refreshError !== null; const installation = useQuery(installationQuery); const { state, refresh, refreshing, failure } = useOverviewData(projects, projectsReady); - const { state: fleetState, refresh: refreshFleet } = useSandboxFleet(); + const { state: fleetState, refresh: refreshFleet, deployment } = useSandboxFleet(); const fleet = fleetSnapshot(fleetState); const data = projectsReady ? state.data : null; const refreshAll = () => { projectsState.refresh(); if (projectsReady) refresh(); refreshFleet(); void installation.refetch(); }; @@ -130,7 +131,7 @@ export function OverviewPage() { capacity, recentFailedSessions: failuresLastHour, }); - const core = coreStatus({ webApiReachable: reachable, maintenance: fleet ? fleet.deployment.maintenance : null }); + const core = coreStatus({ webApiReachable: reachable }); const offline = capacity ? capacity.nodes - capacity.online : 0; const degraded = capacity ? capacity.online - capacity.available : 0; const serviceReason = reachable === false @@ -171,9 +172,10 @@ export function OverviewPage() { actions={} /> + void deployment.refetch()} /> {installation.isError ? void installation.refetch()} partial={installation.data !== undefined} /> : null} - void installation.refetch()} /> + void installation.refetch()} /> {(readFailed || summaryError !== null) && data !== null ? : null}
    = {}): SandboxDeployment => ({ - installation_id: "i", provider: "docker", core_url: "http://core", maintenance: false, owner_epoch: 1, generation: 1, mode: "nodes", + installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, suspension: null, ...overrides, }); const fleet = (value: SandboxDeployment, nodes = [node("n1")]): FleetState => ({ status: "ready", snapshot: { deployment: value, nodes, allocations: [], loadedAt: 0 }, refreshing: false, error: null }); -const sandboxes = (state: FleetState) => gettingStartedSteps({ fleet: state, localOnly: false, projects: [], sessions: 0, harnesses: [] }).sandboxes; +const sandboxes = (state: FleetState) => gettingStartedSteps({ sandboxReset: false, fleet: state, localOnly: false, projects: [], sessions: 0, harnesses: [] }).sandboxes; const provider = { object: "core.model_provider", protocol: "responses", base_url: "https://model.example/v1", api_key_configured: true, updated_at: "2026-09-25T00:00:00Z" } as const; const harness = (id: CoreHarness["id"], fields: Partial = {}): CoreHarness => ({ object: "core.harness", id, enabled: true, default: false, model_provider: null, ...fields }); describe("Getting started steps", () => { + it("does not report ready during reset or an unreadable deployment, even with ready nodes", () => { + for (const backend of ["docker", "e2b"] as const) { + const input = { fleet: fleet(deployment({ provider: backend })), localOnly: false, projects: [project("p")], sessions: 1, + harnesses: [harness("codex", { default: true, model_provider: { ...provider, harness: "codex" } })] }; + for (const sandboxReset of [true, "failed", undefined] as const) { + const steps = gettingStartedSteps({ ...input, sandboxReset }); + expect(steps.sandboxes).toMatchObject({ state: sandboxReset === true ? "todo" : sandboxReset === "failed" ? "unknown" : null, action: "nodes" }); + expect(checklistView([steps.sandboxes.state, steps.model, steps.key.state, steps.session.state], "open")).not.toBe("complete"); + } + expect(gettingStartedSteps({ ...input, sandboxReset: false }).sandboxes.state).toBe("done"); + } + }); + + it("keeps stale fleet readiness unknown after a failed refresh", () => { + const stale = { ...fleet(deployment()), error: new Error("node read failed") } as FleetState; + expect(sandboxes(stale).state).toBe("unknown"); + }); + it("cannot complete onboarding while the installation read is pending or failed", () => { for (const localOnly of [undefined, "failed"] as const) { - const steps = gettingStartedSteps({ localOnly, fleet: fleet(deployment()), projects: [project("p")], sessions: 1, + const steps = gettingStartedSteps({ sandboxReset: false, localOnly, fleet: fleet(deployment()), projects: [project("p")], sessions: 1, harnesses: [harness("codex", { default: true, model_provider: { ...provider, harness: "codex" } })] }); expect(steps.sandboxes.state).toBe(localOnly === "failed" ? "unknown" : null); expect(checklistView([steps.sandboxes.state, steps.model, steps.key.state, steps.session.state], "open")).toBe("full"); @@ -25,7 +43,7 @@ describe("Getting started steps", () => { }); it("keeps local-only installations to do even with a ready node or cloud deployment", () => { for (const provider of ["docker", "e2b"] as const) { - const steps = gettingStartedSteps({ fleet: fleet(deployment({ provider })), projects: [], sessions: 1, harnesses: [], localOnly: true }); + const steps = gettingStartedSteps({ sandboxReset: false, fleet: fleet(deployment({ provider })), projects: [], sessions: 1, harnesses: [], localOnly: true }); expect(steps.sandboxes).toMatchObject({ state: "todo", action: "nodes", cloud: provider === "e2b" }); } }); @@ -44,7 +62,7 @@ describe("Getting started steps", () => { }); it("needs an active project with an active key, and any Session", () => { - const steps = (projects: Parameters[0]["projects"], sessions: number | "failed" | null = 0) => gettingStartedSteps({ fleet: { status: "loading" }, projects, sessions, harnesses: undefined }); + const steps = (projects: Parameters[0]["projects"], sessions: number | "failed" | null = 0) => gettingStartedSteps({ sandboxReset: false, fleet: { status: "loading" }, projects, sessions, harnesses: undefined }); expect(steps([]).key).toEqual({ state: "todo", project: null }); const older = project("p1", { active_key_count: 0, created_at: 1 }); const newer = project("p2", { active_key_count: 0, created_at: 2 }); @@ -56,7 +74,7 @@ describe("Getting started steps", () => { }); it("opens the call samples of the newest active project with a key, else of the newest active project", () => { - const call = (projects: Parameters[0]["projects"]) => gettingStartedSteps({ fleet: { status: "loading" }, projects, sessions: 0, harnesses: undefined }).session.project; + const call = (projects: Parameters[0]["projects"]) => gettingStartedSteps({ sandboxReset: false, fleet: { status: "loading" }, projects, sessions: 0, harnesses: undefined }).session.project; const keyed = project("p1", { created_at: 1 }); const newer = project("p2", { active_key_count: 0, created_at: 2 }); const archived = project("p3", { status: "archived", active_key_count: 0, created_at: 3 }); @@ -66,7 +84,7 @@ describe("Getting started steps", () => { }); it("needs a default model on the default harness, or on any enabled harness when none is default", () => { - const model = (harnesses: Parameters[0]["harnesses"]) => gettingStartedSteps({ fleet: { status: "loading" }, projects: undefined, sessions: null, harnesses }).model; + const model = (harnesses: Parameters[0]["harnesses"]) => gettingStartedSteps({ sandboxReset: false, fleet: { status: "loading" }, projects: undefined, sessions: null, harnesses }).model; const on = (id: CoreHarness["id"]) => ({ ...provider, harness: id }); expect(model([harness("codex", { default: true }), harness("claude_sdk", { model_provider: on("claude_sdk") })])).toBe("todo"); expect(model([harness("codex", { default: true, model_provider: on("codex") })])).toBe("done"); diff --git a/apps/web/src/features/overview/getting-started.ts b/apps/web/src/features/overview/getting-started.ts index 41e5847ae..36f3a8436 100644 --- a/apps/web/src/features/overview/getting-started.ts +++ b/apps/web/src/features/overview/getting-started.ts @@ -26,6 +26,8 @@ export interface GettingStartedSteps { export function gettingStartedSteps(input: { fleet: FleetState; + /** A separate deployment read keeps reset truth available when node reads fail. */ + sandboxReset: boolean | "failed" | undefined; /** Undefined while reading; a failed installation read cannot confirm readiness. */ localOnly?: boolean | "failed"; /** Undefined until the project list is read. */ @@ -37,7 +39,9 @@ export function gettingStartedSteps(input: { }): GettingStartedSteps { const { sessions } = input; return { - sandboxes: input.localOnly === undefined || input.localOnly === "failed" + sandboxes: input.sandboxReset !== false + ? { state: input.sandboxReset === "failed" ? "unknown" : input.sandboxReset ? "todo" : null, action: "nodes", cloud: input.fleet.status === "ready" && input.fleet.snapshot.deployment.provider === "e2b" } + : input.localOnly === undefined || input.localOnly === "failed" ? { state: input.localOnly === "failed" ? "unknown" : null, action: "nodes", cloud: false } : input.localOnly ? { state: "todo", action: "nodes", cloud: input.fleet.status === "ready" && input.fleet.snapshot.deployment.provider === "e2b" } : sandboxStep(input.fleet), model: modelStep(input.harnesses), @@ -60,6 +64,7 @@ function sandboxStep(fleet: FleetState): GettingStartedSteps["sandboxes"] { if (fleet.status !== "ready") { return { state: fleet.status === "failed" || fleet.status === "unconfigured" ? "unknown" : null, action: "nodes", cloud: false }; } + if (fleet.error) return { state: "unknown", action: "nodes", cloud: fleet.snapshot.deployment.provider === "e2b" }; const { deployment, nodes } = fleet.snapshot; if (!deployment.provider) return { state: "todo", action: "setup", cloud: false }; if (deployment.provider === "e2b") { diff --git a/apps/web/src/features/sandbox/NodeEnrollment.tsx b/apps/web/src/features/sandbox/NodeEnrollment.tsx index f4e8a6e77..2b624ee04 100644 --- a/apps/web/src/features/sandbox/NodeEnrollment.tsx +++ b/apps/web/src/features/sandbox/NodeEnrollment.tsx @@ -97,7 +97,11 @@ export function NodeEnrollment({ client, consoleConfig, deployment, nodes, open, // Nodes and their sandboxes reach Core at its public URL, so a loopback one serves no other machine; // and without the provider's node files the installer would fail on the host. Either way no command // is issued, nor before the installation is read: a failed read (an older Core, say) proves nothing. - const blocker: { text: string; failed?: boolean } | null = installation.data === undefined + const blocker: { text: string; failed?: boolean } | null = deployment.reset + ? { text: t("Node enrollment is paused while reset is in progress.") } + : !fresh + ? { text: t("Sandbox state is unconfirmed. Refresh before issuing a node command.") } + : installation.data === undefined ? installation.isError ? { text: t("The installation couldn't be read, so no command can be issued."), failed: true } : { text: t("Checking this installation's public URL…") } : !publicUrl ? { text: t("Nodes need an HTTPS public URL that other machines and their sandboxes can reach: set public_url in config.json and run oac apply") } @@ -200,7 +204,7 @@ export function NodeEnrollment({ client, consoleConfig, deployment, nodes, open, setCopiedMode("sudo"); setNoSudoOpen(false); } async function generate() { - if (request.current || !available || blocker || !limitsReady || activeLimit === null || retainedLimit === null) return; + if (request.current || !fresh || !available || blocker || !limitsReady || activeLimit === null || retainedLimit === null) return; const capacity = { max_active: activeLimit, max_retained: retainedLimit }; generation.current++; const controller = new AbortController(); request.current = controller; @@ -315,10 +319,9 @@ export function NodeEnrollment({ client, consoleConfig, deployment, nodes, open, {copiedMode === "user" ? <>{t("If root ran it, it is a system service:")} : null}
    : null} - {!fresh ?

    {t("Connection status unavailable. Refresh to check your node.")}

    : null} + {deployment.reset ?

    {t("Node enrollment is paused while reset is in progress.")}

    : !fresh ?

    {t("Connection status unavailable. Refresh to check your node.")}

    : null} {!ready ? requirements : null} } , document.body); } - diff --git a/apps/web/src/features/sandbox/SandboxDeploymentSettings.tsx b/apps/web/src/features/sandbox/SandboxDeploymentSettings.tsx index 2f4043148..65ddbe8dd 100644 --- a/apps/web/src/features/sandbox/SandboxDeploymentSettings.tsx +++ b/apps/web/src/features/sandbox/SandboxDeploymentSettings.tsx @@ -1,23 +1,24 @@ import { useState } from "react"; -import type { InitializeSandboxDeployment, SandboxDeployment } from "@agents-core-web/agents-client"; +import type { InitializeSandboxDeployment, SandboxDeployment, StartSandboxReset } from "@agents-core-web/agents-client"; import { useTranslation } from "react-i18next"; import { HelpTip } from "../../components/console-ui"; import { formatBytes, formatPeriod, MISSING } from "../../lib/format"; import type { MessageKey } from "../../lib/locale-strings"; import { sandboxProviderLabel } from "../../lib/sandbox-labels"; import { sandboxSize, templateBuildSize, templateBuildStatus } from "./deployment-specification"; +import { SandboxResetControls } from "./SandboxResetControls"; import { SandboxSetupWizard } from "./SandboxSetupWizard"; const MIB = 2 ** 20; const buildStatusLabel: Record, MessageKey> = { ready: "Ready", notReady: "Not ready", unknown: "Unknown state" }; -export function SandboxDeploymentSettings({ deployment, disabled, fresh, onMaintenance, onUpdate, onRefresh }: { +export function SandboxDeploymentSettings({ deployment, disabled, fresh, onReset, onCancelReset, onUpdate }: { deployment: SandboxDeployment; disabled: boolean; fresh: boolean; - onMaintenance: (maintenance: boolean) => Promise; + onReset: (input: StartSandboxReset) => Promise; + onCancelReset: (expectedGeneration: number) => Promise; onUpdate: (input: InitializeSandboxDeployment) => Promise; - onRefresh: () => void; }) { const { t, i18n } = useTranslation("sandbox"); const locale = i18n.resolvedLanguage?.startsWith("zh") ? "zh" : "en"; @@ -28,12 +29,12 @@ export function SandboxDeploymentSettings({ deployment, disabled, fresh, onMaint const build = deployment.e2b?.template_build; const buildSize = templateBuildSize(deployment); const sizeLabel = (value: { cpus: number; memory_mib: number }) => t("{{cpus}} CPU · {{memory}}", { cpus: value.cpus, memory: formatBytes(value.memory_mib * MIB) }); - const clean = fresh && deployment.resources?.allocations === 0 && deployment.resources?.pending === 0; + const clean = fresh && !deployment.reset && deployment.resources?.allocations === 0 && deployment.resources?.pending === 0; return

    {t("Deployment provider")}

    - {t("One provider serves this entire Core deployment. Switching requires maintenance and verified cleanup of all existing execution resources.")} + {t("One provider serves this deployment. Reset it before choosing a different backend.")} {deployment.provider === "e2b" ? ` ${t("Core creates E2B sandboxes directly. No node enrollment is needed.")} ${t("Saved configuration does not confirm execution readiness. Session and Environment state report actual execution.")}` : ""}
    @@ -59,29 +60,25 @@ export function SandboxDeploymentSettings({ deployment, disabled, fresh, onMaint
    {t("E2B credential")}
    {t(deployment.e2b?.credential_configured ? "Configured" : "Not configured")}
    : null} - {!deployment.maintenance ? : <> -
    {t("Open Sessions")}
    -

    - {t(clean ? "Core reports no remaining execution resources. You can choose the next provider." : "Provider changes are blocked until Core confirms complete cleanup.")} - {t("Maintenance pauses new hosted placement. Clean up existing execution resources before changing provider; historical Sessions and results are preserved by the switch.")} {t("Stopped or offline resources, snapshots, uncertain creates and pending environments still block switching. Deleting a Session alone does not prove cleanup; Core must confirm both counts are zero.")} -

    - {!changing ? : <> + {!deployment.reset ? <> +
    + + {t("Editing keeps the current provider and requires zero allocated resources and pending environments. It is not an online upgrade.")} +
    + {!clean ?

    {t("Changes require Core to confirm that no hosted resources remain.")}

    : null} + {changing ? <> - - - {t(deployment.provider === "e2b" ? "Saving does not migrate Sessions or resume placement automatically." : "Saving retires old node identities and enrollment credentials. It does not migrate Sessions or resume placement automatically.")} - - } - - - {t("Resume is explicit and succeeds only when Core activates the saved configuration. A failed activation leaves maintenance enabled.")} - - } +

    {t(deployment.provider === "e2b" ? "Saving changes only this provider's configuration; it does not confirm execution readiness." : "Saving retires existing node identities and enrollment commands. Add the nodes again after saving.")}

    + + : null} + : null} +
    ; } diff --git a/apps/web/src/features/sandbox/SandboxManagerView.css b/apps/web/src/features/sandbox/SandboxManagerView.css index 7d82b9362..3eeeb1eef 100644 --- a/apps/web/src/features/sandbox/SandboxManagerView.css +++ b/apps/web/src/features/sandbox/SandboxManagerView.css @@ -27,7 +27,7 @@ .sandbox-enrollment-status.connected { color: color-mix(in srgb, var(--success) 65%, var(--fg)); } .sandbox-add-node .sandbox-next-step { display: flex; flex-wrap: wrap; align-items: baseline; gap: 4px; font-size: 13px; } .sandbox-add-node .sandbox-command-expiry { font-size: 12px; } -.sandbox-maintenance { padding: 12px 16px; border: 1px solid var(--line); border-radius: 8px; font-size: 13px; } +.sandbox-notice { padding: 12px 16px; border: 1px solid var(--line); border-radius: 8px; font-size: 13px; } @media (max-width: 600px) { .sandbox-manager { padding: 22px 14px; } .sandbox-enrollment { padding: 18px; } @@ -99,3 +99,14 @@ background: color-mix(in srgb, var(--warning) 9%, var(--surface)); border-color: color-mix(in srgb, var(--warning) 32%, var(--line)); } + +/* Reset stays in the provider panel; only irreversible decisions interrupt with a dialog. */ +.sandbox-reset { border-top: 1px solid var(--line); padding-top: 16px; } +.sandbox-reset h3 { margin: 0; font-size: 15px; font-weight: 600; } +.sandbox-reset-offline { display: grid; gap: 6px; } +.sandbox-reset-offline ul { margin: 0; padding-left: 20px; overflow-wrap: anywhere; } +.sandbox-reset-options { display: grid; gap: 10px; border: 0; padding: 0; margin: 0; } +.sandbox-reset-options legend { font-weight: 600; margin-bottom: 10px; } +.sandbox-reset-options > label:not(.field) { display: flex; align-items: center; gap: 8px; } +.sandbox-reset-options input[type="number"] { max-width: 180px; } +.sandbox-reset-dialog p { line-height: 1.6; } diff --git a/apps/web/src/features/sandbox/SandboxManagerView.tsx b/apps/web/src/features/sandbox/SandboxManagerView.tsx index 91412e694..b6fcc0c7e 100644 --- a/apps/web/src/features/sandbox/SandboxManagerView.tsx +++ b/apps/web/src/features/sandbox/SandboxManagerView.tsx @@ -1,5 +1,5 @@ import { useCallback, useEffect, useRef, useState, type ReactNode, type RefObject } from "react"; -import type { InitializeSandboxDeployment, SandboxDeployment, SandboxNode } from "@agents-core-web/agents-client"; +import { type InitializeSandboxDeployment, type SandboxDeployment, type SandboxNode, type StartSandboxReset } from "@agents-core-web/agents-client"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import { ArrowLeft, Pencil, Plus, Server, Trash2 } from "lucide-react"; import { useTranslation } from "react-i18next"; @@ -13,7 +13,9 @@ import { InstallationNotice } from "../../components/InstallationNotice"; import { installationQuery } from "../../lib/installation"; import { sandboxConfigurationRejection, sandboxRequestError, sandboxWriteUncertain } from "../../lib/sandbox-labels"; import type { SandboxConsoleConfig } from "./console-config"; -import { sandboxAdmin, sandboxConsoleConfigQuery, sandboxDeploymentQuery, sandboxScope, sandboxSnapshotQuery, type SandboxSnapshot } from "./sandbox-queries"; +import { sandboxAdmin, sandboxConsoleConfigQuery } from "./sandbox-queries"; +import { useSandboxManagerState } from "./use-sandbox-manager-state"; +import { writeSandboxDeployment } from "./sandbox-deployment-write"; import { SandboxSetupWizard } from "./SandboxSetupWizard"; import { SandboxDeploymentSettings } from "./SandboxDeploymentSettings"; import { NodeEnrollment } from "./NodeEnrollment"; @@ -62,15 +64,14 @@ function SandboxManager({ consoleConfig }: { consoleConfig: SandboxConsoleConfig const { params, navigate, back: goBack } = useConsoleNavigation(); const client = sandboxAdmin; const queryClient = useQueryClient(); - const query = useQuery(sandboxSnapshotQuery); + const { ownership, deploymentQuery, query, deployment, compatible, inventoryLoading, snapshot } = useSandboxManagerState(); + const { refetch: refetchSnapshot } = query; const installation = useQuery(installationQuery); const localOnly = installation.data?.local_only === true; const { t: tCommon } = useTranslation("common"); - const snapshot: SandboxSnapshot | null = query.data ?? null; - const loading = query.isFetching; - // As before a reload clears the last error, a running read hides it. - const error: unknown = query.isFetching ? null : query.error; - const [busy, setBusy] = useState(false); + const loading = deploymentQuery.isFetching; + const error: unknown = deploymentQuery.error; + const busy = ownership.data.phase === "pending"; const [revision, setRevision] = useState(0); const [removeTarget, setRemoveTarget] = useState(null); const [editTarget, setEditTarget] = useState(null); @@ -80,16 +81,20 @@ function SandboxManager({ consoleConfig }: { consoleConfig: SandboxConsoleConfig const [removeError, setRemoveError] = useState(null); // After a removal, the host's uninstall command; it stays for the closing animation. const [cleanup, setCleanup] = useState<{ node: NodeCleanup; open: boolean } | null>(null); - // When a deployment write last had an uncertain outcome (no response, a timeout or a 5xx); only a read begun after it confirms the state again. - const [uncertainSince, setUncertainSince] = useState(null); - const setupNeedsRefresh = uncertainSince !== null && !(snapshot && snapshot.readAt > uncertainSince); - // The state on screen is Core's last successful read, with no uncertain write since. - const confirmed = snapshot !== null && !query.isError && !setupNeedsRefresh; + // QueryClient owns this across route lifetimes. Only a successful authoritative + // read begun after settlement releases uncertain/pending mutation ownership. + const setupNeedsRefresh = ownership.data.phase === "reconcile"; + const confirmed = deployment !== undefined && !deploymentQuery.isError && ownership.data.phase === "idle"; // Writes additionally wait for any read in flight. const fresh = confirmed && !loading; // A write with an uncertain outcome opens a dialog with the reason; the error stays for the closing animation. const [writeFailure, setWriteFailure] = useState<{ error: unknown; open: boolean } | null>(null); - const { refetch } = query; + const { refetch: refetchDeployment } = deploymentQuery; + const refetch = useCallback(async () => { + const result = await refetchDeployment(); + if (!result.isError) await refetchSnapshot(); + return result; + }, [refetchDeployment, refetchSnapshot]); const refresh = useCallback(() => { // Each refresh starts a new read (cancelling one in flight) and resets the forms, as a reload did. setRevision((value) => value + 1); @@ -121,36 +126,17 @@ function SandboxManager({ consoleConfig }: { consoleConfig: SandboxConsoleConfig async function changeDeployment(operation: (signal: AbortSignal) => Promise, fromWizard = false): Promise { const controller = lifetime.current; if (!controller || busy || loading || setupNeedsRefresh || !fresh) return false; - setBusy(true); setWriteFailure(null); + setWriteFailure(null); try { - const deployment = await operation(controller.signal); - if (!controller.signal.aborted) { - // Core's confirmed response replaces the deployment; nodes of an older generation no longer apply. - queryClient.setQueryData(sandboxSnapshotQuery.queryKey, (current): SandboxSnapshot => { - const same = current !== undefined && deployment.generation === current.deployment.generation; - return { deployment, nodes: same ? current.nodes : [], allocations: same ? current.allocations : [], readAt: current?.readAt ?? performance.now() }; - }); - // A later visit reads the whole snapshot again; other pages re-read the deployment now. - void queryClient.invalidateQueries({ queryKey: sandboxSnapshotQuery.queryKey, refetchType: "none" }); - void queryClient.invalidateQueries({ queryKey: sandboxDeploymentQuery.queryKey }); - // Overview and Sandbox metrics read the fleet separately and lay out by provider. - void queryClient.invalidateQueries({ queryKey: ["sandbox-fleet"] }); - return true; - } + const deployment = await writeSandboxDeployment(queryClient, operation); + return deployment !== null && !controller.signal.aborted; } catch (error) { if (!controller.signal.aborted) { - // Core rejected the configuration and saved nothing: the wizard explains why. if (fromWizard && sandboxConfigurationRejection(error) !== null) throw error; - if (sandboxWriteUncertain(error)) { - setUncertainSince(performance.now()); setWriteFailure({ error, open: true }); - // Nothing re-reads on its own: the operator refreshes to confirm. A later visit reads again. - void queryClient.invalidateQueries({ queryKey: sandboxScope, refetchType: "none" }); - } else { - // Core refused the change, so nothing changed: its reason, and the page stays usable as it was. - toast.show(t("Core rejected the sandbox change"), { tone: "error", detail: sandboxRequestError(error, locale), key: "sandbox-write" }); - } + if (sandboxWriteUncertain(error)) setWriteFailure({ error, open: true }); + else toast.show(t("Core rejected the sandbox change"), { tone: "error", detail: sandboxRequestError(error, locale), key: "sandbox-write" }); } - } finally { if (!controller.signal.aborted) setBusy(false); } + } return false; } /** First setup; own machines continue straight to adding the first node. */ @@ -160,9 +146,8 @@ function SandboxManager({ consoleConfig }: { consoleConfig: SandboxConsoleConfig async function update(input: InitializeSandboxDeployment) { await changeDeployment((signal) => client.updateDeployment({ ...input, expected_generation: snapshot!.deployment.generation }, { signal }), true); } - async function maintenance(maintenance: boolean) { - await changeDeployment((signal) => client.setMaintenance({ maintenance, expected_generation: snapshot!.deployment.generation }, { signal })); - } + const startReset = (input: StartSandboxReset) => changeDeployment((signal) => client.startReset(input, { signal })); + const cancelReset = (expectedGeneration: number) => changeDeployment((signal) => client.cancelReset(expectedGeneration, { signal })); const writeDialog = setAdding(true)); // A node enrolled with another address than Core's current one (config.json's public_url) gets no new sandboxes until it is added again. const staleNodes = hostedNodes && snapshot ? nodes.filter((node) => onOldAddress(node, snapshot.deployment.core_url)).map((node) => node.name || node.id) : []; @@ -216,10 +202,13 @@ function SandboxManager({ consoleConfig }: { consoleConfig: SandboxConsoleConfig const refreshButton = ; const readFailure = error !== null ? sandboxRequestError(error, locale) : null; // A failed refresh keeps the last state on screen and says so in a toast, once while the failure lasts. - useFailureToast(snapshot && query.isError ? sandboxRequestError(query.error, locale) : null, t("Refresh failed; showing the last loaded state."), "sandbox-read"); + useFailureToast(snapshot && deploymentQuery.isError ? sandboxRequestError(deploymentQuery.error, locale) : null, t("Refresh failed; showing the last loaded state."), "sandbox-read"); const status = <> - {!snapshot && (loading || query.isPending) ?

    {t("Loading sandbox state…")}

    : null} + {!snapshot && (loading || deploymentQuery.isPending) ?

    {t("Loading sandbox state…")}

    : null} + {snapshot?.nodesError ? : null} + {snapshot && deploymentQuery.isError ?

    {t("Refresh failed; showing the last loaded state.")}

    : null} + {setupNeedsRefresh ?

    {t("Refresh sandbox state to confirm whether the change was saved before submitting again.")}

    : null} {busy ? {t("Saving sandbox change…")} : null} {!snapshot && readFailure ? : null} ; @@ -243,7 +232,7 @@ function SandboxManager({ consoleConfig }: { consoleConfig: SandboxConsoleConfig window.requestAnimationFrame(() => heading.current?.focus()); }} />; // Rendered first in both the list and a node's page, so an open command outlives the navigation. - const enrollment = hostedNodes && snapshot ? setAdding(false)} onRefresh={refreshNodes} /> : null; + const enrollment = hostedNodes && snapshot ? setAdding(false)} onRefresh={refreshNodes} /> : null; if (params.id && hostedNodes) { const back = () => goBack("nodes"); @@ -255,13 +244,13 @@ function SandboxManager({ consoleConfig }: { consoleConfig: SandboxConsoleConfig title={selected ? selected.name || selected.id : params.id} actions={<> {refreshButton} - {selected ? : null} - {selected ? : null} + {selected ? : null} + {selected ? : null} } />
    {status} - {selected ? : snapshot && !loading ? ( + {selected ? : snapshot && nodesConfirmed && !loading ? ( {t("Back")}} /> ) : null}
    @@ -290,24 +279,23 @@ function SandboxManager({ consoleConfig }: { consoleConfig: SandboxConsoleConfig const actions = <> {localOnly && hostedNodes ? {tCommon("installationNotice.addBlocked")} : null} {refreshButton} - {hostedNodes && snapshot ? : null} + {hostedNodes && snapshot ? : null} ; return <> {enrollment} - +
    {status} - {snapshot && !snapshot.deployment.provider ? : null} + {snapshot && !snapshot.deployment.provider && !snapshot.deployment.reset ? : null} {snapshot?.deployment.provider ? <> - {snapshot.deployment.maintenance ?

    {t("Maintenance is enabled. New sandbox placement is paused.")}

    : null} - {staleNodes.length ?

    {staleNodes.length === 1 + {staleNodes.length ?

    {staleNodes.length === 1 ? t("{{name}} is still bound to an old Core address. Remove it and add it again.", { name: staleNodes[0] }) : t("{{count}} nodes are still bound to an old Core address: {{names}}. Remove them and add them again.", { count: staleNodes.length, names: new Intl.ListFormat(i18n.resolvedLanguage, { type: "conjunction" }).format(staleNodes) })}

    : null} - + {hostedNodes ?
    {nodes.length - ? navigate("nodes", { id: node.id })} onRemove={askRemove} /> - : } + ? navigate("nodes", { id: node.id })} onRemove={askRemove} /> + : nodesConfirmed ? : null}
    : null} : null}
    diff --git a/apps/web/src/features/sandbox/SandboxResetControls.test.tsx b/apps/web/src/features/sandbox/SandboxResetControls.test.tsx new file mode 100644 index 000000000..5ba09fbef --- /dev/null +++ b/apps/web/src/features/sandbox/SandboxResetControls.test.tsx @@ -0,0 +1,38 @@ +import type { SandboxDeployment, SandboxReset } from "@agents-core-web/agents-client"; +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it } from "vitest"; +import { SandboxResetControls } from "./SandboxResetControls"; + +const reset: SandboxReset = { clear: "auto", requested_at: "2020-01-01T10:00:00Z", deadline_at: "2020-01-01T11:00:00Z", forced_at: null, + remaining: { busy: 2, idle: 1, cleanup: 3, on_offline_nodes: 2, offline_nodes: [{ node_id: "offline-a", name: "Offline A", resources: 2 }] } }; +const deployment = (value: SandboxReset | null): SandboxDeployment => ({ installation_id: "i", provider: "docker", core_url: "https://core.example", reset: value, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 5, pending: 1 }, suspension: null }); +const render = (value: SandboxReset | null, stale = false) => renderToStaticMarkup( true} onCancel={async () => true} />); + +describe("authoritative reset progress", () => { + it("uses Core's partition and named offline blockers and never advances from the browser clock", () => { + const html = render(reset); + expect(html).toContain("Busy
    2"); + expect(html).toContain("Idle
    1"); + expect(html).toContain("Awaiting cleanup
    3"); + expect(html).toContain("On offline nodes
    2"); + expect(html).toContain("Offline A"); + expect(html).toContain("Busy work can finish until the deadline"); + expect(html).toContain("Force reset now"); + expect(html).not.toContain("Remaining hosted Sessions are being archived"); + }); + + it("keeps zero remaining resources in progress until Core completes, and force still requires offline cleanup", () => { + const empty = { ...reset, remaining: { busy: 0, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } }; + expect(render(empty)).toContain("Reset in progress"); + const forced = render({ ...reset, clear: "force", forced_at: "2020-01-01T11:00:00Z" }); + expect(forced).toContain("Force reset does not bypass them"); + expect(forced).not.toContain("Force reset now"); + expect(forced).toContain("Cancel reset"); + }); + + it("visibly qualifies stale progress and removes it only when Core returns null", () => { + expect(render(reset, true)).toContain("last confirmed counts"); + expect(render(null)).not.toContain("Reset in progress"); + expect(render(null)).toContain("Reset deployment"); + }); +}); diff --git a/apps/web/src/features/sandbox/SandboxResetControls.tsx b/apps/web/src/features/sandbox/SandboxResetControls.tsx new file mode 100644 index 000000000..2ba4c8fad --- /dev/null +++ b/apps/web/src/features/sandbox/SandboxResetControls.tsx @@ -0,0 +1,94 @@ +import type { SandboxDeployment, StartSandboxReset } from "@agents-core-web/agents-client"; +import { useId, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { HelpTip } from "../../components/console-ui"; +import { Modal } from "../../components/Modal"; +import { formatDateTime, formatInteger } from "../../lib/format"; +import { validResetDeadline } from "./sandbox-reset"; + +type Action = "start" | "force" | "cancel"; + +/** Reset is durable Core work. These controls submit once and only display Core's progress. */ +export function SandboxResetControls({ deployment, disabled, stale, onStart, onCancel }: { + deployment: SandboxDeployment; + disabled: boolean; + stale: boolean; + onStart: (input: StartSandboxReset) => Promise; + onCancel: (expectedGeneration: number) => Promise; +}) { + const { t, i18n } = useTranslation("sandbox"); + const id = useId(); + const reset = deployment.reset; + const [dialog, setDialog] = useState<{ action: Action; generation: number; requestedAt: string | null } | null>(null); + const [clear, setClear] = useState<"auto" | "force">("auto"); + const [deadline, setDeadline] = useState("3600"); + const [submitting, setSubmitting] = useState(false); + const changed = dialog !== null && (dialog.generation !== deployment.generation || dialog.requestedAt !== (reset?.requested_at ?? null)); + const valid = dialog?.action !== "start" || clear === "force" || validResetDeadline(deadline); + const blocked = disabled || submitting || changed || !valid; + const open = (action: Action) => { + setClear("auto"); setDeadline("3600"); + setDialog({ action, generation: deployment.generation, requestedAt: reset?.requested_at ?? null }); + }; + async function confirm() { + if (!dialog || blocked) return; + setSubmitting(true); + try { + const confirmed = dialog.action === "cancel" + ? await onCancel(dialog.generation) + : await onStart({ expected_generation: dialog.generation, clear: dialog.action === "force" ? "force" : clear, ...(dialog.action === "start" && clear === "auto" ? { deadline_seconds: Number(deadline) } : {}) }); + if (confirmed) setDialog(null); + } finally { setSubmitting(false); } + } + const title = t(dialog?.action === "cancel" ? "Cancel reset?" : dialog?.action === "force" ? "Force reset now?" : "Reset sandbox deployment?"); + const label = t(dialog?.action === "cancel" ? "Cancel reset" : dialog?.action === "force" ? "Force reset now" : "Reset deployment"); + return <> + {reset ?
    +

    {t("Reset in progress")}

    {t("Progress comes from Core. Offline resources still need confirmed cleanup, even in force mode.")}
    +

    {t(reset.clear === "auto" ? "Idle sandboxes are being archived. Busy work can finish until the deadline." : "Remaining hosted Sessions are being archived and their work cancelled.")}

    + {stale ?

    {t("Reset progress could not be refreshed. These are the last confirmed counts.")}

    : null} +
    +
    {t("Busy")}
    {formatInteger(reset.remaining.busy, i18n.resolvedLanguage)}
    +
    {t("Idle")}
    {formatInteger(reset.remaining.idle, i18n.resolvedLanguage)}
    +
    {t("Awaiting cleanup")}
    {formatInteger(reset.remaining.cleanup, i18n.resolvedLanguage)}
    +
    {t("On offline nodes")}
    {formatInteger(reset.remaining.on_offline_nodes, i18n.resolvedLanguage)}
    + {reset.clear === "auto" && reset.deadline_at ?
    {t("Force deadline")}
    {formatDateTime(Date.parse(reset.deadline_at) / 1000, i18n.resolvedLanguage)}
    : null} +
    + {reset.remaining.offline_nodes.length ?
    +

    {t("Bring these nodes back online so Core can confirm cleanup. Force reset does not bypass them.")}

    +
      {reset.remaining.offline_nodes.map((node) =>
    • {node.name || node.node_id}{" · "}{t("{{count}} resources", { count: node.resources })}
      • )}
    +

    {t("Remove a node only after Core confirms it holds no resources.")}

    +
    : null} +

    {t("New hosted Sessions and node enrollment are paused. Setup becomes available only after Core completes the reset.")}

    +
    + {reset.clear === "auto" ? : null} + +
    +
    : } + { if (!submitting) setDialog(null); }} footer={<> + + + }> +
    + {dialog?.action === "cancel" ?

    {t("Stop the reset and allow new hosted Sessions again. Sessions already archived stay archived and cannot be resumed.")}

    : <> +

    {t("Reset archives this deployment's hosted Sessions. Archived Sessions cannot be resumed. History, Files and Artifacts are kept; self-hosted Sessions are unchanged.")}

    +

    {t("Unsaved workspace contents may be lost.")}

    +

    {t("After cleanup, Core clears the provider and saved E2B credential, retires nodes and enrollment commands, and returns to setup.")}

    + {dialog?.action === "force" ?

    {t("Force immediately cancels remaining work. Offline resources can still block cleanup.")}

    :
    + {t("Clear hosted work")} + +

    {t("Idle, suspended and queued Sessions are archived immediately. Running or waiting Turns and file writes may finish until the deadline; then Core forces the rest.")}

    + {clear === "auto" ? : null} + + {clear === "force" ?

    {t("Force immediately cancels remaining work. Offline resources can still block cleanup.")}

    : null} +
    } + } + {changed ?

    {t("Sandbox state changed. Close this dialog and review the current state.")}

    : null} +
    +     + ; +} diff --git a/apps/web/src/features/sandbox/SandboxSetupWizard.tsx b/apps/web/src/features/sandbox/SandboxSetupWizard.tsx index 2f5dedb54..8ab6f9a43 100644 --- a/apps/web/src/features/sandbox/SandboxSetupWizard.tsx +++ b/apps/web/src/features/sandbox/SandboxSetupWizard.tsx @@ -76,17 +76,19 @@ function presetOf(provider: SandboxProvider, resources: SandboxResources): Prese * Core's address is config.json's `public_url`: the review only shows it, and * a configuration Core rejects for it is explained here, where it was saved. */ -export function SandboxSetupWizard({ coreUrl, current, disabled, switching = false, onSubmit }: { +export function SandboxSetupWizard({ coreUrl, expectedGeneration, current, disabled, editing = false, onSubmit }: { /** The deployment's read-only Core address. */ coreUrl: string; + expectedGeneration: number; current?: { provider: SandboxProvider; specification?: SandboxSpecification; e2bTemplate?: string }; disabled: boolean; - switching?: boolean; + /** PR-R edits only the saved provider; changing backend requires reset first. */ + editing?: boolean; onSubmit: (input: InitializeSandboxDeployment) => Promise; }) { const { t } = useTranslation("sandbox"); const id = useId(); - const [step, setStep] = useState("where"); + const [step, setStep] = useState(editing ? current?.provider === "e2b" ? "e2b" : "size" : "where"); const [where, setWhere] = useState(current ? (current.provider === "e2b" ? "direct" : "nodes") : null); const [provider, setProvider] = useState(current?.provider ?? null); const saved = provider && current ? savedSpecification(provider, current.provider, current.specification) : null; @@ -119,7 +121,7 @@ export function SandboxSetupWizard({ coreUrl, current, disabled, switching = fal const sizeReady = provider !== null && (!sized || validSandboxResources(provider, resources)); const ready = provider !== null && runtimeReady && e2bReady && sizeReady && !disabled && !busy; - const order: Step[] = where === "direct" ? ["where", "e2b", "review"] : ["where", "backend", "size", "review"]; + const order: Step[] = editing ? where === "direct" ? ["e2b", "review"] : ["size", "review"] : where === "direct" ? ["where", "e2b", "review"] : ["where", "backend", "size", "review"]; const index = Math.max(0, order.indexOf(step === "advanced" ? "review" : step)); const back = () => setStep(step === "advanced" ? "review" : order[Math.max(0, index - 1)]!); @@ -149,6 +151,7 @@ export function SandboxSetupWizard({ coreUrl, current, disabled, switching = fal try { await onSubmit({ provider, + expected_generation: expectedGeneration, ...(sized ? { resources } : {}), ...(needsRuntime ? { runtime: release as SandboxRuntimeRelease } : {}), ...(provider === "e2b" ? { e2b: { api_key: apiKey.trim(), template: template.trim() } } : {}), @@ -273,7 +276,7 @@ export function SandboxSetupWizard({ coreUrl, current, disabled, switching = fal
    @@ -321,7 +324,7 @@ export function SandboxSetupWizard({ coreUrl, current, disabled, switching = fal } return ( -
    +
    {step !== "advanced" ? (
    ); @@ -146,6 +144,7 @@ export function SystemPage() { {reading ?

    {t("loading")}

    : null} {facts} + void deployment.refetch()} /> {body} {about ? : installation.isError ? null : } diff --git a/apps/web/src/i18n/locales/en/metrics.ts b/apps/web/src/i18n/locales/en/metrics.ts index 6c3feac3d..bdf3792fa 100644 --- a/apps/web/src/i18n/locales/en/metrics.ts +++ b/apps/web/src/i18n/locales/en/metrics.ts @@ -116,7 +116,7 @@ export const metrics = { kpiLabel: "Core summary", meta: "{{status}} · {{revision}} · up {{uptime}}", notOwner: "Does not hold the execution lease", - status: { running: "Running", maintenance: "Maintenance", degraded: "Degraded", unknown: "Unknown status" }, + status: { running: "Running", degraded: "Degraded", unknown: "Unknown status" }, slots: "Execution concurrency", slotsHelp: "Sessions Core runs at the same time at most. Turns, environment input and file reads and writes share these slots.", queued: "Queued Turns", diff --git a/apps/web/src/i18n/locales/en/overview.ts b/apps/web/src/i18n/locales/en/overview.ts index e5bb35211..7906b544c 100644 --- a/apps/web/src/i18n/locales/en/overview.ts +++ b/apps/web/src/i18n/locales/en/overview.ts @@ -1,4 +1,11 @@ export const overview = { + reset: { + title: "Sandbox reset in progress", + auto: "Waiting for Sessions and cleanup", + force: "Forced reset · cleanup in progress", + body: "Core is clearing the hosted sandbox backend. New hosted Sessions cannot start until the reset ends. Self-hosted Sessions are unaffected.", + view: "View reset", + }, title: "Overview", description: "Health, capacity, usage and failures across every project of this OpenAgentCore deployment.", gettingStarted: { @@ -98,7 +105,6 @@ export const overview = { coreStatus: { checking: "Checking", running: "Running", - maintenance: "Maintenance", unreachable: "Unreachable", }, nodeHealth: { diff --git a/apps/web/src/i18n/locales/en/system.ts b/apps/web/src/i18n/locales/en/system.ts index 7d03f61b8..93c473134 100644 --- a/apps/web/src/i18n/locales/en/system.ts +++ b/apps/web/src/i18n/locales/en/system.ts @@ -116,8 +116,6 @@ export const system = { suspendAfterHelp: "A sandbox idle this long after its Session's last Turn is suspended into a snapshot on its node. The Session's next Turn resumes it.", keepSuspended: "Keep suspended for", keepSuspendedHelp: "How long a suspended sandbox's snapshot is kept on its node.", - maintenance: "Maintenance", - maintenanceHelp: "While on, no new sandboxes are placed.", }, values: { on: "On", diff --git a/apps/web/src/i18n/locales/zh-CN/metrics.ts b/apps/web/src/i18n/locales/zh-CN/metrics.ts index 106ae2896..1a50040ef 100644 --- a/apps/web/src/i18n/locales/zh-CN/metrics.ts +++ b/apps/web/src/i18n/locales/zh-CN/metrics.ts @@ -116,7 +116,7 @@ export const metrics = { kpiLabel: "Core 概况", meta: "{{status}} · {{revision}} · 已运行 {{uptime}}", notOwner: "未持有执行归属", - status: { running: "运行中", maintenance: "维护中", degraded: "降级", unknown: "状态未知" }, + status: { running: "运行中", degraded: "降级", unknown: "状态未知" }, slots: "执行并发", slotsHelp: "Core 同时最多执行的 Session 数。Turn、环境输入和文件读写共用这些槽位。", queued: "排队 Turn", diff --git a/apps/web/src/i18n/locales/zh-CN/overview.ts b/apps/web/src/i18n/locales/zh-CN/overview.ts index 972228c6a..70b37580f 100644 --- a/apps/web/src/i18n/locales/zh-CN/overview.ts +++ b/apps/web/src/i18n/locales/zh-CN/overview.ts @@ -1,4 +1,11 @@ export const overview = { + reset: { + title: "沙箱重置进行中", + auto: "等待 Session 结束和资源清理", + force: "强制重置 · 正在清理资源", + body: "Core 正在清空托管沙箱后端。重置结束前无法启动新的托管 Session，自托管 Session 不受影响。", + view: "查看重置", + }, title: "概览", description: "这套 OpenAgentCore 部署在所有项目下的健康、容量、用量与故障。", gettingStarted: { @@ -98,7 +105,6 @@ export const overview = { coreStatus: { checking: "检查中", running: "运行中", - maintenance: "维护中", unreachable: "无法访问", }, nodeHealth: { diff --git a/apps/web/src/i18n/locales/zh-CN/system.ts b/apps/web/src/i18n/locales/zh-CN/system.ts index f4d044f81..8a12a07c7 100644 --- a/apps/web/src/i18n/locales/zh-CN/system.ts +++ b/apps/web/src/i18n/locales/zh-CN/system.ts @@ -118,8 +118,6 @@ export const system: TranslationShape = { suspendAfterHelp: "Session 的最后一个 Turn 结束后，沙箱空闲这么久就会被挂起为节点上的快照；Session 的下一个 Turn 会恢复它。", keepSuspended: "挂起后保留", keepSuspendedHelp: "挂起的沙箱快照在节点上保留的时长。", - maintenance: "维护模式", - maintenanceHelp: "开启时不再分配新沙箱。", }, values: { on: "开启", diff --git a/apps/web/src/lib/locale-strings.ts b/apps/web/src/lib/locale-strings.ts index 52e2f40ee..8d8be3bea 100644 --- a/apps/web/src/lib/locale-strings.ts +++ b/apps/web/src/lib/locale-strings.ts @@ -1,6 +1,53 @@ import { consoleAuthChinese } from "./console-auth-strings"; export const chinese = { ...consoleAuthChinese, + "Sandbox state is unconfirmed. Refresh before issuing a node command.": "沙箱状态尚未确认。请刷新后再生成节点命令。", + "Change resources": "修改资源", + "One provider serves this deployment. Reset it before choosing a different backend.": "整个部署使用一个运行后端。请先重置，再选择其他后端。", + "Editing keeps the current provider and requires zero allocated resources and pending environments. It is not an online upgrade.": "修改会保留当前运行后端，并要求已分配资源和待处理 Environment 都为零。这不是在线升级。", + "Changes require Core to confirm that no hosted resources remain.": "只有 Core 确认没有剩余托管资源后，才能修改配置。", + "Saving changes only this provider's configuration; it does not confirm execution readiness.": "保存只修改当前后端的配置，不代表执行已就绪。", + "Saving retires existing node identities and enrollment commands. Add the nodes again after saving.": "保存会作废现有节点身份和注册命令。保存后请重新添加节点。", + "Reset sandbox deployment?": "重置沙箱部署？", + "Reset deployment": "重置部署", + "Reset in progress": "正在重置", + "Progress comes from Core. Offline resources still need confirmed cleanup, even in force mode.": "进度由 Core 提供。即使强制重置，离线资源仍需确认清理完成。", + "Idle sandboxes are being archived. Busy work can finish until the deadline.": "正在归档空闲沙箱。忙碌中的任务可在截止时间前继续完成。", + "Remaining hosted Sessions are being archived and their work cancelled.": "正在归档剩余托管 Session 并取消其中的任务。", + "Reset progress could not be refreshed. These are the last confirmed counts.": "无法刷新重置进度。当前显示的是上次确认的数量。", + "Busy": "忙碌", + "Idle": "空闲", + "Awaiting cleanup": "等待清理", + "On offline nodes": "离线节点上的资源", + "Force deadline": "强制清理截止时间", + "Bring these nodes back online so Core can confirm cleanup. Force reset does not bypass them.": "请让这些节点重新上线，以便 Core 确认清理完成。强制重置也不能跳过它们。", + "{{count}} resources": "{{count}} 个资源", + "Remove a node only after Core confirms it holds no resources.": "只有 Core 确认节点没有保留资源后，才能移除节点。", + "New hosted Sessions and node enrollment are paused. Setup becomes available only after Core completes the reset.": "已暂停创建新的托管 Session 和注册节点。只有 Core 完成重置后，才能重新配置。", + "Force reset now": "立即强制重置", + "Force reset now?": "立即强制重置？", + "Cancel reset": "取消重置", + "Cancel reset?": "取消重置？", + "Stop the reset and allow new hosted Sessions again. Sessions already archived stay archived and cannot be resumed.": "停止重置并重新允许创建托管 Session。已归档的 Session 会保持归档，无法恢复运行。", + "Reset archives this deployment's hosted Sessions. Archived Sessions cannot be resumed. History, Files and Artifacts are kept; self-hosted Sessions are unchanged.": "重置会归档此部署的托管 Session。已归档的 Session 无法恢复运行。历史记录、File 和 Artifact 会保留；自托管 Session 不受影响。", + "After cleanup, Core clears the provider and saved E2B credential, retires nodes and enrollment commands, and returns to setup.": "清理完成后，Core 会清除运行后端和保存的 E2B 凭据，作废节点及注册命令，然后返回配置流程。", + "Force immediately cancels remaining work. Offline resources can still block cleanup.": "强制重置会立即取消剩余任务。离线资源仍可能阻止清理完成。", + "Clear hosted work": "清理托管任务", + "Auto — let busy work finish": "自动 — 等待忙碌任务完成", + "Idle, suspended and queued Sessions are archived immediately. Running or waiting Turns and file writes may finish until the deadline; then Core forces the rest.": "空闲、挂起和排队中的 Session 会立即归档。运行中或等待中的 Turn 及文件写入可在截止时间前继续；之后 Core 会强制清理剩余任务。", + "Force remaining work after (seconds)": "多久后强制清理剩余任务（秒）", + "Default: 1 hour. Choose a whole number from 300 to 86400 seconds (5 minutes to 24 hours).": "默认 1 小时。请输入 300 到 86400 秒之间的整数（5 分钟到 24 小时）。", + "Enter a whole number from 300 to 86400 seconds.": "请输入 300 到 86400 秒之间的整数。", + "Force — cancel remaining work now": "强制 — 立即取消剩余任务", + "Sandbox state changed. Close this dialog and review the current state.": "沙箱状态已改变。请关闭对话框并查看当前状态。", + "Node state could not be read": "无法读取节点状态", + "Node enrollment is paused while reset is in progress.": "重置过程中暂停注册节点。", + "Unsaved workspace contents may be lost.": "尚未保存的工作区内容可能丢失。", + "Core has a newer sandbox configuration. Refresh and review it before submitting again.": "Core 的沙箱配置已更新。请刷新并检查后再提交。", + "Reset the sandbox deployment before changing its backend.": "请先重置沙箱部署，再更换运行后端。", + "A sandbox reset is in progress. Refresh to see its current progress.": "沙箱正在重置。请刷新查看当前进度。", + "The sandbox deployment is not configured. Refresh to start setup.": "沙箱部署尚未配置。请刷新以开始配置。", + "Hosted resources remain. Wait for confirmed cleanup before changing the configuration.": "仍有托管资源。请等待 Core 确认清理完成后再修改配置。", "Change provider or resources": "修改运行后端或资源", "Sandbox backend": "沙箱后端", "Suspended sandboxes keep their state as a snapshot on the node and resume on the Session's next Turn. They count toward the retained limit, not the active one.": "挂起的沙箱把状态存成快照留在节点上，Session 的下一个 Turn 时恢复。它们占用保留上限，不占用同时运行上限。", @@ -30,7 +77,6 @@ export const chinese = { "Status unconfirmed": "状态待确认", "Old address": "地址已过期", "Remove and add again": "移除并重新添加", - "Maintenance is enabled. New sandbox placement is paused.": "维护模式已开启，暂停分配新沙箱。", "This console serves no node installer. For a console deployed by hand, point OAC_WEB_NODE_PAYLOAD_DIR at the distribution's node payload and restart it.": "此控制台没有提供节点安装程序。手动部署的控制台需要把 OAC_WEB_NODE_PAYLOAD_DIR 指向发行包的节点载荷目录，然后重启控制台。", "This console has no node files for {{provider}}. Install Core from the offline bundle, or add the release artifacts and rerun ./install.sh.": "这个控制台没有 {{provider}} 的节点文件。请用离线包安装 Core，或补齐发布制品后重新运行 ./install.sh。", "Run on the host you want to add.": "在需要添加的主机上运行。", @@ -235,7 +281,6 @@ export const chinese = { "Copied": "已复制", "Select the command above and copy it manually.": "请选择上方命令并手动复制。", "Choose the next provider": "选择下一个运行后端", - "One provider serves this entire Core deployment. Switching requires maintenance and verified cleanup of all existing execution resources.": "整个 Core 部署使用一个运行后端。切换前需进入维护模式，并确认所有现有执行资源已清理。", "Where to run sandboxes": "沙箱运行位置", "Choose where to run": "选择运行位置", "E2B cloud": "E2B 云端", @@ -247,7 +292,6 @@ export const chinese = { "Immutable Runtime template": "固定版本 Runtime 模板", "Use the qualified managed Runtime template ID followed by its build UUID. A template alias or a generic E2B template cannot provide Core execution.": "请填写已验证的托管 Runtime 模板 ID 和构建 UUID。模板别名或通用 E2B 模板无法提供 Core 执行能力。", "Core origin stays unchanged": "Core 地址保持不变", - "Save provider and stay in maintenance": "保存运行后端并保持维护模式", "Deployment provider": "部署运行后端", "Allocated resources": "已分配资源", "Pending environments": "待处理环境", @@ -255,8 +299,6 @@ export const chinese = { "Configured": "已配置", "Not configured": "未配置", "Saved configuration does not confirm execution readiness. Session and Environment state report actual execution.": "配置已保存不代表执行已就绪。实际执行情况以会话和环境状态为准。", - "Enter maintenance to change provider": "进入维护模式以更改运行后端", - "Maintenance pauses new hosted placement. Clean up existing execution resources before changing provider; historical Sessions and results are preserved by the switch.": "维护模式会暂停新的托管分配。更改运行后端前请清理现有执行资源；切换会保留历史会话和结果。", "Stopped or offline resources, snapshots, uncertain creates and pending environments still block switching. Deleting a Session alone does not prove cleanup; Core must confirm both counts are zero.": "已停止或离线的资源、快照、创建结果未确认的资源和待处理环境均会阻止切换。删除会话并不代表清理完成，需由 Core 确认两项计数均为零。", "Open Sessions": "打开会话", "Check cleanup": "检查清理状态", @@ -265,7 +307,6 @@ export const chinese = { "Saving retires old node identities and enrollment credentials. It does not migrate Sessions or resume placement automatically.": "保存后旧节点身份和注册凭据将失效。不会迁移会话或自动恢复分配。", "Cancel": "取消", "Resume hosted placement": "恢复托管分配", - "Resume is explicit and succeeds only when Core activates the saved configuration. A failed activation leaves maintenance enabled.": "需手动恢复分配，且仅在 Core 成功激活已保存配置后生效。激活失败时保持维护模式。", "Refresh sandbox state to confirm whether the change was saved before submitting again.": "再次提交前，请刷新沙箱状态以确认更改是否已保存。", "Hosted sandboxes": "托管沙箱", "Manage hosted execution for this Core deployment.": "管理此 Core 部署的托管执行。", @@ -342,7 +383,6 @@ export const chinese = { "microsandbox reference": "microsandbox 引用", "{{cpus}} CPU · {{memory}}": "{{cpus}} 核 · {{memory}}", "Save configuration": "保存配置", - "Save and stay in maintenance": "保存并保持维护模式", "Change the sandbox configuration": "修改沙箱配置", "The address nodes and sandboxes use to reach Core.": "节点和沙箱访问 Core 使用的地址。", "Set by public_url in config.json": "由 config.json 中的 public_url 设置", diff --git a/apps/web/src/lib/sandbox-labels.test.ts b/apps/web/src/lib/sandbox-labels.test.ts index 28441f6c1..9503655d7 100644 --- a/apps/web/src/lib/sandbox-labels.test.ts +++ b/apps/web/src/lib/sandbox-labels.test.ts @@ -1,7 +1,7 @@ import { AgentCoreError } from "@agents-core-web/agents-client"; import { describe, expect, it } from "vitest"; -import { sandboxWriteUncertain } from "./sandbox-labels"; +import { sandboxRequestError, sandboxWriteUncertain } from "./sandbox-labels"; describe("sandbox write outcome", () => { it("is uncertain without a response, on a timeout or a 5xx, and certain on any other 4xx, a withheld E2B reason included", () => { @@ -14,3 +14,16 @@ describe("sandbox write outcome", () => { expect(sandboxWriteUncertain(new AgentCoreError("Changed.", 409, "sandbox_deployment_conflict"))).toBe(false); }); }); + + +describe("reset conflict recovery", () => { + it("gives bilingual safe recovery for known state conflicts without reflecting a server payload", () => { + for (const code of ["sandbox_generation_stale", "sandbox_reset_required", "sandbox_reset_in_progress", "sandbox_not_configured", "sandbox_in_use"]) { + const error = new AgentCoreError("untrusted-secret-like-payload", 409, code); + for (const locale of ["en", "zh"] as const) expect(sandboxRequestError(error, locale)).not.toContain("untrusted-secret-like-payload"); + expect(sandboxWriteUncertain(error)).toBe(false); + } + expect(sandboxRequestError(new AgentCoreError("stale", 409, "sandbox_generation_stale"), "en")).toContain("Refresh and review"); + expect(sandboxRequestError(new AgentCoreError("reset", 409, "sandbox_reset_in_progress"), "zh")).toContain("正在重置"); + }); +}); diff --git a/apps/web/src/lib/sandbox-labels.ts b/apps/web/src/lib/sandbox-labels.ts index bff113df6..fb1511896 100644 --- a/apps/web/src/lib/sandbox-labels.ts +++ b/apps/web/src/lib/sandbox-labels.ts @@ -27,6 +27,11 @@ export function sandboxRequestError(error: unknown, locale: Locale): string { else if (error.status === 401) key = "Sign in to the console again to access sandbox management."; else if (error.code === "sandbox_configuration_unconfirmed") { if (refused) key = "Core rejected the E2B configuration."; } else if (refused) { + if (error.code === "sandbox_generation_stale") return translate(locale, "Core has a newer sandbox configuration. Refresh and review it before submitting again."); + if (error.code === "sandbox_reset_required") return translate(locale, "Reset the sandbox deployment before changing its backend."); + if (error.code === "sandbox_reset_in_progress") return translate(locale, "A sandbox reset is in progress. Refresh to see its current progress."); + if (error.code === "sandbox_not_configured") return translate(locale, "The sandbox deployment is not configured. Refresh to start setup."); + if (error.code === "sandbox_in_use") return translate(locale, "Hosted resources remain. Wait for confirmed cleanup before changing the configuration."); if (error.code === "runtime_node_in_use") return translate(locale, "The node has active allocations or retained resources. Clear allocations, snapshots, reservations and pending cleanup before removal."); if (error.code === "runtime_node_unavailable") return translate(locale, "The selected sandbox node is unavailable or has no capacity."); if (error.message) return error.message; diff --git a/contracts/agents-api/admin-api.md b/contracts/agents-api/admin-api.md index 490f63135..aafe9d933 100644 --- a/contracts/agents-api/admin-api.md +++ b/contracts/agents-api/admin-api.md @@ -92,8 +92,8 @@ reject HEAD with 405, so HEAD never samples a provider or queries telemetry. `POST /projects/{project_id}/sessions/{session_id}/archive` takes `{"expected_generation": N}`, where N is a positive uint64 from the current sandbox deployment. It requires the Core key, a -Web-managed deployment in maintenance and the current generation. Missing -maintenance or a stale generation returns 409. Only Core-managed +Web-managed configured deployment and the current generation. A stale generation +returns 409 `sandbox_generation_stale`; reset is not a precondition. Only Core-managed `openai_hosted` Sessions are eligible; other environment types return 400. A Session outside the selected Project returns the same 404 as a missing Session. @@ -107,7 +107,7 @@ unpersisted workspace contents are lost and the original Session cannot resume. Both POST and `GET /projects/{project_id}/sessions/{session_id}/archive` return `{session_id, environment_id, state}`. GET is read-only and does not require -maintenance or an expected generation. `state` describes current resource +an active reset or an expected generation. `state` describes current resource disposition: `active`, `cleanup_pending`, or `released`. It is not archive provenance: resources may already have expired through their normal lifecycle. `released` does not prove that an active Turn has finished cancellation or @@ -192,3 +192,9 @@ The typed `AdminClient`, `SandboxAdminClient` and `CoreMetricsClient` in `packages/agents-client` use `/core/v1`. Public SDK applications continue to use the existing Agents API client and their own API key. See [design principles](../../docs/design-principles.md). + +Sandbox deployment reset records `reset_start`, explicit `reset_force`, automatic +`reset_deadline`, `reset_cancel` and `reset_complete` in administrator audit. +Background archives retain the reset requester's credential/actor/request/trace +provenance and recover each Session's real Project scope. Audit failure rolls back +the corresponding state transition. Cancel does not undo an archive already committed. diff --git a/contracts/agents-api/core-metrics.md b/contracts/agents-api/core-metrics.md index b8410503a..1879a0ce0 100644 --- a/contracts/agents-api/core-metrics.md +++ b/contracts/agents-api/core-metrics.md @@ -26,7 +26,7 @@ the current partial bucket is excluded from range aggregates and series. Current gauges and range aggregates are intentionally different: current worker, connection pool, Go heap and goroutine values are read when requested; process -CPU, RSS and limits, queue gauges, database size and deployment maintenance are +CPU, RSS and limits, queue gauges, database size are sampled every 30 seconds with bounded I/O. Samples older than 60 seconds are not reported as current. Queue, running and pool and process series report the highest **observed** value in each bucket, not a claim @@ -53,8 +53,8 @@ present. by the standalone builder's `-ldflags`. Manual builds without a valid revision report null. `started_at` records process initialization. `execution_owner` reflects the execution worker's existing lease checks, with unknown ownership - represented as null. `maintenance` means the saved deployment is in maintenance; - measurement or job failures take precedence as `degraded`. + represented as null. Measurement or job failures report `degraded`; sandbox reset + is reported separately by the deployment contract, not a service status. - `execution.slots_in_use` is the worker's active Session reservation set. Its configured capacity is four; environment input, Turns and file work share it. It does not count native harness subprocesses. Worker-disabled installations diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml index d857e9a2d..4ca1b429d 100644 --- a/contracts/agents-api/core.openapi.yaml +++ b/contracts/agents-api/core.openapi.yaml @@ -426,11 +426,15 @@ definitions: them; Core then uses the validated template build's cpus and memory_mib. runtime: $ref: '#/definitions/sandbox.RuntimeRelease' + required: + - expected_generation type: object api.SandboxDeploymentInput: properties: e2b: $ref: '#/definitions/api.SandboxE2BInput' + expected_generation: + type: integer provider: type: string resources: @@ -441,6 +445,8 @@ definitions: them; Core then uses the validated template build's cpus and memory_mib. runtime: $ref: '#/definitions/sandbox.RuntimeRelease' + required: + - expected_generation type: object api.SandboxE2BInput: properties: @@ -921,14 +927,16 @@ definitions: type: integer installation_id: type: string - maintenance: - type: boolean mode: type: string owner_epoch: type: integer provider: type: string + reset: + allOf: + - $ref: '#/definitions/store.SandboxResetView' + x-nullable: true resources: $ref: '#/definitions/store.SandboxDeploymentResources' specification: @@ -1148,12 +1156,62 @@ definitions: - $ref: '#/definitions/store.SandboxE2BTemplateBuildView' description: The fixed template build as Core read it when this selection was saved. type: object - store.SandboxMaintenanceRequest: + store.SandboxResetOfflineNode: + properties: + name: + type: string + node_id: + type: string + resources: + type: integer + type: object + store.SandboxResetRemaining: properties: + busy: + type: integer + cleanup: + type: integer + idle: + type: integer + offline_nodes: + items: + $ref: '#/definitions/store.SandboxResetOfflineNode' + type: array + on_offline_nodes: + type: integer + type: object + store.SandboxResetRequest: + properties: + clear: + enum: + - auto + - force + type: string + deadline_seconds: + maximum: 86400 + minimum: 300 + type: integer expected_generation: + minimum: 0 type: integer - maintenance: - type: boolean + required: + - clear + - expected_generation + type: object + store.SandboxResetView: + properties: + clear: + type: string + deadline_at: + type: string + x-nullable: true + forced_at: + type: string + x-nullable: true + remaining: + $ref: '#/definitions/store.SandboxResetRemaining' + requested_at: + type: string type: object store.SandboxSuspensionView: properties: @@ -4403,7 +4461,7 @@ paths: post: consumes: - application/json - description: Core key only. Requires maintenance and the current deployment generation. Permanently closes execution, requests cancellation and releases sandbox/snapshots through existing cleanup. Session history and persisted files/artifacts remain; unpersisted workspace contents are lost. A cleanup_pending response is not proof of resource release. Does not affect caller-managed Runtime. + description: Core key only. Requires the current deployment generation; no maintenance mode is required. Permanently closes execution, requests cancellation and releases sandbox/snapshots through existing cleanup. Session history and persisted files/artifacts remain; unpersisted workspace contents are lost. A cleanup_pending response is not proof of resource release. Does not affect caller-managed Runtime. parameters: - description: Project ID in: path @@ -5698,7 +5756,7 @@ paths: post: consumes: - application/json - description: Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Exact retries return the existing selection; differing selections and file-managed deployments reject. This does not create compute or execute work. + description: Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Requires explicit expected_generation, including zero at first setup. Stale retries reject before provider validation. An identical selection at the current generation is a no-op; differing selections and file-managed deployments reject. This does not create compute or execute work. parameters: - description: Deployment selection in: body @@ -5741,7 +5799,7 @@ paths: put: consumes: - application/json - description: Requires maintenance, the current generation and verified cleanup of all old resources. Credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory. A core_url member is rejected with 400; the address comes from the installation public URL. Historical records are retained; old node credentials and enrollments are retired. Explicitly resume after success. Never automatically retry an uncertain write. + description: Requires the current generation, the same backend type, no active reset and verified cleanup of all old resources. A backend type change requires reset. Credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory. A core_url member is rejected with 400; the address comes from the installation public URL. Historical records are retained; old node credentials and enrollments are retired. Never automatically retry an uncertain write. parameters: - description: Replacement deployment selection in: body @@ -5781,18 +5839,58 @@ paths: summary: Change a fully drained deployment's sandbox configuration tags: - Sandbox Manager - /core/v1/sandbox/deployment/maintenance: - patch: + /core/v1/sandbox/deployment/reset: + delete: + description: Restores admission but never restores Sessions already archived. With no reset running this is an idempotent read, provided the generation still matches. + parameters: + - description: Current deployment generation + in: query + name: expected_generation + required: true + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/store.RuntimeDeploymentView' + "400": + description: Bad Request + schema: + $ref: '#/definitions/api.CoreErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/api.CoreErrorResponse' + "409": + description: Conflict + schema: + $ref: '#/definitions/api.CoreErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/api.CoreErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/api.CoreErrorResponse' + security: + - DeploymentAdminAuth: [] + summary: Cancel a sandbox deployment reset + tags: + - Sandbox Manager + post: consumes: - application/json - description: Requires the current generation. Maintenance preserves execution, cleanup and history. Resume requires successfully activated provider configuration. + description: Archives hosted Sessions and waits for confirmed provider cleanup, preserving history and Files/Artifacts. Auto waits for started or waiting Turns and file writes until the durable deadline; force cancels them. The same clear is idempotent; force escalates auto. Requires the current generation. Self-hosted Sessions are unchanged. parameters: - - description: Maintenance state + - description: Reset mode and current deployment generation in: body name: body required: true schema: - $ref: '#/definitions/store.SandboxMaintenanceRequest' + $ref: '#/definitions/store.SandboxResetRequest' produces: - application/json responses: @@ -5822,7 +5920,7 @@ paths: $ref: '#/definitions/api.CoreErrorResponse' security: - DeploymentAdminAuth: [] - summary: Pause or resume new hosted allocations + summary: Start or escalate a durable sandbox deployment reset tags: - Sandbox Manager /core/v1/sandbox/enrollment-tokens: diff --git a/contracts/agents-api/harness-selection.md b/contracts/agents-api/harness-selection.md index 6243e6c14..4b7b5870f 100644 --- a/contracts/agents-api/harness-selection.md +++ b/contracts/agents-api/harness-selection.md @@ -55,12 +55,10 @@ another harness does not choose another image or backend; the selected Runtime image must contain and qualify each enabled harness. Capability checks still apply. -Before changing provider, resources or Runtime, enter maintenance at the current -generation, explicitly archive retained hosted Sessions and confirm cleanup is -complete. Then submit the replacement and explicitly resume with the returned -generation. Switching never migrates Sessions or deletes resources automatically. -See the -[deployment procedure](../../services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-maintenance). +Same-provider resource/Runtime edits currently require no reset and verified zero +held resources. Changing provider requires explicit durable reset, confirmed +cleanup and a new setup at the resulting generation. Existing Sessions never move +providers. See the [deployment procedure](../../services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-reset). Each engine has at most one deployment default model provider, stored encrypted in PostgreSQL and managed with the Core key in Web or through diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index 6dfd528b1..a2f646453 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -21,8 +21,9 @@ for the operator workflow. Generated schemas cover the | --- | --- | --- | | `GET /core/v1/sandbox/deployment` | Core key | Read the safe active configuration and retained-resource counts | | `POST /core/v1/sandbox/deployment` | Core key | Select the initial provider, resources and Runtime | -| `PUT /core/v1/sandbox/deployment` | Core key | Replace a fully drained selection while maintenance is enabled | -| `PATCH /core/v1/sandbox/deployment/maintenance` | Core key | Pause or resume fresh hosted admission at the expected generation | +| `PUT /core/v1/sandbox/deployment` | Core key | Update the same provider at zero retained and pending resources | +| `POST /core/v1/sandbox/deployment/reset` | Core key | Start or escalate a durable hosted clear | +| `DELETE /core/v1/sandbox/deployment/reset?expected_generation=N` | Core key | Cancel the remaining clear without restoring archived work | | `GET /api/v1/sandbox-node/configuration` | Enrollment token or retained node credential | Read the active node installation configuration without consuming enrollment | The paired console injects the Core key server-side on every signed-in `/core/v1` @@ -62,11 +63,14 @@ retention, tells roughly when Core reclaims it. ## Selection request -POST and PUT take the same complete selection. PUT also requires a nonzero -`expected_generation` read from the current deployment. +POST and PUT take the same complete selection and require `expected_generation` +from a preceding GET. Zero is valid for the initial unconfigured deployment; +omitted or null is invalid. A stale generation is checked before reset, provider, +resource and same-selection conditions, including an identical old request body. | Field | Meaning | | --- | --- | +| `expected_generation` | Required nonnegative integer from GET; never automatically refresh and replay | | `provider` | Exactly one of `docker`, `microsandbox`, `e2b` | | `resources` | Per-sandbox resource limits described below; required for Docker/microsandbox, optional for E2B | | `runtime` | Required immutable distribution identity for Docker/microsandbox; absent for E2B | @@ -145,13 +149,13 @@ E2B instead uses `e2b.template` in `template-id:build-uuid` form. The build UUID be canonical and nonzero; a mutable template alias alone is insufficient. Omit `runtime`. The API key is encrypted in PostgreSQL and never returned in a safe view, bootstrap configuration, command argument or log. Replacing the key or -build uses the same drained maintenance transition as changing resources. +build uses the same zero-resource update as changing resources. ## Safe response GET and successful mutations return `installation_id`, `provider`, `core_url` (read-only: the installation public URL, present before configuration), `mode`, -`generation`, `owner_epoch`, `maintenance`, `suspension` and resource +`generation`, `owner_epoch`, `reset`, `suspension` and resource accounting. A configured deployment also returns `specification` and `specification_digest`. E2B returns only `e2b.template`, `e2b.credential_configured` and `e2b.template_build`; the `e2b` object is absent @@ -190,56 +194,101 @@ or a wrong type, rejects the whole response with a 502 `invalid_admin_response` error. A node's `diagnostic` is absent or a code, never empty; the client reads an unknown code as `provider_unavailable`. -## Initialization and changes - -POST validates the candidate before persistence and creates no compute, Session -or model request. An exact retry returns the existing generation. A different -selection conflicts once initialization has succeeded. Missing provider -prerequisites or failed candidate validation leave the previous selection intact. - -For a replacement: - -1. Read the deployment and PATCH `{"maintenance":true,"expected_generation":N}`. - Maintenance blocks new hosted Sessions and fresh allocations while preserving - admitted work, reads, receipt retries and explicit cleanup. -2. Verify both response counts are zero before replacing the selection. Stopped - compute, snapshots, uncertain operations, pending cleanup and unallocated - hosted Environments remain blockers. Use the explicit Session archive flow below. -3. PUT the complete replacement selection with `expected_generation: N`, without - `core_url` (a request that contains it gets 400). Core checks the generation and resources, prepares and - validates the candidate, then drains the existing manager calls. A short Store - transaction repeats the guards and commits a changed selection, increments - its generation and retires old nodes and unused enrollment tokens together. - The existing manager publishes the prevalidated configuration after commit. -4. Read the returned generation and explicitly PATCH maintenance to `false`. - Resume requires the committed generation to be active. - -To release a retained hosted Session, explicitly POST -`/core/v1/projects/{project_id}/sessions/{session_id}/archive` with -`{"expected_generation": N}` while maintenance is enabled. Poll GET on the same -path until its resource state is `released`, then recheck the deployment counts. -The [administrator archive contract](admin-api.md#administrative-session-archive) -preserves public history and persisted Files/Artifacts, but discards unpersisted -workspace contents and prevents the original Session from resuming. An active -Turn may still be finalizing after resources are released. Ordinary Session -deletion remains separate. Maintenance and PUT do not clear resources; snapshots, -pending Environments and unknown cleanup remain blockers until the existing -lifecycle confirms release. - -Provider validation, cancellation and draining hold no database transaction or -manager map mutex. One mutation gate, the existing execution lease and the Store -checks serialize changes; there is no second configuration owner. A rejected -candidate does not replace the previous provider or drain its workers. A later -commit failure leaves maintenance enabled and the previous selection retained. -After an interrupted drain, retries and resume must wait for that drain to finish. - -An exact PUT retry at the still-current generation is a no-op when the clean -maintenance guards hold. A successful changed PUT makes the old expected -generation stale. After an uncertain write response, GET the deployment before -choosing another mutation; do not automatically replay writes. No step deletes -compute automatically, migrates an existing Session or rewrites historical -allocation ownership. Revoking an E2B key before cleanup can leave unverifiable -resources that block replacement. +## Initialization, same-provider changes and reset + +POST validates a candidate before persistence and creates no compute, Session or +model request. At the current generation an identical selection is a no-op; an +obsolete generation returns 409 `sandbox_generation_stale`, even for the same body. +Missing prerequisites or failed preparation leave the committed provider intact. +A different backend, or an old node selection without a specification, requires +reset first. POST also initializes after a completed reset, using its new generation. + +PUT currently accepts only the same provider, no active reset and zero unreleased +allocations plus pending hosted Environments. Send the complete selection and the +observed generation once. Core prepares the candidate, drains manager calls outside +any database transaction, repeats guards under the deployment lock, and commits a +changed selection, generation and node/token retirement together. A no-op keeps its +generation. Failed final writes restore the committed provider using the owner's +bounded recovery context before releasing mutation ownership. If recovery fails, +the owner stops with admission fenced. There is no online rollout in this contract; +responses omit `rollout` until that feature exists. + +To change backend, explicitly start reset: + +```json +{"expected_generation": 7, "clear": "auto", "deadline_seconds": 3600} +``` + +`clear` is required: `auto` or `force`. Auto's `deadline_seconds` defaults to 3600 +and accepts 300–86400; force must omit it. Core persists an absolute deadline and +requester audit provenance before closing fresh hosted admission. The existing +execution owner advances the clear after the request ends and across restarts. +Auto archives idle hosted Sessions, including queued or pending work and suspended +sandboxes, but waits for root/subagent Turns in progress or waiting and pending +file writes. It rechecks this condition under the Session lock. At the persisted +deadline it durably escalates to force; force uses the ordinary archive cancellation +and cleanup path for all eligible hosted Sessions. Self-hosted Sessions are excluded. + +Repeated start at the current generation/mode keeps the original deadline. Auto +can escalate to force, but force cannot downgrade to auto. DELETE with the current +`expected_generation` cancels remaining reset work and reopens admission; it does +not undo archives, revive expired Environments or cancel cleanup already requested. +DELETE with no active reset is idempotent. Stale requests still return 409. + +Every deployment response includes `reset: null` when inactive, or: + +```json +{"clear":"auto","requested_at":"2026-09-27T12:00:00Z", + "deadline_at":"2026-09-27T13:00:00Z","forced_at":null, + "remaining":{"busy":2,"idle":1,"cleanup":3,"on_offline_nodes":2, + "offline_nodes":[{"node_id":"node-uuid","name":"worker","resources":2}]}} +``` + +Timestamps are explicit nullable fields where applicable. A single database snapshot +partitions every unreleased allocation and every pending hosted Environment with +no allocation into cleanup first, then busy or idle. +`busy + idle + cleanup == resources.allocations + resources.pending`. Deleted or +expired Sessions with unreleased receipts still count as cleanup. `offline_nodes` +is the untruncated, ID-sorted subset attributed to receipt or active-placement node +identity; its resource sum equals `on_offline_nodes`. Presence uses the current +owner epoch, connection and a heartbeat within 45 seconds, not provider readiness. +Direct E2B resources have no node and do not enter that subset. Offline resources +remain blockers until actual cleanup confirms release. + +Only after both held counts reach zero does the owner drain and atomically clear +provider/mode/specification, E2B credential/template/build metadata and provider +policy, retire nodes and unused enrollment tokens, increment generation and owner +epoch, and record `reset_complete`. Installation identity and history survive. +The manager publishes an unconfigured state immediately; its cache keeps the new +generation even with no provider, preventing delayed old loads from reviving it. +Configure again with POST using the returned generation; no restart is necessary. + +Fresh hosted admission returns 503 `sandbox_reset_in_progress` and leaves no +provisional Session rows. Existing live input, receipt retries, restoration and +cleanup continue. Management writes and new enrollment/configuration return 409 +`sandbox_reset_in_progress`; retained matching nodes can recover for cleanup. +Explicit per-Session archive requires the current generation and remains available +without reset. It preserves history and persisted Files/Artifacts but discards +unpersisted workspace and prevents that Session from resuming. Poll its archive +GET for actual release. Reset never manufactures a release receipt. + +A force archive fences credentials and new work immediately. If its original +hosted delivery is still connected, Core preserves only that delivery's native +cancellation/terminal receipt path until terminal commit or a fixed 20-second +bound from the original cancellation request. Done does not end this bound while +a cancellation acknowledgment or terminal commit is pending. This internal drain +never authorizes reconnect, workspace/MCP access or renewed execution. Explicit +credential revocation ends the exception; repeats do not extend it. Missing or +failed receipts retain honest failure outcomes, and disconnected, expired or +restarted owners fall back to ordinary provider cleanup. There is no new public +state, request field or model/tool timeout. + +One mutation gate serializes setup, PUT, reset, cancel and finalization. Archive +locks Session before deployment; finalization never reverses that order or waits +for itself inside counted manager work. Candidates bind to the reset request time, +so cancel followed by a new reset at the same generation cannot reuse old work. +Never automatically replay a rejected or uncertain write: read current state and +make a new explicit decision. Do not revoke old E2B credentials before cleanup. ## Node configuration and enrollment @@ -249,13 +298,13 @@ by its replacement: `400 invalid_request`, with the message For a new node, send `Authorization: Bearer ` to the configuration GET without `X-OAC-Node-ID`. The token must be valid, unexpired, unconsumed and -belong to this installation. This read does not consume it. Maintenance prevents +belong to this installation. This read does not consume it. An active reset prevents new enrollment configuration reads. An already registered node sends its durable node credential as Bearer and its UUID in `X-OAC-Node-ID`. Its installation, saved generation and specification digest must match the active deployment. This read remains available in -maintenance so the retained node can recover its exact configuration. The old +reset so the retained node can recover its exact configuration. The old enrollment token cannot replace a registered node's credential. The response contains `installation_id`, `provider`, `core_url` (the installation @@ -303,8 +352,11 @@ fields from the administrator node routes; runtime fields from the ## Failure and upgrade boundaries Malformed selections return 400; validated configuration diagnostics use -`invalid_sandbox_configuration`. Stale generations, retained resources or an -incompatible deployment return 409 `sandbox_deployment_conflict`. A node +`invalid_sandbox_configuration`. Stale generation returns 409 `sandbox_generation_stale` with safe `current_generation`; +zero-resource PUT violations return 409 `sandbox_in_use` with allocation/pending +counts; a different backend returns 409 `sandbox_reset_required` with provider names. +Unconfigured mutations return 409 `sandbox_not_configured`. Other incompatible +deployments return 409 `sandbox_deployment_conflict`. A node configuration mismatch returns 409 `sandbox_specification_mismatch`; rejected node credentials return 401 `invalid_node_credential`. Node machine routes check the credential before any deployment state, so a missing or rejected credential, @@ -340,7 +392,7 @@ environment variable alone does not complete that migration. A Web-managed Docker or microsandbox selection saved before specifications has the empty migration default. Core loads it for draining only. GET omits `specification` and `specification_digest`. Fresh hosted sandboxes are refused -with the same error as during maintenance. Enrollment tokens return 409 +until a complete configuration is saved. Enrollment tokens return 409 `sandbox_deployment_conflict`; node configuration reads and enrollment are refused. Retained nodes without a recorded digest or generation authenticate while the deployment is in this state, but those nodes use the removed `/core/v1/sandbox` @@ -348,8 +400,8 @@ node paths. They can reach only a Core that has this drain mode (pull request #114) and still serves those paths. No such release is published: build one from main commit `7b66be236a627246c85658722314285e6b39d9b8`, or another commit with #114 but without `/api/v1/sandbox-node`. Run the maintenance and archive steps there. On this -release, the drain mode matters only for the PUT that records a specification after -draining elsewhere; that PUT retires those nodes. See the +release, reset that drained selection before POSTing its complete specification; +reset completion retires those nodes. See the [operator upgrade notes](../../docs/getting-started/operations.md#data-and-upgrades). Unit tests, database tests and provider inspection are separate from live @@ -365,4 +417,9 @@ release and remove its nodes. The database migration refuses any allocation with no `released_at`, including retained snapshots and uncertain cleanup, then rewrites only the former `parsar-core-runtime@` reference prefix. The specification digest changes, requiring new node enrollment. Historical Session data remains unchanged. -Old E2B templates must be rebuilt with the matching Runtime release. +Old E2B templates must be rebuilt with the matching Runtime release. This release +refuses E2B rename conversion before mutations until a Core-owned safe upgrade +transition is implemented. Ordinary migration retires maintenance and resumes +admission; it never interprets old maintenance as authorization to clear. Downgrade +refuses an active reset or an unconfigured completed reset with generation above +zero; configure a provider before downgrading, never erase the generation. diff --git a/deploy/install/installer_fakes.py b/deploy/install/installer_fakes.py index b23725b23..8e33a501a 100644 --- a/deploy/install/installer_fakes.py +++ b/deploy/install/installer_fakes.py @@ -45,7 +45,7 @@ def __init__(self, test): self.nodes = [] self.remote_core = {} # web-only: origin -> (status, installation_id) self.deployment_core_url = "" # what an old Core reports for its sandbox deployment - self.deployment = {"provider": ""} # what sandbox_setup reads and posts + self.deployment = {"provider": "", "generation": 0, "reset": None, "resources": {"allocations": 0, "pending": 0}} # what sandbox_setup reads and posts self.deployment_posts = [] self.volumes = {} self.project_containers = {} @@ -313,19 +313,25 @@ def sandbox_send(self, req): raise ConnectionRefusedError() if sha256(req.get_header("Authorization", "").removeprefix("Bearer ")) not in digests: return 401, b'{"error": {"message": "Invalid Core key"}}' - if path == "core/v1/sandbox/deployment/maintenance" and req.get_method() == "PATCH": - self.deployment["maintenance"] = json.loads(req.data)["maintenance"] - return 200, json.dumps(dict(self.deployment, installation_id=self.core_installation_id)).encode() if path != "core/v1/sandbox/deployment": return 404, b"" + # The upgraded Core migration resumes admission without a reset. + self.deployment.pop("maintenance", None) + self.deployment.setdefault("reset", None) + runtime = (self.deployment.get("specification") or {}).get("runtime") or {} + reference = runtime.get("microsandbox_ref", "") + if reference.startswith("parsar-core-runtime@"): + runtime["microsandbox_ref"] = "oac-runtime@" + reference.split("@", 1)[1] if req.get_method() in ("POST", "PUT"): selection = json.loads(req.data) self.deployment_posts.append(selection) + if selection.get("expected_generation") != self.deployment.get("generation", 0): + return 409, b'{"error":{"code":"generation_stale","message":"Deployment generation changed"}}' if self.deployment_refusal: return 409, json.dumps({"error": {"message": self.deployment_refusal}}).encode() # E2B adopts the template build's size. self.deployment = {"provider": selection["provider"], "generation": self.deployment.get("generation", 0) + 1, - "maintenance": self.deployment.get("maintenance", False), "resources": {"allocations": 0, "pending": 0}, + "reset": None, "resources": {"allocations": 0, "pending": 0}, "specification": {"runtime": selection.get("runtime"), "resources": selection.get("resources", {"cpus": 2, "memory_mib": 2048})}} native = self.native["active"] and self.native["addr"] == port environment = self.native["environment"] if native else self.core.get("environment", "") diff --git a/deploy/install/node_spec.py b/deploy/install/node_spec.py index 688506e83..a7363c6ba 100644 --- a/deploy/install/node_spec.py +++ b/deploy/install/node_spec.py @@ -117,7 +117,7 @@ def fetch(args, token, retained, open_request, allow_enrollment=False): if error.code == 404: raise SpecificationError("Core node configuration was not found (HTTP 404); route /api/v1 on the Core origin directly to Core, not to Web") from None if error.code == 409: - raise SpecificationError("Core refused node configuration (HTTP 409): the deployment is in maintenance or conflicts with this node's retained specification; inspect the deployment before retrying") from None + raise SpecificationError("Core refused node configuration (HTTP 409): the deployment is resetting or conflicts with this node's retained specification; inspect the deployment before retrying") from None raise SpecificationError("Core rejected the node configuration read (HTTP " + str(error.code) + "); verify the retained or enrollment credential") from None except (urllib.error.URLError, TimeoutError, ConnectionError, http.client.IncompleteRead): raise SpecificationError("Cannot reach Core node configuration; verify the Core origin and that the reverse proxy routes /api/v1 to Core") from None diff --git a/deploy/install/rename.py b/deploy/install/rename.py index e626f42f2..2a2a192ee 100644 --- a/deploy/install/rename.py +++ b/deploy/install/rename.py @@ -127,52 +127,6 @@ def drained(root, config, state, legacy, problems): return deployment, project_ids -class Probe: - """Start only a stopped old Core's dependencies, and undo those starts on refusal.""" - def __init__(self, root, config, state, legacy, run, out): - self.root, self.config, self.state = root, config, state - self.run, self.out = run, out - self.compose = root / ("compose.json" if legacy else "generated/compose.json") - self.started, self.native_started = [], False - - def start(self): - if self.state["mode"] == "web-only" or oac_cli.http(oac_cli.core_base(self.config) + "/healthz")[0] == 200: - return - self.out("Starting the old Core with its existing files to inspect the conversion preconditions.") - result = self.run(["docker", "ps", "--filter", "label=com.docker.compose.project=" + self.state["project"], - "--format", '{{.Label "com.docker.compose.service"}}'], capture_output=True, text=True) - running = result.stdout.split() - native = native_service.is_native(self.state) - # Explicit services and --no-deps avoid running even the old migration job. - for name in (["database"] if native else ["database", "core"]): - if name not in running: - self.started.append(name) # Include a start whose response is lost. - self.run(["docker", "compose", "-f", str(self.compose), "up", "--detach", "--wait", "--no-recreate", "--no-deps", name]) - if native: - unit = convert.legacy_unit_name(self.state) - result = self.run(["systemctl", "--user", "is-active", "--quiet", unit], check=False, - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - if result.returncode: - self.native_started = True - self.run(["systemctl", "--user", "start", unit]) - if oac_cli.wait_status(oac_cli.core_base(self.config) + "/healthz") is None: - raise RenameError("The old Core did not become healthy with its existing files") - - def restore(self): - failures = [] - if self.native_started: - try: - self.run(["systemctl", "--user", "stop", convert.legacy_unit_name(self.state)]) - except (OSError, subprocess.SubprocessError): - failures.append("native Core") - if self.started: - try: - self.run(["docker", "compose", "-f", str(self.compose), "stop", *reversed(self.started)]) - except (OSError, subprocess.SubprocessError): - failures.append("old " + ", ".join(self.started)) - return failures - - def volume(run, name): result = run(["docker", "volume", "inspect", name], capture_output=True, text=True, check=False) if result.returncode: @@ -294,7 +248,6 @@ def preflight(root, target, manifest, public_url, run): problems.append(str(error)) if state and state["mode"] != "web-only" and not problems: try: - room(run, state) target_volume = "oac-" + state["project"][7:] + "_database" if volume(run, target_volume) is not None: problems.append("The target volume already exists without a conversion journal: " + target_volume) @@ -389,31 +342,61 @@ def replace_runtime(root, config, state, manifest): provider = before.get("provider") if current.get("provider") != provider: raise RenameError("The deployment provider changed during conversion; inspect it before continuing") + generation = current.get("generation") + if type(generation) is not int or type(before.get("generation")) is not int: + raise RenameError("Core returned an invalid deployment generation") + def checked(view, expected_generation): + if (view.get("installation_id") != state["installation_id"] or view.get("provider") != provider + or view.get("generation") != expected_generation or "reset" not in view or view["reset"] is not None + or view.get("resources") != {"allocations": 0, "pending": 0}): + raise RenameError("The deployment changed during conversion; inspect its identity, reset and resources") + checked(current, generation) if provider in ("docker", "microsandbox"): desired = {"provider": provider, "resources": before["specification"]["resources"], "runtime": node_spec.release(manifest)} - specification = current.get("specification") or {} - matches = all(specification.get(key) == desired[key] for key in ("resources", "runtime")) - if not matches: - if current.get("generation") != before["generation"] or current.get("maintenance") is not True: - raise RenameError("The deployment changed during conversion; keep maintenance enabled and inspect it") - current = sandbox_setup.request(core, key, "PUT", "deployment", dict(desired, expected_generation=current["generation"])) - if current.get("maintenance"): - current = sandbox_setup.request(core, key, "PATCH", "deployment/maintenance", - {"expected_generation": current["generation"], "maintenance": False}) - record["deployment_generation"] = current["generation"] + expected_specification = {key: desired[key] for key in ("resources", "runtime")} + matches = current.get("specification") == expected_specification + # Migration 78 rewrites only this historical input prefix, without a + # generation change. Compare against that exact, expected source state. + migrated_source = json.loads(json.dumps(before["specification"])) + source_runtime = migrated_source.get("runtime") or {} + reference = source_runtime.get("microsandbox_ref", "") + if reference.startswith("parsar-core-runtime@"): + source_runtime["microsandbox_ref"] = "oac-runtime@" + reference.split("@", 1)[1] + if matches: + # A lost PUT response is reconciled through exact state, not replayed. + if generation not in (before["generation"], before["generation"] + 1): + raise RenameError("The deployment generation changed during conversion") + else: + if generation != before["generation"] or current.get("specification") != migrated_source: + raise RenameError("The deployment specification changed during conversion; inspect it before continuing") + response = sandbox_setup.request(core, key, "PUT", "deployment", dict(desired, expected_generation=generation)) + checked(response, generation + 1) + if response.get("specification") != expected_specification: + raise RenameError("Core did not confirm the exact bundle Runtime specification") + generation += 1 + current = sandbox_setup.request(core, key, "GET", "deployment") + checked(current, generation) + if current.get("specification") != expected_specification: + raise RenameError("Core did not confirm the exact bundle Runtime specification") + record["deployment_generation"] = generation elif provider == "e2b": - if current.get("maintenance") is not True: - raise RenameError("E2B must stay in maintenance until its template is rebuilt") - print("E2B remains in maintenance. Rebuild the template with this release's build-template.py, replace it in Web, then resume admission.") - elif current.get("generation") != before.get("generation"): + refuse_e2b(before) + elif generation != before.get("generation"): raise RenameError("The deployment generation changed during conversion") save(root, state) +def refuse_e2b(deployment): + if deployment and deployment.get("provider") == "e2b": + raise RenameError("E2B rename conversion is not supported by this release: the old template cannot safely resume admission. " + "Keep the previous release until the Core-owned E2B upgrade transition is available; no conversion was performed.") + + def execute(root, target, state, plan, manifest, load_images, run, finish, bundle): record = state["renamed_from"] if record["source_bundle"] != manifest["source_commit"]: raise RenameError("Finish the conversion with bundle " + record["source_bundle"]) + refuse_e2b(record.get("deployment")) if state["installation_id"] != record["installation_id"] or state["source_commit"] not in (record["source_commit"], record["source_bundle"]): raise RenameError("Installation or source identity differs from the recorded conversion") if state.get("format") == 1 and (state["project"] != record["project"] or state["images"] != record["images"]): @@ -512,23 +495,33 @@ def _convert_installation(root, bundle, manifest, load_images, public_url, yes, except oac_cli.OacError: pass refuse(problems) - probe = Probe(root, config, state, bool(plan), run, print) - try: - probe.start() - if plan: - _, updated, _ = convert.preflight(root, manifest, state["images"], public_url, run) - problems += updated.problems - config, plan = updated.config, updated - deployment, ids = (None, None) if state["mode"] == "web-only" else drained(root, config, state, bool(plan), problems) - if problems: - refuse(problems) - confirm(root, target, state, plan, yes, print) - except BaseException: - failed = probe.restore() - if failed: - raise RenameError("No installation conversion was performed, but the preflight could not restore the prior stopped state of " - + ", ".join(failed) + ". Inspect the old services before retrying.") from None - raise + if state["mode"] != "web-only": + # Provider state lives in Core, not installer files. Never start an + # unknown provider to discover whether its conversion is supported. + try: + previous = api(root, config, bool(plan), "sandbox/deployment") + except RenameError: + raise RenameError("The previous Core must already be running and readable before conversion. " + "Start it with the previous release's command, verify its Core key, then rerun --convert. " + "No conversion was performed.") from None + refuse_e2b(previous) + if plan: + _, updated, _ = convert.preflight(root, manifest, state["images"], public_url, run) + problems += updated.problems + config, plan = updated.config, updated + deployment, ids = (None, None) if state["mode"] == "web-only" else drained(root, config, state, bool(plan), problems) + refuse_e2b(deployment) + if problems: + refuse(problems) + if state["mode"] != "web-only": + # Measuring space creates a temporary container, even with read-only + # mounts. Only do it after readable-provider and drain checks permit + # conversion; unsupported E2B and unreadable Core must create nothing. + try: + room(run, state) + except RenameError as error: + refuse([str(error)]) + confirm(root, target, state, plan, yes, print) record = {"install_dir": str(root), "target_dir": str(target), "project": state["project"], "installation_id": state["installation_id"], "source_commit": state["source_commit"], "source_bundle": manifest["source_commit"], "at": oac_cli.now(), "volume_copied": False, "finished": False, "images": state["images"], "deployment": deployment, "projects": ids, diff --git a/deploy/install/sandbox_setup.py b/deploy/install/sandbox_setup.py index 936b0c20d..9f45f436e 100644 --- a/deploy/install/sandbox_setup.py +++ b/deploy/install/sandbox_setup.py @@ -112,4 +112,9 @@ def initialize(root, config, state, request_body): return current if provider: raise SandboxSetupError(f"Core already uses {NAMES.get(provider, provider)} sandboxes") - return request(core, key, "POST", "deployment", request_body) + generation = current.get("generation") + if type(generation) is not int or generation < 0: + raise SandboxSetupError("Core returned an invalid deployment generation") + if current.get("reset") is not None: + raise SandboxSetupError("Core is resetting its sandbox deployment") + return request(core, key, "POST", "deployment", dict(request_body, expected_generation=generation)) diff --git a/deploy/install/test_install.py b/deploy/install/test_install.py index 1669de28f..313e3b72d 100644 --- a/deploy/install/test_install.py +++ b/deploy/install/test_install.py @@ -210,7 +210,7 @@ def test_native_core_migrates_before_enabling_its_generated_unit(self): def test_a_new_installation_selects_microsandbox_at_web_standard_size(self): self.install("--public-url", "https://core.example") standard = json.loads(STANDARD_SIZES.read_text()) - self.assertEqual(self.host.deployment_posts, [{"provider": "microsandbox", "resources": standard["microsandbox"], + self.assertEqual(self.host.deployment_posts, [{"provider": "microsandbox", "expected_generation": 0, "resources": standard["microsandbox"], "runtime": node_spec.release(self.manifest)}]) self.assertIn("\nSandboxes: microsandbox, Standard (2 CPUs, 4 GiB). Its nodes need KVM (/dev/kvm); this host " "needs it only if you add it as a node.\nAdd nodes: in Web, open Nodes and choose Add node, then " @@ -246,11 +246,11 @@ def test_docker_needs_confirmation_before_anything_is_created(self): "without KVM"): self.assertIn(risk, output) standard = json.loads(STANDARD_SIZES.read_text()) - docker = {"provider": "docker", "resources": standard["docker"], "runtime": node_spec.release(self.manifest)} + docker = {"provider": "docker", "expected_generation": 0, "resources": standard["docker"], "runtime": node_spec.release(self.manifest)} prompt = self.install_docker("--core-only", "--accept-docker-risks") prompt.assert_not_called() self.assertEqual(self.host.deployment_posts, [docker]) - self.root, self.host.deployment = self.work / "confirmed", {"provider": ""} + self.root, self.host.deployment = self.work / "confirmed", {"provider": "", "generation": 0, "reset": None} prompt = self.install_docker(answer="y") prompt.assert_called_once_with("Use Docker sandboxes anyway? [y/N] ") self.assertEqual(self.host.deployment_posts, [docker, docker]) @@ -279,7 +279,7 @@ def test_e2b_needs_a_public_address_a_private_key_file_and_an_exact_build(self): self.assertFalse(self.root.exists()) key.chmod(0o600) self.install("--sandbox", "e2b", "--e2b-api-key-file", key, "--e2b-template", BUILD, "--public-url", "https://core.example") - self.assertEqual(self.host.deployment_posts, [{"provider": "e2b", "e2b": {"api_key": secret, "template": BUILD}}]) + self.assertEqual(self.host.deployment_posts, [{"provider": "e2b", "expected_generation": 0, "e2b": {"api_key": secret, "template": BUILD}}]) self.assertIn(f"Sandboxes: E2B template {BUILD} (2 CPUs, 2 GiB). E2B runs them; no nodes are needed.", self.output.getvalue()) self.assertNotIn(secret, self.output.getvalue() + (self.root / "config.json").read_text() diff --git a/deploy/install/test_node_spec.py b/deploy/install/test_node_spec.py index cc2c9a7f5..0f139e693 100644 --- a/deploy/install/test_node_spec.py +++ b/deploy/install/test_node_spec.py @@ -62,7 +62,7 @@ def test_changed_public_url_says_to_generate_a_new_command(self): def test_failures_name_their_cause(self): for failure, message in ((urllib.error.HTTPError("https://core.example", 404, "", {}, None), "route /api/v1"), - (urllib.error.HTTPError("https://core.example", 409, "", {}, None), "maintenance"), + (urllib.error.HTTPError("https://core.example", 409, "", {}, None), "resetting"), (urllib.error.HTTPError("https://core.example", 401, "", {}, None), "credential"), (urllib.error.URLError("refused"), "reverse proxy routes /api/v1")): with self.assertRaisesRegex(node_spec.SpecificationError, message): diff --git a/deploy/install/test_rename.py b/deploy/install/test_rename.py index 98f1269df..6e2fe3534 100644 --- a/deploy/install/test_rename.py +++ b/deploy/install/test_rename.py @@ -88,7 +88,7 @@ def fixture(self, mode="all", native=False, provider="docker", default=False): self.host.deployment = {"provider": provider, "generation": 4, "maintenance": True, "resources": {"allocations": 0, "pending": 0}, "specification": { "resources": {"cpus": 2, "memory_mib": 2048}, - "runtime": {"image": "sha256:" + "9" * 64}}} + "runtime": {"image_id": "sha256:" + "9" * 64, "microsandbox_ref": "parsar-core-runtime@sha256:" + "7" * 64}}} self.before = self.snapshot() self.before_volumes = copy.deepcopy(self.host.volumes) self.identity = state["installation_id"] @@ -107,6 +107,27 @@ def convert(self, *extra, explicit=True, yes=True): def state(self): return json.loads((self.root / "state.json").read_text()) + @contextlib.contextmanager + def no_file_writes(self): + with mock.patch.object(oac_cli, "write_private") as write, \ + mock.patch.object(oac_cli, "create_private") as create, \ + mock.patch.object(oac_cli, "save_state") as save: + yield + write.assert_not_called() + create.assert_not_called() + save.assert_not_called() + + def assert_read_only_commands(self): + # An allowlist also rejects transient Docker run/create operations that + # leave no container behind and therefore escape state snapshots. + prefixes = (("docker", "compose", "version"), ("docker", "info"), + ("docker", "ps"), ("docker", "inspect"), + ("docker", "volume", "inspect"), ("docker", "volume", "ls"), + ("systemctl", "--user", "show", "--property=Version", "--value"), + ("loginctl", "show-user"), ("ldd",)) + for command in self.host.commands: + self.assertTrue(any(tuple(command[:len(prefix)]) == prefix for prefix in prefixes), command) + def assert_preserved(self): state = self.state() self.assertEqual(state["installation_id"], self.identity) @@ -134,7 +155,7 @@ def test_default_moves_custom_stays_preserves_identity_and_replaces_runtime(self self.assertIn("parsar was renamed to oac", (old / "parsar").read_text()) self.assertEqual(len(self.host.deployment_posts), 1 + int(default)) self.assertEqual(self.host.deployment["generation"], 5) - self.assertFalse(self.host.deployment["maintenance"]) + self.assertIsNone(self.host.deployment["reset"]) self.assertEqual(self.host.deployment_posts[-1]["resources"], {"cpus": 2, "memory_mib": 2048}) self.assertEqual(self.host.deployment_posts[-1]["runtime"]["microsandbox_ref"], MANIFEST["runtime_ref"]) @@ -161,34 +182,25 @@ def test_static_refusal_does_not_start_stopped_core(self): self.assertEqual(self.snapshot(), before) self.assertFalse(any("up" in command for command in self.host.commands)) - def test_stopped_core_probe_restores_only_new_starts_on_refusal(self): - self.fixture() - self.host.containers["core"]["running"] = False - self.host.deployment["maintenance"] = False - before = self.host.running() - with self.assertRaisesRegex(rename.RenameError, "maintenance"): - self.convert() - self.assertEqual(self.host.running(), before) - self.assertEqual(self.snapshot(), self.before) - stops = [command[4:] for command in self.host.commands if command[:2] == ["docker", "compose"] and "stop" in command] - self.assertEqual(stops, [["stop", "core"]]) - self.host.deployment["maintenance"] = True - self.convert() - self.assert_preserved() - - def test_restore_failure_is_reported_without_false_no_change_claim(self): - self.fixture() - self.host.containers["core"]["running"] = False - self.host.deployment["maintenance"] = False - original = install.run - def fail_stop(command, **kwargs): - if "stop" in command: - raise subprocess.CalledProcessError(1, command) - return original(command, **kwargs) - with mock.patch.object(install, "run", side_effect=fail_stop), self.assertRaisesRegex(rename.RenameError, "could not restore") as error: - self.convert() - self.assertNotIn("Nothing was changed", str(error.exception)) - self.assertEqual(self.snapshot(), self.before) + def test_stopped_or_unreadable_core_refuses_before_mutations(self): + for unreadable in (False, True): + with self.subTest(unreadable=unreadable): + self.root = self.work / ("unreadable" if unreadable else "stopped") + self.host.containers.clear() + self.host.volumes.clear() + self.fixture() + if unreadable: + (self.root / "secrets/core.key").write_text("wrong-key" * 8) + else: + self.host.containers["core"]["running"] = False + before, running = self.snapshot(), self.host.running() + self.host.commands.clear() + with self.no_file_writes(), self.assertRaisesRegex(rename.RenameError, "previous Core must already be running and readable"): + self.convert() + self.assertEqual(self.snapshot(), before) + self.assertEqual(self.host.running(), running) + self.assertEqual(self.host.volumes, self.before_volumes) + self.assert_read_only_commands() def test_foreign_target_volume_and_container_refuse_without_removal(self): for container in (False, True): @@ -279,15 +291,32 @@ def test_native_unit_is_disabled_before_move_and_new_unit_runs(self): self.assertTrue(self.host.native["active"]) self.assert_preserved() - def test_e2b_stays_in_maintenance_and_web_only_never_copies_a_database(self): + def test_e2b_refuses_before_conversion_mutations(self): self.fixture(provider="e2b") - self.convert() - self.assertTrue(self.host.deployment["maintenance"]) + self.host.commands.clear() + with self.no_file_writes(), self.assertRaisesRegex(rename.RenameError, "E2B rename conversion is not supported"): + self.convert() + self.assertEqual(self.snapshot(), self.before) + self.assertEqual(self.host.volumes, self.before_volumes) self.assertFalse(self.host.deployment_posts) - self.assertIn("E2B remains in maintenance", self.output.getvalue()) - self.root = self.work / "web-only" - self.host.containers.clear() - self.host.volumes.clear() + self.assert_read_only_commands() + + def test_e2b_resume_refuses_before_services_or_journal_changes(self): + self.fixture() + with mock.patch.object(rename, "execute", side_effect=KeyboardInterrupt), self.assertRaises(KeyboardInterrupt): + self.convert() + state = self.state() + state["renamed_from"]["deployment"]["provider"] = "e2b" + oac_cli.save_state(self.root, state) + before, volumes = self.snapshot(), copy.deepcopy(self.host.volumes) + self.host.commands.clear() + with self.no_file_writes(), self.assertRaisesRegex(rename.RenameError, "E2B rename conversion is not supported"): + self.convert() + self.assertEqual(self.snapshot(), before) + self.assertEqual(self.host.volumes, volumes) + self.assert_read_only_commands() + + def test_web_only_never_copies_a_database(self): self.fixture(mode="web-only") self.convert() self.assert_preserved() @@ -310,18 +339,18 @@ def test_old_command_copy_and_old_default_fresh_install_are_refused(self): self.assertFalse(self.new_default.exists()) - def test_stopped_native_probe_restores_unit_and_database(self): + def test_stopped_native_core_refuses_without_start(self): self.fixture(native=True) self.host.native["active"] = False self.host.containers["database"]["running"] = False - self.host.deployment["maintenance"] = False - with self.assertRaisesRegex(rename.RenameError, "maintenance"): + running = self.host.running() + self.host.commands.clear() + with self.no_file_writes(), self.assertRaisesRegex(rename.RenameError, "previous Core must already be running and readable"): self.convert() self.assertFalse(self.host.native["active"]) - self.assertFalse(self.host.containers["database"]["running"]) - self.assertTrue(self.host.containers["web"]["running"]) - self.assertNotIn(["systemctl", "--user", "disable", "--now", "parsar-0123456789-core.service"], self.host.commands) + self.assertEqual(self.host.running(), running) self.assertEqual(self.snapshot(), self.before) + self.assert_read_only_commands() def test_exact_runtime_retry_after_lost_put_response_keeps_one_generation(self): self.fixture(provider="microsandbox") @@ -337,13 +366,30 @@ def uncertain(core, key, method, path, value=None): with mock.patch.object(rename.sandbox_setup, "request", side_effect=uncertain), self.assertRaisesRegex(rename.sandbox_setup.SandboxSetupError, "lost response"): self.convert() self.assertEqual(self.host.deployment["generation"], 5) - self.assertTrue(self.host.deployment["maintenance"]) + self.assertIsNone(self.host.deployment["reset"]) self.convert() self.assertEqual(len(self.host.deployment_posts), 1) self.assertEqual(self.host.deployment["generation"], 5) self.assertEqual(self.host.deployment["specification"]["resources"]["root_disk_mib"], 8192) self.assert_preserved() + def test_new_core_drift_refuses_without_replaying_put(self): + self.fixture() + with mock.patch.object(rename, "execute", side_effect=KeyboardInterrupt), self.assertRaises(KeyboardInterrupt): + self.convert() + state = self.state() + before = state["renamed_from"]["deployment"] + current = dict(copy.deepcopy(before), installation_id=self.identity, reset=None) + current.pop("maintenance") + current["specification"]["runtime"]["microsandbox_ref"] = "oac-runtime@sha256:" + "7" * 64 + for drift in ({"generation": 8}, {"reset": {"clear": "force"}}, + {"resources": {"allocations": 1, "pending": 0}}, + {"specification": {"resources": {"cpus": 99}, "runtime": {}}}): + with self.subTest(drift=drift), mock.patch.object(rename.sandbox_setup, "request", return_value=dict(current, **drift)) as request: + with self.assertRaises(rename.RenameError): + rename.replace_runtime(self.root, oac_cli.load_config(self.root), state, self.manifest) + self.assertEqual([call.args[2] for call in request.call_args_list], ["GET"]) + def test_retry_refuses_foreign_copy_container_even_when_volume_is_owned(self): self.fixture() with interrupt(rename, "save", lambda root, state: state["renamed_from"]["volume_copied"]), self.assertRaises(KeyboardInterrupt): diff --git a/deploy/install/test_sandbox_setup.py b/deploy/install/test_sandbox_setup.py index c685e093a..501192580 100644 --- a/deploy/install/test_sandbox_setup.py +++ b/deploy/install/test_sandbox_setup.py @@ -41,6 +41,22 @@ def test_an_existing_selection_is_kept_or_reported(self): with self.assertRaisesRegex(sandbox_setup.SandboxSetupError, "different installation"): self.initialize(dict(current, installation_id="other", provider=""), docker) + def test_initialization_uses_observed_generation_once(self): + for generation in (0, 8): + with self.subTest(generation=generation): + current = {"installation_id": INSTALLATION, "provider": "", "generation": generation, "reset": None} + selection = {"provider": "docker", "resources": {"cpus": 2, "memory_mib": 2048}, "runtime": {}} + writes = [] + def send(req): + if req.get_method() == "POST": + writes.append(json.loads(req.data)) + return 409, b'{"error":{"code":"generation_stale","message":"Deployment generation changed"}}' + return 200, json.dumps(current).encode() + with mock.patch.object(sandbox_setup, "send", side_effect=send), self.assertRaises(sandbox_setup.SandboxSetupError): + sandbox_setup.initialize(self.root, self.config, self.state, selection) + self.assertEqual(writes, [dict(selection, expected_generation=generation)]) + self.assertNotIn("expected_generation", selection) + def test_e2b_template_is_an_exact_build(self): build = "0f6c1e8e-7d3a-4b8e-9a51-2b7f7f0c9d11" for value in ("base:" + build, "my_template-2:" + build): diff --git a/docs/api/README.md b/docs/api/README.md index e332e8dd6..e9e128db9 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -51,7 +51,7 @@ with the Core key. | `projects/{project_id}/environments/{environment_id}/executor-credentials[/{key_id}]` | Executor credentials for a self-hosted Environment | [Executor credentials](../../contracts/agents-api/environment-executor-credentials.md) | | `installation` | Public URL, API base URL, source commit, the installer's process settings and what is bound to the public URL; available before any deployment | [Installation](../../contracts/agents-api/installation.md) | | `metrics` | Core's own process metrics | [Core metrics](../../contracts/agents-api/core-metrics.md) | -| `sandbox/deployment[/maintenance]`, `sandbox/enrollment-tokens`, `sandbox/nodes[/{node_id}[/allocations]]` | Sandbox deployment, node enrollment tokens and nodes | [Sandbox deployment](../../contracts/agents-api/sandbox-deployment.md), [node operations](../../services/agents-api/HOSTED-SANDBOX-MANAGER.md), [node host history](../../contracts/agents-api/node-host-history.md) | +| `sandbox/deployment[/reset]`, `sandbox/enrollment-tokens`, `sandbox/nodes[/{node_id}[/allocations]]` | Sandbox deployment, node enrollment tokens and nodes | [Sandbox deployment](../../contracts/agents-api/sandbox-deployment.md), [node operations](../../services/agents-api/HOSTED-SANDBOX-MANAGER.md), [node host history](../../contracts/agents-api/node-host-history.md) | | `harnesses`, `harnesses/{harness}/model-provider` | Supported harnesses and each harness's deployment default model provider (write-only key) | [Model execution](../../contracts/agents-api/model-execution.md#deployment-defaults) | ## Machine connection API diff --git a/docs/api/web-management.md b/docs/api/web-management.md index 0b09b6017..3311f5b54 100644 --- a/docs/api/web-management.md +++ b/docs/api/web-management.md @@ -79,15 +79,15 @@ and [generated OpenAPI](../../contracts/agents-api/core.openapi.yaml) define deployment and node operations: - `GET/POST/PUT /core/v1/sandbox/deployment` and - `PATCH /core/v1/sandbox/deployment/maintenance`. + `POST/DELETE /core/v1/sandbox/deployment/reset`. - `GET /core/v1/sandbox/nodes`, `PATCH/DELETE /core/v1/sandbox/nodes/{node_id}`, and `GET /core/v1/sandbox/nodes/{node_id}/allocations`. - `POST /core/v1/sandbox/enrollment-tokens` for a one-time node installation command. Its non-secret `enrollment_id` reappears on the node that command registers. PostgreSQL owns one provider, per-sandbox resource specification and immutable -Runtime selection. POST initializes it; PUT replaces the complete selection using -`expected_generation`. Requests carry `resources` and, for Docker/microsandbox, +Runtime selection. POST initializes it and PUT updates the same provider; both +require the observed `expected_generation`, including zero at first setup. Requests carry `resources` and, for Docker/microsandbox, `runtime`; safe responses return `specification` and `specification_digest`. Response `resources.allocations` and `resources.pending` are cleanup counts, not CPU, memory or disk settings. E2B accepts a write-only key and exact template build @@ -96,23 +96,26 @@ may omit `resources` to adopt the validated build's CPU and memory; responses sh the build as read at selection time in `e2b.template_build`. Microsandbox responses return its idle `suspension` policy; other providers return null. -Provider, resource and Runtime changes all require global maintenance and verified -cleanup of retained/pending resources. Core validates the candidate before commit; -a rejection preserves the previous configuration. A changed commit advances the -generation and retires old nodes and enrollment tokens atomically. Explicitly resume -after success. Neither switching nor editing configuration deletes resources or -migrates existing Sessions. During maintenance, administrators can explicitly -[archive each retained hosted Session](../../contracts/agents-api/admin-api.md#administrative-session-archive) -through the Core administrator API at the current generation, then read its -resource disposition and recheck deployment counts. Archive preserves history -and persisted Files/Artifacts; unpersisted workspace contents are lost and the -original Session cannot resume. This API does not add a console archive control. +Same-provider updates currently require zero retained/pending resources and no +reset. Changing backend requires explicit durable reset before a new POST. +Auto archives idle/queued/suspended hosted Sessions, waits for started work and +file writes, and escalates at its persisted deadline; force requests cancellation +and verified cleanup. The deployment response supplies the authoritative +`reset.remaining` partition and offline-node subset; Web must not derive either +from independently loaded lists. No rollout projection is provided yet. + +Administrators may [archive an individual hosted Session](../../contracts/agents-api/admin-api.md#administrative-session-archive) +at the current generation without reset. History and persisted Files/Artifacts +survive; unpersisted workspace is lost and the original Session cannot resume. +Cancel reset stops further archives, not cleanup already requested. After zero +resources Core clears the selection and advances generation; configure again using +that new generation. Never automatically replay an uncertain write. Public Environment Templates, the `/v1` contract and caller-owned `self_hosted` provisioning remain unchanged. `GET /api/v1/sandbox-node/configuration` uses an enrollment Bearer token, or a retained node Bearer credential with `X-OAC-Node-ID`. This read does not consume -enrollment. Retained matching nodes can read their configuration during maintenance. +enrollment. Retained matching nodes can read their configuration during reset. Installers must verify the returned generation, specification digest and Runtime before registration; local files cannot override the saved limits. A mismatch returns `sandbox_specification_mismatch` without replacing node state. diff --git a/docs/configuration.md b/docs/configuration.md index 4f726d83e..019da0bad 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -125,7 +125,7 @@ Core API with the Core key. | Setting | Where in Web | Core API | Notes | | --- | --- | --- | --- | -| Sandbox backend: Docker, microsandbox or E2B | **Nodes** (**Sandbox backend** with E2B): the setup wizard, ending with **Save configuration** | `/core/v1/sandbox/deployment` | One backend per deployment. `install.sh --sandbox` saves the first choice. To change it, see the Nodes page; that flow will change in a coming release | +| Sandbox backend: Docker, microsandbox or E2B | **Nodes** (**Sandbox backend** with E2B): the setup wizard, ending with **Save configuration** | `/core/v1/sandbox/deployment` | One backend per deployment. `install.sh --sandbox` saves the first choice. Changing backend requires explicit reset and a new setup at the resulting generation | | Sandbox size and Runtime release | **Nodes**: the setup wizard | `/core/v1/sandbox/deployment` | Every sandbox gets the same size. See [Sandbox deployment](#sandbox-deployment) | | E2B API key and template build | **Nodes**: the setup wizard's **E2B cloud** (the page is then called **Sandbox backend**) | `/core/v1/sandbox/deployment` | The key is write-only and encrypted | | Nodes and their capacity | **Nodes**: **Add node**, **Edit node**, **Remove node** on a node's page (**Remove** in its list row) | `/core/v1/sandbox/enrollment-tokens`, `/core/v1/sandbox/nodes` | See [Node capacity](#node-capacity) and the [nodes guide](getting-started/nodes.md) | @@ -162,7 +162,7 @@ Changing the provider, the size or the Runtime applies to the whole deployment; **Nodes** page, and note that this flow will change in a coming release. A successful change retires the old nodes and enrollment commands; history stays, and existing Sessions never move between providers. See the -[operator reference](../services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-maintenance) +[operator reference](../services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-reset) and the [deployment contract](../contracts/agents-api/sandbox-deployment.md). ### Node capacity diff --git a/docs/getting-started/nodes.md b/docs/getting-started/nodes.md index a4d9e4780..60fef0bdc 100644 --- a/docs/getting-started/nodes.md +++ b/docs/getting-started/nodes.md @@ -184,7 +184,7 @@ The installation ID is in the command (`--installation-id`) and on the **System* 1. In Web, open **Nodes** and choose **Remove node** on the node's page, or **Remove** in its list row, then **Confirm removal**. Core refuses while the node still holds sandboxes, snapshots or pending cleanup; let them finish, or - [archive their Sessions](../../services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-maintenance) + [archive their Sessions](../../services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-reset) through the Core API. Removal is permanent: the host can come back only as a new node. 2. Web then shows **Clean up the host** with the uninstall command. Run it on the host: @@ -268,7 +268,7 @@ this flow will change in a coming release. A change applies to the whole deploym and retires every node. Retired nodes drop out of the list, so there is nothing to remove in Web: on each host, run the [uninstall command](#remove-a-node) (Core no longer accepts the node, so it needs no `--force`), then add the host again with a new command. -The [operator reference](../../services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-maintenance) +The [operator reference](../../services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-reset) describes today's procedure and API. diff --git a/docs/getting-started/operations.md b/docs/getting-started/operations.md index a0fd4864d..1f54e8df5 100644 --- a/docs/getting-started/operations.md +++ b/docs/getting-started/operations.md @@ -192,10 +192,12 @@ The latter rename their directory, project and payload without copying a databas Preflight reports generated-file edits, invalid configuration, destination conflicts, insufficient database-copy space, and every available drain blocker. It reads the -old Core with its Core key. When that Core is stopped and static checks pass, it -announces a temporary start using only the existing files. Refusal restores the -services this probe started; a restoration failure is reported explicitly. No -installation conversion happens before confirmation. +old Core with its Core key. Start the previous release yourself before conversion: +a stopped or unreadable Core is refused without starting services. This is required +to identify the provider before any conversion writes. E2B conversion is currently +refused, including interrupted journal resumes; keep the previous release until +the safe Core-owned E2B upgrade transition is available. No installation conversion +happens before confirmation. The confirmation prints the backup command for the old project and the directory and secret moves. For a config.json installation, take the backup before confirming: @@ -220,9 +222,11 @@ The new state format is 2; config.json's schema format remains 1. Ports, public secrets, Core key, installation ID and database contents are preserved. The old `parsar` path becomes a stub naming the new `oac` command. New services use `OAC_*` settings and `io.oac.inputs` labels. Docker and microsandbox deployments retain -their resources, replace their Runtime with the bundle's release through the normal -maintenance-only API, then resume admission. E2B remains in maintenance: rebuild its -template with this release's `build-template.py`, replace it in Web, then resume. +their resources and replace their Runtime with the bundle's exact release through +an expected-generation same-provider PUT. New Core must report reset null and zero +held resources; conversion reads back the exact specification and generation. +Ordinary migration retires old maintenance and resumes admission, which is why E2B +conversion remains blocked until its safe upgrade preparation is implemented. Rerun `./install.sh --convert` with the **same bundle** after any interruption, including after the directory move. For a custom directory, repeat `--install-dir`. @@ -326,10 +330,11 @@ Nodes from releases that used the removed `/core/v1/sandbox/enroll` and microsandbox deployment: 1. Drain with the previous release while its nodes are connected: enter maintenance, - archive retained hosted Sessions and wait until nothing is retained (the - [maintenance procedure](../../services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-maintenance)). + archive retained hosted Sessions and wait until nothing is retained using that + previous release's maintenance procedure. 2. Upgrade Core and Web, and change the routing as above. -3. Save the new release's Runtime, resume, and add the nodes again. On each node host, +3. Reset a drained selection that lacks a specification, then configure the new + release's Runtime and add nodes again. On each node host, stop the old node service and move its state directory aside as a backup first. A Web-selected Docker or microsandbox deployment saved before deployment specifications diff --git a/docs/web/README.md b/docs/web/README.md index 454964fa7..f46787472 100644 --- a/docs/web/README.md +++ b/docs/web/README.md @@ -21,7 +21,7 @@ management client, and it sends nothing to `/v1`. | Monitor | Sandbox metrics | Node capacity and hosted Runtime CPU and memory across Projects | | Monitor | Session log | Every Session, opening one Session's read-only conversation, trace and Turns; a self-hosted Session's page also manages its executor credentials and gives the command that connects a host | | Resources | Agents, Environment templates, Skills, Files, Vaults | Inspection and permitted deletion | -| Platform | Projects and keys, Nodes, System | Project and key lifecycle; sandbox deployment and nodes; System: the installation's public address, API base URL, ID and source commit (read-only), each harness's default model provider (write-only key) beside its read-only startup state, the sandbox configuration every Project shares: provider, sandbox size, Runtime or E2B template build, idle suspension and maintenance, and Core's config.json startup settings with where to change them | +| Platform | Projects and keys, Nodes, System | Project and key lifecycle; sandbox deployment and nodes; System: the installation's public address, API base URL, ID and source commit (read-only), each harness's default model provider (write-only key) beside its read-only startup state, the sandbox configuration every Project shares: provider, sandbox size, Runtime or E2B template build, idle suspension and reset, and Core's config.json startup settings with where to change them | Missing data is shown as missing (—), never as zero. How each figure is read and bounded is recorded in [management interface coverage](protocol-coverage.md). diff --git a/docs/web/admin-metrics-backend-requirements.md b/docs/web/admin-metrics-backend-requirements.md index 100733997..ad1c988f8 100644 --- a/docs/web/admin-metrics-backend-requirements.md +++ b/docs/web/admin-metrics-backend-requirements.md @@ -80,7 +80,7 @@ This replaces the Item fan-out. The Overview shows Core itself beside its sandbox hosts. Core runs no sandboxes, so it has no slots; the operator asked for its CPU and memory instead. Nothing reports them, so the console shows the Web API's reachability -and maintenance state and "Not reported" for CPU and memory. +and "Not reported" for CPU and memory. `GET /core/v1/core-status` diff --git a/docs/web/architecture.md b/docs/web/architecture.md index 6e627327f..9cad3ec53 100644 --- a/docs/web/architecture.md +++ b/docs/web/architecture.md @@ -74,7 +74,7 @@ route. Deployment sandbox management selects one provider at a time: E2B, Docker or microsandbox. E2B uses the deployment's provider integration; Docker and microsandbox -use operator-managed machines. Provider setup, maintenance and node administration +use operator-managed machines. Provider setup, reset and node administration belong to the existing sandbox management surface. An application's `self_hosted` Runtime, including one it provisions in its own E2B diff --git a/docs/web/protocol-coverage.md b/docs/web/protocol-coverage.md index 09b11c628..c14b5d3f0 100644 --- a/docs/web/protocol-coverage.md +++ b/docs/web/protocol-coverage.md @@ -141,8 +141,8 @@ consumed; the list carries each provider. | Operation | Route | Console use | | --- | --- | --- | -| Deployment | `GET`, `POST`, `PUT /core/v1/sandbox/deployment` | Read the provider, the read-only `core_url` (config.json's `public_url`, shown in the setup review and never sent), maintenance state, installation ID and specification; a 409 `sandbox_configuration_error` (E2B with a loopback `public_url`) shows Core's message in the setup wizard, with the installation's config file and apply command, and leaves nothing to confirm; initialize the deployment with `resources` and the Docker or microsandbox `runtime` release, or with the E2B account and no `resources` (Core adopts the template build's CPU and memory); change its settings with the expected generation. E2B's `e2b.template_build` (status, CPU, memory, disk) shows on System, the Sandbox backend summary and Sandbox metrics, and sizes each sandbox when `specification.resources` is missing; microsandbox's `suspension` (idle and retention seconds) shows on System and the Nodes summary | -| Maintenance | `PATCH /core/v1/sandbox/deployment/maintenance` | Enter or leave maintenance to change the provider | +| Deployment | `GET`, `POST`, `PUT /core/v1/sandbox/deployment` | Read the provider, the read-only `core_url` (config.json's `public_url`, shown in the setup review and never sent), reset state, installation ID and specification; a 409 `sandbox_configuration_error` (E2B with a loopback `public_url`) shows Core's message in the setup wizard, with the installation's config file and apply command, and leaves nothing to confirm; initialize the deployment with `resources` and the Docker or microsandbox `runtime` release, or with the E2B account and no `resources` (Core adopts the template build's CPU and memory); change its settings with the expected generation. E2B's `e2b.template_build` (status, CPU, memory, disk) shows on System, the Sandbox backend summary and Sandbox metrics, and sizes each sandbox when `specification.resources` is missing; microsandbox's `suspension` (idle and retention seconds) shows on System and the Nodes summary | +| Reset | `POST/DELETE /core/v1/sandbox/deployment/reset` | Explicitly clear hosted resources or cancel the remaining clear at the observed generation; consume Core’s remaining/offline projection | | Nodes | `GET /core/v1/sandbox/nodes` | Nodes page; fleet on Overview; node capacity on Sandbox metrics. An online node's `diagnostic` (`docker_unavailable`, `docker_limits_unsupported`, `runtime_image_unavailable`, `kvm_unavailable`, `microsandbox_artifacts_unavailable`, `capacity_insufficient`, `provider_unavailable`; any other value reads as `provider_unavailable`) marks it degraded and names the reason and fix in the help tip beside its status on each of these and on the node's page. A node whose `core_url` (the address it enrolled with) differs from the deployment's `core_url` is named on the Nodes page as bound to an old address, to be removed and added again, and its status there and on its page reads Old address instead of its health; an empty `core_url` (a node Core did not enroll) is unknown, not old. **Add node** follows only the node whose `enrollment_id` equals its command's; a node enrolled before Core recorded it reports null and never matches | | Node detail | `GET /core/v1/sandbox/nodes/{node_id}?range=1h\|6h\|24h` | Sandbox metrics node dialog: the host's CPU busy share and memory from its last heartbeat, and their history over the page's range. **Edit node** reads `host.effective_cpu_cores` and `host.total_memory_bytes` to show the host beside each sandbox's size and at most how many of those fit | | Allocations | `GET /core/v1/sandbox/nodes/{node_id}/allocations` | Nodes page; Sandbox metrics. Under microsandbox, a node's page shows from `compute_phase_changed_at` how long each allocation has been in its compute phase and, while suspended, about when Core reclaims it (that time plus the deployment's `suspension.retention_seconds`); a null time shows a dash | diff --git a/internal/agentdaemon/device/cancellation.go b/internal/agentdaemon/device/cancellation.go new file mode 100644 index 000000000..791f8b3b2 --- /dev/null +++ b/internal/agentdaemon/device/cancellation.go @@ -0,0 +1,14 @@ +package device + +import "time" + +// ArchivedCancellationReceiptLimit bounds receipt draining after an administrator +// archive. It is not a model/tool timeout and never restarts on heartbeat or retry. +const ArchivedCancellationReceiptLimit = 20 * time.Second + +// ArchivedCancellationReceipt authorizes only the original delivery's receipt +// drain. It grants neither Runtime credentials nor permission to start work. +type ArchivedCancellationReceipt struct { + RunID string + Deadline time.Time +} diff --git a/internal/agentdaemon/gateway/cancellation.go b/internal/agentdaemon/gateway/cancellation.go new file mode 100644 index 000000000..83937c0bb --- /dev/null +++ b/internal/agentdaemon/gateway/cancellation.go @@ -0,0 +1,122 @@ +package gateway + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +// ArchivedCancellationStore is deliberately separate from RuntimeStore: a +// receipt opportunity cannot authenticate a new connection or authorize work. +type ArchivedCancellationStore interface { + ArchivedCancellationReceipt(context.Context, string, string, []string) (device.ArchivedCancellationReceipt, error) +} + +// TrackExecutionDelivery retains the exact existing Core delivery through its +// terminal database commit. Durable subscriptions may already have seen Done or +// been removed while cancellation ACK/commit is still pending. +func (s *Session) TrackExecutionDelivery(runID string) (func(), error) { + s.receiptMu.Lock() + defer s.receiptMu.Unlock() + if runID == "" || s.IsClosed() || s.receiptDrain.RunID != "" { + return nil, ErrSessionClosed + } + if s.deliveries == nil { + s.deliveries = make(map[string]struct{}) + } + if _, exists := s.deliveries[runID]; exists { + return nil, errors.New("execution delivery already tracked") + } + s.deliveries[runID] = struct{}{} + var once sync.Once + return func() { + once.Do(func() { + s.receiptMu.Lock() + delete(s.deliveries, runID) + closePeer := s.receiptDrain.RunID == runID + s.receiptMu.Unlock() + if closePeer { + s.CloseWithCode(CloseRuntimeDeleted, "archived cancellation settled") + } + }) + }, nil +} + +// DrainArchivedCancellation checks both durable archive identity and this exact +// connection's unfinished delivery. It performs no liveness or credential write. +func (s *Session) DrainArchivedCancellation(ctx context.Context) (bool, error) { + store, ok := s.heartbeat.(ArchivedCancellationStore) + if !ok || s.IsClosed() { + return false, nil + } + s.receiptMu.Lock() + ids := make([]string, 0, len(s.deliveries)) + for id := range s.deliveries { + ids = append(ids, id) + } + s.receiptMu.Unlock() + if len(ids) == 0 { + return false, nil + } + receipt, err := store.ArchivedCancellationReceipt(ctx, s.DeviceID, s.credentialHash, ids) + if err != nil || receipt.RunID == "" { + return false, err + } + s.receiptMu.Lock() + defer s.receiptMu.Unlock() + if _, exists := s.deliveries[receipt.RunID]; !exists || s.IsClosed() || !time.Now().Before(receipt.Deadline) { + return false, nil + } + if s.receiptDrain.RunID != "" { + // Once fenced, neither a later cancel nor a changed response extends it. + return s.receiptDrain.RunID == receipt.RunID && s.receiptDrain.Deadline.Equal(receipt.Deadline), nil + } + s.receiptDrain = receipt + s.receiptTimer = time.AfterFunc(time.Until(receipt.Deadline), func() { s.CloseWithCode(CloseRuntimeDeleted, "archived cancellation deadline") }) + return true, nil +} + +func (s *Session) receiptDraining() bool { + s.receiptMu.Lock() + defer s.receiptMu.Unlock() + return s.receiptDrain.RunID != "" +} + +func (s *Session) allowsReceiptFrame(env proto.Envelope, outbound bool) bool { + s.receiptMu.Lock() + defer s.receiptMu.Unlock() + if s.receiptDrain.RunID == "" { + return true + } + if !time.Now().Before(s.receiptDrain.Deadline) { + return false + } + if !outbound && env.Type == proto.TypeHeartbeat { + return true + } + if env.ID != s.receiptDrain.RunID { + return false + } + if outbound { + return env.Type == proto.TypePromptCancel + } + if env.Type == proto.TypeInteractionDecisionAck { + var ack proto.InteractionDecisionAckPayload + return env.DecodePayload(&ack) == nil && ack.DeliveryID == "cancel:"+env.ID + } + // Existing run events must remain lossless, including child observations and + // Done before ACK. Workspace/preparation replies have other correlation IDs. + return true +} + +func (s *Session) stopReceiptTimer() { + s.receiptMu.Lock() + defer s.receiptMu.Unlock() + if s.receiptTimer != nil { + s.receiptTimer.Stop() + } +} diff --git a/internal/agentdaemon/gateway/cancellation_test.go b/internal/agentdaemon/gateway/cancellation_test.go new file mode 100644 index 000000000..1f0caa264 --- /dev/null +++ b/internal/agentdaemon/gateway/cancellation_test.go @@ -0,0 +1,146 @@ +package gateway + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +type receiptStore struct { + *fakeHeartbeatStore + mu sync.Mutex + receipt device.ArchivedCancellationReceipt +} + +func (r *receiptStore) ArchivedCancellationReceipt(_ context.Context, id, hash string, runs []string) (device.ArchivedCancellationReceipt, error) { + r.mu.Lock() + defer r.mu.Unlock() + if id != "device" || hash != "original-hash" { + return device.ArchivedCancellationReceipt{}, nil + } + for _, run := range runs { + if run == r.receipt.RunID { + return r.receipt, nil + } + } + return device.ArchivedCancellationReceipt{}, nil +} + +func TestArchivedReceiptTracksDeliveryBeyondDoneAndRejectsNewWork(t *testing.T) { + conn := newFakeConn() + peer := NewSession(conn, "device", "", "", NewRegistry(), nil) + defer peer.Close("test complete") + peer.credentialHash = "original-hash" + receipt := device.ArchivedCancellationReceipt{RunID: "run", Deadline: time.Now().Add(time.Second)} + peer.heartbeat = &receiptStore{fakeHeartbeatStore: newFakeHeartbeatStore(), receipt: receipt} + if draining, err := peer.DrainArchivedCancellation(t.Context()); err != nil || draining { + t.Fatal("unowned delivery got drain", draining, err) + } + release, err := peer.TrackExecutionDelivery("run") + if err != nil { + t.Fatal(err) + } + subscription, err := peer.SubscribeDurable("run") + if err != nil { + t.Fatal(err) + } + done, _ := proto.NewEnvelope(proto.TypeDone, "run", proto.DonePayload{}) + peer.dispatch(done) + for range subscription.Events { + } + if draining, err := peer.DrainArchivedCancellation(t.Context()); err != nil || !draining { + t.Fatal("Done discarded in-flight ACK/commit owner", draining, err) + } + if _, err := peer.TrackExecutionDelivery("new-run"); err == nil { + t.Fatal("new delivery admitted while draining") + } + for _, kind := range []string{proto.TypePromptRequest, proto.TypePromptSteer, proto.TypeExecutionPrepare, proto.TypeWorkspaceWrite, proto.TypeWorkspaceRead} { + env, _ := proto.NewEnvelope(kind, "run", nil) + if err := peer.Send(t.Context(), env); err == nil { + t.Fatal("drain permitted new operation", kind) + } + } + cancel, _ := proto.NewEnvelope(proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: "cancel:run"}) + if err := peer.Send(t.Context(), cancel); err != nil { + t.Fatal("drain blocked cancellation", err) + } + release() // The caller has now committed its terminal result. + if !peer.IsClosed() { + t.Fatal("finished delivery retained revoked connection") + } +} + +type blockedControlConn struct { + *fakeConn + entered chan struct{} + resume chan struct{} + wrote chan struct{} + once sync.Once +} + +func (c *blockedControlConn) WriteMessage(kind int, data []byte) error { + c.once.Do(func() { close(c.entered) }) + <-c.resume + defer close(c.wrote) + return c.fakeConn.WriteMessage(kind, data) +} +func (c *blockedControlConn) WriteControl(kind int, data []byte, _ time.Time) error { + return c.fakeConn.WriteMessage(kind, data) +} + +func TestCloseWithCodeUsesConcurrentControlWriter(t *testing.T) { + conn := &blockedControlConn{fakeConn: newFakeConn(), entered: make(chan struct{}), resume: make(chan struct{}), wrote: make(chan struct{})} + peer := NewSession(conn, "device", "", "", NewRegistry(), nil) + peer.Start() + env, _ := proto.NewEnvelope(proto.TypePromptCancel, "run", nil) + if err := peer.Send(t.Context(), env); err != nil { + t.Fatal(err) + } + select { + case <-conn.entered: + case <-time.After(time.Second): + t.Fatal("data writer never entered") + } + finished := make(chan struct{}) + go func() { peer.CloseWithCode(CloseRuntimeDeleted, "retired"); close(finished) }() + select { + case <-finished: + case <-time.After(time.Second): + close(conn.resume) + <-finished + t.Fatal("close used the blocked data writer") + } + close(conn.resume) + select { + case <-conn.wrote: + case <-time.After(time.Second): + t.Fatal("data writer did not exit") + } +} + +func TestArchivedReceiptDeadlineDoesNotRenew(t *testing.T) { + peer := NewSession(newFakeConn(), "device", "", "", NewRegistry(), nil) + defer peer.Close("test complete") + peer.credentialHash = "original-hash" + receipt := device.ArchivedCancellationReceipt{RunID: "run", Deadline: time.Now().Add(100 * time.Millisecond)} + peer.heartbeat = &receiptStore{fakeHeartbeatStore: newFakeHeartbeatStore(), receipt: receipt} + release, err := peer.TrackExecutionDelivery("run") + if err != nil { + t.Fatal(err) + } + defer release() + for range 10 { + if draining, err := peer.DrainArchivedCancellation(t.Context()); err != nil || !draining { + t.Fatal(draining, err) + } + } + select { + case <-peer.Closed(): + case <-time.After(time.Second): + t.Fatal("receipt retries renewed deadline") + } +} diff --git a/internal/agentdaemon/gateway/session.go b/internal/agentdaemon/gateway/session.go index c0cd7572f..7e79ddfdb 100644 --- a/internal/agentdaemon/gateway/session.go +++ b/internal/agentdaemon/gateway/session.go @@ -57,6 +57,7 @@ var ErrSessionClosed = errors.New("agentdaemon gateway: session closed") type WSConn interface { ReadMessage() (int, []byte, error) WriteMessage(messageType int, data []byte) error + WriteControl(messageType int, data []byte, deadline time.Time) error SetReadLimit(limit int64) SetReadDeadline(t time.Time) error SetWriteDeadline(t time.Time) error @@ -125,6 +126,11 @@ type Session struct { // closeOnce guards the shutdown path so concurrent Close calls // collapse into one. + receiptMu sync.Mutex + deliveries map[string]struct{} + receiptDrain device.ArchivedCancellationReceipt + receiptTimer *time.Timer + closeOnce sync.Once closed chan struct{} } @@ -249,6 +255,7 @@ func legacyClaudeCodeKind() device.SupportedAgentKind { // with a synthetic error+done pair, and deregisters. Idempotent. func (s *Session) Close(reason string) { s.closeOnce.Do(func() { + s.stopReceiptTimer() close(s.closed) _ = s.conn.Close() // Synthetic error + done so the connector's translation loop @@ -276,9 +283,8 @@ func (s *Session) Close(reason string) { // permanent conditions (e.g. runtime deleted) can be distinguished // from transient disconnects. func (s *Session) CloseWithCode(code int, reason string) { - _ = s.conn.SetWriteDeadline(time.Now().Add(WriteTimeout)) - _ = s.conn.WriteMessage(websocket.CloseMessage, - websocket.FormatCloseMessage(code, reason)) + // gorilla permits WriteControl concurrently with the sole data writer. + _ = s.conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(code, reason), time.Now().Add(WriteTimeout)) s.Close(reason) } @@ -308,7 +314,7 @@ func (s *Session) deliverSynthetic(runID string, ch chan proto.Envelope, reason // daemon frame inherits the caller's trace_id. Callers that explicitly // set env.Trace win. func (s *Session) Send(ctx context.Context, env proto.Envelope) error { - if s.IsClosed() { + if s.IsClosed() || !s.allowsReceiptFrame(env, true) { return ErrSessionClosed } if env.Trace == "" { @@ -381,6 +387,9 @@ func (s *Session) writeLoop() { if !ok { return } + if !s.allowsReceiptFrame(env, true) { + continue + } raw, err := json.Marshal(env) if err != nil { s.log("agentdaemon gateway: marshal outbound envelope: %v", err) @@ -412,9 +421,11 @@ func (s *Session) readLoop() { s.log("agentdaemon gateway: read frame: %v", err) return } - s.markSeen() - if !s.renewOwnerLease() { - return + if !s.receiptDraining() { + s.markSeen() + if !s.renewOwnerLease() { + return + } } var env proto.Envelope @@ -512,6 +523,10 @@ func (s *Session) handleHeartbeat(env proto.Envelope) { return } if status.Deleted { + draining, drainErr := s.DrainArchivedCancellation(ctx) + if drainErr == nil && draining { + return + } s.log("agentdaemon gateway: runtime retired, closing session device=%s", s.DeviceID) // "retired" rather than "deleted by admin": the row may have // been soft-deleted by sandbox stale-row cleanup or by an @@ -583,6 +598,9 @@ func deviceKindsFromHeartbeat(p proto.HeartbeatPayload) []device.SupportedAgentK } func (s *Session) dispatch(env proto.Envelope) { + if !s.allowsReceiptFrame(env, false) { + return + } switch env.Type { case proto.TypeWorkspaceExportResult: s.dispatchWorkspaceExport(env) diff --git a/internal/agentdaemon/gateway/session_test.go b/internal/agentdaemon/gateway/session_test.go index 778a8eb15..30fc4396f 100644 --- a/internal/agentdaemon/gateway/session_test.go +++ b/internal/agentdaemon/gateway/session_test.go @@ -477,3 +477,7 @@ func TestSession_HeartbeatInfersClaudeCodeFromLegacyFlag(t *testing.T) { func jsonMarshal(v any) ([]byte, error) { return json.Marshal(v) } + +func (c *fakeConn) WriteControl(kind int, data []byte, _ time.Time) error { + return c.WriteMessage(kind, data) +} diff --git a/packages/agents-client/src/core-metrics.ts b/packages/agents-client/src/core-metrics.ts index d650a7e36..502e07d6f 100644 --- a/packages/agents-client/src/core-metrics.ts +++ b/packages/agents-client/src/core-metrics.ts @@ -60,7 +60,7 @@ export interface CoreMetrics { range: { start: string; end: string; resolution_seconds: number }; service: { /** `unknown` stands for a status this client does not recognise; it is never shown as running. */ - status: "running" | "maintenance" | "degraded" | "unknown"; + status: "running" | "degraded" | "unknown"; /** Build revision (source commit) of the running Core. */ revision: string | null; started_at: string | null; @@ -150,7 +150,7 @@ export function projectCoreMetrics(value: unknown): CoreMetrics { const database = optional(body.database); const pool = optional(database.pool); const process = optional(body.process); - const status = service.status === "running" || service.status === "maintenance" || service.status === "degraded" ? service.status : "unknown"; + const status = service.status === "running" || service.status === "degraded" ? service.status : "unknown"; const resolution = number(range.resolution_seconds); if (resolution === null || resolution <= 0) throw new AgentCoreError("Core metrics: range.resolution_seconds is missing.", 0, "invalid_response"); return { diff --git a/packages/agents-client/src/sandbox-client.test.ts b/packages/agents-client/src/sandbox-client.test.ts index 5569102a1..b8c9edbbe 100644 --- a/packages/agents-client/src/sandbox-client.test.ts +++ b/packages/agents-client/src/sandbox-client.test.ts @@ -37,7 +37,7 @@ const allocation = { state: "running", compute_phase: "running", compute_phase_changed_at: null, diagnostic: "", initialization: "ready", created_at: created, }; const runtime = { source_commit: "a".repeat(40), image_id: "sha256:" + "b".repeat(64), image_manifest_digest: "sha256:" + "c".repeat(64), microsandbox_ref: "oac-runtime@sha256:" + "d".repeat(64), runtime_sha256: "e".repeat(64), firmware_sha256: "f".repeat(64) }; -const unconfigured = { installation_id: "", provider: "", core_url: "https://core.example", maintenance: false, owner_epoch: 0, generation: 0, mode: "", resources: { allocations: 0, pending: 0 }, suspension: null }; +const unconfigured = { installation_id: "", provider: "", core_url: "https://core.example", reset: null, owner_epoch: 0, generation: 0, mode: "", resources: { allocations: 0, pending: 0 }, suspension: null }; const docker = { ...unconfigured, installation_id: "94be54a1-138c-4f30-bc87-b13686272dbe", provider: "docker", owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 2, pending: 1 }, specification: { resources: { cpus: 2, memory_mib: 2048 }, runtime }, specification_digest: "0".repeat(64), @@ -154,11 +154,11 @@ describe("Core sandbox credential boundaries", () => { const fetch = vi.fn().mockResolvedValue(response(deployment)); const admin = new SandboxAdminClient({ baseUrl: "/core/v1/sandbox", token: "admin-only", fetch }); const controller = new AbortController(); - expect(await admin.initializeDeployment({ provider: "docker" }, { signal: controller.signal })).toEqual(deployment); + expect(await admin.initializeDeployment({ expected_generation: 0, provider: "docker" }, { signal: controller.signal })).toEqual(deployment); const [url, init] = fetch.mock.calls[0]!; expect(url).toBe("/core/v1/sandbox/deployment"); expect(init?.method).toBe("POST"); - expect(JSON.parse(String(init?.body))).toEqual({ provider: "docker" }); + expect(JSON.parse(String(init?.body))).toEqual({ expected_generation: 0, provider: "docker" }); expect(init?.signal).toBe(controller.signal); expect(new Headers(init?.headers).get("Authorization")).toBe("Bearer admin-only"); expect(new Headers(init?.headers).has("OpenAI-Beta")).toBe(false); @@ -174,13 +174,13 @@ describe("Core sandbox credential boundaries", () => { it.each([409, 503])("does not retry initialization after HTTP %s", async (status) => { const fetch = vi.fn().mockResolvedValue(response({ error: { message: "Setup failed", code: "sandbox_deployment_conflict" } }, status)); const admin = new SandboxAdminClient({ baseUrl: "/core/v1/sandbox", token: "admin", fetch }); - await expect(admin.initializeDeployment({ provider: "microsandbox" })).rejects.toThrow("Setup failed"); + await expect(admin.initializeDeployment({ expected_generation: 0, provider: "microsandbox" })).rejects.toThrow("Setup failed"); expect(fetch).toHaveBeenCalledTimes(1); }); it("does not retry an uncertain initialization transport failure", async () => { const fetch = vi.fn().mockRejectedValue(new TypeError("Connection lost")); const admin = new SandboxAdminClient({ baseUrl: "/core/v1/sandbox", token: "admin", fetch }); - await expect(admin.initializeDeployment({ provider: "docker" })).rejects.toThrow("Connection lost"); + await expect(admin.initializeDeployment({ expected_generation: 0, provider: "docker" })).rejects.toThrow("Connection lost"); expect(fetch).toHaveBeenCalledTimes(1); }); it("uses the explicit admin credential and admin routes without a project beta header", async () => { @@ -218,15 +218,15 @@ describe("hosted provider configuration", () => { const client = new SandboxAdminClient({ baseUrl: "/core/v1/sandbox", fetch }); const controller = new AbortController(); // Omitted E2B resources are filled from the validated template build. - expect(await client.initializeDeployment({ provider: "e2b", e2b })).toEqual(deployment); + expect(await client.initializeDeployment({ expected_generation: 0, provider: "e2b", e2b })).toEqual(deployment); await client.updateDeployment({ provider: "e2b", e2b, expected_generation: 1 }, { signal: controller.signal }); - await client.setMaintenance({ maintenance: false, expected_generation: 2 }); + await client.cancelReset(2); expect(fetch.mock.calls.map(([url, init]) => [url, init?.method])).toEqual([ - ["/core/v1/sandbox/deployment", "POST"], ["/core/v1/sandbox/deployment", "PUT"], ["/core/v1/sandbox/deployment/maintenance", "PATCH"], + ["/core/v1/sandbox/deployment", "POST"], ["/core/v1/sandbox/deployment", "PUT"], ["/core/v1/sandbox/deployment/reset?expected_generation=2", "DELETE"], ]); expect(JSON.parse(String(fetch.mock.calls[1]?.[1]?.body))).toEqual({ provider: "e2b", e2b, expected_generation: 1 }); expect(fetch.mock.calls[1]?.[1]?.signal).toBe(controller.signal); - expect(JSON.parse(String(fetch.mock.calls[2]?.[1]?.body))).toEqual({ maintenance: false, expected_generation: 2 }); + expect(fetch.mock.calls[2]?.[1]?.body).toBeUndefined(); for (const [, init] of fetch.mock.calls) { expect(new Headers(init?.headers).has("Authorization")).toBe(false); expect(new Headers(init?.headers).has("OpenAI-Beta")).toBe(false); @@ -236,7 +236,7 @@ describe("hosted provider configuration", () => { const fetch = vi.fn().mockResolvedValue(response({ error: { message: e2b.api_key, code: e2b.api_key, param: e2b.api_key } }, status)); const client = new SandboxAdminClient({ fetch }); await expect(client.updateDeployment({ provider: "e2b", e2b, expected_generation: 1 })).rejects.toMatchObject({ code: "sandbox_configuration_unconfirmed", status }); - await client.initializeDeployment({ provider: "e2b", e2b }).catch((error) => { + await client.initializeDeployment({ expected_generation: 0, provider: "e2b", e2b }).catch((error) => { expect(JSON.stringify(error)).not.toContain(e2b.api_key); expect(error.message).not.toContain(e2b.api_key); }); @@ -248,16 +248,66 @@ describe("hosted provider configuration", () => { .mockResolvedValueOnce(response({ error: rejection }, 409)) .mockResolvedValueOnce(response({ error: { ...rejection, message: rejection.message + e2b.api_key } }, 409)); const client = new SandboxAdminClient({ fetch }); - const shown = await client.initializeDeployment({ provider: "e2b", e2b }).catch((error: unknown) => error); + const shown = await client.initializeDeployment({ expected_generation: 0, provider: "e2b", e2b }).catch((error: unknown) => error); expect(shown).toMatchObject({ status: 409, code: "sandbox_configuration_error", message: rejection.message, param: null }); expect((shown as AgentCoreError).errorType).toBeUndefined(); - await expect(client.initializeDeployment({ provider: "e2b", e2b })).rejects.toMatchObject({ status: 409, code: "sandbox_configuration_unconfirmed" }); + await expect(client.initializeDeployment({ expected_generation: 0, provider: "e2b", e2b })).rejects.toMatchObject({ status: 409, code: "sandbox_configuration_unconfirmed" }); }); - it("never retries uncertain switch or maintenance writes", async () => { + it("never retries uncertain switch or reset writes", async () => { const fetch = vi.fn().mockRejectedValue(new TypeError("Connection lost")); const client = new SandboxAdminClient({ fetch }); await expect(client.updateDeployment({ provider: "docker", expected_generation: 1 })).rejects.toThrow(); - await expect(client.setMaintenance({ maintenance: true, expected_generation: 1 })).rejects.toThrow(); + await expect(client.startReset({ clear: "auto", expected_generation: 1 })).rejects.toThrow(); expect(fetch).toHaveBeenCalledTimes(2); }); }); + +const resetting = { + ...docker, + reset: { clear: "auto", requested_at: created, deadline_at: "2026-09-25T09:00:00Z", forced_at: null, + remaining: { busy: 1, idle: 1, cleanup: 1, on_offline_nodes: 2, + offline_nodes: [{ node_id: node.id, name: node.name, resources: 2 }] } }, +}; +describe("durable reset projection", () => { + it("preserves Core's complete remaining snapshot and copies blocker identities", async () => { + const result = await read("deployment", resetting) as typeof resetting; + expect(result).toEqual(resetting); + expect(result.reset.remaining.offline_nodes).not.toBe(resetting.reset.remaining.offline_nodes); + }); + it.each([ + { ...resetting, maintenance: true }, + { ...resetting, rollout: { nodes: { ready: 1 } } }, + { ...resetting, reset: { ...resetting.reset, clear: "unknown" } }, + { ...resetting, reset: { ...resetting.reset, deadline_at: null } }, + { ...resetting, reset: { ...resetting.reset, forced_at: created } }, + { ...resetting, reset: { ...resetting.reset, remaining: { ...resetting.reset.remaining, busy: 2 } } }, + { ...resetting, reset: { ...resetting.reset, remaining: { ...resetting.reset.remaining, on_offline_nodes: 1 } } }, + { ...resetting, reset: { ...resetting.reset, remaining: { ...resetting.reset.remaining, offline_nodes: [ + { node_id: node.id, name: node.name, resources: 1 }, { node_id: node.id, name: node.name, resources: 1 }, + ] } } }, + ])("rejects retired/fictional fields or an inconsistent reset snapshot", async value => { + await expect(read("deployment", value)).rejects.toMatchObject({ code: "invalid_admin_response" }); + }); + it("posts the explicit reset once with the requested deadline", async () => { + const fetch = vi.fn().mockResolvedValue(response(resetting)); + const client = new SandboxAdminClient({ fetch }); + await expect(client.startReset({ expected_generation: 1, clear: "auto", deadline_seconds: 300 })).resolves.toEqual(resetting); + expect(fetch).toHaveBeenCalledTimes(1); + expect(String(fetch.mock.calls[0]?.[0])).toBe("/core/v1/sandbox/deployment/reset"); + expect(fetch.mock.calls[0]?.[1]?.method).toBe("POST"); + expect(JSON.parse(String(fetch.mock.calls[0]?.[1]?.body))).toEqual({ expected_generation: 1, clear: "auto", deadline_seconds: 300 }); + }); + it("keeps safe E2B reset errors without reflecting credential-bearing error fields", async () => { + const secret = "test-only-key"; + const fetch = vi.fn().mockResolvedValue(response({ error: { + message: secret, param: secret, type: secret, code: "sandbox_generation_stale", + details: { current_generation: 7, arbitrary: secret }, + } }, 409)); + const error = await new SandboxAdminClient({ fetch }).updateDeployment({ provider: "e2b", expected_generation: 1, + e2b: { api_key: secret, template: "runtime:00000000-0000-0000-0000-000000000001" } }).catch(error => error); + expect(error).toMatchObject({ code: "sandbox_generation_stale", status: 409, details: { current_generation: 7 } }); + expect(JSON.stringify(error)).not.toContain(secret); + expect(error.message).not.toContain(secret); + expect(fetch).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/agents-client/src/sandbox-client.ts b/packages/agents-client/src/sandbox-client.ts index 99fdca795..560e9d622 100644 --- a/packages/agents-client/src/sandbox-client.ts +++ b/packages/agents-client/src/sandbox-client.ts @@ -23,6 +23,8 @@ export interface SandboxSpecification { resources: SandboxResources; runtime?: S /** Core derives the deployment's address from the installation public URL; a `core_url` member is rejected. */ export interface InitializeSandboxDeployment { provider: SandboxProvider; + /** Required, including zero for first setup; read it from GET before submitting once. */ + expected_generation: number; /** Required for Docker/microsandbox. E2B may omit it to adopt its validated template build's CPU and memory. */ resources?: SandboxResources; /** Required for Docker/microsandbox; E2B uses its fixed template build. */ @@ -30,7 +32,17 @@ export interface InitializeSandboxDeployment { e2b?: { api_key: string; template: string }; } export interface UpdateSandboxDeployment extends InitializeSandboxDeployment { expected_generation: number } -export interface SetSandboxMaintenance { maintenance: boolean; expected_generation: number } +export interface StartSandboxReset { expected_generation: number; clear: "auto" | "force"; deadline_seconds?: number } +export interface SandboxReset { + clear: "auto" | "force"; + requested_at: string; + deadline_at: string | null; + forced_at: string | null; + remaining: { + busy: number; idle: number; cleanup: number; on_offline_nodes: number; + offline_nodes: Array<{ node_id: string; name: string; resources: number }>; + }; +} export interface SandboxDeployment { specification?: SandboxSpecification; @@ -39,7 +51,7 @@ export interface SandboxDeployment { provider: SandboxProvider | ""; /** Read-only: the installation public URL, which nodes and sandboxes use to reach Core. Present before configuration. */ readonly core_url: string; - maintenance: boolean; + reset: SandboxReset | null; owner_epoch: number; generation: number; mode: "nodes" | "direct" | ""; @@ -173,20 +185,39 @@ function projectE2B(value: unknown): NonNullable { template_build: { status: build.status as string | null, resources: { ...resources } as SandboxE2BTemplateBuild["resources"] }, }; } +/** Counts and blocker identities are one Core snapshot, never reconstructed from node lists. */ +function projectReset(value: unknown, held: number): SandboxReset | null { + if (value === null) return null; + const reset = members(value, ["clear", "requested_at", "deadline_at", "forced_at", "remaining"]); + const remaining = members(reset.remaining, ["busy", "idle", "cleanup", "on_offline_nodes", "offline_nodes"]); + valid((reset.clear === "auto" || reset.clear === "force") && timestamp(reset.requested_at) && + nullable(timestamp)(reset.deadline_at) && nullable(timestamp)(reset.forced_at) && + (reset.clear === "auto" ? reset.deadline_at !== null && reset.forced_at === null : reset.forced_at !== null) && + [remaining.busy, remaining.idle, remaining.cleanup, remaining.on_offline_nodes].every(isNonnegativeInteger) && Array.isArray(remaining.offline_nodes)); + const nodes = (remaining.offline_nodes as unknown[]).map(value => { + const node = members(value, ["node_id", "name", "resources"]); + valid(strings(node, ["node_id", "name"]) && node.node_id !== "" && isNonnegativeInteger(node.resources) && Number(node.resources) > 0); + return { ...node } as SandboxReset["remaining"]["offline_nodes"][number]; + }); + valid(Number(remaining.busy) + Number(remaining.idle) + Number(remaining.cleanup) === held && + Number(remaining.on_offline_nodes) <= held && nodes.reduce((sum, node) => sum + node.resources, 0) === remaining.on_offline_nodes && + new Set(nodes.map(node => node.node_id)).size === nodes.length); + return { ...reset, remaining: { ...remaining, offline_nodes: nodes } } as unknown as SandboxReset; +} /** `specification` and `specification_digest` appear together once configured; `e2b` appears exactly for E2B. */ function projectDeployment(value: unknown): SandboxDeployment { const e2b = isRecord(value) && value.provider === "e2b"; - const fields = ["installation_id", "provider", "core_url", "maintenance", "owner_epoch", "generation", "mode", "resources", "suspension"]; + const fields = ["installation_id", "provider", "core_url", "reset", "owner_epoch", "generation", "mode", "resources", "suspension"]; const deployment = members(value, e2b ? [...fields, "e2b"] : fields, ["specification", "specification_digest"]); const resources = members(deployment.resources, ["allocations", "pending"]); const suspension = deployment.suspension === null ? null : members(deployment.suspension, ["idle_seconds", "retention_seconds"]); const configured = hasOwn(deployment, "specification"); valid(strings(deployment, ["installation_id", "core_url"]) && providers.has(deployment.provider as string) && modes.has(deployment.mode as string) && - typeof deployment.maintenance === "boolean" && [deployment.owner_epoch, deployment.generation, resources.allocations, resources.pending].every(isNonnegativeInteger) && + [deployment.owner_epoch, deployment.generation, resources.allocations, resources.pending].every(isNonnegativeInteger) && (suspension === null || [suspension.idle_seconds, suspension.retention_seconds].every(isNonnegativeInteger)) && configured === hasOwn(deployment, "specification_digest") && (!configured || (typeof deployment.specification_digest === "string" && deployment.specification_digest !== ""))); return { - ...deployment, resources: { ...resources } as SandboxDeployment["resources"], suspension: suspension && { ...suspension } as SandboxDeployment["suspension"], + ...deployment, reset: projectReset(deployment.reset, Number(resources.allocations) + Number(resources.pending)), resources: { ...resources } as SandboxDeployment["resources"], suspension: suspension && { ...suspension } as SandboxDeployment["suspension"], ...(configured ? { specification: projectSpecification(deployment.specification) } : {}), ...(e2b ? { e2b: projectE2B(deployment.e2b) } : {}), } as unknown as SandboxDeployment; @@ -261,8 +292,11 @@ export class SandboxAdminClient { updateDeployment(input: UpdateSandboxDeployment, options?: ReadOptions): Promise { return this.#writeDeployment("PUT", input, options); } - async setMaintenance(input: SetSandboxMaintenance, options?: ReadOptions): Promise { - return projectDeployment(await this.#core.json("/deployment/maintenance", options, "PATCH", input)); + async startReset(input: StartSandboxReset, options?: ReadOptions): Promise { + return projectDeployment(await this.#core.json("/deployment/reset", options, "POST", input)); + } + async cancelReset(expectedGeneration: number, options?: ReadOptions): Promise { + return projectDeployment(await this.#core.json(`/deployment/reset?expected_generation=${encodeURIComponent(expectedGeneration)}`, options, "DELETE")); } async #writeDeployment(method: "POST" | "PUT", input: InitializeSandboxDeployment | UpdateSandboxDeployment, options?: ReadOptions): Promise { try { @@ -270,6 +304,22 @@ export class SandboxAdminClient { return projectDeployment(await this.#core.json("/deployment", options, method, input)); } catch (error) { if (!input.e2b) throw error; + if (error instanceof AgentCoreError && error.status === 409) { + const messages: Record = { + sandbox_generation_stale: "The sandbox configuration changed. Refresh before submitting again.", + sandbox_reset_required: "Reset the sandbox deployment before changing its backend.", + sandbox_reset_in_progress: "A sandbox reset is in progress.", + sandbox_not_configured: "The sandbox deployment is not configured.", + sandbox_in_use: "Hosted sandbox resources still belong to this deployment.", + }; + if (error.code && messages[error.code]) { + // Credential-bearing errors expose fixed local copy and allowlisted + // numeric facts only; never echo arbitrary message, param or details. + const fields = error.code === "sandbox_generation_stale" ? ["current_generation"] : error.code === "sandbox_in_use" ? ["allocations", "pending"] : []; + const details = Object.fromEntries(fields.filter(field => isNonnegativeInteger(error.details?.[field])).map(field => [field, Number(error.details![field])])); + throw new AgentCoreError(messages[error.code]!, 409, error.code, null, undefined, Object.keys(details).length ? details : undefined); + } + } // The public-URL rejection is safe to show unless it somehow reflects the key. const key = input.e2b.api_key; if (error instanceof AgentCoreError && error.status === 409 && error.code === "sandbox_configuration_error" diff --git a/services/agents-api/HOSTED-SANDBOX-MANAGER.md b/services/agents-api/HOSTED-SANDBOX-MANAGER.md index b6ced42f4..c6136a186 100644 --- a/services/agents-api/HOSTED-SANDBOX-MANAGER.md +++ b/services/agents-api/HOSTED-SANDBOX-MANAGER.md @@ -2,7 +2,7 @@ This reference is for operators who need the details behind Web's **Nodes** page (shown as **Sandbox backend** when the deployment uses E2B): the node protocol, -manual registration, placement, maintenance and failure handling. To add, remove or +manual registration, placement, reset and failure handling. To add, remove or troubleshoot a node, start with the [nodes guide](../../docs/getting-started/nodes.md). One Core execution owner manages sandbox nodes on its own host and on other Linux @@ -194,7 +194,7 @@ for the host: encoded bytes; the installer rejects a longer path before creating node state, so use a service account with a shorter persistent home. -The node file has no `max_active`, `max_retained`, idle-policy or maintenance +The node file has no `max_active`, `max_retained`, idle-policy or reset override, and a node never receives arbitrary host paths from the browser. Editing the downloaded specification does not change Core: the node validates its content and file hashes, and Core compares its digest at enrollment and on every connection. A mismatch @@ -258,7 +258,7 @@ keepalives no longer authorizes cleanup of a node-managed allocation. Reconnect and continue its existing resources. Unknown creation, initialization, capture and restore results do not authorize request replay or a replacement sandbox. -## Removal and maintenance +## Removal and reset There is no per-node drain switch. Online, eligible nodes participate automatically. A node with instances, retained snapshots, pending allocations, @@ -276,45 +276,43 @@ an unreachable Core or a 403 from a proxy in front of it, restart the service ev 5 seconds without a start limit. Ordinary disconnects and host restarts reuse the original identity. -Provider, resource-limit and Runtime changes share one deployment-wide procedure: - -1. Enter maintenance. This blocks new hosted Sessions and allocations while allowing - existing work, queries and cleanup. -2. Verify the deployment's allocation and pending-Environment counts are zero. - Stopped compute, snapshots, unknown creates and pending cleanup still block - switching. Explicitly archive retained Sessions as described below. -3. Once cleanup is verified, submit the complete replacement. Core validates the - candidate before changing the database or draining current workers. It then - drains existing manager calls and repeats the resource/generation checks in the - commit transaction. A changed selection advances the generation and retires old - nodes and unused enrollments atomically; history remains intact. -4. Explicitly resume with the returned generation. A rejected candidate preserves - the old configuration and maintenance state. After an uncertain response, refresh - before another write. Saving never automatically deletes compute. - -During maintenance, explicitly archive each retained Core-managed hosted Session -through `POST /core/v1/projects/{project_id}/sessions/{session_id}/archive` -with the current `expected_generation`. This requests cancellation and revokes -Runtime authority; the existing lifecycle releases compute and snapshots after -provider verification. Poll GET on the same path for `released`, then verify both -deployment counts are zero. Unknown cleanup remains a blocker. Resource release -does not prove that an active Turn has finalized. - -Archive preserves Session history and persisted Files/Artifacts. Unpersisted -workspace contents are lost and the original Session cannot resume. Public Session -deletion has different retention behavior and is not needed for this workflow. -After an uncertain archive response, read its disposition before another explicit -write; never automatically retry. See the -[administrator archive contract](../../contracts/agents-api/admin-api.md#administrative-session-archive). -Maintenance and configuration changes alone perform no cleanup. - -`GET /core/v1/sandbox/deployment` reports the safe configuration, `generation` and -`resources` counts. `PATCH /core/v1/sandbox/deployment/maintenance` takes -`maintenance` and `expected_generation`; `PUT /core/v1/sandbox/deployment` takes -the provider, complete `resources`/`runtime` selection, any E2B input and -`expected_generation`. Both require deployment admin -authority. Neither request takes a Core address. E2B is not -combined with own-machine nodes, and existing Sessions never migrate providers. +Same-provider resource/Runtime or E2B key/build edits currently require zero +unreleased allocations and pending hosted Environments, no reset and the current +generation. PUT the complete selection once. Core validates before draining, then +rechecks and commits the generation and node/token retirement together. A failed +commit recovers the previous provider. There is no online rollout in this release. + +Changing backend requires an explicit reset. POST +`/core/v1/sandbox/deployment/reset` with `expected_generation` and `clear: auto` +(default deadline 3600 seconds, configurable from 300 to 86400) or `clear: force` +(without a deadline). Auto archives idle, queued and suspended hosted Sessions, +waits for work in progress/waiting and pending file writes, then escalates at its +durable deadline. Force requests cancellation and ordinary verified cleanup. +Self-hosted work is excluded. Fresh hosted admission closes during reset; existing +live input, known receipts, restoration and cleanup continue. + +Read `reset.remaining` for busy/idle/cleanup and the authoritative offline-node +subset. Offline or failed cleanup still blocks completion. Bring those nodes back +for cleanup; a node with resources cannot be removed. Once verified counts reach +zero Core clears the selection, retires nodes/tokens and advances generation while +preserving installation identity, history and persisted Files/Artifacts. Then POST +a new selection with the returned generation. Cancel with DELETE on the reset route +and `expected_generation` query; it stops remaining archives, not cleanup already +requested, and does not restore workspace or revive archived Sessions. + +Explicitly archive an individual hosted Session at any time through +`POST /core/v1/projects/{project_id}/sessions/{session_id}/archive` with its current +deployment generation. Poll GET on that path for `released`; resource release does +not prove an active Turn has finished cancellation. History and persisted +Files/Artifacts survive, but unpersisted workspace is lost and the original Session +cannot resume. See the [archive contract](../../contracts/agents-api/admin-api.md#administrative-session-archive). + +After any uncertain mutation, refresh authoritative state before another explicit +write; do not replay automatically. Every setup/update/reset/cancel write uses the +observed generation, including zero for initial setup. The retired maintenance PATCH +returns authenticated 404. See the [sandbox deployment contract](../../contracts/agents-api/sandbox-deployment.md) +for precise reset, concurrency and error semantics. None of these requests changes +the installation's public Core address or migrates existing Sessions. This release has one Core execution owner. It does not add Core multi-active, cross-node snapshot restore, automatic failover, Kubernetes or autoscaling. diff --git a/services/agents-api/README.md b/services/agents-api/README.md index c0557e9e4..5aae79371 100644 --- a/services/agents-api/README.md +++ b/services/agents-api/README.md @@ -640,4 +640,4 @@ not changes to the pinned official protocol. The release includes `oac-node` for local and remote hosts. Add nodes with Web's one-command flow in the [nodes guide](../../docs/getting-started/nodes.md); the [operator reference](HOSTED-SANDBOX-MANAGER.md) covers provider selection, manual -registration, administrator credentials, fixed Session placement and maintenance. +registration, administrator credentials, fixed Session placement and reset. diff --git a/services/agents-api/cmd/server/core_metrics.go b/services/agents-api/cmd/server/core_metrics.go index cb6956ac5..56efb8bb1 100644 --- a/services/agents-api/cmd/server/core_metrics.go +++ b/services/agents-api/cmd/server/core_metrics.go @@ -67,12 +67,7 @@ func (s *coreMetricsSource) Sample(ctx context.Context) coremetrics.Sample { } else { sample.DatabaseSize = &size } - deployment, err := s.store.GetRuntimeDeployment(ctx) - if err != nil { - sample.Healthy = false - } else { - sample.Maintenance = metricPtr(deployment.Maintenance) - } + return sample } func (s *coreMetricsSource) History(ctx context.Context, start, end time.Time, step time.Duration) (coremetrics.History, error) { diff --git a/services/agents-api/cmd/server/daemon_bootstrap_test.go b/services/agents-api/cmd/server/daemon_bootstrap_test.go index 04636a69e..9ded126af 100644 --- a/services/agents-api/cmd/server/daemon_bootstrap_test.go +++ b/services/agents-api/cmd/server/daemon_bootstrap_test.go @@ -28,7 +28,7 @@ func TestBootstrapAddressFollowsAuthenticatedAllocation(t *testing.T) { const publicURL = "wss://private-proxy.example/api/v1/agent-daemon/ws" local := &managedNodes{runtime: &execution.RuntimeProvider{LocalNodeID: "local-node", CoreURL: "http://host.microsandbox.internal:8091/api/v1"}} remote := &managedNodes{setup: &managedSetup{store: &setupStore{value: store.SandboxSetup{Provider: "docker", Generation: 1}}}} - remote.setup.selected.Store(&execution.RuntimeProvider{CoreURL: "https://selected-node-entry.example/api/v1", Generation: 1}) + remote.setup.publish(&execution.RuntimeProvider{CoreURL: "https://selected-node-entry.example/api/v1", Generation: 1}) zero := &managedNodes{setup: &managedSetup{}} for _, tc := range []struct { name, node, want string diff --git a/services/agents-api/cmd/server/main.go b/services/agents-api/cmd/server/main.go index 66634ff64..fae3683fc 100644 --- a/services/agents-api/cmd/server/main.go +++ b/services/agents-api/cmd/server/main.go @@ -233,7 +233,7 @@ func run() error { return err } if managedNodes != nil && managedNodes.setup != nil { - options = append(options, api.WithSandboxDeploymentSetup(worker.InitializeSandboxDeployment), api.WithSandboxDeploymentChanges(worker.UpdateSandboxDeployment, worker.SetSandboxMaintenance)) + options = append(options, api.WithSandboxDeploymentSetup(worker.InitializeSandboxDeployment), api.WithSandboxDeploymentChanges(worker.UpdateSandboxDeployment, worker.StartSandboxReset, worker.CancelSandboxReset)) } workerDone = make(chan error, 1) go func() { workerDone <- worker.Run(ctx) }() diff --git a/services/agents-api/cmd/server/managed_nodes.go b/services/agents-api/cmd/server/managed_nodes.go index 2dfe672a5..daa669867 100644 --- a/services/agents-api/cmd/server/managed_nodes.go +++ b/services/agents-api/cmd/server/managed_nodes.go @@ -83,6 +83,7 @@ func configureManagedNodes(s *store.Store, publicURL string, owner func(context. }) result.setup = &managedSetup{store: s, hub: result.hub, installationID: setupID, publicURL: publicURL} result.runtime = execution.NewDeferredRuntimeProvider(setupID, result.setup.load, result.setup.prepare) + result.runtime.PublishUnconfigured = result.setup.publishUnconfigured success = true return result, nil } diff --git a/services/agents-api/cmd/server/managed_setup.go b/services/agents-api/cmd/server/managed_setup.go index 0eda8c54a..e3d0494fb 100644 --- a/services/agents-api/cmd/server/managed_setup.go +++ b/services/agents-api/cmd/server/managed_setup.go @@ -29,9 +29,33 @@ type managedSetup struct { installationID string // publicURL is OAC_PUBLIC_URL; every sandbox reaches Core through it. publicURL string - selected atomic.Pointer[execution.RuntimeProvider] + selected atomic.Pointer[managedSelection] } +// Empty selections retain their generation so a delayed provider load cannot +// republish a backend retired by reset. +type managedSelection struct { + Generation uint64 + Config *execution.RuntimeProvider +} + +func (s *managedSetup) publishSelection(generation uint64, config *execution.RuntimeProvider) *execution.RuntimeProvider { + next := &managedSelection{Generation: generation, Config: config} + for { + current := s.selected.Load() + if current != nil && current.Generation >= generation { + return current.Config + } + if s.selected.CompareAndSwap(current, next) { + return config + } + } +} +func (s *managedSetup) publish(config *execution.RuntimeProvider) { + s.publishSelection(config.Generation, config) +} +func (s *managedSetup) publishUnconfigured(generation uint64) { s.publishSelection(generation, nil) } + func (s *managedSetup) load(ctx context.Context) (*execution.RuntimeProvider, error) { setup, err := s.store.GetSandboxSetup(ctx) if err != nil { @@ -41,27 +65,17 @@ func (s *managedSetup) load(ctx context.Context) (*execution.RuntimeProvider, er return nil, errors.New("sandbox installation does not match setup") } if setup.Provider == "" { - return nil, nil + return s.publishSelection(setup.Generation, nil), nil } - if selected := s.selected.Load(); selected != nil && selected.Generation == setup.Generation { - return selected, nil + if selected := s.selected.Load(); selected != nil && selected.Generation >= setup.Generation { + return selected.Config, nil } candidate, err := s.configuration(setup) if err != nil { log.Warn(ctx, "Hosted provider is unavailable; administrator recovery remains available", "provider", setup.Provider, "error", err) return nil, fmt.Errorf("%w: %v", execution.ErrExecutionUnavailable, err) } - // A slower read cannot replace a generation that committed while this - // configuration was loading. Publication performs no external work. - for { - selected := s.selected.Load() - if selected != nil && selected.Generation >= setup.Generation { - return selected, nil - } - if s.selected.CompareAndSwap(selected, candidate.Config) { - return candidate.Config, nil - } - } + return s.publishSelection(setup.Generation, candidate.Config), nil } func (s *managedSetup) prepare(ctx context.Context, setup store.SandboxSetup) (execution.PreparedRuntimeDeployment, error) { @@ -110,18 +124,18 @@ func (s *managedSetup) configuration(setup store.SandboxSetup) (execution.Prepar if err != nil { return execution.PreparedRuntimeDeployment{}, fmt.Errorf("%w: %v", execution.ErrExecutionUnavailable, err) } - selected := &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, Maintenance: setup.Maintenance, + selected := &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: s.publicURL + "/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider} if setup.Provider == "microsandbox" { selected.Suspension = &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Duration(setup.IdleSeconds) * time.Second, Retention: time.Duration(setup.RetentionSeconds) * time.Second, MaxActive: 4, MaxRetained: 16} } - return execution.PreparedRuntimeDeployment{Config: selected, Publish: s.selected.Store}, nil + return execution.PreparedRuntimeDeployment{Config: selected, Publish: s.publish}, nil } func (s *managedSetup) ObservationProviderType() string { - if selected := s.selected.Load(); selected != nil { - return selected.ProviderKind + if selected := s.selected.Load(); selected != nil && selected.Config != nil { + return selected.Config.ProviderKind } return "" } diff --git a/services/agents-api/cmd/server/managed_setup_test.go b/services/agents-api/cmd/server/managed_setup_test.go index f0c81d011..127d76b1e 100644 --- a/services/agents-api/cmd/server/managed_setup_test.go +++ b/services/agents-api/cmd/server/managed_setup_test.go @@ -58,7 +58,7 @@ func TestManagedSetupNeverReusesAnotherGenerationOrUnverifiedState(t *testing.T) db := &setupStore{value: store.SandboxSetup{InstallationID: "installation", Provider: "docker", Generation: 1}} s := &managedSetup{store: db, installationID: "installation"} cached := &execution.RuntimeProvider{InstallationID: "installation", ProviderKind: "docker", Generation: 1} - s.selected.Store(cached) + s.publish(cached) if got, err := s.load(t.Context()); err != nil || got != cached { t.Fatal("matching immutable selection was not reused") } @@ -101,18 +101,18 @@ func TestManagedSetupPreparesWithoutPublishing(t *testing.T) { defer hub.Close() s := &managedSetup{installationID: id, hub: hub, store: &setupStore{}, publicURL: "https://core.example"} previous := &execution.RuntimeProvider{InstallationID: id, Generation: 1, ProviderKind: "docker"} - s.selected.Store(previous) + s.publish(previous) candidate, err := s.prepare(t.Context(), store.SandboxSetup{InstallationID: id, Provider: "microsandbox", Mode: "nodes", IdleSeconds: 300, RetentionSeconds: 86400}) if err != nil { t.Fatal(err) } - if s.selected.Load() != previous || candidate.Config.ProviderKind != "microsandbox" || candidate.Config.Suspension == nil || candidate.Config.CoreURL != "https://core.example/api/v1" { + if s.selected.Load().Config != previous || candidate.Config.ProviderKind != "microsandbox" || candidate.Config.Suspension == nil || candidate.Config.CoreURL != "https://core.example/api/v1" { t.Fatal("preparation published or lost candidate configuration") } committed := *candidate.Config - committed.Generation, committed.Maintenance = 2, true + committed.Generation, committed.AdmissionPaused = 2, true candidate.Publish(&committed) - if got := s.selected.Load(); got.Generation != 2 || got.ProviderKind != "microsandbox" || !got.Maintenance { + if got := s.selected.Load(); got.Generation != 2 || got.Config.ProviderKind != "microsandbox" || !got.Config.AdmissionPaused { t.Fatal("commit did not publish the validated selection") } } @@ -123,10 +123,10 @@ func TestManagedSetupRejectedCandidateRetainsSelection(t *testing.T) { t.Setenv("OAC_E2B_STATE_DIR", t.TempDir()) s := &managedSetup{installationID: id} previous := &execution.RuntimeProvider{InstallationID: id, Generation: 1, ProviderKind: "docker"} - s.selected.Store(previous) + s.publish(previous) _, err := s.prepare(t.Context(), store.SandboxSetup{InstallationID: id, Provider: "e2b", Mode: "direct", E2B: &store.SandboxE2BConfiguration{APIKey: "synthetic-key", Template: "runtime:" + uuid.NewString()}}) - if !errors.Is(err, execution.ErrExecutionUnavailable) || s.selected.Load() != previous { + if !errors.Is(err, execution.ErrExecutionUnavailable) || s.selected.Load().Config != previous { t.Fatal("rejected candidate lost the previous selection", err) } } @@ -147,3 +147,50 @@ func TestCoreRejectsFileManagedSandboxConfiguration(t *testing.T) { t.Fatal("accepted a second configuration source") } } + +type delayedSetupStore struct { + setupStore + entered, release chan struct{} +} + +func (s *delayedSetupStore) GetSandboxSetup(ctx context.Context) (store.SandboxSetup, error) { + value := s.value + close(s.entered) + select { + case <-s.release: + return value, nil + case <-ctx.Done(): + return store.SandboxSetup{}, ctx.Err() + } +} +func TestManagedSetupResetTombstoneRejectsDelayedProviderLoad(t *testing.T) { + id := uuid.NewString() + db := &delayedSetupStore{setupStore: setupStore{value: store.SandboxSetup{InstallationID: id, Provider: "docker", Generation: 1}}, entered: make(chan struct{}), release: make(chan struct{})} + s := &managedSetup{installationID: id, store: db} + done := make(chan error, 1) + go func() { + provider, err := s.load(t.Context()) + if err == nil && provider != nil { + err = errors.New("old provider survived reset") + } + done <- err + }() + <-db.entered + s.publishUnconfigured(2) + close(db.release) + if err := <-done; err != nil { + t.Fatal(err) + } + if s.selected.Load().Generation != 2 || s.selected.Load().Config != nil || s.ObservationProviderType() != "" { + t.Fatal("empty publication lost its generation") + } + s.publish(&execution.RuntimeProvider{Generation: 1, ProviderKind: "docker"}) + if s.selected.Load().Config != nil { + t.Fatal("late old publication resurrected provider") + } + next := &execution.RuntimeProvider{Generation: 3, ProviderKind: "microsandbox"} + s.publish(next) + if s.selected.Load().Config != next || s.ObservationProviderType() != "microsandbox" { + t.Fatal("reset blocked subsequent configuration") + } +} diff --git a/services/agents-api/deploy/codex/README.md b/services/agents-api/deploy/codex/README.md index e2b67f96b..0b63baab9 100644 --- a/services/agents-api/deploy/codex/README.md +++ b/services/agents-api/deploy/codex/README.md @@ -157,13 +157,16 @@ Docker socket; the node owns its local Docker access. Node files contain the ins selection and host-specific paths, never a separate provider or resource choice. The Core host joins through the same command as any other host. -Provider, resource and Runtime changes use deployment maintenance, the current -generation and verified zero retained or pending execution resources. Stopped -containers, snapshots, unknown operations and pending cleanup block replacement. -Use the [maintenance procedure](../../HOSTED-SANDBOX-MANAGER.md#removal-and-maintenance). -Configuration changes do not delete resources or migrate Sessions. Do not delete -Sessions to preserve history: that operation removes public access and saved -artifacts. Core rejects `AGENTS_API_MANAGED_RUNTIMES_FILE`; restarting or editing +Same-provider resource and Runtime edits currently require no active reset, +the current generation and verified zero held allocations or pending Environments. +A backend change requires explicit reset and confirmed cleanup, then a new setup. +See the [reset procedure](../../HOSTED-SANDBOX-MANAGER.md#removal-and-reset). +Stopped compute, snapshots and unknown operations remain blockers. Keep the original +node identity, paths and credentials until cleanup is confirmed. Explicit archive +preserves history and persisted Files/Artifacts but discards unpersisted workspace; +ordinary Session deletion has different retention behavior. Existing Sessions never +migrate to another backend. +Core rejects `AGENTS_API_MANAGED_RUNTIMES_FILE`; restarting or editing an old file does not replace database configuration ownership. Node providers use explicit local Unix Docker sockets, ignoring ambient diff --git a/services/agents-api/deploy/e2b/README.md b/services/agents-api/deploy/e2b/README.md index fc92027f1..dfd0400f6 100644 --- a/services/agents-api/deploy/e2b/README.md +++ b/services/agents-api/deploy/e2b/README.md @@ -18,7 +18,7 @@ compute cleanup remains Core's responsibility. Select E2B in Core's hosted deployment setup and supply the account key and an immutable `templateID:build_UUID`. One deployment uses one managed Provider; -E2B needs no physical node enrollment. Switching Providers requires maintenance +E2B needs no physical node enrollment. Changing backend requires explicit reset and confirmed cleanup of every owned allocation, pending creation and snapshot. Do not infer execution readiness from saved configuration or running compute. diff --git a/services/agents-api/deploy/microsandbox/README.md b/services/agents-api/deploy/microsandbox/README.md index 56bd5b106..dc0949b4f 100644 --- a/services/agents-api/deploy/microsandbox/README.md +++ b/services/agents-api/deploy/microsandbox/README.md @@ -32,14 +32,16 @@ specification; they cannot select different resources or a different Runtime. Harness selection is independent. Snapshot suspension is available only on microsandbox; Docker and E2B retain their own supported lifecycle. -Provider, resource and Runtime changes require global maintenance, the current -generation and verified zero retained allocations or pending hosted Environments. -Use the [maintenance procedure](../../HOSTED-SANDBOX-MANAGER.md#removal-and-maintenance). -Stopped compute, snapshots and unknown operations remain blockers. Maintenance -and configuration changes do not delete resources or migrate Sessions. Keep the -original node identity, backend paths and credentials until cleanup is confirmed. -Session deletion removes its saved artifacts and is not a history-preserving -resource-release operation. +Same-provider resource and Runtime edits currently require no active reset, +the current generation and verified zero held allocations or pending Environments. +A backend change requires explicit reset and confirmed cleanup, then a new setup. +See the [reset procedure](../../HOSTED-SANDBOX-MANAGER.md#removal-and-reset). +Stopped compute, snapshots and unknown operations remain blockers. Keep the original +node identity, paths and credentials until cleanup is confirmed. Explicit archive +preserves history and persisted Files/Artifacts but discards unpersisted workspace; +ordinary Session deletion has different retention behavior. Existing Sessions never +migrate to another backend. + Core rejects `AGENTS_API_MANAGED_RUNTIMES_FILE`. It does not automatically adopt an older file-managed database, even after its resources are drained. Keep the diff --git a/services/agents-api/internal/api/admin_session_archive.go b/services/agents-api/internal/api/admin_session_archive.go index b9ccdaa38..66a592a83 100644 --- a/services/agents-api/internal/api/admin_session_archive.go +++ b/services/agents-api/internal/api/admin_session_archive.go @@ -18,7 +18,7 @@ type AdminSessionArchiveRequest struct { } // @Summary Release a managed Session's execution resources while retaining history -// @Description Core key only. Requires maintenance and the current deployment generation. Permanently closes execution, requests cancellation and releases sandbox/snapshots through existing cleanup. Session history and persisted files/artifacts remain; unpersisted workspace contents are lost. A cleanup_pending response is not proof of resource release. Does not affect caller-managed Runtime. +// @Description Core key only. Requires the current deployment generation; no maintenance mode is required. Permanently closes execution, requests cancellation and releases sandbox/snapshots through existing cleanup. Session history and persisted files/artifacts remain; unpersisted workspace contents are lost. A cleanup_pending response is not proof of resource release. Does not affect caller-managed Runtime. // @Tags Core Administration // @Accept json // @Produce json diff --git a/services/agents-api/internal/api/errors.go b/services/agents-api/internal/api/errors.go index d28e126ff..a0e6df808 100644 --- a/services/agents-api/internal/api/errors.go +++ b/services/agents-api/internal/api/errors.go @@ -95,7 +95,22 @@ func writeStoreError(w http.ResponseWriter, r *http.Request, err error, notFound var cursor *store.InvalidCursorError var selection *store.MCPCredentialSelectionError var sandboxConfiguration *store.SandboxConfigurationError + var stale *store.SandboxGenerationStaleError + var resetRequired *store.SandboxResetRequiredError + var inUse *store.SandboxInUseError switch { + case errors.As(err, &stale): + writeCoreError(w, http.StatusConflict, "sandbox_generation_stale", "The sandbox deployment generation changed. Refresh before submitting again.", CoreErrorDetails{"current_generation": CoreErrorNumber(float64(stale.CurrentGeneration))}) + case errors.As(err, &resetRequired): + writeCoreError(w, http.StatusConflict, "sandbox_reset_required", "Reset the sandbox deployment before changing its backend.", CoreErrorDetails{"current_provider": CoreErrorString(resetRequired.CurrentProvider), "requested_provider": CoreErrorString(resetRequired.RequestedProvider)}) + case errors.As(err, &inUse): + writeCoreError(w, http.StatusConflict, "sandbox_in_use", "Hosted sandbox resources still belong to this deployment.", CoreErrorDetails{"allocations": CoreErrorNumber(float64(inUse.Resources.Allocations)), "pending": CoreErrorNumber(float64(inUse.Resources.Pending))}) + case errors.Is(err, store.ErrSandboxResetAdmission): + writeError(w, http.StatusServiceUnavailable, "sandbox_reset_in_progress", "A sandbox reset is in progress.") + case errors.Is(err, store.ErrSandboxResetInProgress): + writeError(w, http.StatusConflict, "sandbox_reset_in_progress", "A sandbox reset is in progress.") + case errors.Is(err, store.ErrSandboxNotConfigured): + writeError(w, http.StatusConflict, "sandbox_not_configured", "The sandbox deployment is not configured.") case errors.As(err, &sandboxConfiguration): writeError(w, http.StatusBadRequest, "invalid_sandbox_configuration", sandboxConfiguration.Message) case errors.Is(err, store.ErrSandboxPublicURLUnreachable): @@ -115,13 +130,13 @@ func writeStoreError(w http.ResponseWriter, r *http.Request, err error, notFound case errors.Is(err, store.ErrSandboxCredentialUnavailable): writeError(w, http.StatusServiceUnavailable, "sandbox_credential_unavailable", "Sandbox credentials are unavailable. Check the service credential encryption configuration.") case errors.Is(err, store.ErrSandboxDeploymentConflict): - writeError(w, http.StatusConflict, "sandbox_deployment_conflict", "The sandbox deployment cannot change in its current state. Refresh the configuration, enter maintenance and finish resource cleanup before switching.") + writeError(w, http.StatusConflict, "sandbox_deployment_conflict", "The sandbox deployment cannot change in its current state. Refresh the configuration and inspect its reset and resource state.") case errors.Is(err, store.ErrRuntimeNodeCredential): writeError(w, http.StatusUnauthorized, "invalid_node_credential", "A valid sandbox node enrollment or node credential is required.") case errors.Is(err, store.ErrRuntimeNodeInUse): writeError(w, http.StatusConflict, "runtime_node_in_use", "The sandbox node retains allocations, snapshots, reservations or pending cleanup.") case errors.Is(err, store.ErrRuntimeLocalNodeConfigured): - writeError(w, http.StatusConflict, "runtime_local_node_configured", "The local sandbox node is enabled in deployment configuration. Disable it through provider maintenance and restart Core before removing it.") + writeError(w, http.StatusConflict, "runtime_local_node_configured", "The local sandbox node is enabled in deployment configuration. Drain it with the previous release and remove its file-managed configuration before replacing it.") case errors.Is(err, store.ErrRuntimeNodeUnavailable): writeError(w, http.StatusServiceUnavailable, "runtime_node_unavailable", "The selected sandbox node is unavailable or has no capacity.") diff --git a/services/agents-api/internal/api/handler.go b/services/agents-api/internal/api/handler.go index 1bef9bdae..4ab72d73f 100644 --- a/services/agents-api/internal/api/handler.go +++ b/services/agents-api/internal/api/handler.go @@ -42,7 +42,8 @@ type Handler struct { deploymentAuth *DeploymentAuthenticator sandboxSetup func(context.Context, store.SandboxDeploymentSetupRequest) (store.RuntimeDeploymentView, error) sandboxUpdate func(context.Context, store.SandboxDeploymentUpdateRequest) (store.RuntimeDeploymentView, error) - sandboxMaintenance func(context.Context, store.SandboxMaintenanceRequest) (store.RuntimeDeploymentView, error) + sandboxReset func(context.Context, store.SandboxResetRequest) (store.RuntimeDeploymentView, error) + sandboxResetCancel func(context.Context, uint64) (store.RuntimeDeploymentView, error) policy execution.Policy store ResourceStore auth *Authenticator diff --git a/services/agents-api/internal/api/installation_test.go b/services/agents-api/internal/api/installation_test.go index de139dcac..e8fa97d69 100644 --- a/services/agents-api/internal/api/installation_test.go +++ b/services/agents-api/internal/api/installation_test.go @@ -66,7 +66,12 @@ func TestInstallationSnapshotCannotCarryASensitiveValue(t *testing.T) { func TestDeploymentAddressIsNotInput(t *testing.T) { project, _ := NewAuthenticator([]APIKey{callerBinding()}) admin, _ := NewDeploymentAuthenticator([]string{device.HashCredential("administrator")}) + initializations := 0 initialize := func(_ context.Context, input store.SandboxDeploymentSetupRequest) (store.RuntimeDeploymentView, error) { + initializations++ + if input.Provider != "e2b" || input.ExpectedGeneration != 0 { + t.Fatal("invalid selection reached initialization", input.Provider, input.ExpectedGeneration) + } return store.RuntimeDeploymentView{}, store.ErrSandboxPublicURLUnreachable } update := func(context.Context, store.SandboxDeploymentUpdateRequest) (store.RuntimeDeploymentView, error) { @@ -74,7 +79,7 @@ func TestDeploymentAddressIsNotInput(t *testing.T) { return store.RuntimeDeploymentView{}, nil } h, err := NewHandler(&recordingStore{}, project, "codex", WithSandboxManager(&store.Store{}, admin), - WithSandboxDeploymentSetup(initialize), WithSandboxDeploymentChanges(update, nil)) + WithSandboxDeploymentSetup(initialize), WithSandboxDeploymentChanges(update, nil, nil)) if err != nil { t.Fatal(err) } @@ -82,9 +87,9 @@ func TestDeploymentAddressIsNotInput(t *testing.T) { method, body, code string status int }{ - {http.MethodPost, `{"provider":"docker","core_url":"https://core.example"}`, `"param":"core_url"`, http.StatusBadRequest}, + {http.MethodPost, `{"provider":"docker","core_url":"https://core.example","expected_generation":0}`, `"param":"core_url"`, http.StatusBadRequest}, {http.MethodPut, `{"provider":"docker","core_url":"https://core.example","expected_generation":1}`, `"param":"core_url"`, http.StatusBadRequest}, - {http.MethodPost, `{"provider":"e2b","e2b":{"api_key":"key","template":"runtime:build"}}`, `"code":"sandbox_configuration_error"`, http.StatusConflict}, + {http.MethodPost, `{"provider":"e2b","expected_generation":0,"e2b":{"api_key":"key","template":"runtime:build"}}`, `"code":"sandbox_configuration_error"`, http.StatusConflict}, } { request := httptest.NewRequest(test.method, "/core/v1/sandbox/deployment", strings.NewReader(test.body)) request.Header.Set("Authorization", "Bearer administrator") @@ -95,4 +100,7 @@ func TestDeploymentAddressIsNotInput(t *testing.T) { t.Fatal(test.method, result.Code, result.Body.String()) } } + if initializations != 1 { + t.Fatal("address rejection did not isolate initialization", initializations) + } } diff --git a/services/agents-api/internal/api/sandbox_deployment_changes_test.go b/services/agents-api/internal/api/sandbox_deployment_changes_test.go index ff8808312..3bdc00238 100644 --- a/services/agents-api/internal/api/sandbox_deployment_changes_test.go +++ b/services/agents-api/internal/api/sandbox_deployment_changes_test.go @@ -14,7 +14,7 @@ import ( func TestSandboxDeploymentChangesAuthenticateAndDecode(t *testing.T) { project, _ := NewAuthenticator([]APIKey{callerBinding()}) admin, _ := NewDeploymentAuthenticator([]string{device.HashCredential("administrator")}) - updates, maintenance := 0, 0 + updates, resets := 0, 0 update := func(_ context.Context, in store.SandboxDeploymentUpdateRequest) (store.RuntimeDeploymentView, error) { updates++ if in.Provider != "e2b" || in.ExpectedGeneration != 2 || in.E2B == nil || in.E2B.APIKey != "synthetic-private-key" { @@ -22,14 +22,16 @@ func TestSandboxDeploymentChangesAuthenticateAndDecode(t *testing.T) { } return store.RuntimeDeploymentView{Provider: in.Provider}, nil } - maintain := func(_ context.Context, in store.SandboxMaintenanceRequest) (store.RuntimeDeploymentView, error) { - maintenance++ + maintain := func(_ context.Context, in store.SandboxResetRequest) (store.RuntimeDeploymentView, error) { + resets++ if in.ExpectedGeneration != 2 { t.Fatal("generation was lost") } - return store.RuntimeDeploymentView{Maintenance: in.Maintenance}, nil + return store.RuntimeDeploymentView{Reset: &store.SandboxResetView{Clear: in.Clear}}, nil } - h, err := NewHandler(&recordingStore{}, project, "codex", WithSandboxManager(&store.Store{}, admin), WithSandboxDeploymentChanges(update, maintain)) + h, err := NewHandler(&recordingStore{}, project, "codex", WithSandboxManager(&store.Store{}, admin), WithSandboxDeploymentChanges(update, maintain, func(context.Context, uint64) (store.RuntimeDeploymentView, error) { + return store.RuntimeDeploymentView{}, nil + })) if err != nil { t.Fatal(err) } @@ -43,10 +45,17 @@ func TestSandboxDeploymentChangesAuthenticateAndDecode(t *testing.T) { {"PUT", "/deployment", "administrator", strings.Replace(selection, `"api_key"`, `"API_KEY"`, 1), 400}, {"PUT", "/deployment", "administrator", strings.Replace(selection, `"expected_generation":2,`, "", 1), 400}, {"PUT", "/deployment", "administrator", selection, 200}, - {"PATCH", "/deployment/maintenance", "administrator", `{"expected_generation":2}`, 400}, - {"PATCH", "/deployment/maintenance", "administrator", `{"maintenance":null,"expected_generation":2}`, 400}, - {"PATCH", "/deployment/maintenance", "administrator", `{"maintenance":true,"expected_generation":2}`, 200}, - {"PATCH", "/deployment/maintenance", "administrator", `{"maintenance":false,"expected_generation":2}`, 200}, + {"POST", "/deployment/reset", "administrator", `{"expected_generation":2}`, 400}, + {"POST", "/deployment/reset", "caller", `{"clear":"force","expected_generation":2}`, 401}, + {"POST", "/deployment/reset", "administrator", `{"clear":"force","deadline_seconds":null,"expected_generation":2}`, 400}, + {"POST", "/deployment/reset", "administrator", `{"clear":"auto","expected_generation":null}`, 400}, + {"DELETE", "/deployment/reset", "administrator", "", 400}, + {"DELETE", "/deployment/reset?expected_generation=2&expected_generation=2", "administrator", "", 400}, + {"POST", "/deployment/reset", "administrator", `{"clear":"auto","deadline_seconds":299,"expected_generation":2}`, 400}, + {"PATCH", "/deployment/maintenance", "administrator", `{"maintenance":true,"expected_generation":2}`, 404}, + {"POST", "/deployment/reset", "administrator", `{"clear":"auto","expected_generation":2}`, 200}, + {"POST", "/deployment/reset", "administrator", `{"clear":"force","expected_generation":2}`, 200}, + {"DELETE", "/deployment/reset?expected_generation=2", "administrator", "", 200}, } { r := httptest.NewRequest(tc.method, "/core/v1/sandbox"+tc.path, strings.NewReader(tc.body)) r.Header.Set("Authorization", "Bearer "+tc.token) @@ -56,8 +65,8 @@ func TestSandboxDeploymentChangesAuthenticateAndDecode(t *testing.T) { t.Fatalf("%s %s status=%d", tc.method, tc.path, w.Code) } } - if updates != 1 || maintenance != 2 { - t.Fatalf("unauthorized or invalid input reached mutation: %d %d", updates, maintenance) + if updates != 1 || resets != 2 { + t.Fatalf("unauthorized or invalid input reached mutation: %d %d", updates, resets) } } @@ -68,7 +77,7 @@ func TestSandboxDeploymentChangesUnavailableWithoutOwner(t *testing.T) { handler http.HandlerFunc }{ {`{"provider":"docker","expected_generation":1}`, h.updateSandboxDeployment}, - {`{"maintenance":true,"expected_generation":1}`, h.setSandboxMaintenance}, + {`{"clear":"auto","expected_generation":1}`, h.startSandboxReset}, } { w := httptest.NewRecorder() tc.handler(w, httptest.NewRequest("PUT", "/", strings.NewReader(tc.body))) @@ -77,3 +86,37 @@ func TestSandboxDeploymentChangesUnavailableWithoutOwner(t *testing.T) { } } } + +func TestSandboxMutationErrorsExposeOnlyTypedCoreFacts(t *testing.T) { + for _, tc := range []struct { + err error + code string + status int + details string + }{ + {&store.SandboxGenerationStaleError{CurrentGeneration: 8}, "sandbox_generation_stale", 409, `"current_generation":8`}, + {&store.SandboxResetRequiredError{CurrentProvider: "docker", RequestedProvider: "e2b"}, "sandbox_reset_required", 409, `"requested_provider":"e2b"`}, + {&store.SandboxInUseError{Resources: store.SandboxDeploymentResources{Allocations: 2, Pending: 1}}, "sandbox_in_use", 409, `"allocations":2`}, + {store.ErrSandboxResetInProgress, "sandbox_reset_in_progress", 409, ""}, + {store.ErrSandboxResetAdmission, "sandbox_reset_in_progress", 503, ""}, + } { + for _, core := range []bool{false, true} { + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { writeStoreError(w, r, tc.err) })) + if core { + handler = coreErrorResponses(handler) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest("POST", "/test", nil)) + body := response.Body.String() + if response.Code != tc.status || !strings.Contains(body, `"code":"`+tc.code+`"`) { + t.Fatal(body) + } + if core && tc.details != "" && !strings.Contains(body, tc.details) { + t.Fatal("typed detail missing", body) + } + if !core && strings.Contains(body, `"details"`) { + t.Fatal("Core facts escaped their router", body) + } + } + } +} diff --git a/services/agents-api/internal/api/sandbox_deployment_setup.go b/services/agents-api/internal/api/sandbox_deployment_setup.go index c9eecbe76..3b31b22b4 100644 --- a/services/agents-api/internal/api/sandbox_deployment_setup.go +++ b/services/agents-api/internal/api/sandbox_deployment_setup.go @@ -3,9 +3,11 @@ package api import ( "context" "encoding/json" - "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" "net/http" + "net/url" + "strconv" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" ) @@ -17,6 +19,7 @@ type SandboxE2BInput struct { } type SandboxDeploymentInput struct { + ExpectedGeneration *uint64 `json:"expected_generation" binding:"required"` // Per-sandbox limits, required for Docker and microsandbox. E2B may omit // them; Core then uses the validated template build's cpus and memory_mib. Resources sandbox.Resources `json:"resources"` @@ -27,11 +30,10 @@ type SandboxDeploymentInput struct { type SandboxDeploymentChangeInput struct { SandboxDeploymentInput - ExpectedGeneration uint64 `json:"expected_generation"` } func (v SandboxDeploymentInput) request() store.SandboxDeploymentSetupRequest { - input := store.SandboxDeploymentSetupRequest{Provider: v.Provider, DeploymentSpec: sandbox.DeploymentSpec{Resources: v.Resources, Runtime: v.Runtime}} + input := store.SandboxDeploymentSetupRequest{ExpectedGeneration: *v.ExpectedGeneration, Provider: v.Provider, DeploymentSpec: sandbox.DeploymentSpec{Resources: v.Resources, Runtime: v.Runtime}} if v.E2B != nil { input.E2B = &store.SandboxE2BConfiguration{APIKey: v.E2B.APIKey, Template: v.E2B.Template} } @@ -58,13 +60,14 @@ func WithSandboxDeploymentSetup(initialize func(context.Context, store.SandboxDe func WithSandboxDeploymentChanges( update func(context.Context, store.SandboxDeploymentUpdateRequest) (store.RuntimeDeploymentView, error), - maintenance func(context.Context, store.SandboxMaintenanceRequest) (store.RuntimeDeploymentView, error), + reset func(context.Context, store.SandboxResetRequest) (store.RuntimeDeploymentView, error), + cancel func(context.Context, uint64) (store.RuntimeDeploymentView, error), ) Option { - return func(h *Handler) { h.sandboxUpdate = update; h.sandboxMaintenance = maintenance } + return func(h *Handler) { h.sandboxUpdate = update; h.sandboxReset = reset; h.sandboxResetCancel = cancel } } // @Summary Initialize the deployment sandbox provider -// @Description Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Exact retries return the existing selection; differing selections and file-managed deployments reject. This does not create compute or execute work. +// @Description Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Requires explicit expected_generation, including zero at first setup. Stale retries reject before provider validation. An identical selection at the current generation is a no-op; differing selections and file-managed deployments reject. This does not create compute or execute work. // @Tags Sandbox Manager // @Produce json // @Security DeploymentAdminAuth @@ -82,7 +85,7 @@ func (h *Handler) initializeSandboxDeployment(w http.ResponseWriter, r *http.Req return } var input SandboxDeploymentInput - if decodeInputObject(raw, &input, "provider", "e2b", "resources", "runtime") != nil { + if decodeInputObject(raw, &input, "provider", "e2b", "resources", "runtime", "expected_generation") != nil || input.ExpectedGeneration == nil { writeStoreError(w, r, store.ErrInvalidInput) return } @@ -99,7 +102,7 @@ func (h *Handler) initializeSandboxDeployment(w http.ResponseWriter, r *http.Req } // @Summary Change a fully drained deployment's sandbox configuration -// @Description Requires maintenance, the current generation and verified cleanup of all old resources. Credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory. A core_url member is rejected with 400; the address comes from the installation public URL. Historical records are retained; old node credentials and enrollments are retired. Explicitly resume after success. Never automatically retry an uncertain write. +// @Description Requires the current generation, the same backend type, no active reset and verified cleanup of all old resources. A backend type change requires reset. Credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory. A core_url member is rejected with 400; the address comes from the installation public URL. Historical records are retained; old node credentials and enrollments are retired. Never automatically retry an uncertain write. // @Tags Sandbox Manager // @Produce json // @Security DeploymentAdminAuth @@ -117,7 +120,7 @@ func (h *Handler) updateSandboxDeployment(w http.ResponseWriter, r *http.Request return } var input SandboxDeploymentChangeInput - if decodeInputObject(raw, &input, "provider", "e2b", "resources", "runtime", "expected_generation") != nil || input.ExpectedGeneration == 0 { + if decodeInputObject(raw, &input, "provider", "e2b", "resources", "runtime", "expected_generation") != nil || input.ExpectedGeneration == nil { writeStoreError(w, r, store.ErrInvalidInput) return } @@ -125,7 +128,7 @@ func (h *Handler) updateSandboxDeployment(w http.ResponseWriter, r *http.Request writeStoreError(w, r, store.ErrSandboxDeploymentConflict) return } - result, err := h.sandboxUpdate(r.Context(), store.SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input.request(), ExpectedGeneration: input.ExpectedGeneration}) + result, err := h.sandboxUpdate(r.Context(), store.SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input.request(), ExpectedGeneration: *input.ExpectedGeneration}) if err != nil { writeStoreError(w, r, err) return @@ -133,38 +136,88 @@ func (h *Handler) updateSandboxDeployment(w http.ResponseWriter, r *http.Request writeJSON(w, http.StatusOK, result) } -// @Summary Pause or resume new hosted allocations -// @Description Requires the current generation. Maintenance preserves execution, cleanup and history. Resume requires successfully activated provider configuration. +// @Summary Start or escalate a durable sandbox deployment reset +// @Description Archives hosted Sessions and waits for confirmed provider cleanup, preserving history and Files/Artifacts. Auto waits for started or waiting Turns and file writes until the durable deadline; force cancels them. The same clear is idempotent; force escalates auto. Requires the current generation. Self-hosted Sessions are unchanged. // @Tags Sandbox Manager // @Produce json // @Security DeploymentAdminAuth // @Accept json -// @Param body body store.SandboxMaintenanceRequest true "Maintenance state" +// @Param body body store.SandboxResetRequest true "Reset mode and current deployment generation" // @Success 200 {object} store.RuntimeDeploymentView // @Failure 400,401,409,500,503 {object} CoreErrorResponse -// @Router /core/v1/sandbox/deployment/maintenance [patch] -func (h *Handler) setSandboxMaintenance(w http.ResponseWriter, r *http.Request) { - raw, ok := readJSONBody(w, r) +// @Router /core/v1/sandbox/deployment/reset [post] +func (h *Handler) startSandboxReset(w http.ResponseWriter, r *http.Request) { + raw, ok := readJSONBodyLimit(w, r, 4096, "Reset request is too large.") if !ok { return } - // Omitted or null maintenance is not an instruction to resume. var input struct { - Maintenance *bool `json:"maintenance"` - ExpectedGeneration uint64 `json:"expected_generation"` + ExpectedGeneration *uint64 `json:"expected_generation"` + Clear string `json:"clear"` + DeadlineSeconds *int32 `json:"deadline_seconds"` } - if decodeInputObject(raw, &input, "maintenance", "expected_generation") != nil || input.Maintenance == nil || input.ExpectedGeneration == 0 { - writeStoreError(w, r, store.ErrInvalidInput) + if decodeInputObject(raw, &input, "expected_generation", "clear", "deadline_seconds") != nil || input.ExpectedGeneration == nil { + writeError(w, http.StatusBadRequest, "invalid_request_error", "A current expected_generation is required.", "expected_generation") + return + } + if input.Clear != "auto" && input.Clear != "force" { + writeError(w, http.StatusBadRequest, "invalid_request_error", "Choose auto or force for clear.", "clear") + return + } + var fields map[string]json.RawMessage + _ = json.Unmarshal(raw, &fields) + if value, present := fields["deadline_seconds"]; present && string(value) == "null" { + writeError(w, http.StatusBadRequest, "invalid_request_error", "deadline_seconds must be an integer when supplied.", "deadline_seconds") + return + } + if input.DeadlineSeconds != nil && (input.Clear != "auto" || *input.DeadlineSeconds < 300 || *input.DeadlineSeconds > 86400) { + writeCoreError(w, http.StatusBadRequest, "invalid_request_error", "deadline_seconds applies only to auto and must be between 300 and 86400.", CoreErrorDetails{"min": CoreErrorNumber(300), "max": CoreErrorNumber(86400)}, "deadline_seconds") return } - if h.sandboxMaintenance == nil { + if h.sandboxReset == nil { writeStoreError(w, r, store.ErrSandboxDeploymentConflict) return } - result, err := h.sandboxMaintenance(r.Context(), store.SandboxMaintenanceRequest{Maintenance: *input.Maintenance, ExpectedGeneration: input.ExpectedGeneration}) + setAdminAuditSource(r, "") + result, err := h.sandboxReset(r.Context(), store.SandboxResetRequest{ExpectedGeneration: *input.ExpectedGeneration, Clear: input.Clear, DeadlineSeconds: input.DeadlineSeconds}) if err != nil { writeStoreError(w, r, err) return } writeJSON(w, http.StatusOK, result) } + +// @Summary Cancel a sandbox deployment reset +// @Description Restores admission but never restores Sessions already archived. With no reset running this is an idempotent read, provided the generation still matches. +// @Tags Sandbox Manager +// @Produce json +// @Security DeploymentAdminAuth +// @Param expected_generation query integer true "Current deployment generation" +// @Success 200 {object} store.RuntimeDeploymentView +// @Failure 400,401,409,500,503 {object} CoreErrorResponse +// @Router /core/v1/sandbox/deployment/reset [delete] +func (h *Handler) cancelSandboxReset(w http.ResponseWriter, r *http.Request) { + query, err := parseResetGeneration(r) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid_request_error", "A single current expected_generation is required.", "expected_generation") + return + } + if h.sandboxResetCancel == nil { + writeStoreError(w, r, store.ErrSandboxDeploymentConflict) + return + } + setAdminAuditSource(r, "") + result, err := h.sandboxResetCancel(r.Context(), query) + if err != nil { + writeStoreError(w, r, err) + return + } + writeJSON(w, http.StatusOK, result) +} +func parseResetGeneration(r *http.Request) (uint64, error) { + query, err := url.ParseQuery(r.URL.RawQuery) + if err != nil || len(query) != 1 || len(query["expected_generation"]) != 1 { + return 0, store.ErrInvalidInput + } + return strconv.ParseUint(query.Get("expected_generation"), 10, 64) +} diff --git a/services/agents-api/internal/api/sandbox_deployment_setup_test.go b/services/agents-api/internal/api/sandbox_deployment_setup_test.go index a7fd195ef..ab7f140f0 100644 --- a/services/agents-api/internal/api/sandbox_deployment_setup_test.go +++ b/services/agents-api/internal/api/sandbox_deployment_setup_test.go @@ -30,12 +30,14 @@ func TestSandboxDeploymentSetupRequiresAdministratorAndStrictBody(t *testing.T) token, body string status, calls int }{ - {"caller", `{"provider":"docker"}`, 401, 0}, - {"node-credential", `{"provider":"docker"}`, 401, 0}, - {"enrollment-token", `{"provider":"docker"}`, 401, 0}, + {"caller", `{"provider":"docker","expected_generation":0}`, 401, 0}, + {"node-credential", `{"provider":"docker","expected_generation":0}`, 401, 0}, + {"enrollment-token", `{"provider":"docker","expected_generation":0}`, 401, 0}, {"administrator", `{"provider":"docker","unexpected":true}`, 400, 0}, - {"administrator", `{"provider":"docker"}`, 200, 1}, - {"administrator", `{"provider":"microsandbox"}`, 409, 2}, + {"administrator", `{"provider":"docker"}`, 400, 0}, + {"administrator", `{"provider":"docker","expected_generation":null}`, 400, 0}, + {"administrator", `{"provider":"docker","expected_generation":0}`, 200, 1}, + {"administrator", `{"provider":"microsandbox","expected_generation":0}`, 409, 2}, } { request := httptest.NewRequest(http.MethodPost, "/core/v1/sandbox/deployment", strings.NewReader(test.body)) request.Header.Set("Authorization", "Bearer "+test.token) @@ -49,7 +51,7 @@ func TestSandboxDeploymentSetupRequiresAdministratorAndStrictBody(t *testing.T) func TestSandboxDeploymentSetupFileModeReturnsConflict(t *testing.T) { h := &Handler{} - request := httptest.NewRequest(http.MethodPost, "/core/v1/sandbox/deployment", strings.NewReader(`{"provider":"docker"}`)) + request := httptest.NewRequest(http.MethodPost, "/core/v1/sandbox/deployment", strings.NewReader(`{"provider":"docker","expected_generation":0}`)) result := httptest.NewRecorder() h.initializeSandboxDeployment(result, request) if result.Code != http.StatusConflict || !strings.Contains(result.Body.String(), "sandbox_deployment_conflict") { diff --git a/services/agents-api/internal/api/sandbox_manager.go b/services/agents-api/internal/api/sandbox_manager.go index c24b9197a..6a31eee6b 100644 --- a/services/agents-api/internal/api/sandbox_manager.go +++ b/services/agents-api/internal/api/sandbox_manager.go @@ -55,7 +55,8 @@ func (h *Handler) registerSandboxManagerRoutes(r chi.Router) { r.Get("/sandbox/deployment", h.sandboxDeployment) r.Post("/sandbox/deployment", h.initializeSandboxDeployment) r.Put("/sandbox/deployment", h.updateSandboxDeployment) - r.Patch("/sandbox/deployment/maintenance", h.setSandboxMaintenance) + r.Post("/sandbox/deployment/reset", h.startSandboxReset) + r.Delete("/sandbox/deployment/reset", h.cancelSandboxReset) r.Get("/sandbox/nodes", h.sandboxNodes) r.Get("/sandbox/nodes/{node_id}", h.sandboxNodeDetail) r.Patch("/sandbox/nodes/{node_id}", h.updateSandboxNode) diff --git a/services/agents-api/internal/api/sandbox_manager_test.go b/services/agents-api/internal/api/sandbox_manager_test.go index 6cfc1b024..2c119aaf7 100644 --- a/services/agents-api/internal/api/sandbox_manager_test.go +++ b/services/agents-api/internal/api/sandbox_manager_test.go @@ -74,7 +74,7 @@ func TestSandboxLocalNodeRemovalExplainsDeploymentBinding(t *testing.T) { request := httptest.NewRequest(http.MethodDelete, "/core/v1/sandbox/nodes/local", nil) response := httptest.NewRecorder() writeStoreError(response, request, store.ErrRuntimeLocalNodeConfigured) - if response.Code != http.StatusConflict || !strings.Contains(response.Body.String(), "runtime_local_node_configured") || !strings.Contains(response.Body.String(), "maintenance") { + if response.Code != http.StatusConflict || !strings.Contains(response.Body.String(), "runtime_local_node_configured") || !strings.Contains(response.Body.String(), "previous release") { t.Fatal(response.Code, response.Body.String()) } } diff --git a/services/agents-api/internal/coremetrics/service.go b/services/agents-api/internal/coremetrics/service.go index 2c878d036..87fb79d1e 100644 --- a/services/agents-api/internal/coremetrics/service.go +++ b/services/agents-api/internal/coremetrics/service.go @@ -144,9 +144,7 @@ func (s *Service) Read(ctx context.Context, name string) (View, error) { view.Execution.OldestQueuedSeconds = latest.OldestQueuedSeconds view.Database.SizeBytes = latest.DatabaseSize view.Process = latest.Process - if latest.Maintenance != nil && *latest.Maintenance && view.Service.Status == "running" { - view.Service.Status = "maintenance" - } + } for _, id := range jobIDs { j := s.jobs[id] diff --git a/services/agents-api/internal/coremetrics/service_test.go b/services/agents-api/internal/coremetrics/service_test.go index fb6daf187..44a3ab294 100644 --- a/services/agents-api/internal/coremetrics/service_test.go +++ b/services/agents-api/internal/coremetrics/service_test.go @@ -120,10 +120,10 @@ func TestAllRangesAndBoundedRetention(t *testing.T) { } func TestJobResultsAndConcurrentReads(t *testing.T) { s, _, now := fixtureService(t) - s.record(Sample{At: now, Healthy: true, Maintenance: ptr(true)}) + s.record(Sample{At: now, Healthy: true}) s.ReportJob("audit_cleanup", now, ptr(int64(12)), ptr(int64(0)), nil) got, _ := s.Read(t.Context(), "1h") - if got.Service.Status != "maintenance" || *got.Jobs[3].Processed != 12 { + if got.Service.Status != "running" || *got.Jobs[3].Processed != 12 { t.Fatal(got.Service, got.Jobs) } s.ReportJob("audit_cleanup", now, ptr(int64(2)), ptr(int64(1)), errors.New("failure")) diff --git a/services/agents-api/internal/coremetrics/types.go b/services/agents-api/internal/coremetrics/types.go index a0cfc19a7..3cc899d16 100644 --- a/services/agents-api/internal/coremetrics/types.go +++ b/services/agents-api/internal/coremetrics/types.go @@ -94,7 +94,6 @@ type Sample struct { OldestQueuedSeconds *float64 PingMS *float64 PoolInUse, DatabaseSize *int64 - Maintenance *bool Healthy bool Process Process } diff --git a/services/agents-api/internal/db/queries/devices.sql b/services/agents-api/internal/db/queries/devices.sql index d7379b580..80262106e 100644 --- a/services/agents-api/internal/db/queries/devices.sql +++ b/services/agents-api/internal/db/queries/devices.sql @@ -12,7 +12,7 @@ LEFT JOIN runtime_allocations a ON a.device_id = d.id WHERE d.id = $1; -- name: RevokeDevice :execrows -UPDATE devices SET revoked_at = COALESCE(revoked_at, clock_timestamp()) +UPDATE devices SET revoked_at = COALESCE(revoked_at, clock_timestamp()), archive_cancel_turn_id = NULL WHERE tenant_id = $1 AND id = $2; -- name: TouchDevice :execrows diff --git a/services/agents-api/internal/db/queries/runtime_cancellation.sql b/services/agents-api/internal/db/queries/runtime_cancellation.sql new file mode 100644 index 000000000..600b246b7 --- /dev/null +++ b/services/agents-api/internal/db/queries/runtime_cancellation.sql @@ -0,0 +1,32 @@ +-- name: GetArchivedCancellationReceipt :one +SELECT t.id, t.cancel_requested_at +FROM devices d +JOIN runtime_allocations a ON a.device_id = d.id +JOIN environments e ON e.id = a.environment_id AND e.id = d.environment_id +JOIN sessions s ON s.id = e.session_id AND s.tenant_id = d.tenant_id +JOIN turns t ON t.session_id = s.id AND t.id = d.archive_cancel_turn_id +WHERE d.id = sqlc.arg(device_id) + AND d.credential_hash = sqlc.arg(credential_hash) + AND d.executor_key_id IS NULL AND d.revoked_at IS NOT NULL + AND a.state = 'cleanup_pending' AND a.released_at IS NULL + AND e.status = 'expired' AND s.deleted_at IS NULL + AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND t.status IN ('in_progress', 'waiting') + AND t.id::text = ANY(sqlc.arg(run_ids)::text[]) + AND t.cancel_requested_at IS NOT NULL + AND d.revoked_at >= t.cancel_requested_at + AND t.cancel_requested_at > clock_timestamp() - sqlc.arg(limit_seconds)::int * interval '1 second'; + +-- name: RevokeArchivedRuntimeDevice :execrows +UPDATE devices d +SET archive_cancel_turn_id = CASE WHEN d.revoked_at IS NULL THEN ( + SELECT t.id FROM turns t + WHERE t.session_id = sqlc.arg(session_id) + AND t.status IN ('in_progress', 'waiting') AND t.cancel_requested_at IS NOT NULL +) ELSE d.archive_cancel_turn_id END, + revoked_at = COALESCE(d.revoked_at, clock_timestamp()) +WHERE d.tenant_id = sqlc.arg(tenant_id) AND d.id = sqlc.arg(device_id); + +-- name: RevokeRuntimeCleanupDevice :execrows +UPDATE devices SET revoked_at = COALESCE(revoked_at, clock_timestamp()) +WHERE tenant_id = sqlc.arg(tenant_id) AND id = sqlc.arg(device_id); diff --git a/services/agents-api/internal/db/queries/runtime_deployment.sql b/services/agents-api/internal/db/queries/runtime_deployment.sql index cd22b1420..36f23dc1c 100644 --- a/services/agents-api/internal/db/queries/runtime_deployment.sql +++ b/services/agents-api/internal/db/queries/runtime_deployment.sql @@ -3,7 +3,7 @@ SELECT * FROM runtime_deployment WHERE singleton = true FOR UPDATE; -- name: SetRuntimeDeployment :exec UPDATE runtime_deployment SET installation_id = $1, backend_fingerprint = $2, -maintenance = $3, updated_at = clock_timestamp() WHERE singleton = true; +admission_paused = $3, updated_at = clock_timestamp() WHERE singleton = true; -- name: CountRuntimeDeploymentResources :one SELECT diff --git a/services/agents-api/internal/db/queries/runtime_lifecycle_nodes.sql b/services/agents-api/internal/db/queries/runtime_lifecycle_nodes.sql index 3b6d7302f..403741dce 100644 --- a/services/agents-api/internal/db/queries/runtime_lifecycle_nodes.sql +++ b/services/agents-api/internal/db/queries/runtime_lifecycle_nodes.sql @@ -20,7 +20,7 @@ FROM environments e JOIN sessions s ON s.id=e.session_id LEFT JOIN runtime_placements p ON p.environment_id=e.id WHERE p.node_id IS NOT DISTINCT FROM sqlc.narg(node_id)::uuid AND p.released_at IS NULL - AND NOT (SELECT maintenance FROM runtime_deployment) + AND NOT (SELECT admission_paused FROM runtime_deployment) AND e.id > sqlc.arg(after_id)::uuid AND s.deleted_at IS NULL AND e.status='pending' AND s.configuration->'environment'->>'type'='openai_hosted' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id=e.id) diff --git a/services/agents-api/internal/db/queries/sandbox_deployment_setup.sql b/services/agents-api/internal/db/queries/sandbox_deployment_setup.sql index c96fd093d..871e74eba 100644 --- a/services/agents-api/internal/db/queries/sandbox_deployment_setup.sql +++ b/services/agents-api/internal/db/queries/sandbox_deployment_setup.sql @@ -14,9 +14,6 @@ UPDATE runtime_deployment SET e2b_template_build_status=sqlc.narg(e2b_template_b e2b_template_memory_mib=sqlc.narg(e2b_template_memory_mib), e2b_template_root_disk_mib=sqlc.narg(e2b_template_root_disk_mib), updated_at=clock_timestamp() WHERE singleton=true AND provider_kind='e2b'; --- name: SetSandboxMaintenance :exec -UPDATE runtime_deployment SET maintenance=$1,updated_at=clock_timestamp() WHERE singleton=true; - -- name: RetireSandboxNodes :exec UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL,provider_ready=false WHERE removed_at IS NULL; diff --git a/services/agents-api/internal/db/queries/sandbox_reset.sql b/services/agents-api/internal/db/queries/sandbox_reset.sql new file mode 100644 index 000000000..3ca3458fc --- /dev/null +++ b/services/agents-api/internal/db/queries/sandbox_reset.sql @@ -0,0 +1,100 @@ +-- name: StartSandboxReset :exec +UPDATE runtime_deployment SET admission_paused = true, reset_clear = sqlc.arg(clear), + reset_requested_at = clock_timestamp(), + reset_deadline_at = CASE WHEN sqlc.arg(clear)::text = 'auto' + THEN clock_timestamp() + make_interval(secs => sqlc.arg(deadline_seconds)::int) END, + reset_forced_at = CASE WHEN sqlc.arg(clear)::text = 'force' THEN clock_timestamp() END, + reset_audit = sqlc.arg(audit)::jsonb, updated_at = clock_timestamp() +WHERE singleton = true; + +-- name: ForceSandboxReset :exec +UPDATE runtime_deployment SET reset_clear = 'force', reset_forced_at = clock_timestamp(), updated_at = clock_timestamp() +WHERE singleton = true AND reset_clear = 'auto'; + +-- name: CancelSandboxReset :exec +UPDATE runtime_deployment SET admission_paused = false, reset_clear = NULL, + reset_requested_at = NULL, reset_deadline_at = NULL, reset_forced_at = NULL, + reset_audit = NULL, updated_at = clock_timestamp() +WHERE singleton = true; + +-- name: CompleteSandboxReset :exec +UPDATE runtime_deployment SET provider_kind = '', backend_fingerprint = '', mode = '', + specification = '{}', idle_seconds = 0, retention_seconds = 0, + e2b_template = '', e2b_credential = NULL, e2b_template_build_status = NULL, + e2b_template_cpus = NULL, e2b_template_memory_mib = NULL, e2b_template_root_disk_mib = NULL, + generation = generation + 1, owner_epoch = owner_epoch + 1, + admission_paused = false, reset_clear = NULL, reset_requested_at = NULL, + reset_deadline_at = NULL, reset_forced_at = NULL, reset_audit = NULL, + updated_at = clock_timestamp() +WHERE singleton = true; + +-- name: SessionBlocksAutoReset :one +SELECT ( + EXISTS (SELECT 1 FROM turns t WHERE t.session_id = sqlc.arg(session_id) AND status IN ('in_progress', 'waiting')) + OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = sqlc.arg(session_id) AND status IN ('in_progress', 'waiting')) + OR EXISTS (SELECT 1 FROM environment_file_writes f JOIN environments e ON e.id = f.environment_id + WHERE e.session_id = sqlc.arg(session_id) AND f.state = 'pending') +)::boolean; + +-- name: ListSandboxResetSessions :many +SELECT s.id, s.tenant_id +FROM sessions s JOIN environments e ON e.session_id = s.id +WHERE s.deleted_at IS NULL AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND e.status NOT IN ('failed', 'expired') + AND s.id > sqlc.arg(after_id)::uuid + AND (EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state NOT IN ('released', 'cleanup_pending')) + OR (e.status = 'pending' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id))) + AND (sqlc.arg(force)::boolean OR NOT ( + EXISTS (SELECT 1 FROM turns t WHERE t.session_id = s.id AND t.status IN ('in_progress', 'waiting')) + OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = s.id AND t.status IN ('in_progress', 'waiting')) + OR EXISTS (SELECT 1 FROM environment_file_writes f WHERE f.environment_id = e.id AND f.state = 'pending') + )) +ORDER BY s.id LIMIT 32; + +-- name: GetSandboxResetProject :one +SELECT id FROM projects WHERE tenant_id = $1; + +-- name: GetSandboxDeploymentSnapshot :one +WITH deployment AS MATERIALIZED (SELECT * FROM runtime_deployment WHERE singleton = true LIMIT 1), +observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), +held AS ( + SELECT a.node_id, s.id AS session_id, e.id AS environment_id, false AS pending, + (a.state = 'cleanup_pending' OR s.deleted_at IS NOT NULL OR e.status IN ('failed', 'expired') + OR CASE WHEN a.compute_phase NOT IN ('disabled', 'running') + THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= observed.as_of + ELSE a.node_id IS NULL AND d.mode <> 'direct' AND a.kept_at <= observed.as_of - interval '1 hour' END) AS cleanup + FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id + JOIN sessions s ON s.id = e.session_id CROSS JOIN deployment d CROSS JOIN observed + WHERE a.state <> 'released' + UNION ALL + SELECT p.node_id, s.id, e.id, true, false + FROM environments e JOIN sessions s ON s.id = e.session_id + LEFT JOIN runtime_placements p ON p.environment_id = e.id AND p.released_at IS NULL + WHERE s.deleted_at IS NULL AND e.status = 'pending' + AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id) +), classified AS ( + SELECT h.*, CASE WHEN h.cleanup THEN 'cleanup' + WHEN EXISTS (SELECT 1 FROM turns t WHERE t.session_id = h.session_id AND t.status IN ('in_progress', 'waiting')) + OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = h.session_id AND t.status IN ('in_progress', 'waiting')) + OR EXISTS (SELECT 1 FROM environment_file_writes f WHERE f.environment_id = h.environment_id AND f.state = 'pending') + THEN 'busy' ELSE 'idle' END AS category, + n.name, (h.node_id IS NOT NULL AND NOT COALESCE(n.removed_at IS NULL AND n.connection_id IS NOT NULL + AND n.connected_epoch = d.owner_epoch AND n.last_seen_at > observed.as_of - interval '45 seconds', false)) AS offline + FROM held h LEFT JOIN runtime_nodes n ON n.id = h.node_id CROSS JOIN deployment d CROSS JOIN observed +), offline AS ( + SELECT node_id, name, count(*)::bigint AS resources FROM classified WHERE offline GROUP BY node_id, name +) +SELECT sqlc.embed(d), + (SELECT count(*) FROM classified WHERE NOT pending)::bigint AS allocations, + (SELECT count(*) FROM classified WHERE pending)::bigint AS pending, + jsonb_build_object( + 'busy', (SELECT count(*) FROM classified WHERE category = 'busy'), + 'idle', (SELECT count(*) FROM classified WHERE category = 'idle'), + 'cleanup', (SELECT count(*) FROM classified WHERE category = 'cleanup'), + 'on_offline_nodes', (SELECT count(*) FROM classified WHERE offline), + 'offline_nodes', COALESCE((SELECT jsonb_agg(jsonb_build_object('node_id', node_id, 'name', name, 'resources', resources) ORDER BY node_id) FROM offline), '[]'::jsonb) + )::jsonb AS remaining +FROM runtime_deployment d +WHERE d.singleton = true +LIMIT 1; diff --git a/services/agents-api/internal/db/sqlc/devices.sql.go b/services/agents-api/internal/db/sqlc/devices.sql.go index 7a2ff38a6..fbe8cbd2c 100644 --- a/services/agents-api/internal/db/sqlc/devices.sql.go +++ b/services/agents-api/internal/db/sqlc/devices.sql.go @@ -195,7 +195,7 @@ func (q *Queries) RememberNativeSession(ctx context.Context, arg RememberNativeS } const revokeDevice = `-- name: RevokeDevice :execrows -UPDATE devices SET revoked_at = COALESCE(revoked_at, clock_timestamp()) +UPDATE devices SET revoked_at = COALESCE(revoked_at, clock_timestamp()), archive_cancel_turn_id = NULL WHERE tenant_id = $1 AND id = $2 ` diff --git a/services/agents-api/internal/db/sqlc/models.go b/services/agents-api/internal/db/sqlc/models.go index e07fa711b..c7abcc44f 100644 --- a/services/agents-api/internal/db/sqlc/models.go +++ b/services/agents-api/internal/db/sqlc/models.go @@ -64,15 +64,16 @@ type DeploymentModelProvider struct { } type Device struct { - ID pgtype.UUID `json:"id"` - TenantID pgtype.UUID `json:"tenant_id"` - Name string `json:"name"` - CredentialHash pgtype.Text `json:"credential_hash"` - CreatedAt pgtype.Timestamptz `json:"created_at"` - LastSeenAt pgtype.Timestamptz `json:"last_seen_at"` - RevokedAt pgtype.Timestamptz `json:"revoked_at"` - EnvironmentID pgtype.UUID `json:"environment_id"` - ExecutorKeyID pgtype.UUID `json:"executor_key_id"` + ID pgtype.UUID `json:"id"` + TenantID pgtype.UUID `json:"tenant_id"` + Name string `json:"name"` + CredentialHash pgtype.Text `json:"credential_hash"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + LastSeenAt pgtype.Timestamptz `json:"last_seen_at"` + RevokedAt pgtype.Timestamptz `json:"revoked_at"` + EnvironmentID pgtype.UUID `json:"environment_id"` + ExecutorKeyID pgtype.UUID `json:"executor_key_id"` + ArchiveCancelTurnID pgtype.UUID `json:"archive_cancel_turn_id"` } type Environment struct { @@ -246,7 +247,7 @@ type RuntimeDeployment struct { Singleton bool `json:"singleton"` InstallationID pgtype.UUID `json:"installation_id"` BackendFingerprint string `json:"backend_fingerprint"` - Maintenance bool `json:"maintenance"` + AdmissionPaused bool `json:"admission_paused"` UpdatedAt pgtype.Timestamptz `json:"updated_at"` ProviderKind string `json:"provider_kind"` LocalNodeID pgtype.UUID `json:"local_node_id"` @@ -263,6 +264,11 @@ type RuntimeDeployment struct { E2bTemplateCpus pgtype.Int4 `json:"e2b_template_cpus"` E2bTemplateMemoryMib pgtype.Int4 `json:"e2b_template_memory_mib"` E2bTemplateRootDiskMib pgtype.Int4 `json:"e2b_template_root_disk_mib"` + ResetClear pgtype.Text `json:"reset_clear"` + ResetRequestedAt pgtype.Timestamptz `json:"reset_requested_at"` + ResetDeadlineAt pgtype.Timestamptz `json:"reset_deadline_at"` + ResetForcedAt pgtype.Timestamptz `json:"reset_forced_at"` + ResetAudit []byte `json:"reset_audit"` } type RuntimeDeviceAuthority struct { diff --git a/services/agents-api/internal/db/sqlc/runtime_cancellation.sql.go b/services/agents-api/internal/db/sqlc/runtime_cancellation.sql.go new file mode 100644 index 000000000..dbe9b5ba2 --- /dev/null +++ b/services/agents-api/internal/db/sqlc/runtime_cancellation.sql.go @@ -0,0 +1,99 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: runtime_cancellation.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getArchivedCancellationReceipt = `-- name: GetArchivedCancellationReceipt :one +SELECT t.id, t.cancel_requested_at +FROM devices d +JOIN runtime_allocations a ON a.device_id = d.id +JOIN environments e ON e.id = a.environment_id AND e.id = d.environment_id +JOIN sessions s ON s.id = e.session_id AND s.tenant_id = d.tenant_id +JOIN turns t ON t.session_id = s.id AND t.id = d.archive_cancel_turn_id +WHERE d.id = $1 + AND d.credential_hash = $2 + AND d.executor_key_id IS NULL AND d.revoked_at IS NOT NULL + AND a.state = 'cleanup_pending' AND a.released_at IS NULL + AND e.status = 'expired' AND s.deleted_at IS NULL + AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND t.status IN ('in_progress', 'waiting') + AND t.id::text = ANY($3::text[]) + AND t.cancel_requested_at IS NOT NULL + AND d.revoked_at >= t.cancel_requested_at + AND t.cancel_requested_at > clock_timestamp() - $4::int * interval '1 second' +` + +type GetArchivedCancellationReceiptParams struct { + DeviceID pgtype.UUID `json:"device_id"` + CredentialHash pgtype.Text `json:"credential_hash"` + RunIds []string `json:"run_ids"` + LimitSeconds int32 `json:"limit_seconds"` +} + +type GetArchivedCancellationReceiptRow struct { + ID pgtype.UUID `json:"id"` + CancelRequestedAt pgtype.Timestamptz `json:"cancel_requested_at"` +} + +func (q *Queries) GetArchivedCancellationReceipt(ctx context.Context, arg GetArchivedCancellationReceiptParams) (GetArchivedCancellationReceiptRow, error) { + row := q.db.QueryRow(ctx, getArchivedCancellationReceipt, + arg.DeviceID, + arg.CredentialHash, + arg.RunIds, + arg.LimitSeconds, + ) + var i GetArchivedCancellationReceiptRow + err := row.Scan(&i.ID, &i.CancelRequestedAt) + return i, err +} + +const revokeArchivedRuntimeDevice = `-- name: RevokeArchivedRuntimeDevice :execrows +UPDATE devices d +SET archive_cancel_turn_id = CASE WHEN d.revoked_at IS NULL THEN ( + SELECT t.id FROM turns t + WHERE t.session_id = $1 + AND t.status IN ('in_progress', 'waiting') AND t.cancel_requested_at IS NOT NULL +) ELSE d.archive_cancel_turn_id END, + revoked_at = COALESCE(d.revoked_at, clock_timestamp()) +WHERE d.tenant_id = $2 AND d.id = $3 +` + +type RevokeArchivedRuntimeDeviceParams struct { + SessionID pgtype.UUID `json:"session_id"` + TenantID pgtype.UUID `json:"tenant_id"` + DeviceID pgtype.UUID `json:"device_id"` +} + +func (q *Queries) RevokeArchivedRuntimeDevice(ctx context.Context, arg RevokeArchivedRuntimeDeviceParams) (int64, error) { + result, err := q.db.Exec(ctx, revokeArchivedRuntimeDevice, arg.SessionID, arg.TenantID, arg.DeviceID) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const revokeRuntimeCleanupDevice = `-- name: RevokeRuntimeCleanupDevice :execrows +UPDATE devices SET revoked_at = COALESCE(revoked_at, clock_timestamp()) +WHERE tenant_id = $1 AND id = $2 +` + +type RevokeRuntimeCleanupDeviceParams struct { + TenantID pgtype.UUID `json:"tenant_id"` + DeviceID pgtype.UUID `json:"device_id"` +} + +func (q *Queries) RevokeRuntimeCleanupDevice(ctx context.Context, arg RevokeRuntimeCleanupDeviceParams) (int64, error) { + result, err := q.db.Exec(ctx, revokeRuntimeCleanupDevice, arg.TenantID, arg.DeviceID) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/services/agents-api/internal/db/sqlc/runtime_deployment.sql.go b/services/agents-api/internal/db/sqlc/runtime_deployment.sql.go index 97dec6ea4..556579dd3 100644 --- a/services/agents-api/internal/db/sqlc/runtime_deployment.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_deployment.sql.go @@ -55,7 +55,7 @@ func (q *Queries) CountRuntimeDeploymentResources(ctx context.Context) (CountRun } const lockRuntimeDeployment = `-- name: LockRuntimeDeployment :one -SELECT singleton, installation_id, backend_fingerprint, maintenance, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, idle_seconds, retention_seconds, generation, mode, e2b_template, e2b_credential, specification, e2b_template_build_status, e2b_template_cpus, e2b_template_memory_mib, e2b_template_root_disk_mib FROM runtime_deployment WHERE singleton = true FOR UPDATE +SELECT singleton, installation_id, backend_fingerprint, admission_paused, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, idle_seconds, retention_seconds, generation, mode, e2b_template, e2b_credential, specification, e2b_template_build_status, e2b_template_cpus, e2b_template_memory_mib, e2b_template_root_disk_mib, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit FROM runtime_deployment WHERE singleton = true FOR UPDATE ` func (q *Queries) LockRuntimeDeployment(ctx context.Context) (RuntimeDeployment, error) { @@ -65,7 +65,7 @@ func (q *Queries) LockRuntimeDeployment(ctx context.Context) (RuntimeDeployment, &i.Singleton, &i.InstallationID, &i.BackendFingerprint, - &i.Maintenance, + &i.AdmissionPaused, &i.UpdatedAt, &i.ProviderKind, &i.LocalNodeID, @@ -82,22 +82,27 @@ func (q *Queries) LockRuntimeDeployment(ctx context.Context) (RuntimeDeployment, &i.E2bTemplateCpus, &i.E2bTemplateMemoryMib, &i.E2bTemplateRootDiskMib, + &i.ResetClear, + &i.ResetRequestedAt, + &i.ResetDeadlineAt, + &i.ResetForcedAt, + &i.ResetAudit, ) return i, err } const setRuntimeDeployment = `-- name: SetRuntimeDeployment :exec UPDATE runtime_deployment SET installation_id = $1, backend_fingerprint = $2, -maintenance = $3, updated_at = clock_timestamp() WHERE singleton = true +admission_paused = $3, updated_at = clock_timestamp() WHERE singleton = true ` type SetRuntimeDeploymentParams struct { InstallationID pgtype.UUID `json:"installation_id"` BackendFingerprint string `json:"backend_fingerprint"` - Maintenance bool `json:"maintenance"` + AdmissionPaused bool `json:"admission_paused"` } func (q *Queries) SetRuntimeDeployment(ctx context.Context, arg SetRuntimeDeploymentParams) error { - _, err := q.db.Exec(ctx, setRuntimeDeployment, arg.InstallationID, arg.BackendFingerprint, arg.Maintenance) + _, err := q.db.Exec(ctx, setRuntimeDeployment, arg.InstallationID, arg.BackendFingerprint, arg.AdmissionPaused) return err } diff --git a/services/agents-api/internal/db/sqlc/runtime_lifecycle_nodes.sql.go b/services/agents-api/internal/db/sqlc/runtime_lifecycle_nodes.sql.go index a96e83b53..f2ccd5149 100644 --- a/services/agents-api/internal/db/sqlc/runtime_lifecycle_nodes.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_lifecycle_nodes.sql.go @@ -150,7 +150,7 @@ FROM environments e JOIN sessions s ON s.id=e.session_id LEFT JOIN runtime_placements p ON p.environment_id=e.id WHERE p.node_id IS NOT DISTINCT FROM $1::uuid AND p.released_at IS NULL - AND NOT (SELECT maintenance FROM runtime_deployment) + AND NOT (SELECT admission_paused FROM runtime_deployment) AND e.id > $2::uuid AND s.deleted_at IS NULL AND e.status='pending' AND s.configuration->'environment'->>'type'='openai_hosted' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id=e.id) diff --git a/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go b/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go index 75d616d83..71e03a966 100644 --- a/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go @@ -162,7 +162,7 @@ func (q *Queries) DisconnectRuntimeNode(ctx context.Context, arg DisconnectRunti } const getRuntimeDeployment = `-- name: GetRuntimeDeployment :one -SELECT singleton, installation_id, backend_fingerprint, maintenance, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, idle_seconds, retention_seconds, generation, mode, e2b_template, e2b_credential, specification, e2b_template_build_status, e2b_template_cpus, e2b_template_memory_mib, e2b_template_root_disk_mib FROM runtime_deployment WHERE singleton=true +SELECT singleton, installation_id, backend_fingerprint, admission_paused, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, idle_seconds, retention_seconds, generation, mode, e2b_template, e2b_credential, specification, e2b_template_build_status, e2b_template_cpus, e2b_template_memory_mib, e2b_template_root_disk_mib, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit FROM runtime_deployment WHERE singleton=true ` func (q *Queries) GetRuntimeDeployment(ctx context.Context) (RuntimeDeployment, error) { @@ -172,7 +172,7 @@ func (q *Queries) GetRuntimeDeployment(ctx context.Context) (RuntimeDeployment, &i.Singleton, &i.InstallationID, &i.BackendFingerprint, - &i.Maintenance, + &i.AdmissionPaused, &i.UpdatedAt, &i.ProviderKind, &i.LocalNodeID, @@ -189,6 +189,11 @@ func (q *Queries) GetRuntimeDeployment(ctx context.Context) (RuntimeDeployment, &i.E2bTemplateCpus, &i.E2bTemplateMemoryMib, &i.E2bTemplateRootDiskMib, + &i.ResetClear, + &i.ResetRequestedAt, + &i.ResetDeadlineAt, + &i.ResetForcedAt, + &i.ResetAudit, ) return i, err } diff --git a/services/agents-api/internal/db/sqlc/sandbox_deployment_setup.sql.go b/services/agents-api/internal/db/sqlc/sandbox_deployment_setup.sql.go index c02e9fde7..e48c7210b 100644 --- a/services/agents-api/internal/db/sqlc/sandbox_deployment_setup.sql.go +++ b/services/agents-api/internal/db/sqlc/sandbox_deployment_setup.sql.go @@ -113,12 +113,3 @@ func (q *Queries) RetireSandboxNodes(ctx context.Context) error { _, err := q.db.Exec(ctx, retireSandboxNodes) return err } - -const setSandboxMaintenance = `-- name: SetSandboxMaintenance :exec -UPDATE runtime_deployment SET maintenance=$1,updated_at=clock_timestamp() WHERE singleton=true -` - -func (q *Queries) SetSandboxMaintenance(ctx context.Context, maintenance bool) error { - _, err := q.db.Exec(ctx, setSandboxMaintenance, maintenance) - return err -} diff --git a/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go b/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go new file mode 100644 index 000000000..355795a6d --- /dev/null +++ b/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go @@ -0,0 +1,234 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: sandbox_reset.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const cancelSandboxReset = `-- name: CancelSandboxReset :exec +UPDATE runtime_deployment SET admission_paused = false, reset_clear = NULL, + reset_requested_at = NULL, reset_deadline_at = NULL, reset_forced_at = NULL, + reset_audit = NULL, updated_at = clock_timestamp() +WHERE singleton = true +` + +func (q *Queries) CancelSandboxReset(ctx context.Context) error { + _, err := q.db.Exec(ctx, cancelSandboxReset) + return err +} + +const completeSandboxReset = `-- name: CompleteSandboxReset :exec +UPDATE runtime_deployment SET provider_kind = '', backend_fingerprint = '', mode = '', + specification = '{}', idle_seconds = 0, retention_seconds = 0, + e2b_template = '', e2b_credential = NULL, e2b_template_build_status = NULL, + e2b_template_cpus = NULL, e2b_template_memory_mib = NULL, e2b_template_root_disk_mib = NULL, + generation = generation + 1, owner_epoch = owner_epoch + 1, + admission_paused = false, reset_clear = NULL, reset_requested_at = NULL, + reset_deadline_at = NULL, reset_forced_at = NULL, reset_audit = NULL, + updated_at = clock_timestamp() +WHERE singleton = true +` + +func (q *Queries) CompleteSandboxReset(ctx context.Context) error { + _, err := q.db.Exec(ctx, completeSandboxReset) + return err +} + +const forceSandboxReset = `-- name: ForceSandboxReset :exec +UPDATE runtime_deployment SET reset_clear = 'force', reset_forced_at = clock_timestamp(), updated_at = clock_timestamp() +WHERE singleton = true AND reset_clear = 'auto' +` + +func (q *Queries) ForceSandboxReset(ctx context.Context) error { + _, err := q.db.Exec(ctx, forceSandboxReset) + return err +} + +const getSandboxDeploymentSnapshot = `-- name: GetSandboxDeploymentSnapshot :one +WITH deployment AS MATERIALIZED (SELECT singleton, installation_id, backend_fingerprint, admission_paused, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, idle_seconds, retention_seconds, generation, mode, e2b_template, e2b_credential, specification, e2b_template_build_status, e2b_template_cpus, e2b_template_memory_mib, e2b_template_root_disk_mib, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit FROM runtime_deployment WHERE singleton = true LIMIT 1), +observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), +held AS ( + SELECT a.node_id, s.id AS session_id, e.id AS environment_id, false AS pending, + (a.state = 'cleanup_pending' OR s.deleted_at IS NOT NULL OR e.status IN ('failed', 'expired') + OR CASE WHEN a.compute_phase NOT IN ('disabled', 'running') + THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= observed.as_of + ELSE a.node_id IS NULL AND d.mode <> 'direct' AND a.kept_at <= observed.as_of - interval '1 hour' END) AS cleanup + FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id + JOIN sessions s ON s.id = e.session_id CROSS JOIN deployment d CROSS JOIN observed + WHERE a.state <> 'released' + UNION ALL + SELECT p.node_id, s.id, e.id, true, false + FROM environments e JOIN sessions s ON s.id = e.session_id + LEFT JOIN runtime_placements p ON p.environment_id = e.id AND p.released_at IS NULL + WHERE s.deleted_at IS NULL AND e.status = 'pending' + AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id) +), classified AS ( + SELECT h.node_id, h.session_id, h.environment_id, h.pending, h.cleanup, CASE WHEN h.cleanup THEN 'cleanup' + WHEN EXISTS (SELECT 1 FROM turns t WHERE t.session_id = h.session_id AND t.status IN ('in_progress', 'waiting')) + OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = h.session_id AND t.status IN ('in_progress', 'waiting')) + OR EXISTS (SELECT 1 FROM environment_file_writes f WHERE f.environment_id = h.environment_id AND f.state = 'pending') + THEN 'busy' ELSE 'idle' END AS category, + n.name, (h.node_id IS NOT NULL AND NOT COALESCE(n.removed_at IS NULL AND n.connection_id IS NOT NULL + AND n.connected_epoch = d.owner_epoch AND n.last_seen_at > observed.as_of - interval '45 seconds', false)) AS offline + FROM held h LEFT JOIN runtime_nodes n ON n.id = h.node_id CROSS JOIN deployment d CROSS JOIN observed +), offline AS ( + SELECT node_id, name, count(*)::bigint AS resources FROM classified WHERE offline GROUP BY node_id, name +) +SELECT d.singleton, d.installation_id, d.backend_fingerprint, d.admission_paused, d.updated_at, d.provider_kind, d.local_node_id, d.owner_epoch, d.web_managed, d.idle_seconds, d.retention_seconds, d.generation, d.mode, d.e2b_template, d.e2b_credential, d.specification, d.e2b_template_build_status, d.e2b_template_cpus, d.e2b_template_memory_mib, d.e2b_template_root_disk_mib, d.reset_clear, d.reset_requested_at, d.reset_deadline_at, d.reset_forced_at, d.reset_audit, + (SELECT count(*) FROM classified WHERE NOT pending)::bigint AS allocations, + (SELECT count(*) FROM classified WHERE pending)::bigint AS pending, + jsonb_build_object( + 'busy', (SELECT count(*) FROM classified WHERE category = 'busy'), + 'idle', (SELECT count(*) FROM classified WHERE category = 'idle'), + 'cleanup', (SELECT count(*) FROM classified WHERE category = 'cleanup'), + 'on_offline_nodes', (SELECT count(*) FROM classified WHERE offline), + 'offline_nodes', COALESCE((SELECT jsonb_agg(jsonb_build_object('node_id', node_id, 'name', name, 'resources', resources) ORDER BY node_id) FROM offline), '[]'::jsonb) + )::jsonb AS remaining +FROM runtime_deployment d +WHERE d.singleton = true +LIMIT 1 +` + +type GetSandboxDeploymentSnapshotRow struct { + RuntimeDeployment RuntimeDeployment `json:"runtime_deployment"` + Allocations int64 `json:"allocations"` + Pending int64 `json:"pending"` + Remaining []byte `json:"remaining"` +} + +func (q *Queries) GetSandboxDeploymentSnapshot(ctx context.Context) (GetSandboxDeploymentSnapshotRow, error) { + row := q.db.QueryRow(ctx, getSandboxDeploymentSnapshot) + var i GetSandboxDeploymentSnapshotRow + err := row.Scan( + &i.RuntimeDeployment.Singleton, + &i.RuntimeDeployment.InstallationID, + &i.RuntimeDeployment.BackendFingerprint, + &i.RuntimeDeployment.AdmissionPaused, + &i.RuntimeDeployment.UpdatedAt, + &i.RuntimeDeployment.ProviderKind, + &i.RuntimeDeployment.LocalNodeID, + &i.RuntimeDeployment.OwnerEpoch, + &i.RuntimeDeployment.WebManaged, + &i.RuntimeDeployment.IdleSeconds, + &i.RuntimeDeployment.RetentionSeconds, + &i.RuntimeDeployment.Generation, + &i.RuntimeDeployment.Mode, + &i.RuntimeDeployment.E2bTemplate, + &i.RuntimeDeployment.E2bCredential, + &i.RuntimeDeployment.Specification, + &i.RuntimeDeployment.E2bTemplateBuildStatus, + &i.RuntimeDeployment.E2bTemplateCpus, + &i.RuntimeDeployment.E2bTemplateMemoryMib, + &i.RuntimeDeployment.E2bTemplateRootDiskMib, + &i.RuntimeDeployment.ResetClear, + &i.RuntimeDeployment.ResetRequestedAt, + &i.RuntimeDeployment.ResetDeadlineAt, + &i.RuntimeDeployment.ResetForcedAt, + &i.RuntimeDeployment.ResetAudit, + &i.Allocations, + &i.Pending, + &i.Remaining, + ) + return i, err +} + +const getSandboxResetProject = `-- name: GetSandboxResetProject :one +SELECT id FROM projects WHERE tenant_id = $1 +` + +func (q *Queries) GetSandboxResetProject(ctx context.Context, tenantID pgtype.UUID) (pgtype.UUID, error) { + row := q.db.QueryRow(ctx, getSandboxResetProject, tenantID) + var id pgtype.UUID + err := row.Scan(&id) + return id, err +} + +const listSandboxResetSessions = `-- name: ListSandboxResetSessions :many +SELECT s.id, s.tenant_id +FROM sessions s JOIN environments e ON e.session_id = s.id +WHERE s.deleted_at IS NULL AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND e.status NOT IN ('failed', 'expired') + AND s.id > $1::uuid + AND (EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state NOT IN ('released', 'cleanup_pending')) + OR (e.status = 'pending' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id))) + AND ($2::boolean OR NOT ( + EXISTS (SELECT 1 FROM turns t WHERE t.session_id = s.id AND t.status IN ('in_progress', 'waiting')) + OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = s.id AND t.status IN ('in_progress', 'waiting')) + OR EXISTS (SELECT 1 FROM environment_file_writes f WHERE f.environment_id = e.id AND f.state = 'pending') + )) +ORDER BY s.id LIMIT 32 +` + +type ListSandboxResetSessionsParams struct { + AfterID pgtype.UUID `json:"after_id"` + Force bool `json:"force"` +} + +type ListSandboxResetSessionsRow struct { + ID pgtype.UUID `json:"id"` + TenantID pgtype.UUID `json:"tenant_id"` +} + +func (q *Queries) ListSandboxResetSessions(ctx context.Context, arg ListSandboxResetSessionsParams) ([]ListSandboxResetSessionsRow, error) { + rows, err := q.db.Query(ctx, listSandboxResetSessions, arg.AfterID, arg.Force) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListSandboxResetSessionsRow{} + for rows.Next() { + var i ListSandboxResetSessionsRow + if err := rows.Scan(&i.ID, &i.TenantID); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const sessionBlocksAutoReset = `-- name: SessionBlocksAutoReset :one +SELECT ( + EXISTS (SELECT 1 FROM turns t WHERE t.session_id = $1 AND status IN ('in_progress', 'waiting')) + OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = $1 AND status IN ('in_progress', 'waiting')) + OR EXISTS (SELECT 1 FROM environment_file_writes f JOIN environments e ON e.id = f.environment_id + WHERE e.session_id = $1 AND f.state = 'pending') +)::boolean +` + +func (q *Queries) SessionBlocksAutoReset(ctx context.Context, sessionID pgtype.UUID) (bool, error) { + row := q.db.QueryRow(ctx, sessionBlocksAutoReset, sessionID) + var column_1 bool + err := row.Scan(&column_1) + return column_1, err +} + +const startSandboxReset = `-- name: StartSandboxReset :exec +UPDATE runtime_deployment SET admission_paused = true, reset_clear = $1, + reset_requested_at = clock_timestamp(), + reset_deadline_at = CASE WHEN $1::text = 'auto' + THEN clock_timestamp() + make_interval(secs => $2::int) END, + reset_forced_at = CASE WHEN $1::text = 'force' THEN clock_timestamp() END, + reset_audit = $3::jsonb, updated_at = clock_timestamp() +WHERE singleton = true +` + +type StartSandboxResetParams struct { + Clear pgtype.Text `json:"clear"` + DeadlineSeconds int32 `json:"deadline_seconds"` + Audit []byte `json:"audit"` +} + +func (q *Queries) StartSandboxReset(ctx context.Context, arg StartSandboxResetParams) error { + _, err := q.db.Exec(ctx, startSandboxReset, arg.Clear, arg.DeadlineSeconds, arg.Audit) + return err +} diff --git a/services/agents-api/internal/execution/archive_cancellation_cleanup_test.go b/services/agents-api/internal/execution/archive_cancellation_cleanup_test.go new file mode 100644 index 000000000..4ccfa7374 --- /dev/null +++ b/services/agents-api/internal/execution/archive_cancellation_cleanup_test.go @@ -0,0 +1,197 @@ +package execution + +import ( + "context" + "encoding/json" + runtimegateway "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/runtime" + "github.com/gorilla/websocket" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + v1 "github.com/MiniMax-AI-Dev/parsar/contracts/agents-api/v1" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/adminaudit" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/identity" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" +) + +// Provider callbacks inspect the real database at the instant destructive +// cleanup is invoked. They do not create containers or claim native evidence. +type waitingCleanupProvider struct { + sandbox.Provider + beforeKill func() +} + +func (p waitingCleanupProvider) GetInfo(_ context.Context, r sandbox.Reference) (sandbox.Info, error) { + return sandbox.Info{Reference: r, ProviderID: r.AllocationID, State: "running", BootstrapComplete: true, CreateSettled: true}, nil +} +func (p waitingCleanupProvider) Kill(context.Context, sandbox.Reference) error { + p.beforeKill() + return nil +} + +type waitingCleanupCheckpoint struct { + sandbox.CheckpointProvider + beforeKill func() +} + +func (p waitingCleanupCheckpoint) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { + p.beforeKill() + return nil +} + +func TestArchiveWaitingCleanupReceiptBarrier(t *testing.T) { + for _, scenario := range []struct { + name string + checkpoint, delivery bool + }{{"Kill_no_delivery", false, false}, {"KillCompute_no_delivery", true, false}, {"Kill_live_delivery", false, true}, {"KillCompute_live_delivery", true, true}} { + t.Run(scenario.name, func(t *testing.T) { + checkpoint := scenario.checkpoint + s, lease := resetManagerStore(t) + writer := lease.Store() + installation := uuid.NewString() + if err := writer.ClaimWebSandboxDeployment(t.Context(), installation); err != nil { + t.Fatal(err) + } + selection := store.SandboxDeploymentSetupRequest{Provider: "e2b", E2B: &store.SandboxE2BConfiguration{APIKey: "fixture-key", Template: "runtime:" + uuid.NewString()}} + selection.Resources.CPUs = 2 + selection.Resources.MemoryMiB = 2048 + if _, err := writer.InitializeSandboxDeployment(t.Context(), installation, selection); err != nil { + t.Fatal(err) + } + projectID := uuid.NewString() + audit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", ProjectID: projectID, RequestID: uuid.NewString(), TraceID: uuid.NewString()}) + project, err := s.CreateProject(audit, projectID, "Cleanup diagnosis") + if err != nil { + t.Fatal(err) + } + session, err := s.CreateSession(t.Context(), project.TenantID, store.CreateSessionInput{Creator: identity.Subject{Kind: "service_account", ID: "fixture"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"disabled"}}}`), ModelProvider: &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://model.fixture.example/v1", APIKey: "fixture-key"}, ModelProviderSource: v1.ModelProviderSourceSession}) + if err != nil { + t.Fatal(err) + } + secret := uuid.NewString() + owner, err := writer.ReserveRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID, installation, device.HashCredential(secret)) + if err != nil { + t.Fatal(err) + } + owner, err = writer.ObserveRuntimeRunning(t.Context(), owner) + if err != nil { + t.Fatal(err) + } + input, err := s.SubmitMessage(t.Context(), project.TenantID, session.ID, "start", json.RawMessage(`{"text":"run"}`)) + if err != nil { + t.Fatal(err) + } + // This fixture isolates lifecycle ordering. Protocol-driven waiting is + // independently exercised in TestArchiveWaitingCancellationReceipts. + for _, transition := range []store.TurnTransition{{ExpectedStatus: store.TurnQueued, Status: store.TurnInProgress}, {ExpectedStatus: store.TurnInProgress, Status: store.TurnWaiting}} { + if _, err := writer.TransitionTurn(t.Context(), project.TenantID, session.ID, input.TurnID, transition); err != nil { + t.Fatal(err) + } + } + currentCompute := sandbox.Compute{ID: uuid.NewString(), Name: owner.ID + "-g0"} + if checkpoint { + state, _ := json.Marshal(runtimeCompute{Current: currentCompute}) + owner, err = writer.SetRuntimeCompute(t.Context(), owner, "running", state, nil, 0) + if err != nil { + t.Fatal(err) + } + } + registry := gateway.NewRegistry() + if scenario.delivery { + server := httptest.NewUnstartedServer(nil) + wsURL := "ws://" + server.Listener.Addr().String() + "/api/v1/agent-daemon/ws" + handler, liveRegistry, err := runtimegateway.NewGateway(s, wsURL) + if err != nil { + t.Fatal(err) + } + registry = liveRegistry + server.Config.Handler = handler + server.Start() + t.Cleanup(func() { runtimegateway.CloseConnections(registry); server.Close() }) + u, _ := url.Parse(wsURL) + u.RawQuery = url.Values{"device_id": {owner.DeviceID}, "version": {proto.Version}}.Encode() + conn, _, err := websocket.DefaultDialer.Dial(u.String(), http.Header{"Authorization": {"Bearer " + secret}}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + var peer *gateway.Session + for end := time.Now().Add(3 * time.Second); ; { + peer, err = registry.LookupDevice(owner.DeviceID) + if err == nil { + break + } + if time.Now().After(end) { + t.Fatal(err) + } + time.Sleep(time.Millisecond) + } + release, err := peer.TrackExecutionDelivery(input.TurnID) + if err != nil { + t.Fatal(err) + } + defer release() + } + if _, err := writer.ArchiveManagedSession(audit, project.TenantID, session.ID, 1); err != nil { + t.Fatal(err) + } + owner, err = s.GetRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID) + if err != nil { + t.Fatal(err) + } + kills := 0 + expectedStatus := store.TurnWaiting + provider := waitingCleanupProvider{beforeKill: func() { + kills++ + turn, err := s.GetTurn(t.Context(), project.TenantID, session.ID, input.TurnID) + if err != nil || turn.Status != expectedStatus || turn.CancelRequestedAt.IsZero() || (turn.CompletedAt.IsZero() != (expectedStatus == store.TurnWaiting)) { + t.Fatal("cleanup observed unexpected terminal state", turn, err) + } + allocation, err := s.GetRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID) + if err != nil || allocation.State != "cleanup_pending" { + t.Fatal("Kill bypassed durable cleanup ownership", allocation, err) + } + }} + lifecycle := &runtimeLifecycle{store: writer, registry: registry, config: RuntimeProvider{InstallationID: installation, Provider: provider}, connections: map[string]*runtimeConnection{}} + if checkpoint { + lifecycle.config.Provider = waitingCleanupCheckpoint{beforeKill: provider.beforeKill} + } + if err := lifecycle.observe(t.Context(), owner); err != nil { + t.Fatal(err) + } + if scenario.delivery { + if kills != 0 { + t.Fatal("destroyed compute before cancellation committed") + } + pending, err := s.GetRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID) + if err != nil || pending.State != "cleanup_pending" { + t.Fatal(pending, err) + } + // Controlled terminal receipt fixture; no native cancellation claim. + if _, err := writer.TransitionTurn(t.Context(), project.TenantID, session.ID, input.TurnID, store.TurnTransition{ExpectedStatus: store.TurnWaiting, Status: store.TurnCancelled}); err != nil { + t.Fatal(err) + } + expectedStatus = store.TurnCancelled + if err := lifecycle.observe(t.Context(), owner); err != nil { + t.Fatal(err) + } + } + after, err := s.GetRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID) + if err != nil || after.State != "released" || kills != 1 { + t.Fatal("cleanup did not release", after, kills, err) + } + turn, err := s.GetTurn(t.Context(), project.TenantID, session.ID, input.TurnID) + if err != nil || turn.Status != expectedStatus || (turn.CompletedAt.IsZero() != (expectedStatus == store.TurnWaiting)) { + t.Fatal("cleanup should not fabricate cancellation", turn, err) + } + }) + } +} diff --git a/services/agents-api/internal/execution/device_authority.go b/services/agents-api/internal/execution/device_authority.go index 8cd0ff502..b5cc877af 100644 --- a/services/agents-api/internal/execution/device_authority.go +++ b/services/agents-api/internal/execution/device_authority.go @@ -19,7 +19,11 @@ func authorizedRuntimePeer(ctx context.Context, s *store.Store, registry *gatewa return nil, err } if !found || !peer.AuthenticatedWith(credential.CredentialHash) { - peer.Close("Runtime authorization changed") + // Reject new work even while this exact old delivery drains its receipt. + draining, drainErr := peer.DrainArchivedCancellation(ctx) + if drainErr != nil || !draining { + peer.Close("Runtime authorization changed") + } return nil, store.ErrNotFound } if peer.IsClosed() { diff --git a/services/agents-api/internal/execution/dispatcher.go b/services/agents-api/internal/execution/dispatcher.go index 11ec43c75..ba92fc92c 100644 --- a/services/agents-api/internal/execution/dispatcher.go +++ b/services/agents-api/internal/execution/dispatcher.go @@ -94,6 +94,11 @@ func (d *Dispatcher) Run(ctx context.Context, tenantID, sessionID, turnID string } req.ConversationID, req.RunID, req.Input = sessionID, turnID, text req.WorkDir, req.DisableExecutionEnvironment = workDir, noEnvironment + release, err := peer.TrackExecutionDelivery(req.RunID) + if err != nil { + return d.finishRun(tenantID, sessionID, turnID, snapshot.Agent.Model, Result{ErrorCode: "delivery_unknown", AppliedThrough: through}, store.TurnFailed) + } + defer release() result, status := d.deliver(ctx, tenantID, sessionID, peer, req, through, nil) return d.finishRun(tenantID, sessionID, turnID, snapshot.Agent.Model, result, status) } diff --git a/services/agents-api/internal/execution/prepared_dispatch.go b/services/agents-api/internal/execution/prepared_dispatch.go index 453d85745..f13d3f48d 100644 --- a/services/agents-api/internal/execution/prepared_dispatch.go +++ b/services/agents-api/internal/execution/prepared_dispatch.go @@ -106,6 +106,12 @@ func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, tenantID, sessionI req.RunID = run.Reservation.Receipts[0].TurnID req.Input = messages through := run.Reservation.Receipts[len(run.Reservation.Receipts)-1].Sequence + releaseDelivery, err := peer.TrackExecutionDelivery(req.RunID) + if err != nil { + run.Turn, err = d.finishRun(tenantID, sessionID, req.RunID, snapshot.Agent.Model, Result{ErrorCode: "delivery_unknown", AppliedThrough: through}, store.TurnFailed) + return run, err + } + defer releaseDelivery() result, status := d.deliver(owner, tenantID, sessionID, peer, req, through, prepared) result, status = d.captureCompletedArtifacts(owner, peer, session, environment, bound.Device, req.RunID, result, status) run.Turn, err = d.finishRun(tenantID, sessionID, req.RunID, snapshot.Agent.Model, result, status) diff --git a/services/agents-api/internal/execution/runtime_cancellation.go b/services/agents-api/internal/execution/runtime_cancellation.go new file mode 100644 index 000000000..dcf627782 --- /dev/null +++ b/services/agents-api/internal/execution/runtime_cancellation.go @@ -0,0 +1,76 @@ +package execution + +import "context" + +// cancelLifecycles leaves the manager mutex free while the bounded lease gate +// drains. The cancellation itself runs synchronously under that gate; waiting +// for provider operations and lifecycle accounting happens only after release. +func (m *runtimeManager) cancelLifecycles(nodes []*runtimeNode) error { + if len(nodes) == 0 { + return nil + } + cancel := func() { + for _, n := range nodes { + n.lifecycle.cancelOperations() + } + } + // Managers in pure lifecycle tests have no Store or execution connection. + if m.store == nil { + cancel() + return nil + } + err := m.store.CancelExecutionOperations(m.ctx, cancel) + if err != nil { + // A manual reconcile may have no running coordinator to consume failed. + // Close admission synchronously; Worker shutdown still owns cancellation. + m.mu.Lock() + m.closed = true + m.mu.Unlock() + select { + case m.failed <- err: + default: + } + } + return err +} + +func (r *runtimeLifecycle) cancelOperations() { + r.cancelMu.Lock() + defer r.cancelMu.Unlock() + r.stop() + // A manual reconcile has an independently parented context. AfterFunc is + // asynchronous, so it cannot be the only cancellation under the lease gate. + if r.reconcileCancel != nil { + r.reconcileCancel() + } +} + +func (r *runtimeLifecycle) beginReconcile(parent context.Context) (context.Context, func(), error) { + ctx, cancel := context.WithCancel(parent) + detach := context.AfterFunc(r.ctx, cancel) + if err := r.lock(ctx); err != nil { + detach() + cancel() + return nil, nil, err + } + r.cancelMu.Lock() + if err := r.ctx.Err(); err != nil { + r.cancelMu.Unlock() + <-r.gate + detach() + cancel() + return nil, nil, err + } + // The lifecycle gate admits only one active reconcile. Register cancellation + // before its first leased query, and retain it until its last query returns. + r.reconcileCancel = cancel + r.cancelMu.Unlock() + return ctx, func() { + r.cancelMu.Lock() + r.reconcileCancel = nil + r.cancelMu.Unlock() + <-r.gate + detach() + cancel() + }, nil +} diff --git a/services/agents-api/internal/execution/runtime_lifecycle.go b/services/agents-api/internal/execution/runtime_lifecycle.go index 7fac5f727..cf00b5d29 100644 --- a/services/agents-api/internal/execution/runtime_lifecycle.go +++ b/services/agents-api/internal/execution/runtime_lifecycle.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "net/url" + "sync" "time" "github.com/google/uuid" @@ -20,6 +21,8 @@ import ( // RuntimeProvider binds one deployment to one sandbox installation. // BackendFingerprint identifies its namespace independently of mutable sizing. type RuntimeProvider struct { + // PublishUnconfigured updates the shared observation cache after reset commit. + PublishUnconfigured func(uint64) Generation uint64 Mode string loadDeployment func(context.Context) (*RuntimeProvider, error) @@ -33,23 +36,25 @@ type RuntimeProvider struct { InstallationID string BackendFingerprint string Provider sandbox.Provider - Maintenance bool + AdmissionPaused bool Suspension *RuntimeSuspensionPolicy } type runtimeLifecycle struct { - store *store.Store - registry *gateway.Registry - config RuntimeProvider - nodeID string - gate chan struct{} - ctx context.Context - stop context.CancelFunc - cursor string - pendingCursor string - connections map[string]*runtimeConnection - initializing *runtimeInitialization - wakeHints chan struct{} + store *store.Store + registry *gateway.Registry + config RuntimeProvider + nodeID string + gate chan struct{} + ctx context.Context + stop context.CancelFunc + cancelMu sync.Mutex + reconcileCancel context.CancelFunc + cursor string + pendingCursor string + connections map[string]*runtimeConnection + initializing *runtimeInitialization + wakeHints chan struct{} } func newRuntimeManager(s *store.Store, registry *gateway.Registry, config *RuntimeProvider) (*runtimeManager, error) { @@ -70,7 +75,7 @@ func newRuntimeManager(s *store.Store, registry *gateway.Registry, config *Runti } } ctx, stop := context.WithCancel(context.Background()) - return &runtimeManager{store: s, registry: registry, config: copied, setupInstallationID: config.InstallationID, loadDeployment: config.loadDeployment, prepareDeployment: config.prepareDeployment, setupGate: make(chan struct{}, 1), mutationGate: make(chan struct{}, 1), ctx: ctx, cancel: stop, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1)}, nil + return &runtimeManager{store: s, registry: registry, config: copied, setupInstallationID: config.InstallationID, loadDeployment: config.loadDeployment, prepareDeployment: config.prepareDeployment, publishUnconfigured: config.PublishUnconfigured, setupGate: make(chan struct{}, 1), mutationGate: make(chan struct{}, 1), ctx: ctx, cancel: stop, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1)}, nil } func validatedRuntimeProvider(config *RuntimeProvider, registry *gateway.Registry) (RuntimeProvider, error) { @@ -173,7 +178,7 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p return store.RuntimeAllocation{}, sandbox.ErrInvalid } if _, err := r.store.GetRuntimeAllocation(ctx, tenant, environment); errors.Is(err, store.ErrNotFound) { - if r.config.Maintenance && r.config.Generation == 0 { + if r.config.AdmissionPaused && r.config.Generation == 0 { return store.RuntimeAllocation{}, ErrExecutionUnavailable } if err := r.computeFreshCapacity(ctx, providerKey); err != nil { @@ -260,13 +265,11 @@ func (w *Worker) ReconcileManagedRuntimes(ctx context.Context) error { } func (r *runtimeLifecycle) reconcile(ctx context.Context) error { - ctx, cancel := context.WithCancel(ctx) - detach := context.AfterFunc(r.ctx, cancel) - defer func() { detach(); cancel() }() - if err := r.lock(ctx); err != nil { + ctx, finish, err := r.beginReconcile(ctx) + if err != nil { return err } - defer func() { <-r.gate }() + defer finish() rows, err := r.store.ListRuntimeAllocationsForNode(ctx, r.nodeID, r.cursor) if err != nil { return err @@ -324,6 +327,15 @@ func (r *runtimeLifecycle) observe(ctx context.Context, owner store.RuntimeAlloc if err != nil { return err } + if peer, err := r.registry.LookupDevice(owner.DeviceID); err == nil { + draining, err := peer.DrainArchivedCancellation(ctx) + if err != nil { + return err + } + if draining { + return nil + } + } r.clearRuntimeState(owner) } else if owner.ComputePhase == "disabled" || owner.ComputePhase == "running" { if err := r.observeConnection(ctx, owner); err != nil { diff --git a/services/agents-api/internal/execution/runtime_manager.go b/services/agents-api/internal/execution/runtime_manager.go index da39e79b4..f6af84cd7 100644 --- a/services/agents-api/internal/execution/runtime_manager.go +++ b/services/agents-api/internal/execution/runtime_manager.go @@ -22,10 +22,13 @@ type runtimeManager struct { setupInstallationID string loadDeployment func(context.Context) (*RuntimeProvider, error) prepareDeployment RuntimeDeploymentPreparer + publishUnconfigured func(uint64) + resetCursor string + resetRequestedAt time.Time setupGate chan struct{} mutationGate chan struct{} switching bool - switchDrained chan struct{} + switchDrained *deploymentDrain ctx context.Context cancel context.CancelFunc mu sync.Mutex @@ -176,8 +179,17 @@ func (m *runtimeManager) applyInventory(previous map[string]*runtimeNode, ids [] m.mu.Unlock() // Inventory includes offline nodes. Only explicit removal retires a lane; // the Store requires all retained resources to be cleaned before removal. + if err := m.cancelLifecycles(retired); err != nil { + // No cancellation was authorized. Keep each retiring identity and gate; + // acquiring its gate later cannot substitute for successful cancellation. + // Undo only the retirement tasks that will not be started. Existing callers + // and workers retain their accounting until ordinary owner shutdown settles. + for range retired { + m.active.Done() + } + return nil, err + } for _, n := range retired { - n.lifecycle.stop() go m.retire(n) } return nodes, nil @@ -221,6 +233,9 @@ func (m *runtimeManager) run(ctx context.Context) error { } m.running = true m.mu.Unlock() + if err := m.resetStep(ctx); err != nil { + return err + } if _, err := m.syncNodes(ctx); err != nil && !errors.Is(err, errRuntimeTransition) { return err } @@ -235,6 +250,9 @@ func (m *runtimeManager) run(ctx context.Context) error { case err := <-m.failed: return err case <-ticker.C: + if err := m.resetStep(ctx); err != nil { + return err + } if _, err := m.syncNodes(ctx); err != nil && !errors.Is(err, errRuntimeTransition) { return err } diff --git a/services/agents-api/internal/execution/runtime_pending.go b/services/agents-api/internal/execution/runtime_pending.go index 407da7af7..2cac9afa3 100644 --- a/services/agents-api/internal/execution/runtime_pending.go +++ b/services/agents-api/internal/execution/runtime_pending.go @@ -10,7 +10,7 @@ import ( // provisionPending shares the existing lifecycle owner and serial gate. This // also recovers idle Session creation interrupted after its database commit. func (r *runtimeLifecycle) provisionPending(ctx context.Context) error { - if r.config.Maintenance && r.config.Generation == 0 { + if r.config.AdmissionPaused && r.config.Generation == 0 { return nil } rows, err := r.store.ListUnallocatedHostedEnvironmentsForNode(ctx, r.nodeID, r.pendingCursor) diff --git a/services/agents-api/internal/execution/runtime_retirement_failure_test.go b/services/agents-api/internal/execution/runtime_retirement_failure_test.go new file mode 100644 index 000000000..18096d7ee --- /dev/null +++ b/services/agents-api/internal/execution/runtime_retirement_failure_test.go @@ -0,0 +1,268 @@ +package execution + +import ( + "context" + "errors" + "net" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// Each case has its own real advisory-lock namespace. The delayed driver read +// lets the cancellation fence hit its own deadline without shortening production +// timeouts or depending on an unrelated SQL failure to exercise this branch. +func retirementFailureLease(t *testing.T, armed *atomic.Bool, reading chan struct{}, release <-chan struct{}) (*store.ExecutionLease, *pgxpool.Pool) { + t.Helper() + dsn := os.Getenv("OAC_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("dedicated PostgreSQL required") + } + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(cfg.ConnConfig.Database, "oac_") || !strings.HasSuffix(cfg.ConnConfig.Database, "_tests") { + t.Fatal("dedicated test database required") + } + admin, err := pgxpool.NewWithConfig(t.Context(), cfg.Copy()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(admin.Close) + name := "oac_retirement_" + uuid.NewString()[:8] + "_tests" + quoted := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(t.Context(), "CREATE DATABASE "+quoted); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := admin.Exec(ctx, "DROP DATABASE "+quoted+" WITH (FORCE)"); err != nil { + t.Error(err) + } + }) + cfg.ConnConfig.Database = name + dial := cfg.ConnConfig.DialFunc + cfg.ConnConfig.DialFunc = func(ctx context.Context, network, address string) (net.Conn, error) { + c, err := dial(ctx, network, address) + if err != nil { + return nil, err + } + return &delayedLeaseRead{Conn: c, armed: armed, reading: reading, release: release}, nil + } + pool, err := pgxpool.NewWithConfig(t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + lease, err := store.New(pool).AcquireExecutionLease(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := lease.Close(ctx); err != nil { + t.Error(err) + } + }) + return lease, pool +} + +func TestFailedInventoryRetirementClosesAdmissionAndRetainsGate(t *testing.T) { + for _, mode := range []string{"gate_timeout", "lease_loss"} { + t.Run(mode, func(t *testing.T) { + var armed atomic.Bool + reading, releaseRead := make(chan struct{}), make(chan struct{}) + var readOnce sync.Once + unblockRead := func() { readOnce.Do(func() { close(releaseRead) }) } + defer unblockRead() + lease, pool := retirementFailureLease(t, &armed, reading, releaseRead) + m := testRuntimeManager(t) + m.store = lease.Store() + m.loadDeployment = func(context.Context) (*RuntimeProvider, error) { return nil, nil } + m.mutationGate = make(chan struct{}, 1) + // This fixture models an already loaded node deployment; its provider is + // needed only for node admission, not for external sandbox operations. + m.config.Provider = &drainFixtureProvider{} + original, err := m.node("retiring") + if err != nil { + t.Fatal(err) + } + operation, finish, err := original.lifecycle.beginReconcile(t.Context()) + if err != nil { + t.Fatal(err) + } + releaseProvider := make(chan struct{}) + var providerOnce sync.Once + unblockProvider := func() { providerOnce.Do(func() { close(releaseProvider) }) } + t.Cleanup(unblockProvider) + m.active.Add(1) + go func() { defer m.active.Done(); defer finish(); <-operation.Done(); <-releaseProvider }() + var queryDone chan error + if mode == "gate_timeout" { + // This independent owner caller has its own bounded query context. Unlike + // a lifecycle scan, it is not canceled by the manager's shutdown context. + queryCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + queryDone = make(chan error, 1) + armed.Store(true) + go func() { queryDone <- lease.Ping(queryCtx) }() + select { + case <-reading: + case <-time.After(2 * time.Second): + t.Fatal("owner query did not reach delayed read") + } + defer func() { + if queryDone != nil { + unblockRead() + select { + case <-queryDone: + case <-time.After(2 * time.Second): + t.Error("owned query did not settle") + } + } + }() + } else { + var killed bool + err := pool.QueryRow(t.Context(), `SELECT pg_terminate_backend(pid,1000) FROM pg_locks WHERE locktype='advisory' AND database=(SELECT oid FROM pg_database WHERE datname=current_database())`).Scan(&killed) + if err != nil || !killed { + t.Fatal("could not terminate this test's owner", killed, err) + } + } + _, err = m.applyInventory(m.snapshotNodes(), nil) + if err == nil { + t.Fatal("failed owner accepted inventory retirement") + } + if mode == "gate_timeout" && !errors.Is(err, context.DeadlineExceeded) { + t.Fatal("did not exercise cancellation gate timeout", err) + } + if _, finish, err := m.enter(t.Context()); err == nil { + finish() + t.Error("manual caller admitted before coordinator consumed failure") + } + if _, err := m.node("retiring"); err == nil { + t.Error("failed retirement admitted a replacement gate") + } + if _, err := m.node("new-node"); err == nil { + t.Error("failed retirement admitted another lifecycle") + } + if unlock, err := m.lockMutation(t.Context()); err == nil { + unlock() + t.Error("failed retirement admitted a configuration mutation") + } + m.mu.Lock() + retained := m.nodes["retiring"] == original && original.retiring + m.mu.Unlock() + if !retained { + t.Error("failed retirement discarded its original lifecycle") + } + if original.lifecycle.ctx.Err() != nil { + t.Error("failed fence claimed successful lifecycle cancellation") + } + select { + case failure := <-m.failed: + if failure == nil { + t.Error("empty owner failure") + } + default: + t.Error("owner failure was not published") + } + if queryDone != nil { + unblockRead() + select { + case err := <-queryDone: + if err != nil { + t.Fatal("fence timeout damaged unrelated owner read", err) + } + case <-time.After(2 * time.Second): + t.Fatal("owner read did not settle") + } + queryDone = nil + } + // Shutdown can now cancel the original work, but cannot call it settled + // merely because cancellation was requested or a serial gate became free. + m.stop() + drained := make(chan struct{}) + go func() { m.drain(); close(drained) }() + select { + case <-drained: + t.Fatal("shutdown skipped retained provider settlement") + case <-time.After(25 * time.Millisecond): + } + unblockProvider() + select { + case <-drained: + case <-time.After(2 * time.Second): + t.Fatal("shutdown retirement accounting did not settle") + } + m.mu.Lock() + retained = m.nodes["retiring"] == original + m.mu.Unlock() + if !retained { + t.Fatal("failed-retirement entry was removed by a task that never canceled it") + } + }) + } +} + +// No provider method is called by the retirement ownership fixture. +type drainFixtureProvider struct{ sandbox.Provider } + +// Done is evaluated only after lockMutation's initial admission check, letting +// this test close admission while a request is already waiting for its gate. +type mutationWaitContext struct { + context.Context + waiting chan struct{} + once sync.Once +} + +func (c *mutationWaitContext) Done() <-chan struct{} { + c.once.Do(func() { close(c.waiting) }) + return c.Context.Done() +} + +func TestFailedManagerRejectsAlreadyWaitingMutation(t *testing.T) { + m := testRuntimeManager(t) + m.loadDeployment = func(context.Context) (*RuntimeProvider, error) { return nil, nil } + m.mutationGate = make(chan struct{}, 1) + m.mutationGate <- struct{}{} + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + waiting := &mutationWaitContext{Context: ctx, waiting: make(chan struct{})} + result := make(chan error, 1) + go func() { + unlock, err := m.lockMutation(waiting) + if unlock != nil { + unlock() + } + result <- err + }() + select { + case <-waiting.waiting: + case <-time.After(time.Second): + t.Fatal("mutation never reached gate wait") + } + m.mu.Lock() + m.closed = true + m.mu.Unlock() + <-m.mutationGate + select { + case err := <-result: + if !errors.Is(err, ErrExecutionUnavailable) { + t.Fatal("waiting mutation bypassed synchronous admission closure", err) + } + case <-time.After(time.Second): + t.Fatal("waiting mutation did not settle") + } +} diff --git a/services/agents-api/internal/execution/sandbox_deployment_drain_test.go b/services/agents-api/internal/execution/sandbox_deployment_drain_test.go new file mode 100644 index 000000000..f7d7cfff1 --- /dev/null +++ b/services/agents-api/internal/execution/sandbox_deployment_drain_test.go @@ -0,0 +1,287 @@ +package execution + +import ( + "bytes" + "context" + "net" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/node" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// delayedLeaseRead holds one real PostgreSQL response while the deployment drain +// begins. Cancellation during this driver read closes the advisory-lock owner. +type delayedLeaseRead struct { + net.Conn + armed *atomic.Bool + held atomic.Bool + reading chan struct{} + release <-chan struct{} +} + +func (c *delayedLeaseRead) Write(p []byte) (int, error) { + if bytes.Contains(p, []byte("-- ping")) && c.armed.CompareAndSwap(true, false) { + c.held.Store(true) + } + return c.Conn.Write(p) +} +func (c *delayedLeaseRead) Read(p []byte) (int, error) { + if c.held.CompareAndSwap(true, false) { + close(c.reading) + <-c.release + } + return c.Conn.Read(p) +} + +func TestSandboxDeploymentDrainPreservesLeaseInFlightRead(t *testing.T) { + for _, mode := range []string{"deployment", "inventory", "manual"} { + t.Run(mode, func(t *testing.T) { + testLifecycleCancellationPreservesLease(t, mode) + }) + } +} + +func testLifecycleCancellationPreservesLease(t *testing.T, mode string) { + dsn := os.Getenv("OAC_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("dedicated PostgreSQL required") + } + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(cfg.ConnConfig.Database, "oac_") || !strings.HasSuffix(cfg.ConnConfig.Database, "_tests") { + t.Fatal("dedicated test database required") + } + admin, err := pgxpool.NewWithConfig(t.Context(), cfg.Copy()) + if err != nil { + t.Fatal(err) + } + defer admin.Close() + database := pgx.Identifier{"oac_drain_" + uuid.NewString()[:8] + "_tests"}.Sanitize() + if _, err := admin.Exec(t.Context(), "CREATE DATABASE "+database); err != nil { + t.Fatal(err) + } + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := admin.Exec(ctx, "DROP DATABASE "+database+" WITH (FORCE)"); err != nil { + t.Error(err) + } + }() + cfg.ConnConfig.Database = strings.Trim(database, `"`) + var armed atomic.Bool + reading, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + defer unblock() + dial := cfg.ConnConfig.DialFunc + cfg.ConnConfig.DialFunc = func(ctx context.Context, network, address string) (net.Conn, error) { + c, err := dial(ctx, network, address) + if err != nil { + return nil, err + } + return &delayedLeaseRead{Conn: c, armed: &armed, reading: reading, release: release}, nil + } + pool, err := pgxpool.NewWithConfig(t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + lease, err := store.New(pool).AcquireExecutionLease(t.Context()) + if err != nil { + t.Fatal(err) + } + defer lease.Close(context.Background()) + hub := node.NewHub(node.HubOptions{}) + defer hub.Close() + id := uuid.NewString() + configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker")} + m, err := newRuntimeManager(lease.Store(), gateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil })) + if err != nil { + t.Fatal(err) + } + defer func() { unblock(); m.stop(); m.drain() }() + if _, err := m.ensureDeployment(t.Context()); err != nil { + t.Fatal(err) + } + n, err := m.node(uuid.NewString()) + if err != nil { + t.Fatal(err) + } + var delayed *delayedCancellationContext + if mode == "manual" { + delayed = &delayedCancellationContext{Context: n.lifecycle.ctx, release: make(chan struct{}), finished: make(chan struct{})} + n.lifecycle.ctx = delayed + defer delayed.unblock() + } + m.active.Add(1) + queryDone := make(chan error, 1) + leaseFree := make(chan error, 1) + armed.Store(true) + go func() { + defer m.active.Done() + parent := n.lifecycle.ctx + if mode == "manual" { + parent = t.Context() + } + ctx, finish, err := n.lifecycle.beginReconcile(parent) + if err != nil { + queryDone <- err + return + } + defer finish() + queryDone <- lease.Store().CheckExecutionOwnership(ctx) + <-ctx.Done() + // Provider settlement remains outside the cancellation fence. A fresh owner + // read must proceed even before this tracked lifecycle operation returns. + leaseFree <- lease.Store().CheckExecutionOwnership(t.Context()) + }() + select { + case <-reading: + case <-time.After(2 * time.Second): + t.Fatal("lease query did not reach its driver read") + } + paused := make(chan error, 1) + go func() { + if mode == "inventory" { + _, err := m.applyInventory(map[string]*runtimeNode{n.lifecycle.nodeID: n}, nil) + paused <- err + } else { + paused <- m.pauseDeployment(t.Context()) + } + }() + // The cancellation is only legal after the leased query has returned. The + // small observation window is not a delay used to make the operation succeed. + select { + case <-n.lifecycle.ctx.Done(): + t.Log("drain cancelled a live leased operation") + case <-time.After(100 * time.Millisecond): + } + unblock() + queryErr := <-queryDone + select { + case err := <-leaseFree: + if err != nil { + t.Fatal("cancellation held or lost the lease during provider settlement", err) + } + case <-time.After(2 * time.Second): + t.Fatal("lifecycle cancellation or provider settlement did not proceed beyond the fence") + } + select { + case err := <-paused: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("cancellation depended on delayed AfterFunc or drain did not finish") + } + if mode == "inventory" { + settled := make(chan struct{}) + go func() { m.active.Wait(); close(settled) }() + select { + case <-settled: + case <-time.After(2 * time.Second): + t.Fatal("retired lifecycle did not settle") + } + } + if delayed != nil { + delayed.unblock() + } + if err := lease.Store().CheckExecutionOwnership(t.Context()); err != nil { + t.Fatalf("deployment drain destroyed the owner connection (in-flight query: %v): %v", queryErr, err) + } + if queryErr != nil { + t.Fatal("drain interrupted the leased read", queryErr) + } +} + +// Force the manual-reconcile AfterFunc to run after the synchronous fence. Value +// intentionally hides cancelCtx internals so context uses this AfterFunc method. +type delayedCancellationContext struct { + context.Context + release chan struct{} + finished chan struct{} + releaseOnce sync.Once + finishOnce sync.Once +} + +func (c *delayedCancellationContext) Value(any) any { return nil } +func (c *delayedCancellationContext) AfterFunc(f func()) func() bool { + stop := context.AfterFunc(c.Context, func() { + <-c.release + f() + c.finishOnce.Do(func() { close(c.finished) }) + }) + return func() bool { + stopped := stop() + if stopped { + c.finishOnce.Do(func() { close(c.finished) }) + } + return stopped + } +} +func (c *delayedCancellationContext) unblock() { + c.releaseOnce.Do(func() { close(c.release) }) + select { + case <-c.finished: + case <-time.After(time.Second): + } +} + +func TestSandboxDeploymentDrainFailureCannotReactivate(t *testing.T) { + _, lease := resetManagerStore(t) + hub := node.NewHub(node.HubOptions{}) + defer hub.Close() + id := uuid.NewString() + configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker")} + m, err := newRuntimeManager(lease.Store(), gateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil })) + if err != nil { + t.Fatal(err) + } + defer func() { m.stop(); m.drain() }() + if _, err := m.ensureDeployment(t.Context()); err != nil { + t.Fatal(err) + } + n, err := m.node(uuid.NewString()) + if err != nil { + t.Fatal(err) + } + if err := lease.Close(t.Context()); err != nil { + t.Fatal(err) + } + first := m.pauseDeployment(t.Context()) + if first == nil { + t.Fatal("drain accepted lost ownership") + } + if n.lifecycle.ctx.Err() != nil { + t.Fatal("failed fence canceled outside the lease gate") + } + if err := m.pauseDeployment(t.Context()); err == nil { + t.Fatal("repeated drain forgot its failure") + } + if err := m.activateDeployment(t.Context(), store.RuntimeDeploymentView{InstallationID: id, Generation: 1, Mode: "nodes", Provider: "docker"}); err == nil { + t.Fatal("failed drain reopened the provider") + } + if _, _, err := m.enter(t.Context()); err == nil { + t.Fatal("failed drain admitted new work") + } + select { + case err := <-m.failed: + if err == nil { + t.Fatal("missing owner failure") + } + default: + t.Fatal("failed fence did not stop the owner") + } +} diff --git a/services/agents-api/internal/execution/sandbox_deployment_setup.go b/services/agents-api/internal/execution/sandbox_deployment_setup.go index 1f94e795c..eeeb8b922 100644 --- a/services/agents-api/internal/execution/sandbox_deployment_setup.go +++ b/services/agents-api/internal/execution/sandbox_deployment_setup.go @@ -35,7 +35,7 @@ type RuntimeDeploymentPreparer func(context.Context, store.SandboxSetup) (Prepar // NewDeferredRuntimeProvider enables Web setup for one fixed installation. The // loader returns nil until selection, then the committed immutable generation. -// Replacement is serialized by the deployment maintenance and drain flow. +// Replacement is serialized by the deployment mutation gate and drain flow. func NewDeferredRuntimeProvider(installationID string, load func(context.Context) (*RuntimeProvider, error), prepare ...RuntimeDeploymentPreparer) *RuntimeProvider { config := &RuntimeProvider{InstallationID: installationID, loadDeployment: load} if len(prepare) == 1 { @@ -51,6 +51,9 @@ func (w *Worker) InitializeSandboxDeployment(ctx context.Context, input store.Sa } defer unlock() m := w.runtimes + if err := m.store.CheckSandboxDeploymentSetup(ctx, m.setupInstallationID, input); err != nil { + return store.RuntimeDeploymentView{}, err + } candidate, err := m.prepareCandidate(ctx, input) if err != nil { return store.RuntimeDeploymentView{}, err @@ -166,7 +169,7 @@ func (m *runtimeManager) publishDeployment(candidate PreparedRuntimeDeployment, m.mu.Lock() defer m.mu.Unlock() config := *candidate.Config - config.Generation, config.Maintenance = committed.Generation, committed.Maintenance + config.Generation, config.AdmissionPaused = committed.Generation, committed.Reset != nil if m.switching { m.nodes = make(map[string]*runtimeNode) } diff --git a/services/agents-api/internal/execution/sandbox_deployment_setup_test.go b/services/agents-api/internal/execution/sandbox_deployment_setup_test.go index 7670b5a7e..213230c0a 100644 --- a/services/agents-api/internal/execution/sandbox_deployment_setup_test.go +++ b/services/agents-api/internal/execution/sandbox_deployment_setup_test.go @@ -201,9 +201,9 @@ func TestCommittedSandboxCandidatePublishesAfterShutdown(t *testing.T) { var published *RuntimeProvider candidate := PreparedRuntimeDeployment{Config: config, Publish: func(value *RuntimeProvider) { published = value }} m.stop() - m.publishDeployment(candidate, store.RuntimeDeploymentView{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b", Maintenance: true}) + m.publishDeployment(candidate, store.RuntimeDeploymentView{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b", Reset: &store.SandboxResetView{}}) m.drain() - if m.config.Generation != 2 || !m.config.Maintenance || published == nil || published.Generation != 2 || !published.Maintenance || m.switching { + if m.config.Generation != 2 || !m.config.AdmissionPaused || published == nil || published.Generation != 2 || !published.AdmissionPaused || m.switching { t.Fatal("committed candidate was lost during shutdown") } if _, _, err := m.enter(t.Context()); !errors.Is(err, ErrExecutionUnavailable) { diff --git a/services/agents-api/internal/execution/sandbox_deployment_switch.go b/services/agents-api/internal/execution/sandbox_deployment_switch.go index ee8c5f74f..f6e06a7ad 100644 --- a/services/agents-api/internal/execution/sandbox_deployment_switch.go +++ b/services/agents-api/internal/execution/sandbox_deployment_switch.go @@ -2,6 +2,9 @@ package execution import ( "context" + "errors" + "time" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" ) @@ -10,8 +13,21 @@ func (m *runtimeManager) lockMutation(ctx context.Context) (func(), error) { if m == nil || m.loadDeployment == nil { return nil, store.ErrSandboxDeploymentConflict } + m.mu.Lock() + closed := m.closed + m.mu.Unlock() + if closed { + return nil, ErrExecutionUnavailable + } select { case m.mutationGate <- struct{}{}: + m.mu.Lock() + closed := m.closed + m.mu.Unlock() + if closed { + <-m.mutationGate + return nil, ErrExecutionUnavailable + } return func() { <-m.mutationGate }, nil case <-ctx.Done(): return nil, ctx.Err() @@ -30,14 +46,14 @@ func (m *runtimeManager) pauseDeployment(ctx context.Context) error { } if !m.switching { m.switching = true - m.switchDrained = make(chan struct{}) + m.switchDrained = &deploymentDrain{done: make(chan struct{})} go m.drainDeployment(m.switchDrained) } drained := m.switchDrained m.mu.Unlock() select { - case <-drained: - return nil + case <-drained.done: + return drained.err case <-ctx.Done(): return ctx.Err() case <-m.ctx.Done(): @@ -47,8 +63,13 @@ func (m *runtimeManager) pauseDeployment(ctx context.Context) error { // One drain outlives the initiating HTTP request. A cancelled waiter cannot // permit a replacement generation while an old caller is still completing. -func (m *runtimeManager) drainDeployment(done chan struct{}) { - defer close(done) +type deploymentDrain struct { + done chan struct{} + err error // Published by closing done; immutable afterwards. +} + +func (m *runtimeManager) drainDeployment(result *deploymentDrain) { + defer close(result.done) for _, gate := range []chan struct{}{m.setupGate, m.inventory} { select { case gate <- struct{}{}: @@ -57,10 +78,14 @@ func (m *runtimeManager) drainDeployment(done chan struct{}) { } } m.mu.Lock() + nodes := make([]*runtimeNode, 0, len(m.nodes)) for _, n := range m.nodes { - n.lifecycle.stop() + nodes = append(nodes, n) } m.mu.Unlock() + if result.err = m.cancelLifecycles(nodes); result.err != nil { + return + } m.active.Wait() } @@ -87,6 +112,10 @@ func (m *runtimeManager) activateDeployment(ctx context.Context, expected store. if err != nil { return err } + if config == nil && expected.Provider == "" { + m.publishEmptyDeployment(expected) + return nil + } if config == nil || config.InstallationID != expected.InstallationID || config.Generation != expected.Generation || config.Mode != expected.Mode || config.ProviderKind != expected.Provider || config.loadDeployment != nil || config.LocalNodeID != "" { return sandbox.ErrInvalid } @@ -124,35 +153,45 @@ func (w *Worker) UpdateSandboxDeployment(ctx context.Context, input store.Sandbo return store.RuntimeDeploymentView{}, err } if err := m.pauseDeployment(ctx); err != nil { - return store.RuntimeDeploymentView{}, err + return store.RuntimeDeploymentView{}, errors.Join(err, m.restoreCommittedDeployment()) } input.SandboxDeploymentSetupRequest = withTemplateBuild(input.SandboxDeploymentSetupRequest, candidate) result, err := m.store.UpdateSandboxDeployment(ctx, m.setupInstallationID, input) if err != nil { - return store.RuntimeDeploymentView{}, err + return store.RuntimeDeploymentView{}, errors.Join(err, m.restoreCommittedDeployment()) } m.publishDeployment(candidate, result) return result, nil } -func (w *Worker) SetSandboxMaintenance(ctx context.Context, input store.SandboxMaintenanceRequest) (store.RuntimeDeploymentView, error) { - unlock, err := w.runtimes.lockMutation(ctx) - if err != nil { - return store.RuntimeDeploymentView{}, err +// Recovery belongs to the owner, not a cancelled HTTP request. Failure keeps the +// drain barrier closed and stops the owner rather than admitting an unknown provider. +func (m *runtimeManager) restoreCommittedDeployment() error { + ctx, cancel := context.WithTimeout(m.ctx, 5*time.Second) + defer cancel() + committed, err := m.store.GetRuntimeDeployment(ctx) + if err == nil { + err = m.activateDeployment(ctx, committed) } - defer unlock() - m := w.runtimes - if !input.Maintenance { - expected, err := m.store.GetRuntimeDeployment(ctx) - if err != nil { - return store.RuntimeDeploymentView{}, err - } - if expected.Generation != input.ExpectedGeneration { - return store.RuntimeDeploymentView{}, store.ErrSandboxDeploymentConflict - } - if err := m.activateDeployment(ctx, expected); err != nil { - return store.RuntimeDeploymentView{}, err + if err != nil { + select { + case m.failed <- err: + default: } } - return m.store.SetSandboxMaintenance(ctx, m.setupInstallationID, input) + return err +} + +// Empty-state publication is infallible after commit, even if the request was +// cancelled. Loader, preparer and installation ownership stay on the manager. +func (m *runtimeManager) publishEmptyDeployment(committed store.RuntimeDeploymentView) { + m.mu.Lock() + defer m.mu.Unlock() + m.config = RuntimeProvider{InstallationID: committed.InstallationID, Generation: committed.Generation} + m.nodes = make(map[string]*runtimeNode) + if m.publishUnconfigured != nil { + m.publishUnconfigured(committed.Generation) + } + m.switching = false + m.switchDrained = nil } diff --git a/services/agents-api/internal/execution/sandbox_deployment_switch_test.go b/services/agents-api/internal/execution/sandbox_deployment_switch_test.go index 53c493a56..2d78d268d 100644 --- a/services/agents-api/internal/execution/sandbox_deployment_switch_test.go +++ b/services/agents-api/internal/execution/sandbox_deployment_switch_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" - "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/node" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" "github.com/google/uuid" @@ -151,7 +150,7 @@ func TestSandboxManagerCancelledSwitchCannotResumeBeforeDrain(t *testing.T) { } } -func TestSandboxSetupRetryCannotBypassOutstandingDrain(t *testing.T) { +func TestSandboxActivationCannotBypassOutstandingDrain(t *testing.T) { hub := node.NewHub(node.HubOptions{}) defer hub.Close() id := uuid.NewString() @@ -176,9 +175,8 @@ func TestSandboxSetupRetryCannotBypassOutstandingDrain(t *testing.T) { cancel() ctx, cancel = context.WithTimeout(t.Context(), 10*time.Millisecond) defer cancel() - w := &Worker{runtimes: m} - _, err = w.InitializeSandboxDeployment(ctx, store.SandboxDeploymentSetupRequest{Provider: "e2b", E2B: &store.SandboxE2BConfiguration{APIKey: "fixture-key", Template: "runtime:" + uuid.NewString()}, DeploymentSpec: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 1024}}}) + err = m.activateDeployment(ctx, store.RuntimeDeploymentView{InstallationID: id, Generation: 1, Provider: "e2b", Mode: "direct"}) if !errors.Is(err, context.DeadlineExceeded) { - t.Fatal("setup retry bypassed the unfinished drain", err) + t.Fatal("activation bypassed the unfinished drain", err) } } diff --git a/services/agents-api/internal/execution/sandbox_reset.go b/services/agents-api/internal/execution/sandbox_reset.go new file mode 100644 index 000000000..13e8e6f9a --- /dev/null +++ b/services/agents-api/internal/execution/sandbox_reset.go @@ -0,0 +1,120 @@ +package execution + +import ( + "context" + "errors" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/obs/log" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +func (w *Worker) StartSandboxReset(ctx context.Context, input store.SandboxResetRequest) (store.RuntimeDeploymentView, error) { + unlock, err := w.runtimes.lockMutation(ctx) + if err != nil { + return store.RuntimeDeploymentView{}, err + } + defer unlock() + return w.runtimes.store.StartSandboxReset(ctx, w.runtimes.setupInstallationID, input) +} + +func (w *Worker) CancelSandboxReset(ctx context.Context, generation uint64) (store.RuntimeDeploymentView, error) { + unlock, err := w.runtimes.lockMutation(ctx) + if err != nil { + return store.RuntimeDeploymentView{}, err + } + defer unlock() + return w.runtimes.store.CancelSandboxReset(ctx, w.runtimes.setupInstallationID, generation) +} + +// resetStep runs only on the manager's uncounted coordinator loop. It must never +// enter m.active, hold a Session/deployment transaction, or call a provider while +// waiting for a deployment drain. Each page has both a row and time bound. +func (m *runtimeManager) resetStep(parent context.Context) error { + if m.loadDeployment == nil { + return nil + } + ctx, cancel := context.WithTimeout(parent, 5*time.Second) + defer cancel() + return m.resetPage(parent, ctx) +} + +func (m *runtimeManager) resetPage(parent, ctx context.Context) error { + unlock, err := m.lockMutation(ctx) + if err != nil { + if ctx.Err() != nil && parent.Err() == nil { + return nil + } + return err + } + defer unlock() + if err := m.store.AdvanceSandboxResetDeadline(ctx); err != nil { + return err + } + current, err := m.store.GetRuntimeDeployment(ctx) + if err != nil { + return err + } + if current.Reset == nil { + m.resetCursor = "" + m.resetRequestedAt = time.Time{} + return nil + } + if !current.Reset.RequestedAt.Equal(m.resetRequestedAt) { + m.resetCursor = "" + m.resetRequestedAt = current.Reset.RequestedAt + } + candidates, err := m.store.ListSandboxResetSessions(ctx, m.resetCursor, current.Reset.Clear == "force") + if err != nil { + return err + } + for _, candidate := range candidates { + if ctx.Err() != nil { + return nil + } + _, err := m.store.ArchiveSandboxResetSession(ctx, candidate.TenantID, candidate.SessionID, current.Generation, current.Reset.RequestedAt) + m.resetCursor = candidate.SessionID + if err != nil && !errors.Is(err, store.ErrSandboxResetSessionBusy) && !errors.Is(err, store.ErrNotFound) { + // Do not log a provider body, request, credential or stored provenance. + log.Warn(ctx, "Sandbox reset archive remains pending", "session_id", candidate.SessionID) + } + } + if len(candidates) < 32 { + m.resetCursor = "" + } + if ctx.Err() != nil { + return nil + } + current, err = m.store.GetRuntimeDeployment(ctx) + if err != nil { + return err + } + if current.Reset == nil || current.Resources.Allocations != 0 || current.Resources.Pending != 0 { + return nil + } + if err := m.pauseDeployment(ctx); err != nil { + if recovery := m.restoreCommittedDeployment(); recovery != nil { + return errors.Join(err, recovery) + } + if parent.Err() == nil && ctx.Err() != nil && errors.Is(err, ctx.Err()) { + // A bounded page may expire during drain. Successful owner-context + // recovery keeps this durable reset available for the next tick. + return nil + } + return err + } + committed, err := m.store.CompleteSandboxReset(ctx, m.setupInstallationID, current.Generation, current.Reset.RequestedAt) + if err != nil { + recovery := m.restoreCommittedDeployment() + if recovery != nil { + return errors.Join(err, recovery) + } + // The transaction rechecks counts after the drain. A losing recheck + // leaves the durable clear intact for the next owner tick. + log.Warn(parent, "Sandbox reset completion remains pending", "generation", current.Generation) + return nil + } + m.publishEmptyDeployment(committed) + m.resetCursor = "" + return nil +} diff --git a/services/agents-api/internal/execution/sandbox_reset_test.go b/services/agents-api/internal/execution/sandbox_reset_test.go new file mode 100644 index 000000000..f2fcca3b5 --- /dev/null +++ b/services/agents-api/internal/execution/sandbox_reset_test.go @@ -0,0 +1,178 @@ +package execution + +import ( + "bytes" + "context" + "database/sql" + "os" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/adminaudit" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/credentialcrypto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/node" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" +) + +// The execution lease is database-scoped, so this manager test owns a database. +func resetManagerStore(t *testing.T) (*store.Store, *store.ExecutionLease) { + t.Helper() + return resetManagerStoreConfig(t, nil) +} + +func resetManagerStoreConfig(t *testing.T, configure func(*pgxpool.Config)) (*store.Store, *store.ExecutionLease) { + t.Helper() + url := os.Getenv("OAC_TEST_DATABASE_URL") + if url == "" { + t.Skip("OAC_TEST_DATABASE_URL is required") + } + admin, err := pgxpool.New(t.Context(), url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(admin.Close) + name := "oac_reset_" + uuid.NewString()[:8] + "_tests" + quoted := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(t.Context(), "CREATE DATABASE "+quoted); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := admin.Exec(ctx, "DROP DATABASE "+quoted+" WITH (FORCE)"); err != nil { + t.Error(err) + } + }) + cfg := admin.Config().Copy() + cfg.ConnConfig.Database = name + db := sql.OpenDB(stdlib.GetConnector(*cfg.ConnConfig)) + provider, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) + if err != nil { + t.Fatal(err) + } + _, err = provider.Up(t.Context()) + _ = db.Close() + if err != nil { + t.Fatal(err) + } + if configure != nil { + configure(cfg) + } + pool, err := pgxpool.NewWithConfig(t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + cipher, err := credentialcrypto.New(bytes.Repeat([]byte{8}, 32)) + if err != nil { + t.Fatal(err) + } + s := store.NewWithCredentialCipher(pool, cipher) + lease, err := s.AcquireExecutionLease(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = lease.Close(context.Background()) }) + return s, lease +} + +func TestSandboxResetPageTimeoutRecoversCommittedOwner(t *testing.T) { + s, lease := resetManagerStore(t) + w := lease.Store() + id := uuid.NewString() + if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + t.Fatal(err) + } + selection := store.SandboxDeploymentSetupRequest{Provider: "e2b", E2B: &store.SandboxE2BConfiguration{APIKey: "fixture-key", Template: "runtime:" + uuid.NewString()}} + selection.Resources.CPUs = 2 + selection.Resources.MemoryMiB = 2048 + if _, err := w.InitializeSandboxDeployment(t.Context(), id, selection); err != nil { + t.Fatal(err) + } + hub := node.NewHub(node.HubOptions{}) + defer hub.Close() + loads := 0 + config := NewDeferredRuntimeProvider(id, func(ctx context.Context) (*RuntimeProvider, error) { + if ctx.Err() != nil { + t.Error("recovery inherited cancelled page") + } + loads++ + setup, err := s.GetSandboxSetup(ctx) + if err != nil || setup.Provider == "" { + return nil, err + } + return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Mode: setup.Mode, Generation: setup.Generation, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker")}, nil + }) + m, err := newRuntimeManager(w, gateway.NewRegistry(), config) + if err != nil { + t.Fatal(err) + } + defer func() { m.stop(); m.drain() }() + if _, err := m.ensureDeployment(t.Context()); err != nil { + t.Fatal(err) + } + old, err := m.node("") + if err != nil { + t.Fatal(err) + } + _, finish, err := m.enter(t.Context()) + if err != nil { + t.Fatal(err) + } + released := false + defer func() { + if !released { + finish() + } + }() + audit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "reset-test", ActorLabel: "operator", RequestID: "request", TraceID: "trace"}) + reset, err := w.StartSandboxReset(audit, id, store.SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}) + if err != nil { + t.Fatal(err) + } + page, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { done <- m.resetPage(t.Context(), page) }() + select { + case <-old.lifecycle.ctx.Done(): + case <-time.After(5 * time.Second): + t.Fatal("reset never reached drain") + } + cancel() // Expire this page only after its drain barrier is established. + finish() + released = true + select { + case err := <-done: + if err != nil { + t.Fatal("recoverable page cancellation stopped owner", err) + } + case <-time.After(5 * time.Second): + t.Fatal("owner recovery blocked") + } + current, err := s.GetRuntimeDeployment(t.Context()) + if err != nil || current.Reset == nil || current.Generation != 1 || !current.Reset.RequestedAt.Equal(reset.Reset.RequestedAt) { + t.Fatal("page timeout lost durable reset", current, err) + } + if loads < 2 { + t.Fatal("committed provider was not restored") + } + _, finishNext, err := m.enter(t.Context()) + if err != nil { + t.Fatal("recovery left barrier closed", err) + } + finishNext() + if err := m.resetStep(t.Context()); err != nil { + t.Fatal(err) + } + current, err = s.GetRuntimeDeployment(t.Context()) + if err != nil || current.Reset != nil || current.Provider != "" || current.Generation != 2 { + t.Fatal("next tick did not finish", current, err) + } +} diff --git a/services/agents-api/internal/execution/sandbox_snapshot_budget_test.go b/services/agents-api/internal/execution/sandbox_snapshot_budget_test.go new file mode 100644 index 000000000..443613fb7 --- /dev/null +++ b/services/agents-api/internal/execution/sandbox_snapshot_budget_test.go @@ -0,0 +1,111 @@ +package execution + +import ( + "context" + "runtime/debug" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/adminaudit" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/node" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +type snapshotBudgetKey struct{} +type snapshotBudgetQuery struct { + started time.Time + ordinal int64 +} +type snapshotBudget struct { + t *testing.T + armed atomic.Bool + reads atomic.Int64 +} + +func (d *snapshotBudget) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context { + q := snapshotBudgetQuery{started: time.Now()} + if d.armed.Load() && strings.Contains(data.SQL, "GetSandboxDeploymentSnapshot") && strings.Contains(string(debug.Stack()), "runtimeManager).resetPage") { + q.ordinal = d.reads.Add(1) + } + return context.WithValue(ctx, snapshotBudgetKey{}, q) +} +func (d *snapshotBudget) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + q, _ := ctx.Value(snapshotBudgetKey{}).(snapshotBudgetQuery) + if q.ordinal == 0 { + return + } + d.t.Logf("snapshot=%d query_elapsed=%s err=%v ctx=%v", q.ordinal, time.Since(q.started), data.Err, ctx.Err()) + // Model scheduling pressure on the first two reads without changing the real + // five-second page deadline. The final snapshot uses the lease connection. + if q.ordinal <= 2 && data.Err == nil { + delay := 2200*time.Millisecond - time.Since(q.started) + if delay > 0 { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-timer.C: + case <-ctx.Done(): + } + } + } +} + +func TestSandboxResetSnapshotFitsPageBudget(t *testing.T) { + budget := &snapshotBudget{t: t} + s, lease := resetManagerStoreConfig(t, func(cfg *pgxpool.Config) { + cfg.ConnConfig.RuntimeParams["jit"] = "on" + cfg.ConnConfig.Tracer = budget + }) + w := lease.Store() + id := uuid.NewString() + if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + t.Fatal(err) + } + selection := store.SandboxDeploymentSetupRequest{Provider: "e2b", E2B: &store.SandboxE2BConfiguration{APIKey: "fixture-key", Template: "runtime:" + uuid.NewString()}} + selection.Resources.CPUs = 2 + selection.Resources.MemoryMiB = 2048 + if _, err := w.InitializeSandboxDeployment(t.Context(), id, selection); err != nil { + t.Fatal(err) + } + hub := node.NewHub(node.HubOptions{}) + defer hub.Close() + configuration := NewDeferredRuntimeProvider(id, func(ctx context.Context) (*RuntimeProvider, error) { + setup, err := s.GetSandboxSetup(ctx) + if err != nil || setup.Provider == "" { + return nil, err + } + return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker")}, nil + }) + m, err := newRuntimeManager(w, gateway.NewRegistry(), configuration) + if err != nil { + t.Fatal(err) + } + defer func() { m.stop(); m.drain() }() + if _, err = m.ensureDeployment(t.Context()); err != nil { + t.Fatal(err) + } + if _, err = w.StartSandboxReset(adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "reset-test", ActorLabel: "operator", RequestID: "request", TraceID: "trace"}), id, store.SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err != nil { + t.Fatal(err) + } + budget.armed.Store(true) + started := time.Now() + err = m.resetStep(t.Context()) + ping := lease.Ping(t.Context()) + t.Logf("reset_elapsed=%s reset_error=%v lease_ping=%v", time.Since(started), err, ping) + if err != nil || ping != nil { + t.Fatal("bounded snapshot lost execution ownership", err, ping) + } + if budget.reads.Load() < 3 { + t.Fatal("did not exercise final leased snapshot") + } + view, err := s.GetRuntimeDeployment(t.Context()) + if err != nil || view.Generation != 2 || view.Provider != "" || view.Reset != nil { + t.Fatal("reset did not commit", view, err) + } +} diff --git a/services/agents-api/internal/execution/worker.go b/services/agents-api/internal/execution/worker.go index cbdd976d7..832ab8229 100644 --- a/services/agents-api/internal/execution/worker.go +++ b/services/agents-api/internal/execution/worker.go @@ -49,7 +49,7 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher) (*Worker, error) { if worker.runtimes != nil && worker.runtimes.loadDeployment == nil { config := worker.runtimes.config verify = config.VerifyLegacyOwnership - deployment = &store.RuntimeDeployment{ProviderKind: config.ProviderKind, LocalNodeID: config.LocalNodeID, LocalCredentialSHA256: config.LocalCredentialSHA256, LocalMaxActive: config.LocalMaxActive, LocalMaxRetained: config.LocalMaxRetained, InstallationID: config.InstallationID, BackendFingerprint: config.BackendFingerprint, Maintenance: config.Maintenance} + deployment = &store.RuntimeDeployment{ProviderKind: config.ProviderKind, LocalNodeID: config.LocalNodeID, LocalCredentialSHA256: config.LocalCredentialSHA256, LocalMaxActive: config.LocalMaxActive, LocalMaxRetained: config.LocalMaxRetained, InstallationID: config.InstallationID, BackendFingerprint: config.BackendFingerprint, AdmissionPaused: config.AdmissionPaused} } if worker.runtimes != nil && worker.runtimes.loadDeployment != nil { err = owned.Store.ClaimWebSandboxDeployment(ctx, worker.runtimes.setupInstallationID) diff --git a/services/agents-api/internal/sandbox/node/hub_lifetime_test.go b/services/agents-api/internal/sandbox/node/hub_lifetime_test.go index 283614a93..d72021c21 100644 --- a/services/agents-api/internal/sandbox/node/hub_lifetime_test.go +++ b/services/agents-api/internal/sandbox/node/hub_lifetime_test.go @@ -194,6 +194,7 @@ func TestSlowOpeningAndClosingLeaveOtherNodesResponsive(t *testing.T) { live := connectRawNode(t, server.URL, fast) defer live.Close() go serveRawInfo(live) + wait(t, func() bool { return hub.Online(fast.NodeID) }) beginRawNode(t, server.URL, slow) select { case <-opening: diff --git a/services/agents-api/internal/sandbox/node/hub_postgres_test.go b/services/agents-api/internal/sandbox/node/hub_postgres_test.go index ac29125e1..38a751663 100644 --- a/services/agents-api/internal/sandbox/node/hub_postgres_test.go +++ b/services/agents-api/internal/sandbox/node/hub_postgres_test.go @@ -113,6 +113,7 @@ func TestHubPostgresBlockedOpeningIsBounded(t *testing.T) { live := connectRawNode(t, server.URL, fast) defer live.Close() go serveRawInfo(live) + wait(t, func() bool { return hub.Online(fast.NodeID) }) start := time.Now() beginRawNode(t, server.URL, slow) var pid uint32 diff --git a/services/agents-api/internal/store/admin_session_archive.go b/services/agents-api/internal/store/admin_session_archive.go index 2835b00f1..3235c8bdc 100644 --- a/services/agents-api/internal/store/admin_session_archive.go +++ b/services/agents-api/internal/store/admin_session_archive.go @@ -3,6 +3,8 @@ package store import ( "context" "errors" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/adminaudit" + "time" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" "github.com/jackc/pgx/v5" @@ -21,6 +23,18 @@ type ManagedSessionArchive struct { // public Session, history and persisted files. The lifecycle owner performs the // external cleanup; only its existing confirmation can release an allocation. func (s *Store) ArchiveManagedSession(ctx context.Context, tenantID, sessionID string, expectedGeneration uint64) (ManagedSessionArchive, error) { + return s.archiveManagedSession(ctx, tenantID, sessionID, expectedGeneration, nil) +} + +// A reset instance is identified by its persisted request time as well as its +// generation, preventing a cancelled clear's candidates from affecting its successor. +func (s *Store) ArchiveSandboxResetSession(ctx context.Context, tenantID, sessionID string, generation uint64, requestedAt time.Time) (ManagedSessionArchive, error) { + return s.archiveManagedSession(ctx, tenantID, sessionID, generation, &requestedAt) +} + +var ErrSandboxResetSessionBusy = errors.New("the hosted Session is busy") + +func (s *Store) archiveManagedSession(ctx context.Context, tenantID, sessionID string, expectedGeneration uint64, resetRequestedAt *time.Time) (ManagedSessionArchive, error) { if s.executionLease == nil { return ManagedSessionArchive{}, ErrInvalidInput } @@ -30,6 +44,20 @@ func (s *Store) ArchiveManagedSession(ctx context.Context, tenantID, sessionID s } var result ManagedSessionArchive err = s.withPublicSession(ctx, tenantID, sessionID, func(ctx context.Context, q *sqlc.Queries, session pgtype.UUID) error { + // Session precedes deployment, matching Turn, allocation and input admission. + deployment, err := q.LockRuntimeDeployment(ctx) + if err != nil { + return err + } + if uint64(deployment.Generation) != expectedGeneration { + return &SandboxGenerationStaleError{uint64(deployment.Generation)} + } + if !deployment.WebManaged || !deployment.InstallationID.Valid { + return ErrSandboxDeploymentConflict + } + if deployment.ProviderKind == "" { + return ErrSandboxNotConfigured + } environment, err := q.GetSessionEnvironment(ctx, sqlc.GetSessionEnvironmentParams{TenantID: tenant, ID: session}) if errors.Is(err, pgx.ErrNoRows) { return ErrInvalidInput @@ -41,13 +69,33 @@ func (s *Store) ArchiveManagedSession(ctx context.Context, tenantID, sessionID s if err != nil || kind != "openai_hosted" { return ErrInvalidInput } - // Allocation and input admission take these locks in the same order. - deployment, err := q.LockRuntimeDeployment(ctx) - if err != nil { - return err - } - if !deployment.WebManaged || !deployment.InstallationID.Valid || deployment.ProviderKind == "" || !deployment.Maintenance || uint64(deployment.Generation) != expectedGeneration { - return ErrSandboxDeploymentConflict + if resetRequestedAt != nil { + if !deployment.ResetClear.Valid || !deployment.ResetRequestedAt.Time.Equal(*resetRequestedAt) { + return ErrSandboxDeploymentConflict + } + if deployment.ResetClear.String == "auto" { + busy, err := q.SessionBlocksAutoReset(ctx, session) + if err != nil { + return err + } + if busy { + return ErrSandboxResetSessionBusy + } + } + if environment.Environment.Status == "failed" || environment.Environment.Status == "expired" { + result, err = getManagedSessionArchive(ctx, q, tenant, session) + return err + } + source, err := sandboxResetAudit(deployment) + if err != nil { + return err + } + project, err := q.GetSandboxResetProject(ctx, tenant) + if err != nil { + return err + } + source.ProjectID = runtimeUUID(project) + ctx = adminaudit.WithSource(ctx, source) } allocation, err := q.GetRuntimeAllocation(ctx, sqlc.GetRuntimeAllocationParams{TenantID: tenant, EnvironmentID: environment.Environment.ID}) allocated := err == nil @@ -68,7 +116,7 @@ func (s *Store) ArchiveManagedSession(ctx context.Context, tenantID, sessionID s return err } if allocated && allocation.RuntimeAllocation.State != "released" { - if _, err := q.RevokeDevice(ctx, sqlc.RevokeDeviceParams{TenantID: tenant, ID: allocation.RuntimeAllocation.DeviceID}); err != nil { + if _, err := q.RevokeArchivedRuntimeDevice(ctx, sqlc.RevokeArchivedRuntimeDeviceParams{TenantID: tenant, DeviceID: allocation.RuntimeAllocation.DeviceID, SessionID: session}); err != nil { return err } if _, err := q.RequestRuntimeCleanup(ctx, allocation.RuntimeAllocation.ID); err != nil { diff --git a/services/agents-api/internal/store/admin_session_archive_race_test.go b/services/agents-api/internal/store/admin_session_archive_race_test.go index c0f610fdc..bb8915a12 100644 --- a/services/agents-api/internal/store/admin_session_archive_race_test.go +++ b/services/agents-api/internal/store/admin_session_archive_race_test.go @@ -30,7 +30,6 @@ func TestManagedSessionArchiveReleasesPendingNodePlacement(t *testing.T) { } onlineManagerNode(t, s, nodeID) tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) - archiveMaintenance(t, w, installation, true) result, err := w.ArchiveManagedSession(adminDeleteContext(t.Context(), tenant, uuid.NewString()), tenant, session.ID, 1) if err != nil || result.State != "released" { t.Fatal(result, err) @@ -45,11 +44,9 @@ func TestManagedSessionArchiveReleasesPendingNodePlacement(t *testing.T) { } func TestManagedSessionArchiveOrdersConcurrentInput(t *testing.T) { - s, w, installation := managedArchiveFixture(t) + s, w, _ := managedArchiveFixture(t) for range 8 { - archiveMaintenance(t, w, installation, false) tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) - archiveMaintenance(t, w, installation, true) start := make(chan struct{}) var wg sync.WaitGroup wg.Add(2) diff --git a/services/agents-api/internal/store/admin_session_archive_test.go b/services/agents-api/internal/store/admin_session_archive_test.go index 52e57baec..4e9908715 100644 --- a/services/agents-api/internal/store/admin_session_archive_test.go +++ b/services/agents-api/internal/store/admin_session_archive_test.go @@ -48,13 +48,6 @@ func managedArchiveSession(t *testing.T, s *Store, input CreateSessionInput) (st return tenant, session } -func archiveMaintenance(t *testing.T, w *Store, installation string, enabled bool) { - t.Helper() - if _, err := w.SetSandboxMaintenance(t.Context(), installation, SandboxMaintenanceRequest{Maintenance: enabled, ExpectedGeneration: 1}); err != nil { - t.Fatal(err) - } -} - func archiveAllocation(t *testing.T, w *Store, tenant string, session Session, installation string) RuntimeAllocation { t.Helper() owner, err := w.ReserveRuntimeAllocation(t.Context(), tenant, session.Environment.ID, installation, device.HashCredential(uuid.NewString())) @@ -74,10 +67,6 @@ func TestManagedSessionArchiveUnallocatedAndGuards(t *testing.T) { if err != nil || active.State != "active" || active.SessionID != session.ID || active.EnvironmentID != session.Environment.ID { t.Fatal("unallocated Session status", active, err) } - if _, err := w.ArchiveManagedSession(ctx, tenant, session.ID, 1); !errors.Is(err, ErrSandboxDeploymentConflict) { - t.Fatal("archive bypassed maintenance", err) - } - archiveMaintenance(t, w, installation, true) for _, generation := range []uint64{0, 2, ^uint64(0)} { if _, err := w.ArchiveManagedSession(ctx, tenant, session.ID, generation); !errors.Is(err, ErrSandboxDeploymentConflict) { t.Fatal("archive accepted wrong generation", generation, err) @@ -120,9 +109,8 @@ func TestManagedSessionArchiveUnallocatedAndGuards(t *testing.T) { if status, err := s.GetManagedSessionArchive(ctx, tenant, session.ID); err != nil || status != result { t.Fatal("status differs from committed archive", status, err) } - archiveMaintenance(t, w, installation, false) if _, err := w.ReserveRuntimeAllocation(t.Context(), tenant, session.Environment.ID, installation, device.HashCredential(uuid.NewString())); !errors.Is(err, ErrInvalidInput) { - t.Fatal("archived Environment allocated after maintenance", err) + t.Fatal("archived Environment allocated after archive", err) } if _, err := s.ReserveEnvironmentInput(t.Context(), tenant, session.ID, "later", []Input{{Kind: "message", Payload: json.RawMessage(`{"text":"later"}`)}}); !errors.Is(err, ErrEnvironmentUnavailable) { t.Fatal("archived Environment accepted new input", err) @@ -152,7 +140,6 @@ func TestManagedSessionArchiveRetainsHistoryAndSettledResources(t *testing.T) { } transition(t, w, tenant, session.ID, input.TurnID, TurnInProgress, TurnCompleted) history := adminMutationSnapshot(t, s, "sessions", "turns", "session_items", "session_artifacts", "source_files", "pg_largeobject", "pg_largeobject_metadata") - archiveMaintenance(t, w, installation, true) request := uuid.NewString() result, err := w.ArchiveManagedSession(adminDeleteContext(t.Context(), tenant, request), tenant, session.ID, 1) if err != nil || result.State != "cleanup_pending" { @@ -218,7 +205,6 @@ func TestManagedSessionArchiveAuditFailureRollsBack(t *testing.T) { archiveAllocation(t, w, tenant, session, installation) input := submitMessage(t, s, tenant, session.ID, "running") transition(t, w, tenant, session.ID, input.TurnID, TurnQueued, TurnInProgress) - archiveMaintenance(t, w, installation, true) rejectAdminAuditInsert(t, s) tables := []string{"sessions", "environments", "turns", "session_events", "devices", "runtime_allocations", "runtime_placements", "environment_input_reservations", "admin_audit_log"} before := adminMutationSnapshot(t, s, tables...) @@ -249,7 +235,6 @@ func TestManagedSessionArchivePreservesFailuresAndRejectsSelfHosted(t *testing.T t.Fatal("failure fixture", err) } otherTenant, selfHosted := managedArchiveSession(t, s, environmentInput(uuid.NewString(), "self_hosted", "/workspace")) - archiveMaintenance(t, w, installation, true) if _, err := w.ArchiveManagedSession(adminDeleteContext(t.Context(), tenant, uuid.NewString()), tenant, session.ID, 1); err != nil { t.Fatal(err) } diff --git a/services/agents-api/internal/store/admin_session_archive_worker_http_test.go b/services/agents-api/internal/store/admin_session_archive_worker_http_test.go index c84f659c2..ac25698d7 100644 --- a/services/agents-api/internal/store/admin_session_archive_worker_http_test.go +++ b/services/agents-api/internal/store/admin_session_archive_worker_http_test.go @@ -34,7 +34,7 @@ func TestAdminSessionArchiveWorkerHTTPPostgres(t *testing.T) { installation := uuid.NewString() provider := &lifecycleProvider{resources: map[string]sandbox.Info{}} providerConfig := func(setup store.SandboxSetup) *execution.RuntimeProvider { - return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, Maintenance: setup.Maintenance, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider} + return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider} } configuration := execution.NewDeferredRuntimeProvider(installation, func(ctx context.Context) (*execution.RuntimeProvider, error) { setup, err := s.GetSandboxSetup(ctx) @@ -85,9 +85,6 @@ func TestAdminSessionArchiveWorkerHTTPPostgres(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := worker.SetSandboxMaintenance(t.Context(), store.SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 1}); err != nil { - t.Fatal(err) - } if _, err := s.ArchiveManagedSession(ctx, project.TenantID, active.ID, 1); !errors.Is(err, store.ErrInvalidInput) { t.Fatal("fixture admission Store unexpectedly holds execution ownership", err) } diff --git a/services/agents-api/internal/store/archive_cancellation_test.go b/services/agents-api/internal/store/archive_cancellation_test.go new file mode 100644 index 000000000..ad4044573 --- /dev/null +++ b/services/agents-api/internal/store/archive_cancellation_test.go @@ -0,0 +1,318 @@ +package store_test + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/adminaudit" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/runtime" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" + "github.com/gorilla/websocket" +) + +// Controlled protocol fixtures, not native/model acceptance. Allocation setup +// uses the real execution lease; the real dispatcher consumes a function request +// through the real authenticated WebSocket and persists waiting before archive. +func TestArchiveWaitingCancellationReceipts(t *testing.T) { + for _, scenario := range []string{"receipt_without_heartbeat", "heartbeat_before_receipt", "done_heartbeat_ack", "ack_commit_blocked", "rotated", "expired", "transport_lost", "negative_ack", "missing_outcome", "revoke_before_archive", "cancel_revoke_archive", "revoke_after_archive", "revoke_concurrent_archive"} { + t.Run(scenario, func(t *testing.T) { + heartbeat := scenario != "receipt_without_heartbeat" + s, pool := store.NewManagedTestStore(t) + lease, err := s.AcquireExecutionLease(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := lease.Close(context.Background()); err != nil { + t.Error(err) + } + }) + writer := lease.Store() + installation := uuid.NewString() + if err := writer.ClaimWebSandboxDeployment(t.Context(), installation); err != nil { + t.Fatal(err) + } + if _, err := writer.InitializeSandboxDeployment(t.Context(), installation, store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("e2b"), Provider: "e2b", E2B: &store.SandboxE2BConfiguration{APIKey: "fixture", Template: "runtime:" + uuid.NewString()}}); err != nil { + t.Fatal(err) + } + projectID := uuid.NewString() + auditCtx := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", ProjectID: projectID, RequestID: uuid.NewString(), TraceID: uuid.NewString()}) + project, err := s.CreateProject(auditCtx, projectID, "Archive diagnosis") + if err != nil { + t.Fatal(err) + } + configuration := strings.Replace(functionConfiguration, `"type":"none"`, `"type":"openai_hosted","network":{"access":"disabled"}`, 1) + session, err := s.CreateSession(t.Context(), project.TenantID, store.WithFixtureModelProvider(store.CreateSessionInput{Creator: store.FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(configuration)})) + if err != nil { + t.Fatal(err) + } + secret := uuid.NewString() + owner, err := writer.ReserveRuntimeAllocation(t.Context(), project.TenantID, session.Environment.ID, installation, device.HashCredential(secret)) + if err != nil { + t.Fatal(err) + } + for _, step := range []func(context.Context, store.RuntimeAllocation) (store.RuntimeAllocation, error){writer.ObserveRuntimeRunning, writer.SettleRuntimeCreation} { + owner, err = step(t.Context(), owner) + if err != nil { + t.Fatal(err) + } + } + if err := writer.BindSessionDevice(t.Context(), project.TenantID, session.ID, owner.DeviceID); err != nil { + t.Fatal(err) + } + generation := uuid.NewString() + if err := writer.ReplaceEnvironmentConnection(t.Context(), project.TenantID, session.Environment.ID, generation); err != nil { + t.Fatal(err) + } + if err := writer.ObserveEnvironmentConnection(t.Context(), project.TenantID, session.Environment.ID, generation, 1, true); err != nil { + t.Fatal(err) + } + server := httptest.NewUnstartedServer(nil) + wsURL := "ws://" + server.Listener.Addr().String() + "/api/v1/agent-daemon/ws" + handler, registry, err := runtime.NewGateway(s, wsURL) + if err != nil { + t.Fatal(err) + } + server.Config.Handler = handler + server.Start() + t.Cleanup(func() { runtime.CloseConnections(registry); server.Close() }) + u, _ := url.Parse(wsURL) + u.RawQuery = url.Values{"device_id": {owner.DeviceID}, "version": {proto.Version}}.Encode() + conn, _, err := websocket.DefaultDialer.Dial(u.String(), http.Header{"Authorization": {"Bearer " + secret}}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + h := &dispatchHarness{t: t, s: s, tenant: project.TenantID, session: session, conn: conn, registry: registry, d: &execution.Dispatcher{Store: writer, Registry: registry}} + capabilities := workerEnvironmentCapabilities() + capabilities.FunctionTools = true + h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: capabilities}}}) + var peer *gateway.Session + for deadline := time.Now().Add(3 * time.Second); ; { + peer, err = registry.LookupDevice(owner.DeviceID) + if err == nil { + info, _, known := peer.AgentKindStatus("codex") + if known && info.Capabilities.FunctionTools { + break + } + } + if time.Now().After(deadline) { + t.Fatal("initial heartbeat missing") + } + time.Sleep(time.Millisecond) + } + pending, err := s.ReserveEnvironmentInput(t.Context(), h.tenant, session.ID, "pending", []store.Input{{Kind: "message", Payload: json.RawMessage(`{"text":"first"}`)}, {Kind: "message", Payload: json.RawMessage(`{"text":"second"}`)}}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + result := runPreparedDispatch(h, ctx, pending) + frame := h.read(proto.TypeExecutionPrepare) + handle := acknowledgePreparation(h, frame.ID) + start := readyPreparedDispatch(t, h, frame.ID, handle) + h.write(frame.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 3, State: "started", RunID: start.RunID}) + input := store.InputReceipt{TurnID: start.RunID} + h.write(input.TurnID, proto.TypeFunctionCall, proto.FunctionCallPayload{CallID: "pending", Name: "lookup_ticket", Arguments: json.RawMessage(`{"ticket":"42"}`)}) + state := functionState(t, h, 1) + if state.LastTurn.Status != store.TurnWaiting { + t.Fatal(state.LastTurn) + } + + if scenario == "cancel_revoke_archive" { + if _, err := s.RequestCancel(t.Context(), h.tenant, session.ID, "ordinary-cancel"); err != nil { + t.Fatal(err) + } + } + if scenario == "revoke_before_archive" || scenario == "cancel_revoke_archive" { + if err := s.RevokeDevice(t.Context(), h.tenant, owner.DeviceID); err != nil { + t.Fatal(err) + } + } + var revokeDone chan error + if scenario == "revoke_concurrent_archive" { + revokeDone = make(chan error, 1) + go func() { revokeDone <- s.RevokeDevice(t.Context(), h.tenant, owner.DeviceID) }() + } + + archived, err := writer.ArchiveManagedSession(auditCtx, h.tenant, session.ID, 1) + if err != nil || archived.State != "cleanup_pending" { + t.Fatal(archived, err) + } + if revokeDone != nil { + if err := <-revokeDone; err != nil { + t.Fatal(err) + } + } + if scenario == "revoke_after_archive" { + if err := s.RevokeDevice(t.Context(), h.tenant, owner.DeviceID); err != nil { + t.Fatal(err) + } + } + // A repeat archive and cleanup must not recreate an explicitly + // cleared marker, nor erase the marker from a fresh archive. + repeatAudit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", ProjectID: projectID, RequestID: uuid.NewString(), TraceID: uuid.NewString()}) + if _, err := writer.ArchiveManagedSession(repeatAudit, h.tenant, session.ID, 1); err != nil { + t.Fatal(err) + } + if _, err := writer.RequestRuntimeCleanup(t.Context(), owner); err != nil { + t.Fatal(err) + } + + current, err := s.GetTurn(t.Context(), h.tenant, session.ID, input.TurnID) + if err != nil || current.Status != store.TurnWaiting || current.CancelRequestedAt.IsZero() { + t.Fatal("archive must request rather than invent cancellation", current, err) + } + if _, err := gateway.NewAuthenticator(s).AuthenticateBearer(t.Context(), owner.DeviceID, secret); !errors.Is(err, gateway.ErrAuthUnknownDevice) { + t.Fatal("archive allowed renewed authority", err) + } + rejected, response, dialErr := websocket.DefaultDialer.Dial(u.String(), http.Header{"Authorization": {"Bearer " + secret}}) + if rejected != nil { + rejected.Close() + } + if response != nil { + response.Body.Close() + } + if dialErr == nil || response == nil || response.StatusCode != http.StatusUnauthorized { + t.Fatal("revoked Runtime reconnected") + } + drain, err := s.ArchivedCancellationReceipt(t.Context(), owner.DeviceID, secret, nil) + if err != nil || drain.RunID != "" { + t.Fatal("unowned delivery got receipt permission", drain, err) + } + drain, err = s.ArchivedCancellationReceipt(t.Context(), owner.DeviceID, device.HashCredential(secret), []string{input.TurnID}) + if err != nil || (drain.RunID == input.TurnID) == strings.Contains(scenario, "revoke") { + t.Fatal("archive revocation causality lost", drain, err) + } + if drain.RunID != "" && !drain.Deadline.Equal(current.CancelRequestedAt.Add(device.ArchivedCancellationReceiptLimit)) { + t.Fatal("archive renewed cancellation deadline") + } + // Explicitly observe cancel delivery before inducing transport loss. This + // proves even a sent cancellation can lose its receipt; no ticker timing guess. + var request proto.PromptCancelPayload + if err := h.read(proto.TypePromptCancel).DecodePayload(&request); err != nil { + t.Fatal(err) + } + if request.DeliveryID == "" { + t.Fatal("missing cancel delivery identity") + } + if scenario == "rotated" { + if _, err := pool.Exec(t.Context(), "UPDATE devices SET credential_hash=$2 WHERE id=$1", owner.DeviceID, device.HashCredential(uuid.NewString())); err != nil { + t.Fatal(err) + } + } + if scenario == "expired" { + if _, err := pool.Exec(t.Context(), "UPDATE turns SET cancel_requested_at=clock_timestamp()-interval '21 seconds' WHERE id=$1", input.TurnID); err != nil { + t.Fatal(err) + } + } + var unlockCommit func() + if scenario == "ack_commit_blocked" { + tx, err := pool.Begin(t.Context()) + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(t.Context(), "SELECT session_id FROM session_devices WHERE session_id=$1 FOR UPDATE", session.ID); err != nil { + t.Fatal(err) + } + unlockCommit = func() { + if err := tx.Rollback(context.Background()); err != nil { + t.Fatal(err) + } + } + defer func() { _ = tx.Rollback(context.Background()) }() + } + + if scenario == "done_heartbeat_ack" { + h.write(input.TurnID, proto.TypeDone, proto.DonePayload{}) + } + closedCase := scenario == "rotated" || scenario == "expired" || scenario == "transport_lost" || strings.Contains(scenario, "revoke") + if scenario == "transport_lost" { + h.conn.Close() + } else if heartbeat && scenario != "ack_commit_blocked" { + h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{}) + } + if !closedCase { + ack := proto.InteractionDecisionAckPayload{DeliveryID: request.DeliveryID, Applied: true, Outcome: &proto.DonePayload{Usage: proto.Usage{InputTokens: 17}, Metadata: map[string]any{proto.DoneMetaAgentSessionID: "cancelled-native"}}} + if scenario == "negative_ack" { + ack.Applied = false + ack.ErrorCode = "cancel_failed" + ack.Outcome = nil + } + if scenario == "missing_outcome" { + ack.Outcome = nil + } + h.write(input.TurnID, proto.TypeInteractionDecisionAck, ack) + } + if unlockCommit != nil { + // Observe actual SQL lock contention, not an assumed timing delay. + for deadline := time.Now().Add(3 * time.Second); ; { + var blocked bool + if err := pool.QueryRow(t.Context(), "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock' AND query ILIKE '%session_devices%')").Scan(&blocked); err != nil { + t.Fatal(err) + } + if blocked { + break + } + if time.Now().After(deadline) { + t.Fatal("terminal commit never blocked") + } + time.Sleep(time.Millisecond) + } + h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{}) + draining, err := peer.DrainArchivedCancellation(t.Context()) + if err != nil || !draining || peer.IsClosed() { + t.Fatal("ACK lost drain before terminal commit", draining, err) + } + unlockCommit() + } + + got := awaitPreparedDispatch(t, result) + wantStatus := store.TurnCancelled + if closedCase || scenario == "negative_ack" || scenario == "missing_outcome" { + wantStatus = store.TurnFailed + } + if got.err != nil || got.run.Turn.Status != wantStatus { + t.Fatal(got.run.Turn.Status, got.err, string(got.run.Turn.Outcome)) + } + var outcome execution.Result + if err := json.Unmarshal(got.run.Turn.Outcome, &outcome); err != nil || (wantStatus == store.TurnCancelled && outcome.Done.Usage.InputTokens != 17) { + t.Fatal("receipt lost usage", string(got.run.Turn.Outcome), err) + } + + var receipts int + if err := pool.QueryRow(t.Context(), "SELECT count(*) FROM turn_events WHERE turn_id=$1 AND kind='cancel_receipt'", input.TurnID).Scan(&receipts); err != nil { + t.Fatal(err) + } + wantReceipts := 1 + if closedCase { + wantReceipts = 0 + } + if receipts != wantReceipts { + t.Fatal("durable cancellation receipt mismatch", receipts, wantReceipts) + } + // The original cleanup owner survives every delivery outcome; only + // provider receipts can release its resources. + allocation, err := s.GetRuntimeAllocation(t.Context(), h.tenant, session.Environment.ID) + if err != nil || allocation.State != "cleanup_pending" { + t.Fatal(allocation, err) + } + var revoked bool + if err := pool.QueryRow(t.Context(), "SELECT revoked_at IS NOT NULL FROM devices WHERE id=$1", owner.DeviceID).Scan(&revoked); err != nil || !revoked { + t.Fatal(revoked, err) + } + }) + } +} diff --git a/services/agents-api/internal/store/archived_cancellation_migration_test.go b/services/agents-api/internal/store/archived_cancellation_migration_test.go new file mode 100644 index 000000000..dbf72c05e --- /dev/null +++ b/services/agents-api/internal/store/archived_cancellation_migration_test.go @@ -0,0 +1,48 @@ +package store + +import ( + "database/sql" + "os" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" +) + +func TestArchivedCancellationMigrationDoesNotAdoptOldRevocations(t *testing.T) { + s, w, installation := managedArchiveFixture(t) + tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + owner := archiveAllocation(t, w, tenant, session, installation) + input := submitMessage(t, s, tenant, session.ID, "waiting") + transition(t, w, tenant, session.ID, input.TurnID, TurnQueued, TurnInProgress) + transition(t, w, tenant, session.ID, input.TurnID, TurnInProgress, TurnWaiting) + if _, err := w.ArchiveManagedSession(adminDeleteContext(t.Context(), tenant, uuid.NewString()), tenant, session.ID, 1); err != nil { + t.Fatal(err) + } + db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) + defer db.Close() + provider, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) + if err != nil { + t.Fatal(err) + } + if _, err := provider.DownTo(t.Context(), 79); err == nil { + t.Fatal("downgrade discarded unsettled cancellation marker") + } + if _, err := w.SettleRuntimeCreation(t.Context(), owner); err != nil { + t.Fatal(err) + } + if _, err := w.ReleaseRuntimeAllocation(t.Context(), owner); err != nil { + t.Fatal(err) + } + if _, err := provider.DownTo(t.Context(), 79); err != nil { + t.Fatal(err) + } + if _, err := provider.Up(t.Context()); err != nil { + t.Fatal(err) + } + var marker *string + if err := s.pool.QueryRow(t.Context(), "SELECT archive_cancel_turn_id::text FROM devices WHERE id=$1", owner.DeviceID).Scan(&marker); err != nil || marker != nil { + t.Fatal("upgrade adopted historical revoked device", marker, err) + } +} diff --git a/services/agents-api/internal/store/credential_matrix_http_test.go b/services/agents-api/internal/store/credential_matrix_http_test.go index 37583d57d..3589053a5 100644 --- a/services/agents-api/internal/store/credential_matrix_http_test.go +++ b/services/agents-api/internal/store/credential_matrix_http_test.go @@ -96,7 +96,7 @@ func TestCredentialNamespaceMatrix(t *testing.T) { } return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, BackendFingerprint: setup.BackendFingerprint, CoreURL: "https://core.example/api/v1", Provider: provider}, nil }, func(_ context.Context, setup store.SandboxSetup) (execution.PreparedRuntimeDeployment, error) { - return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, Maintenance: setup.Maintenance, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider}}, nil + return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider}}, nil }) worker, err := execution.StartWorker(ctx, &execution.Dispatcher{Store: s, Registry: gateway.NewRegistry(), ManagedRuntimes: runtimes}) if err != nil { diff --git a/services/agents-api/internal/store/execution_cancellation_test.go b/services/agents-api/internal/store/execution_cancellation_test.go new file mode 100644 index 000000000..695bfb59c --- /dev/null +++ b/services/agents-api/internal/store/execution_cancellation_test.go @@ -0,0 +1,87 @@ +package store + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestExecutionCancellationFenceHonorsBounds(t *testing.T) { + s, _ := testStore(t) + lease := executionLease(t, s) + for _, test := range []struct { + name string + limit, maximum time.Duration + }{ + {"caller", 25 * time.Millisecond, time.Second}, + {"operation", 8 * time.Second, 6 * time.Second}, + } { + t.Run(test.name, func(t *testing.T) { + if err := lease.lock(t.Context()); err != nil { + t.Fatal(err) + } + held := true + defer func() { + if held { + lease.unlock() + } + }() + operation, stopOperation := context.WithCancel(t.Context()) + defer stopOperation() + ctx, cancel := context.WithTimeout(t.Context(), test.limit) + defer cancel() + started := time.Now() + err := lease.Store().CancelExecutionOperations(ctx, stopOperation) + if !errors.Is(err, context.DeadlineExceeded) || time.Since(started) > test.maximum { + t.Fatal("cancellation fence did not preserve its deadline", err, time.Since(started)) + } + if operation.Err() != nil { + t.Fatal("timed-out fence canceled operations outside the lease gate") + } + lease.unlock() + held = false + if err := lease.Store().CheckExecutionOwnership(t.Context()); err != nil { + t.Fatal("gate timeout damaged the healthy owner", err) + } + if err := lease.Store().CancelExecutionOperations(t.Context(), stopOperation); err != nil { + t.Fatal(err) + } + if !errors.Is(operation.Err(), context.Canceled) { + t.Fatal("successful fence did not cancel synchronously") + } + if err := lease.Store().CheckExecutionOwnership(t.Context()); err != nil { + t.Fatal("cancellation damaged the healthy owner", err) + } + }) + } +} + +func TestExecutionCancellationFenceRejectsLostLease(t *testing.T) { + s, pool := testStore(t) + lease := executionLease(t, s) + var killed bool + if err := pool.QueryRow(t.Context(), "SELECT pg_terminate_backend($1, 1000)", lease.conn.Conn().PgConn().PID()).Scan(&killed); err != nil || !killed { + t.Fatal(killed, err) + } + operation, cancel := context.WithCancel(t.Context()) + defer cancel() + if err := lease.Store().CancelExecutionOperations(t.Context(), cancel); err == nil { + t.Fatal("lost lease accepted cancellation fence") + } + if operation.Err() != nil { + t.Fatal("lost lease ran the cancellation callback") + } + if err := lease.Store().CheckExecutionOwnership(t.Context()); err == nil { + t.Fatal("lost owner became writable") + } + if err := lease.Close(t.Context()); err != nil { + t.Fatal(err) + } + if err := lease.Store().CancelExecutionOperations(t.Context(), cancel); err == nil { + t.Fatal("closed lease accepted cancellation fence") + } + if err := s.CancelExecutionOperations(t.Context(), cancel); err == nil { + t.Fatal("pooled Store accepted execution cancellation") + } +} diff --git a/services/agents-api/internal/store/execution_lease.go b/services/agents-api/internal/store/execution_lease.go index 0dd91c559..f7d502c8c 100644 --- a/services/agents-api/internal/store/execution_lease.go +++ b/services/agents-api/internal/store/execution_lease.go @@ -56,6 +56,26 @@ func (s *Store) CheckExecutionOwnership(ctx context.Context) error { return s.executionLease.Ping(ctx) } +// CancelExecutionOperations cancels coordinator-owned contexts between leased +// operations. Canceling an in-flight pgx operation can close the connection that +// owns the execution advisory lock. The callback must only invoke synchronous +// context cancel functions; it must not perform database, provider or wait work. +// Caller cancellation and operation deadlines keep their existing semantics. +func (s *Store) CancelExecutionOperations(ctx context.Context, cancelOperations context.CancelFunc) error { + if s.executionLease == nil || cancelOperations == nil { + return ErrInvalidInput + } + ctx, cancel := context.WithTimeout(ctx, executionTransactionTimeout) + defer cancel() + return s.executionLease.withConn(ctx, func(conn *pgxpool.Conn) error { + if err := conn.Ping(ctx); err != nil { + return err + } + cancelOperations() + return nil + }) +} + func (l *ExecutionLease) lock(ctx context.Context) error { select { case l.gate <- struct{}{}: diff --git a/services/agents-api/internal/store/runtime_allocation_state.go b/services/agents-api/internal/store/runtime_allocation_state.go index 50535992a..10d7e390a 100644 --- a/services/agents-api/internal/store/runtime_allocation_state.go +++ b/services/agents-api/internal/store/runtime_allocation_state.go @@ -63,7 +63,7 @@ func (s *Store) requestRuntimeCleanup(ctx context.Context, owner RuntimeAllocati return row, nil } tenant, _ := parseID(owner.TenantID) - if _, err := q.RevokeDevice(ctx, sqlc.RevokeDeviceParams{TenantID: tenant, ID: row.DeviceID}); err != nil { + if _, err := q.RevokeRuntimeCleanupDevice(ctx, sqlc.RevokeRuntimeCleanupDeviceParams{TenantID: tenant, DeviceID: row.DeviceID}); err != nil { return sqlc.RuntimeAllocation{}, err } current, err := q.GetRuntimeAllocation(ctx, sqlc.GetRuntimeAllocationParams{TenantID: tenant, EnvironmentID: row.EnvironmentID}) diff --git a/services/agents-api/internal/store/runtime_cancellation.go b/services/agents-api/internal/store/runtime_cancellation.go new file mode 100644 index 000000000..7a64eb08a --- /dev/null +++ b/services/agents-api/internal/store/runtime_cancellation.go @@ -0,0 +1,35 @@ +package store + +import ( + "context" + "errors" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +// ArchivedCancellationReceipt is a read-only exception for the exact already +// authenticated delivery. The ordinary credential view remains revoked; this +// cannot authorize bootstrap, reconnect, dispatch, workspace access or renewal. +// A marker records that archive caused the first revocation; timestamps alone +// cannot distinguish an earlier ordinary cancel/revoke followed by archive. +func (s *Store) ArchivedCancellationReceipt(ctx context.Context, deviceID, credentialHash string, runIDs []string) (device.ArchivedCancellationReceipt, error) { + if len(runIDs) == 0 || credentialHash == "" { + return device.ArchivedCancellationReceipt{}, nil + } + id, err := parseID(deviceID) + if err != nil { + return device.ArchivedCancellationReceipt{}, err + } + row, err := s.queries.GetArchivedCancellationReceipt(ctx, sqlc.GetArchivedCancellationReceiptParams{DeviceID: id, CredentialHash: pgtype.Text{String: credentialHash, Valid: true}, RunIds: runIDs, LimitSeconds: int32(device.ArchivedCancellationReceiptLimit.Seconds())}) + if errors.Is(err, pgx.ErrNoRows) { + return device.ArchivedCancellationReceipt{}, nil + } + if err != nil { + return device.ArchivedCancellationReceipt{}, err + } + return device.ArchivedCancellationReceipt{RunID: uuid.UUID(row.ID.Bytes).String(), Deadline: row.CancelRequestedAt.Time.Add(device.ArchivedCancellationReceiptLimit)}, nil +} diff --git a/services/agents-api/internal/store/runtime_deployment.go b/services/agents-api/internal/store/runtime_deployment.go index a2bf62c5a..84e69952a 100644 --- a/services/agents-api/internal/store/runtime_deployment.go +++ b/services/agents-api/internal/store/runtime_deployment.go @@ -20,11 +20,11 @@ type RuntimeDeployment struct { LocalMaxActive, LocalMaxRetained int InstallationID string BackendFingerprint string - Maintenance bool + AdmissionPaused bool } // ConfigureRuntimeDeployment runs before Worker startup under its execution lease. -// Maintenance must be committed for the old installation before any switch. +// AdmissionPaused must be committed for the old installation before any switch. // A nil selection never forgets the previous identity or unresolved resources. func (s *Store) ConfigureRuntimeDeployment(ctx context.Context, selected *RuntimeDeployment, verify RuntimeOwnershipVerifier) error { if s.executionLease == nil { @@ -44,7 +44,7 @@ func (s *Store) ConfigureRuntimeDeployment(ctx context.Context, selected *Runtim if err != nil || len(digest) != 32 || strings.ToLower(selected.BackendFingerprint) != selected.BackendFingerprint { return fmt.Errorf("%w: invalid backend identity fingerprint", ErrInvalidInput) } - update = sqlc.SetRuntimeDeploymentParams{InstallationID: id, BackendFingerprint: selected.BackendFingerprint, Maintenance: selected.Maintenance} + update = sqlc.SetRuntimeDeploymentParams{InstallationID: id, BackendFingerprint: selected.BackendFingerprint, AdmissionPaused: selected.AdmissionPaused} } plan, err := s.verifyLegacyRuntimeAdoption(ctx, selected, verify) if err != nil { @@ -82,7 +82,7 @@ func (s *Store) ConfigureRuntimeDeployment(ctx context.Context, selected *Runtim return fmt.Errorf("cannot adopt sandbox installation: %d existing unreleased allocations (including retained snapshots and pending cleanup) have no verified backend identity", resources.Allocations) } } else { - if !previous.Maintenance || !selected.Maintenance { + if !previous.AdmissionPaused || !selected.AdmissionPaused { return fmt.Errorf("cannot switch sandbox installation: persist maintenance on the previous installation and keep the new installation in maintenance") } if resources.Allocations != 0 || resources.Pending != 0 { @@ -106,7 +106,10 @@ func checkRuntimeDeploymentAdmission(ctx context.Context, q *sqlc.Queries, insta if !current.InstallationID.Valid { return nil } - if current.Maintenance { + if current.ResetClear.Valid { + return ErrSandboxResetAdmission + } + if current.AdmissionPaused { return fmt.Errorf("%w: sandbox creation is paused for provider maintenance", ErrEnvironmentUnavailable) } if unspecifiedNodeDeployment(current) { diff --git a/services/agents-api/internal/store/runtime_deployment_test.go b/services/agents-api/internal/store/runtime_deployment_test.go index 276eea333..b09fe85a2 100644 --- a/services/agents-api/internal/store/runtime_deployment_test.go +++ b/services/agents-api/internal/store/runtime_deployment_test.go @@ -56,32 +56,32 @@ func TestRuntimeDeploymentRequiresMaintenanceBeforeIdentityChange(t *testing.T) } else { next.BackendFingerprint = strings.Repeat("b", 64) } - next.Maintenance = true + next.AdmissionPaused = true if err := w.ConfigureRuntimeDeployment(t.Context(), &next, nil); err == nil || !strings.Contains(err.Error(), "maintenance") { t.Fatal("identity changed before prior maintenance", err) } } - old.Maintenance = true + old.AdmissionPaused = true deploymentConfigure(t, w, &old) next := old next.BackendFingerprint = strings.Repeat("b", 64) - next.Maintenance = false + next.AdmissionPaused = false if err := w.ConfigureRuntimeDeployment(t.Context(), &next, nil); err == nil { t.Fatal("switch reopened creation in same operation") } - next.Maintenance = true + next.AdmissionPaused = true deploymentConfigure(t, w, &next) var id, fingerprint string var maintenance bool - if err := pool.QueryRow(t.Context(), "SELECT installation_id::text,backend_fingerprint,maintenance FROM runtime_deployment").Scan(&id, &fingerprint, &maintenance); err != nil || id != next.InstallationID || fingerprint != next.BackendFingerprint || !maintenance { + if err := pool.QueryRow(t.Context(), "SELECT installation_id::text,backend_fingerprint,admission_paused FROM runtime_deployment").Scan(&id, &fingerprint, &maintenance); err != nil || id != next.InstallationID || fingerprint != next.BackendFingerprint || !maintenance { t.Fatal("switch identity not durable", id, fingerprint, maintenance, err) } deploymentConfigure(t, w, nil) // Disabling the configured adapter must not forget the old maintenance state. - next.Maintenance = false + next.AdmissionPaused = false deploymentConfigure(t, w, &next) another := deploymentSelection() - another.Maintenance = true + another.AdmissionPaused = true if err := w.ConfigureRuntimeDeployment(t.Context(), &another, nil); err == nil { t.Fatal("nil selection erased the maintenance prerequisite") } @@ -95,10 +95,10 @@ func TestRuntimeDeploymentPendingSessionsCannotMigrate(t *testing.T) { old := deploymentSelection() // First selection is allowed for work that has never had an installation. deploymentConfigure(t, w, &old) - old.Maintenance = true + old.AdmissionPaused = true deploymentConfigure(t, w, &old) next := deploymentSelection() - next.Maintenance = true + next.AdmissionPaused = true if err := w.ConfigureRuntimeDeployment(t.Context(), &next, nil); err == nil || !strings.Contains(err.Error(), "1 pending hosted") { t.Fatal("pending Session migrated", err) } @@ -148,10 +148,10 @@ func TestRuntimeDeploymentUnknownAllocationsBlockAdoptionAndSwitch(t *testing.T) if err != nil { t.Fatal(err) } - old.Maintenance = true + old.AdmissionPaused = true deploymentConfigure(t, w, &old) next := deploymentSelection() - next.Maintenance = true + next.AdmissionPaused = true if err := w.ConfigureRuntimeDeployment(t.Context(), &next, nil); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { t.Fatal("unknown creation did not block switch", err) } @@ -178,7 +178,7 @@ func TestRuntimeDeploymentMaintenancePreservesCreationRetriesAndOtherPlacements( if err != nil { t.Fatal(err) } - old.Maintenance = true + old.AdmissionPaused = true deploymentConfigure(t, w, &old) replay, err := s.CreateSession(t.Context(), tenant, input) if err != nil || replay.ID != existing.ID { @@ -202,7 +202,7 @@ func TestRuntimeDeploymentMaintenancePreservesCreationRetriesAndOtherPlacements( t.Fatal("maintenance blocked unrelated placement", kind, err) } } - old.Maintenance = false + old.AdmissionPaused = false deploymentConfigure(t, w, &old) if _, err := w.ReserveRuntimeAllocation(t.Context(), tenant, existing.Environment.ID, uuid.NewString(), device.HashCredential(uuid.NewString())); !errors.Is(err, ErrEnvironmentUnavailable) { t.Fatal("wrong installation reserved resource", err) @@ -235,7 +235,7 @@ func TestRuntimeDeploymentMaintenanceSerializesHostedCreation(t *testing.T) { done <- err }() runtimeSuspensionWaitBlocked(t, ctx, pool, blocker, done) - if _, err := tx.Exec(ctx, "UPDATE runtime_deployment SET maintenance=true"); err != nil { + if _, err := tx.Exec(ctx, "UPDATE runtime_deployment SET admission_paused=true"); err != nil { t.Fatal(err) } if err := tx.Commit(ctx); err != nil { @@ -271,7 +271,7 @@ func TestRuntimeDeploymentRetainedResourcesBlockSwitchWithoutMutation(t *testing if err != nil { t.Fatal(err) } - old.Maintenance = true + old.AdmissionPaused = true deploymentConfigure(t, w, &old) var before, after, oldIdentity, newIdentity string if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(a)::text FROM runtime_allocations a WHERE id=$1", owner.ID).Scan(&before); err != nil { @@ -281,7 +281,7 @@ func TestRuntimeDeploymentRetainedResourcesBlockSwitchWithoutMutation(t *testing t.Fatal(err) } next := deploymentSelection() - next.Maintenance = true + next.AdmissionPaused = true if err := w.ConfigureRuntimeDeployment(t.Context(), &next, nil); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { t.Fatal("retained resource allowed switch", state, err) } @@ -324,7 +324,7 @@ func TestRuntimeDeploymentAllocationBeforeMaintenanceRetainsOwnership(t *testing runtimeSuspensionWaitBlocked(t, ctx, pool, blocker, allocated) maintaining := make(chan error, 1) maintenance := config - maintenance.Maintenance = true + maintenance.AdmissionPaused = true go func() { maintaining <- w.ConfigureRuntimeDeployment(ctx, &maintenance, nil) }() if err := tx.Commit(ctx); err != nil { t.Fatal(err) @@ -340,7 +340,7 @@ func TestRuntimeDeploymentAllocationBeforeMaintenanceRetainsOwnership(t *testing t.Fatal("maintenance changed uncertain receipt", owner, err) } next := deploymentSelection() - next.Maintenance = true + next.AdmissionPaused = true if err := w.ConfigureRuntimeDeployment(ctx, &next, nil); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { t.Fatal("earlier in-flight allocation omitted from switch guard", err) } diff --git a/services/agents-api/internal/store/runtime_deployment_worker_test.go b/services/agents-api/internal/store/runtime_deployment_worker_test.go index e50ccc67c..c8ac3c352 100644 --- a/services/agents-api/internal/store/runtime_deployment_worker_test.go +++ b/services/agents-api/internal/store/runtime_deployment_worker_test.go @@ -23,7 +23,7 @@ func TestManagedDeploymentStartupRejectsSwitchBeforeBackendAccess(t *testing.T) } stop() replacement := &lifecycleProvider{resources: map[string]sandbox.Info{}} - config := &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("b", 64), Provider: replacement, Maintenance: true} + config := &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("b", 64), Provider: replacement, AdmissionPaused: true} start := func(config *execution.RuntimeProvider) error { _, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: s, Registry: gateway.NewRegistry(), ManagedRuntimes: config}) return err diff --git a/services/agents-api/internal/store/runtime_lifecycle_test.go b/services/agents-api/internal/store/runtime_lifecycle_test.go index 1134fa4b1..58da0521d 100644 --- a/services/agents-api/internal/store/runtime_lifecycle_test.go +++ b/services/agents-api/internal/store/runtime_lifecycle_test.go @@ -75,7 +75,7 @@ func managedWorker(t *testing.T, s *store.Store, key string, p sandbox.Provider) func managedWorkerMode(t *testing.T, s *store.Store, key string, p sandbox.Provider, maintenance bool) (*execution.Worker, func()) { t.Helper() - w, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: s, Registry: gateway.NewRegistry(), ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: key, BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Provider: p, Maintenance: maintenance}}) + w, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: s, Registry: gateway.NewRegistry(), ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: key, BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Provider: p, AdmissionPaused: maintenance}}) if err != nil { t.Fatal(err) } diff --git a/services/agents-api/internal/store/runtime_node_capacity_test.go b/services/agents-api/internal/store/runtime_node_capacity_test.go index 38b109cc8..3434ed0e6 100644 --- a/services/agents-api/internal/store/runtime_node_capacity_test.go +++ b/services/agents-api/internal/store/runtime_node_capacity_test.go @@ -9,7 +9,7 @@ import ( ) func TestRuntimeEnrollmentApprovedCapacity(t *testing.T) { - s, _, view, _ := webSpecificationFixture(t, "microsandbox") + s, w, view, _ := webSpecificationFixture(t, "microsandbox") for _, capacity := range []RuntimeNodeCapacity{{0, 8}, {3, 2}, {1, 1000001}} { if _, err := s.CreateRuntimeEnrollment(t.Context(), capacity); !errors.Is(err, ErrInvalidInput) { t.Fatal("invalid capacity accepted", err) @@ -50,10 +50,14 @@ func TestRuntimeEnrollmentApprovedCapacity(t *testing.T) { if err != nil || config.MaxActive != 2 || config.MaxRetained != 5 { t.Fatal("configuration read ignored admin update", config, err) } - // Invalid capacity is reported before the deployment's maintenance conflict. - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_deployment SET maintenance=true"); err != nil { + // Invalid capacity is reported before the active reset conflict. Enter reset + // through its owner transaction so the fixture obeys the admission invariant. + if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxResetRequest{ExpectedGeneration: view.Generation, Clear: "auto"}); err != nil { t.Fatal(err) } + if _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{1, 3}); !errors.Is(err, ErrSandboxResetInProgress) { + t.Fatal("valid capacity did not reach the reset guard", err) + } if _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{3, 2}); !errors.Is(err, ErrInvalidInput) { t.Fatal("invalid capacity reported as a conflict", err) } diff --git a/services/agents-api/internal/store/runtime_node_configuration.go b/services/agents-api/internal/store/runtime_node_configuration.go index 306dbb2f7..d864a17f6 100644 --- a/services/agents-api/internal/store/runtime_node_configuration.go +++ b/services/agents-api/internal/store/runtime_node_configuration.go @@ -54,6 +54,9 @@ func (s *Store) RuntimeNodeConfiguration(ctx context.Context, nodeID, token stri if !runtimeDeploymentInitialized(d) { return ErrRuntimeNodeUnavailable } + if node == nil && d.ResetClear.Valid { + return ErrSandboxResetInProgress + } if d.Mode != "nodes" { return ErrSandboxDeploymentConflict } @@ -61,7 +64,7 @@ func (s *Store) RuntimeNodeConfiguration(ctx context.Context, nodeID, token stri if err != nil { return err } - if node == nil && d.Maintenance { + if node == nil && d.AdmissionPaused { return ErrSandboxDeploymentConflict } if node != nil && (node.DeploymentGeneration != d.Generation || node.SpecificationDigest != spec.Digest(d.ProviderKind)) { diff --git a/services/agents-api/internal/store/runtime_node_deployment.go b/services/agents-api/internal/store/runtime_node_deployment.go index 9c5446b81..6eb8b275b 100644 --- a/services/agents-api/internal/store/runtime_node_deployment.go +++ b/services/agents-api/internal/store/runtime_node_deployment.go @@ -48,7 +48,7 @@ func configureRuntimeManager(ctx context.Context, q *sqlc.Queries, previous sqlc if err != nil { return err } - if resources.Allocations != 0 || resources.Pending != 0 || !previous.Maintenance || !selected.Maintenance { + if resources.Allocations != 0 || resources.Pending != 0 || !previous.AdmissionPaused || !selected.AdmissionPaused { return fmt.Errorf("local sandbox node identity changed: restore its original state directory; replacement requires maintenance and no retained resources") } } diff --git a/services/agents-api/internal/store/runtime_node_types.go b/services/agents-api/internal/store/runtime_node_types.go index b78c971c0..83decebc7 100644 --- a/services/agents-api/internal/store/runtime_node_types.go +++ b/services/agents-api/internal/store/runtime_node_types.go @@ -136,7 +136,7 @@ type RuntimeDeploymentView struct { Suspension *SandboxSuspensionView `json:"suspension" extensions:"x-nullable"` InstallationID string `json:"installation_id"` Provider string `json:"provider"` - Maintenance bool `json:"maintenance"` + Reset *SandboxResetView `json:"reset" extensions:"x-nullable"` OwnerEpoch uint64 `json:"owner_epoch"` // Read-only: the installation public URL (OAC_PUBLIC_URL), which nodes and sandboxes use to reach Core. The deployment API does not accept it. CoreURL string `json:"core_url"` diff --git a/services/agents-api/internal/store/runtime_nodes.go b/services/agents-api/internal/store/runtime_nodes.go index 99549bea2..d5bd9b3af 100644 --- a/services/agents-api/internal/store/runtime_nodes.go +++ b/services/agents-api/internal/store/runtime_nodes.go @@ -71,18 +71,24 @@ func (s *Store) GetRuntimeDeployment(ctx context.Context) (RuntimeDeploymentView // deploymentView reports the public URL as the deployment's read-only core_url. func (s *Store) deploymentView(ctx context.Context, q *sqlc.Queries) (RuntimeDeploymentView, error) { - d, err := q.GetRuntimeDeployment(ctx) - if err != nil { - return RuntimeDeploymentView{}, err - } - resources, err := q.CountRuntimeDeploymentResources(ctx) + row, err := q.GetSandboxDeploymentSnapshot(ctx) if err != nil { return RuntimeDeploymentView{}, err } + d := row.RuntimeDeployment result := runtimeDeploymentView(d, s.publicURL) - result.Resources = SandboxDeploymentResources{Allocations: resources.Allocations, Pending: resources.Pending} + result.Resources = SandboxDeploymentResources{Allocations: row.Allocations, Pending: row.Pending} + if d.ResetClear.Valid { + remaining := SandboxResetRemaining{} + if err := json.Unmarshal(row.Remaining, &remaining); err != nil { + return RuntimeDeploymentView{}, err + } + result.Reset = &SandboxResetView{Clear: d.ResetClear.String, RequestedAt: d.ResetRequestedAt.Time, + DeadlineAt: resetTimestamp(d.ResetDeadlineAt), ForcedAt: resetTimestamp(d.ResetForcedAt), Remaining: remaining} + } return result, nil } + func (s *Store) RuntimeOwnerEpoch(ctx context.Context) (uint64, error) { d, err := s.queries.GetRuntimeDeployment(ctx) return uint64(d.OwnerEpoch), err @@ -145,7 +151,10 @@ func (s *Store) CreateRuntimeEnrollment(ctx context.Context, capacity RuntimeNod if err := validateRuntimeNode("enrollment", capacity.MaxActive, capacity.MaxRetained); err != nil { return err } - if d.Mode != "nodes" || d.Maintenance || unspecifiedNodeDeployment(d) { + if d.ResetClear.Valid { + return ErrSandboxResetInProgress + } + if d.Mode != "nodes" || d.AdmissionPaused || unspecifiedNodeDeployment(d) { return ErrSandboxDeploymentConflict } if err := q.CreateRuntimeEnrollment(ctx, sqlc.CreateRuntimeEnrollmentParams{ID: pgtype.UUID{Bytes: id, Valid: true}, TokenSha256: runtimeTokenDigest(result.Token), InstallationID: d.InstallationID, MaxActive: int32(capacity.MaxActive), MaxRetained: int32(capacity.MaxRetained)}); err != nil { @@ -183,7 +192,10 @@ func (s *Store) EnrollRuntimeNode(ctx context.Context, token string, input Runti if !runtimeDeploymentInitialized(d) { return ErrRuntimeNodeUnavailable } - if d.Mode != "nodes" || d.Maintenance || input.Provider != d.ProviderKind { + if d.ResetClear.Valid { + return ErrSandboxResetInProgress + } + if d.Mode != "nodes" || d.AdmissionPaused || input.Provider != d.ProviderKind { return ErrInvalidInput } spec, err := deploymentSpecification(d) diff --git a/services/agents-api/internal/store/runtime_nodes_test.go b/services/agents-api/internal/store/runtime_nodes_test.go index a844bdb9e..bf1ef1f22 100644 --- a/services/agents-api/internal/store/runtime_nodes_test.go +++ b/services/agents-api/internal/store/runtime_nodes_test.go @@ -271,7 +271,7 @@ func TestRuntimeNodesLegacyAdoptionAndRetention(t *testing.T) { if _, err := s.AuthenticateRuntimeNode(t.Context(), next.LocalNodeID, "node"); err != nil { t.Fatal("rejected removal changed local credentials", err) } - next.Maintenance = true + next.AdmissionPaused = true deploymentConfigure(t, w, &next) detached := next detached.LocalNodeID = "" diff --git a/services/agents-api/internal/store/runtime_placements.go b/services/agents-api/internal/store/runtime_placements.go index 3ae225e96..fa52ffe46 100644 --- a/services/agents-api/internal/store/runtime_placements.go +++ b/services/agents-api/internal/store/runtime_placements.go @@ -15,8 +15,11 @@ func reserveRuntimePlacement(ctx context.Context, q *sqlc.Queries, session pgtyp if err != nil { return err } + if d.ResetClear.Valid { + return ErrSandboxResetAdmission + } if d.Mode == "direct" { - if d.Maintenance { + if d.AdmissionPaused { return ErrRuntimeNodeUnavailable } // E2B guests reach Core over the internet. A selection saved before the @@ -33,7 +36,7 @@ func reserveRuntimePlacement(ctx context.Context, q *sqlc.Queries, session pgtyp } return nil } - if d.Maintenance { + if d.AdmissionPaused { return ErrRuntimeNodeUnavailable } rows, err := q.ListRuntimeNodes(ctx, pgtype.UUID{}) diff --git a/services/agents-api/internal/store/sandbox_deployment_mutations.go b/services/agents-api/internal/store/sandbox_deployment_mutations.go index e9d8e8985..9ddb287be 100644 --- a/services/agents-api/internal/store/sandbox_deployment_mutations.go +++ b/services/agents-api/internal/store/sandbox_deployment_mutations.go @@ -38,10 +38,6 @@ type SandboxDeploymentUpdateRequest struct { SandboxDeploymentSetupRequest ExpectedGeneration uint64 `json:"expected_generation"` } -type SandboxMaintenanceRequest struct { - Maintenance bool `json:"maintenance"` - ExpectedGeneration uint64 `json:"expected_generation"` -} // E2BResourcesPending reports an E2B selection that omitted resources. Core // fills them from the validated template build before any write. @@ -183,7 +179,13 @@ func (s *Store) InitializeSandboxDeployment(ctx context.Context, installationID if err != nil { return err } - if !d.WebManaged || d.InstallationID != id { + if err := checkSandboxGeneration(d, installationID, input.ExpectedGeneration); err != nil { + return err + } + if d.ResetClear.Valid { + return ErrSandboxResetInProgress + } + if d.InstallationID != id { return ErrSandboxDeploymentConflict } if d.ProviderKind != "" { @@ -227,15 +229,24 @@ func (s *Store) CheckSandboxDeploymentSwitch(ctx context.Context, installation s }) } func checkSandboxSwitch(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment, installation string, input SandboxDeploymentUpdateRequest) error { - if !d.WebManaged || runtimeUUID(d.InstallationID) != installation || d.ProviderKind == "" || !d.Maintenance || uint64(d.Generation) != input.ExpectedGeneration { - return ErrSandboxDeploymentConflict + if err := checkSandboxGeneration(d, installation, input.ExpectedGeneration); err != nil { + return err + } + if d.ResetClear.Valid { + return ErrSandboxResetInProgress + } + if d.ProviderKind == "" { + return ErrSandboxNotConfigured + } + if d.ProviderKind != input.Provider || unspecifiedNodeDeployment(d) { + return &SandboxResetRequiredError{CurrentProvider: d.ProviderKind, RequestedProvider: input.Provider} } resources, err := q.CountRuntimeDeploymentResources(ctx) if err != nil { return err } if resources.Allocations != 0 || resources.Pending != 0 { - return ErrSandboxDeploymentConflict + return &SandboxInUseError{SandboxDeploymentResources{Allocations: resources.Allocations, Pending: resources.Pending}} } return nil } @@ -285,28 +296,22 @@ func (s *Store) UpdateSandboxDeployment(ctx context.Context, installation string return result, err } -// The Worker verifies activation before calling this with maintenance=false. -func (s *Store) SetSandboxMaintenance(ctx context.Context, installation string, input SandboxMaintenanceRequest) (RuntimeDeploymentView, error) { - if s.executionLease == nil { - return RuntimeDeploymentView{}, ErrInvalidInput - } - ctx, cancel := context.WithTimeout(ctx, executionTransactionTimeout) - defer cancel() - var result RuntimeDeploymentView - err := s.executionLease.transaction(ctx, func(tx pgx.Tx) error { - q := s.queries.WithTx(tx) - d, err := q.LockRuntimeDeployment(ctx) - if err != nil { +// CheckSandboxDeploymentSetup rejects stale/reset state before provider preparation; +// InitializeSandboxDeployment repeats the check in its committing transaction. +func (s *Store) CheckSandboxDeploymentSetup(ctx context.Context, installation string, input SandboxDeploymentSetupRequest) error { + return s.resetTransaction(ctx, func(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment) error { + if err := checkSandboxGeneration(d, installation, input.ExpectedGeneration); err != nil { return err } - if !d.WebManaged || runtimeUUID(d.InstallationID) != installation || d.ProviderKind == "" || uint64(d.Generation) != input.ExpectedGeneration { - return ErrSandboxDeploymentConflict + if d.ResetClear.Valid { + return ErrSandboxResetInProgress } - if err := q.SetSandboxMaintenance(ctx, input.Maintenance); err != nil { + if err := validateSandboxSelection(input); err != nil { return err } - result, err = s.deploymentView(ctx, q) - return err + if d.ProviderKind != "" && (d.ProviderKind != input.Provider || unspecifiedNodeDeployment(d)) { + return &SandboxResetRequiredError{CurrentProvider: d.ProviderKind, RequestedProvider: input.Provider} + } + return nil }) - return result, err } diff --git a/services/agents-api/internal/store/sandbox_deployment_resources_test.go b/services/agents-api/internal/store/sandbox_deployment_resources_test.go index 25ea8006d..412584321 100644 --- a/services/agents-api/internal/store/sandbox_deployment_resources_test.go +++ b/services/agents-api/internal/store/sandbox_deployment_resources_test.go @@ -38,12 +38,19 @@ func TestSandboxDeploymentMutationViewsIncludeActualResources(t *testing.T) { } want := SandboxDeploymentResources{Allocations: 1, Pending: 1} // Replaying setup must report the current resources rather than initial zeros. + selection.ExpectedGeneration = 1 replay, err := w.InitializeSandboxDeployment(t.Context(), installation, selection) if err != nil || replay.Resources != want { t.Fatalf("setup replay resources = %+v, error = %v", replay.Resources, err) } for _, maintenance := range []bool{true, false} { - response, err := w.SetSandboxMaintenance(t.Context(), installation, SandboxMaintenanceRequest{Maintenance: maintenance, ExpectedGeneration: 1}) + var response RuntimeDeploymentView + var err error + if maintenance { + response, err = w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + } else { + response, err = w.CancelSandboxReset(SandboxResetTestContext(t.Context()), installation, 1) + } if err != nil { t.Fatal(err) } @@ -51,7 +58,7 @@ func TestSandboxDeploymentMutationViewsIncludeActualResources(t *testing.T) { if err != nil { t.Fatal(err) } - if response.Maintenance != maintenance || response.Resources != want || response.Resources != current.Resources { + if (response.Reset != nil) != maintenance || response.Resources != want || response.Resources != current.Resources { t.Fatalf("maintenance %v response = %+v, current resources = %+v", maintenance, response, current.Resources) } } diff --git a/services/agents-api/internal/store/sandbox_deployment_setup.go b/services/agents-api/internal/store/sandbox_deployment_setup.go index 3c797263a..ea328cfb7 100644 --- a/services/agents-api/internal/store/sandbox_deployment_setup.go +++ b/services/agents-api/internal/store/sandbox_deployment_setup.go @@ -19,8 +19,9 @@ var ErrSandboxDeploymentConflict = errors.New("sandbox deployment is already con type SandboxDeploymentSetupRequest struct { sandbox.DeploymentSpec - Provider string `json:"provider"` - E2B *SandboxE2BConfiguration `json:"e2b,omitempty"` + ExpectedGeneration uint64 `json:"expected_generation"` + Provider string `json:"provider"` + E2B *SandboxE2BConfiguration `json:"e2b,omitempty"` } // SandboxSetup is the immutable configuration selected by the deployment admin. @@ -30,7 +31,7 @@ type SandboxSetup struct { InstallationID, Provider, BackendFingerprint string Generation uint64 Mode string - Maintenance bool + AdmissionPaused bool E2B *SandboxE2BConfiguration IdleSeconds, RetentionSeconds int64 } @@ -45,7 +46,7 @@ func (s *Store) GetSandboxSetup(ctx context.Context) (SandboxSetup, error) { if !d.WebManaged { return SandboxSetup{}, ErrSandboxDeploymentConflict } - result := SandboxSetup{InstallationID: runtimeUUID(d.InstallationID), Provider: d.ProviderKind, BackendFingerprint: d.BackendFingerprint, IdleSeconds: d.IdleSeconds, RetentionSeconds: d.RetentionSeconds, Generation: uint64(d.Generation), Mode: d.Mode, Maintenance: d.Maintenance} + result := SandboxSetup{InstallationID: runtimeUUID(d.InstallationID), Provider: d.ProviderKind, BackendFingerprint: d.BackendFingerprint, IdleSeconds: d.IdleSeconds, RetentionSeconds: d.RetentionSeconds, Generation: uint64(d.Generation), Mode: d.Mode, AdmissionPaused: d.AdmissionPaused} if err := json.Unmarshal(d.Specification, &result.Specification); err != nil { return SandboxSetup{}, err } @@ -151,7 +152,7 @@ func LoopbackOrigin(value string) bool { } func runtimeDeploymentView(d sqlc.RuntimeDeployment, publicURL string) RuntimeDeploymentView { - result := RuntimeDeploymentView{InstallationID: runtimeUUID(d.InstallationID), Provider: d.ProviderKind, CoreURL: publicURL, Maintenance: d.Maintenance, OwnerEpoch: uint64(d.OwnerEpoch), Generation: uint64(d.Generation), Mode: d.Mode} + result := RuntimeDeploymentView{InstallationID: runtimeUUID(d.InstallationID), Provider: d.ProviderKind, CoreURL: publicURL, OwnerEpoch: uint64(d.OwnerEpoch), Generation: uint64(d.Generation), Mode: d.Mode} if len(d.Specification) > 0 && string(d.Specification) != "{}" { var spec sandbox.DeploymentSpec if json.Unmarshal(d.Specification, &spec) == nil { diff --git a/services/agents-api/internal/store/sandbox_deployment_setup_test.go b/services/agents-api/internal/store/sandbox_deployment_setup_test.go index 6a2409db7..5d8c3ede8 100644 --- a/services/agents-api/internal/store/sandbox_deployment_setup_test.go +++ b/services/agents-api/internal/store/sandbox_deployment_setup_test.go @@ -51,6 +51,7 @@ func TestSandboxDeploymentSetupPersistsWithoutExecution(t *testing.T) { if err != nil || selected.Provider != input.Provider || selected.CoreURL != "https://core.example" || selected.OwnerEpoch != before.OwnerEpoch { t.Fatal(selected, err) } + input.ExpectedGeneration = selected.Generation replay, err := w.InitializeSandboxDeployment(t.Context(), id, input) if err != nil || !reflect.DeepEqual(replay, selected) { t.Fatal("identical retry changed selection", replay, err) @@ -122,7 +123,7 @@ func TestSandboxDeploymentSetupConcurrentSelection(t *testing.T) { t.Fatal(err) } } - if conflicts != 10 { + if conflicts != 19 { t.Fatal("both provider selections won", conflicts) } winner, err := s.GetSandboxSetup(t.Context()) diff --git a/services/agents-api/internal/store/sandbox_deployment_switch_test.go b/services/agents-api/internal/store/sandbox_deployment_switch_test.go index b1c8e6a0b..fece61d4d 100644 --- a/services/agents-api/internal/store/sandbox_deployment_switch_test.go +++ b/services/agents-api/internal/store/sandbox_deployment_switch_test.go @@ -73,12 +73,12 @@ func TestSandboxDirectDeploymentOwnershipAndCleanSwitch(t *testing.T) { if err != nil || observed.Expired { t.Fatal("cloud inherited legacy node-less expiry", err) } - if _, err := w.SetSandboxMaintenance(t.Context(), id, SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 1}); err != nil { + if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), id, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { t.Fatal(err) } update := SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}, ExpectedGeneration: 1} - if _, err := w.UpdateSandboxDeployment(t.Context(), id, update); !errors.Is(err, ErrSandboxDeploymentConflict) { - t.Fatal("unknown allocation allowed switch", err) + if _, err := w.UpdateSandboxDeployment(t.Context(), id, update); !errors.Is(err, ErrSandboxResetInProgress) { + t.Fatal("reset allowed switch", err) } if _, err := w.RequestRuntimeCleanup(t.Context(), owner); err != nil { t.Fatal(err) @@ -92,11 +92,11 @@ func TestSandboxDirectDeploymentOwnershipAndCleanSwitch(t *testing.T) { if _, err := w.ReleaseRuntimeAllocation(t.Context(), owner); err != nil { t.Fatal(err) } - changed, err := w.UpdateSandboxDeployment(t.Context(), id, update) - if err != nil || changed.Generation != 2 || changed.Mode != "nodes" || !changed.Maintenance || changed.E2B != nil || changed.Resources != (SandboxDeploymentResources{}) { + changed, err := resetAndSelect(t, w, id, update.ExpectedGeneration, update.SandboxDeploymentSetupRequest) + if err != nil || changed.Generation != 3 || changed.Mode != "nodes" || changed.Reset != nil || changed.E2B != nil || changed.Resources != (SandboxDeploymentResources{}) { t.Fatal("clean switch", changed, err) } - if _, err := w.SetSandboxMaintenance(t.Context(), id, SandboxMaintenanceRequest{ExpectedGeneration: 1}); !errors.Is(err, ErrSandboxDeploymentConflict) { + if _, err := w.CancelSandboxReset(SandboxResetTestContext(t.Context()), id, 1); !errors.Is(err, ErrSandboxDeploymentConflict) { t.Fatal("stale resume accepted", err) } if _, err := s.GetSession(t.Context(), tenant, session.ID); err != nil { @@ -128,15 +128,15 @@ func TestSandboxSwitchRetiresNodesAndEnrollment(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := w.SetSandboxMaintenance(t.Context(), id, SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 1}); err != nil { + if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), id, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { t.Fatal(err) } - // Maintenance rejects a valid enrollment without consuming it. Authentication + // AdmissionPaused rejects a valid enrollment without consuming it. Authentication // still precedes deployment details for invalid or retired credentials. spareNode := node spareNode.NodeID = uuid.NewString() - if _, err := s.EnrollRuntimeNode(t.Context(), unused, spareNode); !errors.Is(err, ErrInvalidInput) { - t.Fatal("maintenance accepted enrollment", err) + if _, err := s.EnrollRuntimeNode(t.Context(), unused, spareNode); !errors.Is(err, ErrSandboxResetInProgress) { + t.Fatal("reset accepted enrollment", err) } var consumed bool if err := pool.QueryRow(t.Context(), "SELECT consumed_at IS NOT NULL FROM runtime_node_enrollments WHERE token_sha256=$1", runtimeTokenDigest(unused)).Scan(&consumed); err != nil || consumed { @@ -148,7 +148,7 @@ func TestSandboxSwitchRetiresNodesAndEnrollment(t *testing.T) { } spareNode.Provider = "docker" input := e2bSelection() - if _, err := w.UpdateSandboxDeployment(t.Context(), id, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}); err != nil { + if _, err := resetAndSelect(t, w, id, 1, input); err != nil { t.Fatal(err) } @@ -158,13 +158,13 @@ func TestSandboxSwitchRetiresNodesAndEnrollment(t *testing.T) { if _, err := s.AuthenticateRuntimeNode(t.Context(), node.NodeID, node.Credential); !errors.Is(err, ErrRuntimeNodeCredential) { t.Fatal("old node credential survived", err) } - if _, err := w.SetSandboxMaintenance(t.Context(), id, SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 2}); err != nil { + if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), id, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 3}); err != nil { t.Fatal(err) } - if _, err := w.UpdateSandboxDeployment(t.Context(), id, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}, ExpectedGeneration: 2}); err != nil { + if _, err := resetAndSelect(t, w, id, 3, SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { t.Fatal(err) } - if _, err := w.SetSandboxMaintenance(t.Context(), id, SandboxMaintenanceRequest{ExpectedGeneration: 3}); err != nil { + if _, err := w.CancelSandboxReset(SandboxResetTestContext(t.Context()), id, 5); err != nil { t.Fatal(err) } node.NodeID = uuid.NewString() @@ -177,7 +177,7 @@ func TestSandboxSwitchRetiresNodesAndEnrollment(t *testing.T) { } } -func TestSandboxMaintenanceSerializesFreshDirectSessions(t *testing.T) { +func TestSandboxResetSerializesFreshDirectSessions(t *testing.T) { _, pool := newManagedTestStore(t) cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{6}, 32)) s := NewWithCredentialCipher(pool, cipher) @@ -196,18 +196,18 @@ func TestSandboxMaintenanceSerializesFreshDirectSessions(t *testing.T) { go func() { defer wg.Done() _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())) - if err != nil && !errors.Is(err, ErrEnvironmentUnavailable) && !errors.Is(err, ErrRuntimeNodeUnavailable) { + if err != nil && !errors.Is(err, ErrSandboxResetAdmission) && !errors.Is(err, ErrRuntimeNodeUnavailable) { t.Error(err) } }() } - if _, err := w.SetSandboxMaintenance(t.Context(), id, SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 1}); err != nil { + if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), id, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { t.Fatal(err) } wg.Wait() for range 3 { - if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); !errors.Is(err, ErrEnvironmentUnavailable) { - t.Fatal("fresh creation bypassed maintenance", err) + if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); !errors.Is(err, ErrSandboxResetAdmission) { + t.Fatal("fresh creation bypassed reset", err) } } view, err := s.GetRuntimeDeployment(t.Context()) @@ -215,7 +215,7 @@ func TestSandboxMaintenanceSerializesFreshDirectSessions(t *testing.T) { t.Fatal(err) } _, err = w.UpdateSandboxDeployment(t.Context(), id, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}, ExpectedGeneration: 1}) - if view.Resources.Pending > 0 && !errors.Is(err, ErrSandboxDeploymentConflict) { + if view.Resources.Pending > 0 && !errors.Is(err, ErrSandboxResetInProgress) { t.Fatal("committed pending Session bypassed switch guard", err) } } @@ -272,10 +272,10 @@ func TestSandboxSwitchPreservesReleasedAllocationAndItemHistory(t *testing.T) { if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(a) FROM runtime_allocations a WHERE id=$1", owner.ID).Scan(&allocationBefore); err != nil { t.Fatal(err) } - if _, err := w.SetSandboxMaintenance(t.Context(), installation, SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 1}); err != nil { + if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { t.Fatal(err) } - if _, err := w.UpdateSandboxDeployment(t.Context(), installation, SandboxDeploymentUpdateRequest{ExpectedGeneration: 1, SandboxDeploymentSetupRequest: SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}}); err != nil { + if _, err := resetAndSelect(t, w, installation, 1, SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { t.Fatal(err) } items, err = s.ListItems(t.Context(), tenant, history.ID, "", 100, true) @@ -350,11 +350,11 @@ func TestUnspecifiedNodeDeploymentDrainsBeforeReplacement(t *testing.T) { t.Fatal("unspecified deployment issued an enrollment token", err) } - if _, err := w.SetSandboxMaintenance(t.Context(), id, SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 1}); err != nil { + if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), id, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { t.Fatal(err) } - view, err := w.UpdateSandboxDeployment(t.Context(), id, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: selection, ExpectedGeneration: 1}) - if err != nil || view.Generation != 2 || view.Specification == nil { + view, err := resetAndSelect(t, w, id, 1, selection) + if err != nil || view.Generation != 3 || view.Specification == nil { t.Fatal("replacement did not record the specification", view, err) } if _, err := s.AuthenticateRuntimeNode(t.Context(), node.NodeID, node.Credential); !errors.Is(err, ErrRuntimeNodeCredential) { diff --git a/services/agents-api/internal/store/sandbox_deployment_switch_worker_test.go b/services/agents-api/internal/store/sandbox_deployment_switch_worker_test.go index 23b0acc15..a3558311a 100644 --- a/services/agents-api/internal/store/sandbox_deployment_switch_worker_test.go +++ b/services/agents-api/internal/store/sandbox_deployment_switch_worker_test.go @@ -25,17 +25,19 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { id := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}} var fail atomic.Bool + var preparations atomic.Int32 configuration := execution.NewDeferredRuntimeProvider(id, func(ctx context.Context) (*execution.RuntimeProvider, error) { setup, err := s.GetSandboxSetup(ctx) if err != nil || setup.Provider == "" { return nil, err } - return &execution.RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, Maintenance: setup.Maintenance, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}, nil + return &execution.RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}, nil }, func(ctx context.Context, setup store.SandboxSetup) (execution.PreparedRuntimeDeployment, error) { + preparations.Add(1) if fail.Load() { return execution.PreparedRuntimeDeployment{}, errors.New("fixture provider unavailable") } - return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, Maintenance: setup.Maintenance, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}}, nil + return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}}, nil }) w, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: s, Registry: gateway.NewRegistry(), ManagedRuntimes: configuration}) if err != nil { @@ -64,6 +66,10 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { if _, err := w.InitializeSandboxDeployment(t.Context(), store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { t.Fatal(err) } + prepared := preparations.Load() + if _, err := w.InitializeSandboxDeployment(t.Context(), store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err == nil || preparations.Load() != prepared { + t.Fatal("stale identical POST reached provider preparation", err) + } enrollment, err := store.EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), store.RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4})) if err != nil { t.Fatal(err) @@ -72,14 +78,13 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { if _, err := s.EnrollRuntimeNode(t.Context(), enrollment, node); err != nil { t.Fatal(err) } - if _, err := w.SetSandboxMaintenance(t.Context(), store.SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 1}); err != nil { - t.Fatal(err) - } previous, err := s.GetRuntimeDeployment(t.Context()) if err != nil { t.Fatal(err) } - input := store.SandboxDeploymentUpdateRequest{ExpectedGeneration: 1, SandboxDeploymentSetupRequest: store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("e2b"), Provider: "e2b", E2B: &store.SandboxE2BConfiguration{APIKey: "fixture-api-key", Template: "runtime:" + uuid.NewString()}}} + changed := store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"} + changed.Resources.CPUs++ + input := store.SandboxDeploymentUpdateRequest{ExpectedGeneration: 1, SandboxDeploymentSetupRequest: changed} fail.Store(true) if _, err := w.UpdateSandboxDeployment(t.Context(), input); err == nil { t.Fatal("failed activation reported success") @@ -91,25 +96,61 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { if _, err := s.AuthenticateRuntimeNode(t.Context(), node.NodeID, node.Credential); err != nil { t.Fatal("rejected candidate retired the previous node", err) } - if _, err := w.SetSandboxMaintenance(t.Context(), store.SandboxMaintenanceRequest{ExpectedGeneration: 2}); err == nil { - t.Fatal("failed activation resumed") - } fail.Store(false) - if _, err := w.UpdateSandboxDeployment(t.Context(), input); err != nil { + // Inject a real final SQL failure after successful provider preparation and + // manager drain; the surviving owner must recover before another mutation. + if _, err := pool.Exec(t.Context(), `CREATE FUNCTION reject_reset_fixture_update() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.generation > OLD.generation THEN RAISE EXCEPTION 'fixture commit rejected'; END IF; RETURN NEW; END $$; CREATE TRIGGER reject_reset_fixture_update BEFORE UPDATE ON runtime_deployment FOR EACH ROW EXECUTE FUNCTION reject_reset_fixture_update()`); err != nil { t.Fatal(err) } - if _, err := w.SetSandboxMaintenance(t.Context(), store.SandboxMaintenanceRequest{ExpectedGeneration: 2}); err != nil { + if _, err := w.UpdateSandboxDeployment(t.Context(), input); err == nil { + t.Fatal("failed final transaction accepted") + } + if _, err := pool.Exec(t.Context(), `DROP TRIGGER reject_reset_fixture_update ON runtime_deployment; DROP FUNCTION reject_reset_fixture_update()`); err != nil { + t.Fatal(err) + } + if err := w.ReconcileManagedRuntimes(t.Context()); err != nil { + t.Fatal("failed commit left manager barrier closed", err) + } + if view, err := s.GetRuntimeDeployment(t.Context()); err != nil || view.Generation != 1 || view.Provider != "docker" { + t.Fatal("failed commit replaced provider", view, err) + } + if _, err := w.UpdateSandboxDeployment(t.Context(), input); err != nil { t.Fatal(err) } + reset := func(generation uint64) store.RuntimeDeploymentView { + t.Helper() + if _, err := w.StartSandboxReset(store.SandboxResetTestContext(t.Context()), store.SandboxResetRequest{ExpectedGeneration: generation, Clear: "force"}); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(12 * time.Second) + for time.Now().Before(deadline) { + view, err := s.GetRuntimeDeployment(t.Context()) + if err != nil { + t.Fatal(err) + } + if view.Provider == "" && view.Reset == nil && view.Generation == generation+1 { + return view + } + select { + case err := <-done: + t.Fatalf("reset stopped owner: %v", err) + case <-time.After(20 * time.Millisecond): + } + } + t.Fatal("reset completion blocked (including possible manager self-drain)") + return store.RuntimeDeploymentView{} + } + empty = reset(2) + cloud := store.SandboxDeploymentSetupRequest{ExpectedGeneration: empty.Generation, DeploymentSpec: store.SandboxDeploymentTestSpec("e2b"), Provider: "e2b", E2B: &store.SandboxE2BConfiguration{APIKey: "fixture-api-key", Template: "runtime:" + uuid.NewString()}} + if _, err := w.InitializeSandboxDeployment(t.Context(), cloud); err != nil { + t.Fatal("setup after unconfigured publication", err) + } tenant, session, environment := managedSession(t, s) allocation, err := w.ProvisionEnvironment(t.Context(), tenant, environment.ID, id) if err != nil || allocation.NodeID != "" || allocation.State != "running" { t.Fatal("direct provider not available after resume", allocation, err) } - if _, err := w.SetSandboxMaintenance(t.Context(), store.SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 2}); err != nil { - t.Fatal(err) - } - next := store.SandboxDeploymentUpdateRequest{ExpectedGeneration: 2, SandboxDeploymentSetupRequest: store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"}} + next := store.SandboxDeploymentUpdateRequest{ExpectedGeneration: 4, SandboxDeploymentSetupRequest: store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"}} if _, err := w.UpdateSandboxDeployment(t.Context(), next); !errors.Is(err, store.ErrSandboxDeploymentConflict) { t.Fatal("dirty switch accepted", err) } @@ -117,11 +158,10 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { t.Fatal(err) } reconcileManagedState(t, w, s, tenant, environment.ID, "released") - if _, err := w.UpdateSandboxDeployment(t.Context(), next); err != nil { - t.Fatal("clean switch failed", err) - } - if _, err := w.SetSandboxMaintenance(t.Context(), store.SandboxMaintenanceRequest{ExpectedGeneration: 3}); err != nil { - t.Fatal(err) + empty = reset(4) + next.SandboxDeploymentSetupRequest.ExpectedGeneration = empty.Generation + if _, err := w.InitializeSandboxDeployment(t.Context(), next.SandboxDeploymentSetupRequest); err != nil { + t.Fatal("clean setup after reset", err) } select { case err := <-done: diff --git a/services/agents-api/internal/store/sandbox_deployment_view_test.go b/services/agents-api/internal/store/sandbox_deployment_view_test.go index e6c3b0ecb..ea97f9341 100644 --- a/services/agents-api/internal/store/sandbox_deployment_view_test.go +++ b/services/agents-api/internal/store/sandbox_deployment_view_test.go @@ -61,14 +61,12 @@ func TestSandboxDeploymentViewRecordsTemplateBuildAndSuspension(t *testing.T) { if err != nil || !bytes.Contains(raw, []byte(`"template_build":{"status":null,"resources":{"cpus":null,"memory_mib":null,"root_disk_mib":null}}`)) { t.Fatalf("unknown build was not null: %s %v", raw, err) } + input.ExpectedGeneration = 1 view, err = w.InitializeSandboxDeployment(t.Context(), id, input) raw, _ = json.Marshal(view) if err != nil || view.Generation != 1 || !bytes.Contains(raw, recorded) { t.Fatalf("identical POST did not record the build: %s %v", raw, err) } - if _, err := w.SetSandboxMaintenance(t.Context(), id, SandboxMaintenanceRequest{Maintenance: true, ExpectedGeneration: 1}); err != nil { - t.Fatal(err) - } forget() view, err = w.UpdateSandboxDeployment(t.Context(), id, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}) raw, _ = json.Marshal(view) @@ -77,7 +75,7 @@ func TestSandboxDeploymentViewRecordsTemplateBuildAndSuspension(t *testing.T) { } update := SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: SandboxDeploymentSetupRequest{ DeploymentSpec: SandboxDeploymentTestSpec("microsandbox"), Provider: "microsandbox"}, ExpectedGeneration: 1} - view, err = w.UpdateSandboxDeployment(t.Context(), id, update) + view, err = resetAndSelect(t, w, id, update.ExpectedGeneration, update.SandboxDeploymentSetupRequest) if err != nil || view.E2B != nil || view.Suspension == nil || view.Suspension.IdleSeconds != 300 || view.Suspension.RetentionSeconds != 86400 { t.Fatalf("microsandbox suspension view = %+v %v", view, err) } diff --git a/services/agents-api/internal/store/sandbox_deployment_worker_test.go b/services/agents-api/internal/store/sandbox_deployment_worker_test.go index 32e4a932a..ddb60ff50 100644 --- a/services/agents-api/internal/store/sandbox_deployment_worker_test.go +++ b/services/agents-api/internal/store/sandbox_deployment_worker_test.go @@ -27,7 +27,7 @@ func TestSandboxDeploymentWorkerActivatesWithoutRestart(t *testing.T) { return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, BackendFingerprint: setup.BackendFingerprint, CoreURL: "https://core.example/api/v1", Provider: p}, nil }, func(ctx context.Context, setup store.SandboxSetup) (execution.PreparedRuntimeDeployment, error) { - return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, Maintenance: setup.Maintenance, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}}, nil + return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}}, nil }) start := func() (*execution.Worker, func()) { t.Helper() diff --git a/services/agents-api/internal/store/sandbox_reset.go b/services/agents-api/internal/store/sandbox_reset.go new file mode 100644 index 000000000..50ac5060d --- /dev/null +++ b/services/agents-api/internal/store/sandbox_reset.go @@ -0,0 +1,267 @@ +package store + +import ( + "context" + "encoding/json" + "errors" + "math" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/adminaudit" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +var ErrSandboxResetAdmission = errors.New("hosted admission is paused for a sandbox reset") +var ErrSandboxResetInProgress = errors.New("a sandbox reset is in progress") +var ErrSandboxNotConfigured = errors.New("the sandbox deployment is not configured") + +type SandboxGenerationStaleError struct{ CurrentGeneration uint64 } + +func (e *SandboxGenerationStaleError) Error() string { + return "the sandbox deployment generation changed" +} +func (e *SandboxGenerationStaleError) Unwrap() error { return ErrSandboxDeploymentConflict } + +type SandboxResetRequiredError struct{ CurrentProvider, RequestedProvider string } + +func (e *SandboxResetRequiredError) Error() string { + return "reset the sandbox deployment before changing its backend" +} +func (e *SandboxResetRequiredError) Unwrap() error { return ErrSandboxDeploymentConflict } + +type SandboxInUseError struct{ Resources SandboxDeploymentResources } + +func (e *SandboxInUseError) Error() string { + return "hosted sandbox resources still belong to this deployment" +} +func (e *SandboxInUseError) Unwrap() error { return ErrSandboxDeploymentConflict } + +type SandboxResetRequest struct { + ExpectedGeneration uint64 `json:"expected_generation" binding:"required" minimum:"0"` + Clear string `json:"clear" binding:"required" enums:"auto,force"` + DeadlineSeconds *int32 `json:"deadline_seconds,omitempty" minimum:"300" maximum:"86400"` +} + +func checkSandboxGeneration(d sqlc.RuntimeDeployment, installation string, generation uint64) error { + if uint64(d.Generation) != generation { + return &SandboxGenerationStaleError{uint64(d.Generation)} + } + if !d.WebManaged || runtimeUUID(d.InstallationID) != installation { + return ErrSandboxDeploymentConflict + } + return nil +} + +func (s *Store) resetTransaction(ctx context.Context, apply func(context.Context, *sqlc.Queries, sqlc.RuntimeDeployment) error) error { + if s.executionLease == nil { + return ErrInvalidInput + } + ctx, cancel := context.WithTimeout(ctx, executionTransactionTimeout) + defer cancel() + return s.executionLease.transaction(ctx, func(tx pgx.Tx) error { + q := s.queries.WithTx(tx) + d, err := q.LockRuntimeDeployment(ctx) + if err != nil { + return err + } + return apply(ctx, q, d) + }) +} + +func (s *Store) StartSandboxReset(ctx context.Context, installation string, input SandboxResetRequest) (RuntimeDeploymentView, error) { + if input.Clear != "auto" && input.Clear != "force" { + return RuntimeDeploymentView{}, ErrInvalidInput + } + deadline := int32(3600) + if input.DeadlineSeconds != nil { + if input.Clear != "auto" || *input.DeadlineSeconds < 300 || *input.DeadlineSeconds > 86400 { + return RuntimeDeploymentView{}, ErrInvalidInput + } + deadline = *input.DeadlineSeconds + } + var result RuntimeDeploymentView + err := s.resetTransaction(ctx, func(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment) error { + if err := checkSandboxGeneration(d, installation, input.ExpectedGeneration); err != nil { + return err + } + if d.ProviderKind == "" { + return ErrSandboxNotConfigured + } + if d.ResetClear.Valid { + if d.ResetClear.String != input.Clear { + if input.Clear != "force" { + return ErrSandboxResetInProgress + } + if err := q.ForceSandboxReset(ctx); err != nil { + return err + } + if err := recordDeploymentMutation(ctx, q, "reset_force", "sandbox_deployment", installation); err != nil { + return err + } + } + } else { + source, ok := adminaudit.FromContext(ctx) + if !ok { + return ErrInvalidInput + } + // The audit insert validates the provenance before this transaction + // can commit. Persist only the same non-secret typed source. + audit, err := json.Marshal(source) + if err != nil { + return err + } + if err := q.StartSandboxReset(ctx, sqlc.StartSandboxResetParams{Clear: pgtype.Text{String: input.Clear, Valid: true}, DeadlineSeconds: deadline, Audit: audit}); err != nil { + return err + } + if err := recordDeploymentMutation(ctx, q, "reset_start", "sandbox_deployment", installation); err != nil { + return err + } + } + var err error + result, err = s.deploymentView(ctx, q) + return err + }) + return result, err +} + +func (s *Store) CancelSandboxReset(ctx context.Context, installation string, generation uint64) (RuntimeDeploymentView, error) { + var result RuntimeDeploymentView + err := s.resetTransaction(ctx, func(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment) error { + if err := checkSandboxGeneration(d, installation, generation); err != nil { + return err + } + if d.ResetClear.Valid { + if err := q.CancelSandboxReset(ctx); err != nil { + return err + } + if err := recordDeploymentMutation(ctx, q, "reset_cancel", "sandbox_deployment", installation); err != nil { + return err + } + } + var err error + result, err = s.deploymentView(ctx, q) + return err + }) + return result, err +} + +// AdvanceSandboxResetDeadline makes force escalation durable before selecting +// work. A restart cannot extend an administrator's original absolute deadline. +func (s *Store) AdvanceSandboxResetDeadline(ctx context.Context) error { + return s.resetTransaction(ctx, func(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment) error { + if d.ResetClear.String != "auto" || !d.ResetDeadlineAt.Valid || time.Now().Before(d.ResetDeadlineAt.Time) { + return nil + } + source, err := sandboxResetAudit(d) + if err != nil { + return err + } + if err := q.ForceSandboxReset(ctx); err != nil { + return err + } + return recordDeploymentMutation(adminaudit.WithSource(ctx, source), q, "reset_deadline", "sandbox_deployment", runtimeUUID(d.InstallationID)) + }) +} + +func sandboxResetAudit(d sqlc.RuntimeDeployment) (adminaudit.Source, error) { + var source adminaudit.Source + if !d.ResetClear.Valid || json.Unmarshal(d.ResetAudit, &source) != nil { + return source, ErrInvalidInput + } + return source, nil +} + +// CompleteSandboxReset must run after the manager has drained. It does not take +// Session locks: archive and admission always lock Session before deployment. +func (s *Store) CompleteSandboxReset(ctx context.Context, installation string, generation uint64, requestedAt time.Time) (RuntimeDeploymentView, error) { + var result RuntimeDeploymentView + err := s.resetTransaction(ctx, func(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment) error { + if err := checkSandboxGeneration(d, installation, generation); err != nil { + return err + } + if !d.ResetClear.Valid || !d.ResetRequestedAt.Time.Equal(requestedAt) { + return ErrSandboxDeploymentConflict + } + if d.Generation == math.MaxInt64 || d.OwnerEpoch == math.MaxInt64 { + return ErrSandboxDeploymentConflict + } + resources, err := q.CountRuntimeDeploymentResources(ctx) + if err != nil { + return err + } + if resources.Allocations != 0 || resources.Pending != 0 { + return &SandboxInUseError{SandboxDeploymentResources{Allocations: resources.Allocations, Pending: resources.Pending}} + } + source, err := sandboxResetAudit(d) + if err != nil { + return err + } + if err := q.CompleteSandboxReset(ctx); err != nil { + return err + } + if err := q.RetireSandboxNodes(ctx); err != nil { + return err + } + if err := q.RetireSandboxEnrollments(ctx); err != nil { + return err + } + if err := recordDeploymentMutation(adminaudit.WithSource(ctx, source), q, "reset_complete", "sandbox_deployment", installation); err != nil { + return err + } + result, err = s.deploymentView(ctx, q) + return err + }) + return result, err +} + +// SandboxResetView contains only durable state and a single-snapshot resource partition. +type SandboxResetView struct { + Clear string `json:"clear"` + RequestedAt time.Time `json:"requested_at"` + DeadlineAt *time.Time `json:"deadline_at" extensions:"x-nullable"` + ForcedAt *time.Time `json:"forced_at" extensions:"x-nullable"` + Remaining SandboxResetRemaining `json:"remaining"` +} +type SandboxResetRemaining struct { + Busy int64 `json:"busy"` + Idle int64 `json:"idle"` + Cleanup int64 `json:"cleanup"` + OnOfflineNodes int64 `json:"on_offline_nodes"` + OfflineNodes []SandboxResetOfflineNode `json:"offline_nodes"` +} +type SandboxResetOfflineNode struct { + NodeID string `json:"node_id"` + Name string `json:"name"` + Resources int64 `json:"resources"` +} + +func resetTimestamp(value pgtype.Timestamptz) *time.Time { + if !value.Valid { + return nil + } + return &value.Time +} + +type SandboxResetSession struct{ SessionID, TenantID string } + +func (s *Store) ListSandboxResetSessions(ctx context.Context, after string, force bool) ([]SandboxResetSession, error) { + cursor := pgtype.UUID{Valid: true} + if after != "" { + var err error + cursor, err = parseID(after) + if err != nil { + return nil, err + } + } + rows, err := s.queries.ListSandboxResetSessions(ctx, sqlc.ListSandboxResetSessionsParams{AfterID: cursor, Force: force}) + if err != nil { + return nil, err + } + result := make([]SandboxResetSession, 0, len(rows)) + for _, row := range rows { + result = append(result, SandboxResetSession{SessionID: runtimeUUID(row.ID), TenantID: runtimeUUID(row.TenantID)}) + } + return result, nil +} diff --git a/services/agents-api/internal/store/sandbox_reset_migration_test.go b/services/agents-api/internal/store/sandbox_reset_migration_test.go new file mode 100644 index 000000000..8a8bf3ce6 --- /dev/null +++ b/services/agents-api/internal/store/sandbox_reset_migration_test.go @@ -0,0 +1,52 @@ +package store + +import ( + "strings" + "testing" +) + +func TestSandboxResetMigrationResumesAdmissionAndGuardsDowngrade(t *testing.T) { + db, provider := runtimeNamesMigrationSchema(t) + ctx := t.Context() + if _, err := provider.UpTo(ctx, 78); err != nil { + t.Fatal(err) + } + exec := func(q string, args ...any) { + t.Helper() + if _, err := db.ExecContext(ctx, q, args...); err != nil { + t.Fatal(err) + } + } + for _, paused := range []bool{false, true} { + exec(`UPDATE runtime_deployment SET web_managed=true,provider_kind='docker',mode='nodes',generation=1,maintenance=$1`, paused) + if _, err := provider.UpTo(ctx, 79); err != nil { + t.Fatal(err) + } + var resumed bool + if err := db.QueryRowContext(ctx, `SELECT NOT admission_paused AND reset_clear IS NULL AND generation=1 FROM runtime_deployment`).Scan(&resumed); err != nil || !resumed { + t.Fatal("migration invented a reset or kept maintenance", err) + } + if _, err := provider.DownTo(ctx, 78); err != nil { + t.Fatal(err) + } + } + if _, err := provider.UpTo(ctx, 79); err != nil { + t.Fatal(err) + } + exec(`UPDATE runtime_deployment SET admission_paused=true,reset_clear='auto',reset_requested_at=clock_timestamp(),reset_deadline_at=clock_timestamp()+interval '1 hour',reset_audit='{}'`) + if _, err := provider.DownTo(ctx, 78); err == nil || !strings.Contains(err.Error(), "reset is active") { + t.Fatal("active reset downgrade", err) + } + exec(`UPDATE runtime_deployment SET admission_paused=false,reset_clear=NULL,reset_requested_at=NULL,reset_deadline_at=NULL,reset_audit=NULL,provider_kind='',mode='',generation=2`) + if _, err := provider.DownTo(ctx, 78); err == nil || !strings.Contains(err.Error(), "Configure a sandbox provider") { + t.Fatal("completed reset lost generation", err) + } + var generation int64 + if err := db.QueryRowContext(ctx, `SELECT generation FROM runtime_deployment`).Scan(&generation); err != nil || generation != 2 { + t.Fatal(generation, err) + } + exec(`UPDATE runtime_deployment SET provider_kind='docker',mode='nodes',generation=3`) + if _, err := provider.DownTo(ctx, 78); err != nil { + t.Fatal("configured downgrade", err) + } +} diff --git a/services/agents-api/internal/store/sandbox_reset_test.go b/services/agents-api/internal/store/sandbox_reset_test.go new file mode 100644 index 000000000..83e903403 --- /dev/null +++ b/services/agents-api/internal/store/sandbox_reset_test.go @@ -0,0 +1,381 @@ +package store + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/adminaudit" + "github.com/google/uuid" +) + +func SandboxResetTestContext(ctx context.Context) context.Context { + return adminaudit.WithSource(ctx, adminaudit.Source{CredentialID: "reset-fixture", ActorLabel: "operator", RequestID: "reset-request", TraceID: "reset-trace"}) +} +func resetAndSelect(t *testing.T, w *Store, installation string, generation uint64, input SandboxDeploymentSetupRequest) (RuntimeDeploymentView, error) { + t.Helper() + ctx := SandboxResetTestContext(t.Context()) + reset, err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{Clear: "auto", ExpectedGeneration: generation}) + if err != nil { + return RuntimeDeploymentView{}, err + } + empty, err := w.CompleteSandboxReset(ctx, installation, generation, reset.Reset.RequestedAt) + if err != nil { + return RuntimeDeploymentView{}, err + } + input.ExpectedGeneration = empty.Generation + return w.InitializeSandboxDeployment(ctx, installation, input) +} + +func assertResetPartition(t *testing.T, view RuntimeDeploymentView) { + t.Helper() + if view.Reset == nil { + t.Fatal("missing reset") + } + remaining := view.Reset.Remaining + if remaining.Busy+remaining.Idle+remaining.Cleanup != view.Resources.Allocations+view.Resources.Pending { + t.Fatalf("inconsistent reset partition: %+v", view) + } + var offline int64 + for _, node := range remaining.OfflineNodes { + offline += node.Resources + } + if offline != remaining.OnOfflineNodes || offline > view.Resources.Allocations+view.Resources.Pending { + t.Fatalf("inconsistent offline subset: %+v", remaining) + } +} + +func TestSandboxResetAutoUsesStartedWorkAndLockedRecheck(t *testing.T) { + for _, kind := range []string{"idle", "queued", "in_progress", "waiting", "subagent_queued", "subagent_in_progress", "subagent_waiting", "file_write", "suspended", "pending"} { + t.Run(kind, func(t *testing.T) { + s, w, installation := managedArchiveFixture(t) + tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + var owner RuntimeAllocation + if kind != "pending" { + owner = archiveAllocation(t, w, tenant, session, installation) + } + busy := kind == "in_progress" || kind == "waiting" || kind == "subagent_in_progress" || kind == "subagent_waiting" || kind == "file_write" + switch kind { + case "queued", "in_progress", "waiting": + runtimeSuspensionSQL(t, s.pool, `INSERT INTO turns(id,session_id,status) VALUES($1,$2,$3)`, uuid.NewString(), session.ID, kind) + case "subagent_queued", "subagent_in_progress", "subagent_waiting": + parent, child := uuid.NewString(), uuid.NewString() + runtimeSuspensionSQL(t, s.pool, `INSERT INTO turns(id,session_id,status,completed_at) VALUES($1,$2,'completed',clock_timestamp())`, parent, session.ID) + runtimeSuspensionSQL(t, s.pool, `INSERT INTO turn_events(session_id,turn_id,ordinal,kind,payload) VALUES($1,$2,1,'subagent','{}')`, session.ID, parent) + runtimeSuspensionSQL(t, s.pool, `INSERT INTO subagent_identities(id,session_id,device_id,engine,native_id,parent_native_id,native_created_at,first_turn_id,first_event_ordinal) VALUES($1,$2,$3,'codex','child','root',1,$4,1)`, child, session.ID, owner.DeviceID, parent) + runtimeSuspensionSQL(t, s.pool, `INSERT INTO subagent_turns(id,session_id,subagent_id,native_id,status,created_at) VALUES($1,$2,$3,'child-turn',$4,clock_timestamp())`, uuid.NewString(), session.ID, child, strings.TrimPrefix(kind, "subagent_")) + case "file_write": + runtimeSuspensionSQL(t, s.pool, `INSERT INTO environment_file_writes(id,environment_id,device_id,request_sha256) VALUES($1,$2,$3,$4)`, uuid.NewString(), session.Environment.ID, owner.DeviceID, strings.Repeat("a", 64)) + case "suspended": + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_allocations SET compute_phase='suspended', compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, owner.ID) + } + reset, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + if err != nil { + t.Fatal(err) + } + assertResetPartition(t, reset) + assertSandboxSnapshotEquivalent(t, s.pool) + if (reset.Reset.Remaining.Busy == 1) != busy { + t.Fatalf("wrong busy classification: %+v", reset.Reset.Remaining) + } + page, err := w.ListSandboxResetSessions(t.Context(), "", false) + if err != nil || (len(page) == 0) != busy { + t.Fatal("auto eligibility", page, err) + } + _, err = w.ArchiveSandboxResetSession(t.Context(), tenant, session.ID, 1, reset.Reset.RequestedAt) + if busy { + if !errors.Is(err, ErrSandboxResetSessionBusy) { + t.Fatal("auto cut active work", err) + } + if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err != nil { + t.Fatal(err) + } + _, err = w.ArchiveSandboxResetSession(t.Context(), tenant, session.ID, 1, reset.Reset.RequestedAt) + } + if err != nil { + t.Fatal("archive", err) + } + archived, err := s.GetSession(t.Context(), tenant, session.ID) + if err != nil || archived.Environment.Status != "expired" { + t.Fatal("archive not durable", archived, err) + } + var auditTenant, auditProject, credential, request string + if err := s.pool.QueryRow(t.Context(), `SELECT tenant_id::text,project_id::text,admin_credential_id,request_id FROM admin_audit_log WHERE action='archive' AND resource_id=$1`, session.ID).Scan(&auditTenant, &auditProject, &credential, &request); err != nil || auditTenant != tenant || auditProject != tenant || credential != "reset-fixture" || request != "reset-request" { + t.Fatal("background provenance was not reconstructed", err) + } + }) + } +} + +func TestSandboxResetCancellationABADeadlineAndGeneration(t *testing.T) { + s, w, installation := managedArchiveFixture(t) + tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + ctx := SandboxResetTestContext(t.Context()) + first, err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + if err != nil { + t.Fatal(err) + } + if first.Reset.DeadlineAt == nil || first.Reset.DeadlineAt.Sub(first.Reset.RequestedAt) < 3599*time.Second { + t.Fatal("default deadline", first) + } + shorter := int32(300) + replay, err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto", DeadlineSeconds: &shorter}) + if err != nil || !replay.Reset.RequestedAt.Equal(first.Reset.RequestedAt) || !replay.Reset.DeadlineAt.Equal(*first.Reset.DeadlineAt) { + t.Fatal("retry moved durable deadline", replay, err) + } + if _, err = w.CancelSandboxReset(ctx, installation, 1); err != nil { + t.Fatal(err) + } + second, err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + if err != nil { + t.Fatal(err) + } + if _, err = w.ArchiveSandboxResetSession(t.Context(), tenant, session.ID, 1, first.Reset.RequestedAt); !errors.Is(err, ErrSandboxDeploymentConflict) { + t.Fatal("cancelled reset archived successor work", err) + } + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_deployment SET reset_deadline_at=clock_timestamp()-interval '1 second'`) + if err := w.AdvanceSandboxResetDeadline(t.Context()); err != nil { + t.Fatal(err) + } + forced, err := s.GetRuntimeDeployment(t.Context()) + if err != nil || forced.Reset.Clear != "force" || forced.Reset.ForcedAt == nil || !forced.Reset.RequestedAt.Equal(second.Reset.RequestedAt) { + t.Fatal("deadline not durable", forced, err) + } + if _, err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}); !errors.Is(err, ErrSandboxResetInProgress) { + t.Fatal("force downgraded", err) + } + if _, err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{ExpectedGeneration: 0, Clear: "force"}); !errors.Is(err, ErrSandboxDeploymentConflict) { + t.Fatal("stale reset precedence", err) + } + if _, err := w.ArchiveSandboxResetSession(t.Context(), tenant, session.ID, 1, second.Reset.RequestedAt); err != nil { + t.Fatal(err) + } + if _, err := w.CompleteSandboxReset(ctx, installation, 1, first.Reset.RequestedAt); !errors.Is(err, ErrSandboxDeploymentConflict) { + t.Fatal("cancelled reset finalized successor", err) + } + empty, err := w.CompleteSandboxReset(ctx, installation, 1, second.Reset.RequestedAt) + if err != nil || empty.Provider != "" || empty.Generation != 2 || empty.Reset != nil || empty.InstallationID != installation || empty.E2B != nil || empty.Specification != nil { + t.Fatal("reset commit", empty, err) + } + if _, err := w.CancelSandboxReset(ctx, installation, 1); !errors.Is(err, ErrSandboxDeploymentConflict) { + t.Fatal("stale cancel", err) + } + if _, err := w.CancelSandboxReset(ctx, installation, 2); err != nil { + t.Fatal("cancel without reset", err) + } +} + +func TestSandboxResetAutoRechecksTurnStartedAfterListing(t *testing.T) { + s, w, installation := managedArchiveFixture(t) + tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + archiveAllocation(t, w, tenant, session, installation) + reset, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + if err != nil { + t.Fatal(err) + } + page, err := w.ListSandboxResetSessions(t.Context(), "", false) + if err != nil || len(page) != 1 { + t.Fatal(page, err) + } + // Existing live input remains admitted; Turn transition takes the same + // Session lock that the later conditional archive must reacquire. + turn := submitMessage(t, s, tenant, session.ID, "after-list") + transition(t, w, tenant, session.ID, turn.TurnID, TurnQueued, TurnInProgress) + if _, err := w.ArchiveSandboxResetSession(t.Context(), tenant, session.ID, 1, reset.Reset.RequestedAt); !errors.Is(err, ErrSandboxResetSessionBusy) { + t.Fatal("listed idle candidate cut a new Turn", err) + } + before := adminMutationSnapshot(t, s, "sessions", "environments", "environment_input_reservations", "turns", "runtime_placements") + if _, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())); !errors.Is(err, ErrSandboxResetAdmission) { + t.Fatal("new hosted admission during reset", err) + } + after := adminMutationSnapshot(t, s, "sessions", "environments", "environment_input_reservations", "turns", "runtime_placements") + if !reflect.DeepEqual(before, after) { + t.Fatal("rejected admission left provisional rows") + } + if _, err := w.CancelSandboxReset(SandboxResetTestContext(t.Context()), installation, 1); err != nil { + t.Fatal(err) + } + if _, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())); err != nil { + t.Fatal("cancel did not restore admission", err) + } +} + +func TestSandboxResetAuditFailureRollsBackPauseAndCompletion(t *testing.T) { + s, w, installation := managedArchiveFixture(t) + rejectAdminAuditInsert(t, s) + rejected := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "reset-fixture", ActorLabel: "operator", RequestID: rejectedAdminRequest, TraceID: "reset-trace"}) + if _, err := w.StartSandboxReset(rejected, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err == nil { + t.Fatal("reset committed without audit") + } + view, err := s.GetRuntimeDeployment(t.Context()) + if err != nil || view.Reset != nil || view.Generation != 1 { + t.Fatal("failed audit left reset state", view, err) + } + reset, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}) + if err != nil { + t.Fatal(err) + } + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_deployment SET reset_audit=jsonb_set(reset_audit,'{RequestID}',to_jsonb($1::text))`, rejectedAdminRequest) + if _, err := w.CompleteSandboxReset(t.Context(), installation, 1, reset.Reset.RequestedAt); err == nil { + t.Fatal("completion committed without audit") + } + view, err = s.GetRuntimeDeployment(t.Context()) + if err != nil || view.Reset == nil || view.Provider != "e2b" || view.Generation != 1 || view.OwnerEpoch != reset.OwnerEpoch { + t.Fatal("failed completion destroyed committed provider", view, err) + } +} + +func TestSandboxResetSnapshotCountsOfflineOwnershipOnce(t *testing.T) { + s, w, deployment := managerFixture(t, 10, 10) + // Reuse the real placement fixture, then adopt its selection as Web-managed. + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_deployment SET web_managed=true,local_node_id=NULL`) + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET deployment_generation=1,specification_digest=$1`, SandboxDeploymentTestSpec("docker").Digest("docker")) + _, pending := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + tenant, suspended := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + allocation := archiveAllocation(t, w, tenant, suspended, deployment.InstallationID) + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_allocations SET compute_phase='suspended',compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, allocation.ID) + tenant, deleted := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + archiveAllocation(t, w, tenant, deleted, deployment.InstallationID) + if err := s.DeleteSession(t.Context(), tenant, deleted.ID); err != nil { + t.Fatal(err) + } + reset, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), deployment.InstallationID, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + if err != nil { + t.Fatal(err) + } + assertResetPartition(t, reset) + if reset.Resources != (SandboxDeploymentResources{Allocations: 2, Pending: 1}) || reset.Reset.Remaining.Idle != 2 || reset.Reset.Remaining.Cleanup != 1 { + t.Fatal("duplicated placement or missing deleted receipt", reset) + } + for _, state := range []string{"preparing", "stale", "epoch", "disconnected"} { + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET connected_epoch=(SELECT owner_epoch FROM runtime_deployment)`) + onlineManagerNode(t, s, deployment.LocalNodeID) + switch state { + case "preparing": + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET provider_ready=false`) + case "stale": + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET last_seen_at=clock_timestamp()-interval '46 seconds'`) + case "epoch": + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET connected_epoch=connected_epoch+1`) + case "disconnected": + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET connection_id=NULL`) + } + view, err := s.GetRuntimeDeployment(t.Context()) + if err != nil { + t.Fatal(err) + } + assertResetPartition(t, view) + assertSandboxSnapshotEquivalent(t, s.pool) + want := int64(3) + if state == "preparing" { + want = 0 + } + if view.Reset.Remaining.OnOfflineNodes != want { + t.Fatalf("%s presence: %+v", state, view.Reset.Remaining) + } + if want > 0 && (len(view.Reset.Remaining.OfflineNodes) != 1 || view.Reset.Remaining.OfflineNodes[0].NodeID != deployment.LocalNodeID || view.Reset.Remaining.OfflineNodes[0].Resources != 3) { + t.Fatal("offline ownership projection", view.Reset.Remaining) + } + } + if _, err := w.CompleteSandboxReset(t.Context(), deployment.InstallationID, 1, reset.Reset.RequestedAt); err == nil { + t.Fatal("offline resources were treated as cleaned") + } + if err := s.RemoveRuntimeNode(t.Context(), deployment.LocalNodeID); !errors.Is(err, ErrRuntimeNodeInUse) { + t.Fatal("removed node with reset resources", err) + } + if row, err := s.GetEnvironment(t.Context(), pending.TenantID, pending.Environment.ID); err == nil && row.Status == "expired" { + t.Fatal("projection mutated pending resource") + } +} + +func TestSandboxResetPaginationSkipsBusyPrefixAndPreservesSelfHosted(t *testing.T) { + s, w, installation := managedArchiveFixture(t) + // Use deterministic ordered UUIDs so more than one page of busy rows precedes + // idle work. Listing must not repeatedly return the busy prefix. + for i := 1; i <= 35; i++ { + tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + id := fmt.Sprintf("00000000-0000-4000-8000-%012d", i) + // IDs are immutable API identities, so seed this bounded scheduling fixture + // directly instead of modifying an existing Session primary key. + runtimeSuspensionSQL(t, s.pool, `INSERT INTO sessions(id,tenant_id,engine,idempotency_key,request_hash,configuration) SELECT $1::uuid,tenant_id,engine,$1::text,$1::text,configuration FROM sessions WHERE id=$2`, id, session.ID) + runtimeSuspensionSQL(t, s.pool, `INSERT INTO environments(id,session_id) VALUES($1,$2)`, uuid.NewString(), id) + if i <= 34 { + runtimeSuspensionSQL(t, s.pool, `INSERT INTO turns(id,session_id,status) VALUES($1,$2,'in_progress')`, uuid.NewString(), id) + } + if err := s.DeleteSession(t.Context(), tenant, session.ID); err != nil { + t.Fatal(err) + } + } + selfTenant, self := managedArchiveSession(t, s, environmentInput(uuid.NewString(), "self_hosted", "/workspace")) + reset, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + if err != nil { + t.Fatal(err) + } + page, err := w.ListSandboxResetSessions(t.Context(), "", false) + if err != nil || len(page) != 1 || page[0].SessionID != "00000000-0000-4000-8000-000000000035" { + t.Fatal("busy prefix starved idle work", page, err) + } + if _, err := w.ArchiveSandboxResetSession(t.Context(), page[0].TenantID, page[0].SessionID, 1, reset.Reset.RequestedAt); err != nil { + t.Fatal(err) + } + if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err != nil { + t.Fatal(err) + } + first, err := w.ListSandboxResetSessions(t.Context(), "", true) + if err != nil || len(first) != 32 { + t.Fatal(first, err) + } + second, err := w.ListSandboxResetSessions(t.Context(), first[31].SessionID, true) + if err != nil || len(second) != 2 { + t.Fatal(second, err) + } + if _, err := w.ArchiveSandboxResetSession(t.Context(), selfTenant, self.ID, 1, reset.Reset.RequestedAt); !errors.Is(err, ErrInvalidInput) { + t.Fatal("self-hosted reset archive", err) + } + view, err := s.GetSession(t.Context(), selfTenant, self.ID) + if err != nil || view.Environment.Status == "expired" { + t.Fatal("reset changed self-hosted Session", view, err) + } +} + +func TestSandboxResetOwnerRestartRetainsDeadlineAndProvenance(t *testing.T) { + s, w, installation := managedArchiveFixture(t) + tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + reset, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + if err != nil { + t.Fatal(err) + } + // Model an abrupt owner loss and wait for PostgreSQL to terminate that backend, + // rather than assuming local TCP cleanup acknowledges advisory-lock release. + var stopped bool + if err := s.pool.QueryRow(t.Context(), `SELECT pg_terminate_backend($1,1000)`, w.executionLease.conn.Conn().PgConn().PID()).Scan(&stopped); err != nil || !stopped { + t.Fatal(stopped, err) + } + successor := executionLease(t, s).Store() + current, err := s.GetRuntimeDeployment(t.Context()) + if err != nil || !current.Reset.RequestedAt.Equal(reset.Reset.RequestedAt) || !current.Reset.DeadlineAt.Equal(*reset.Reset.DeadlineAt) { + t.Fatal("restart moved reset deadline", current, err) + } + if _, err := w.CancelSandboxReset(SandboxResetTestContext(t.Context()), installation, 1); err == nil { + t.Fatal("detached writer cancelled successor reset") + } + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_deployment SET reset_deadline_at=clock_timestamp()-interval '1 second'`) + if err := successor.AdvanceSandboxResetDeadline(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := successor.ArchiveSandboxResetSession(t.Context(), tenant, session.ID, 1, reset.Reset.RequestedAt); err != nil { + t.Fatal(err) + } + empty, err := successor.CompleteSandboxReset(t.Context(), installation, 1, reset.Reset.RequestedAt) + if err != nil || empty.Generation != 2 || empty.Provider != "" { + t.Fatal(empty, err) + } + var actions []string + if err := s.pool.QueryRow(t.Context(), `SELECT array_agg(action ORDER BY action) FROM admin_audit_log WHERE action IN ('reset_start','reset_deadline','reset_complete') AND request_id='reset-request' AND admin_credential_id='reset-fixture'`).Scan(&actions); err != nil || len(actions) != 3 { + t.Fatal("restart lost requester audit", actions, err) + } +} diff --git a/services/agents-api/internal/store/sandbox_snapshot_plan_test.go b/services/agents-api/internal/store/sandbox_snapshot_plan_test.go new file mode 100644 index 000000000..feb3e86df --- /dev/null +++ b/services/agents-api/internal/store/sandbox_snapshot_plan_test.go @@ -0,0 +1,106 @@ +package store + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func sandboxSnapshotSQL(t *testing.T) string { + t.Helper() + raw, err := os.ReadFile("../db/queries/sandbox_reset.sql") + if err != nil { + t.Fatal(err) + } + query := strings.Split(string(raw), "-- name: GetSandboxDeploymentSnapshot :one\n")[1] + return strings.TrimSuffix(strings.TrimSpace(strings.ReplaceAll(strings.Split(query, "\n-- name:")[0], "sqlc.embed(d)", "d.*")), ";") +} + +func TestSandboxSnapshotFreshPlan(t *testing.T) { + _, pool := newManagedTestStore(t) + tx, err := pool.Begin(t.Context()) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(t.Context()) + if _, err = tx.Exec(t.Context(), "SET LOCAL jit=on"); err != nil { + t.Fatal(err) + } + var stats int + if err = tx.QueryRow(t.Context(), "SELECT count(*) FROM pg_stats WHERE tablename='runtime_deployment'").Scan(&stats); err != nil || stats != 0 { + t.Fatal("fixture must exercise fresh cardinality", stats, err) + } + query := sandboxSnapshotSQL(t) + var raw []byte + if err = tx.QueryRow(t.Context(), "EXPLAIN (ANALYZE, FORMAT JSON) "+query).Scan(&raw); err != nil { + t.Fatal(err) + } + var plan []map[string]any + if err = json.Unmarshal(raw, &plan); err != nil { + t.Fatal(err) + } + if _, jit := plan[0]["JIT"]; jit { + t.Fatal("fresh singleton snapshot unexpectedly requires JIT", string(raw)) + } + bounded := false + var visit func(any) + visit = func(v any) { + switch v := v.(type) { + case map[string]any: + if v["Subplan Name"] == "CTE deployment" { + bounded = v["Plan Rows"] == float64(1) + } + for _, child := range v { + visit(child) + } + case []any: + for _, child := range v { + visit(child) + } + } + } + visit(plan[0]) + if !bounded { + t.Fatal("planner lost singleton bound", string(raw)) + } + t.Logf("fresh snapshot execution_ms=%v planning_ms=%v; singleton rows=1, no JIT", plan[0]["Execution Time"], plan[0]["Planning Time"]) + // A legal empty singleton table must not fabricate a deployment. + if _, err = tx.Exec(t.Context(), "DELETE FROM runtime_deployment"); err != nil { + t.Fatal(err) + } + var empty any + if err = tx.QueryRow(t.Context(), query).Scan(&empty); err != pgx.ErrNoRows { + t.Fatal("empty deployment produced a row", err) + } +} + +// Compare both complete projections in one MVCC statement and at one as_of, +// including every resource/rollout field. Only the cardinality annotation differs. +func assertSandboxSnapshotEquivalent(t *testing.T, pool *pgxpool.Pool) { + t.Helper() + query := sandboxSnapshotSQL(t) + legacy := strings.Replace(query, "WITH deployment AS MATERIALIZED (SELECT * FROM runtime_deployment WHERE singleton = true LIMIT 1),\nobserved AS", "WITH observed AS", 1) + legacy = strings.ReplaceAll(legacy, "CROSS JOIN deployment d", "CROSS JOIN runtime_deployment d") + legacy = strings.TrimSuffix(legacy, "\nWHERE d.singleton = true\nLIMIT 1") + query = strings.Replace(query, "clock_timestamp() AS as_of", "(SELECT as_of FROM snapshot_time) AS as_of", 1) + legacy = strings.Replace(legacy, "clock_timestamp() AS as_of", "(SELECT as_of FROM snapshot_time) AS as_of", 1) + tx, err := pool.Begin(t.Context()) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(t.Context()) + // Avoid paying the legacy planner's JIT cost in this result-equivalence check. + // The separate fresh-plan test keeps JIT enabled. + if _, err = tx.Exec(t.Context(), "SET LOCAL jit=off"); err != nil { + t.Fatal(err) + } + var same bool + err = tx.QueryRow(t.Context(), "WITH snapshot_time AS MATERIALIZED (SELECT clock_timestamp() AS as_of) SELECT (SELECT to_jsonb(s) FROM ("+query+") s) IS NOT DISTINCT FROM (SELECT to_jsonb(s) FROM ("+legacy+") s)").Scan(&same) + if err != nil || !same { + t.Fatal("singleton annotation changed snapshot", same, err) + } +} diff --git a/services/agents-api/internal/store/sandbox_specification_store_test.go b/services/agents-api/internal/store/sandbox_specification_store_test.go index 03bb4c481..1fdcea2c5 100644 --- a/services/agents-api/internal/store/sandbox_specification_store_test.go +++ b/services/agents-api/internal/store/sandbox_specification_store_test.go @@ -60,6 +60,7 @@ func TestSandboxSpecificationRoundTripAndFileConfigurationCannotOverride(t *test if err != nil || !reflect.DeepEqual(setup.Specification, input.DeploymentSpec) || view.Specification == nil || !reflect.DeepEqual(*view.Specification, input.DeploymentSpec) || view.SpecificationDigest != input.DeploymentSpec.Digest(provider) { t.Fatal("saved deployment lost its resources or Runtime provenance", err) } + input.ExpectedGeneration = view.Generation retry, err := w.InitializeSandboxDeployment(t.Context(), view.InstallationID, input) if err != nil || !reflect.DeepEqual(retry, view) { t.Fatal("identical specification changed the generation", err) @@ -69,7 +70,7 @@ func TestSandboxSpecificationRoundTripAndFileConfigurationCannotOverride(t *test if _, err := w.InitializeSandboxDeployment(t.Context(), view.InstallationID, changed); !errors.Is(err, ErrSandboxDeploymentConflict) { t.Fatal("initial setup silently resized a configured deployment", err) } - file := RuntimeDeployment{InstallationID: view.InstallationID, BackendFingerprint: setup.BackendFingerprint, ProviderKind: provider, Maintenance: true} + file := RuntimeDeployment{InstallationID: view.InstallationID, BackendFingerprint: setup.BackendFingerprint, ProviderKind: provider, AdmissionPaused: true} for _, candidate := range []*RuntimeDeployment{nil, &file} { if err := w.ConfigureRuntimeDeployment(t.Context(), candidate, nil); !errors.Is(err, ErrSandboxDeploymentConflict) { t.Fatal("file configuration replaced database ownership", err) @@ -115,7 +116,7 @@ func TestSandboxSpecificationBootstrapReadDoesNotConsumeEnrollment(t *testing.T) if _, err := s.RuntimeNodeConfiguration(t.Context(), "", token); !errors.Is(err, ErrRuntimeNodeCredential) { t.Fatal("consumed enrollment still authorized bootstrap", err) } - if _, err := w.SetSandboxMaintenance(t.Context(), view.InstallationID, SandboxMaintenanceRequest{ExpectedGeneration: view.Generation, Maintenance: true}); err != nil { + if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxResetRequest{Clear: "auto", ExpectedGeneration: view.Generation}); err != nil { t.Fatal(err) } config, err := s.RuntimeNodeConfiguration(t.Context(), node.NodeID, node.Credential) @@ -191,7 +192,7 @@ func TestSandboxSpecificationChangesWaitForEveryRetainedResource(t *testing.T) { } } } - before, err := w.SetSandboxMaintenance(t.Context(), view.InstallationID, SandboxMaintenanceRequest{ExpectedGeneration: view.Generation, Maintenance: true}) + before, err := s.GetRuntimeDeployment(t.Context()) if err != nil { t.Fatal(err) } @@ -253,7 +254,7 @@ func TestSandboxSpecificationChangesWaitForEveryRetainedResource(t *testing.T) { runtime.SourceCommit = strings.Repeat("2", 40) changed.Runtime = &runtime committed, err := w.UpdateSandboxDeployment(t.Context(), view.InstallationID, SandboxDeploymentUpdateRequest{ExpectedGeneration: view.Generation, SandboxDeploymentSetupRequest: changed}) - if err != nil || committed.Generation != view.Generation+1 || !committed.Maintenance || committed.Resources != (SandboxDeploymentResources{}) || committed.Specification == nil || !reflect.DeepEqual(*committed.Specification, changed.DeploymentSpec) { + if err != nil || committed.Generation != view.Generation+1 || committed.Reset != nil || committed.Resources != (SandboxDeploymentResources{}) || committed.Specification == nil || !reflect.DeepEqual(*committed.Specification, changed.DeploymentSpec) { t.Fatal("completed cleanup did not permit the replacement", err) } if _, err := s.AuthenticateRuntimeNode(t.Context(), node.NodeID, node.Credential); !errors.Is(err, ErrRuntimeNodeCredential) { @@ -307,7 +308,7 @@ func TestSandboxSpecificationAllocationRaceWithMaintenance(t *testing.T) { } go func() { <-start - _, err := w.SetSandboxMaintenance(t.Context(), view.InstallationID, SandboxMaintenanceRequest{ExpectedGeneration: view.Generation, Maintenance: true}) + _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxResetRequest{Clear: "auto", ExpectedGeneration: view.Generation}) maintenance <- err }() close(start) @@ -324,20 +325,20 @@ func TestSandboxSpecificationAllocationRaceWithMaintenance(t *testing.T) { t.Fatal("maintenance changed an admitted allocation retry", err) } } else { - if !errors.Is(result.err, ErrEnvironmentUnavailable) { + if !errors.Is(result.err, ErrSandboxResetAdmission) { t.Fatal("allocation race failed outside admission", result.err) } - if _, err := w.ReserveRuntimeAllocation(t.Context(), tenant, result.session.Environment.ID, view.InstallationID, device.HashCredential(uuid.NewString())); !errors.Is(err, ErrEnvironmentUnavailable) { + if _, err := w.ReserveRuntimeAllocation(t.Context(), tenant, result.session.Environment.ID, view.InstallationID, device.HashCredential(uuid.NewString())); !errors.Is(err, ErrSandboxResetAdmission) { t.Fatal("fresh allocation passed committed maintenance", err) } } } after, err := s.GetRuntimeDeployment(t.Context()) - if err != nil || !after.Maintenance || after.Generation != view.Generation || after.Resources.Allocations != allocated || after.Resources.Pending != int64(len(sessions))-allocated { + if err != nil || after.Reset == nil || after.Generation != view.Generation || after.Resources.Allocations != allocated || after.Resources.Pending != int64(len(sessions))-allocated { t.Fatal("concurrent maintenance lost resource accounting", after.Resources, err) } input.Resources.CPUs++ - if _, err := w.UpdateSandboxDeployment(t.Context(), view.InstallationID, SandboxDeploymentUpdateRequest{ExpectedGeneration: view.Generation, SandboxDeploymentSetupRequest: input}); !errors.Is(err, ErrSandboxDeploymentConflict) { + if _, err := w.UpdateSandboxDeployment(t.Context(), view.InstallationID, SandboxDeploymentUpdateRequest{ExpectedGeneration: view.Generation, SandboxDeploymentSetupRequest: input}); !errors.Is(err, ErrSandboxResetInProgress) { t.Fatal("allocation race bypassed replacement guard", err) } } diff --git a/services/agents-api/migrations/000079_sandbox_reset.sql b/services/agents-api/migrations/000079_sandbox_reset.sql new file mode 100644 index 000000000..7c71dde23 --- /dev/null +++ b/services/agents-api/migrations/000079_sandbox_reset.sql @@ -0,0 +1,63 @@ +-- +goose Up +ALTER TABLE runtime_deployment RENAME COLUMN maintenance TO admission_paused; +-- Old Web-managed maintenance is retired; ordinary upgrades resume admission. +UPDATE runtime_deployment SET admission_paused = false WHERE web_managed; +ALTER TABLE runtime_deployment + ADD COLUMN reset_clear text, + ADD COLUMN reset_requested_at timestamptz, + ADD COLUMN reset_deadline_at timestamptz, + ADD COLUMN reset_forced_at timestamptz, + ADD COLUMN reset_audit jsonb, + ADD CONSTRAINT runtime_deployment_reset_check CHECK ( + (reset_clear IS NULL AND reset_requested_at IS NULL AND reset_deadline_at IS NULL + AND reset_forced_at IS NULL AND reset_audit IS NULL) + OR (reset_clear IS NOT NULL AND web_managed AND provider_kind <> '' + AND reset_requested_at IS NOT NULL AND reset_audit IS NOT NULL + AND jsonb_typeof(reset_audit) = 'object' + AND ((reset_clear = 'auto' AND reset_deadline_at IS NOT NULL AND reset_forced_at IS NULL) + OR (reset_clear = 'force' AND reset_forced_at IS NOT NULL))) + ), + ADD CONSTRAINT runtime_deployment_reset_admission_check CHECK ( + NOT web_managed OR admission_paused = (reset_clear IS NOT NULL) + ); + +ALTER TABLE runtime_deployment DROP CONSTRAINT runtime_deployment_setup_check; +ALTER TABLE runtime_deployment ADD CONSTRAINT runtime_deployment_setup_check CHECK ( + NOT web_managed OR (local_node_id IS NULL AND ( + (provider_kind='' AND mode='' AND generation>=0 AND idle_seconds=0 AND retention_seconds=0) OR + (provider_kind='docker' AND mode='nodes' AND generation>0 AND idle_seconds=0 AND retention_seconds=0) OR + (provider_kind='microsandbox' AND mode='nodes' AND generation>0 AND idle_seconds>0 AND retention_seconds>0) OR + (provider_kind='e2b' AND mode='direct' AND generation>0 AND idle_seconds=0 AND retention_seconds=0) + )) +); + +-- +goose Down +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM runtime_deployment WHERE reset_clear IS NOT NULL) THEN + RAISE EXCEPTION 'Cannot downgrade while a sandbox reset is active'; + END IF; + IF EXISTS (SELECT 1 FROM runtime_deployment WHERE web_managed AND provider_kind = '' AND generation > 0) THEN + RAISE EXCEPTION 'Configure a sandbox provider before downgrading a completed reset'; + END IF; +END $$; +-- +goose StatementEnd +ALTER TABLE runtime_deployment + DROP CONSTRAINT runtime_deployment_reset_admission_check, + DROP CONSTRAINT runtime_deployment_reset_check, + DROP COLUMN reset_clear, + DROP COLUMN reset_requested_at, + DROP COLUMN reset_deadline_at, + DROP COLUMN reset_forced_at, + DROP COLUMN reset_audit; +ALTER TABLE runtime_deployment RENAME COLUMN admission_paused TO maintenance; +ALTER TABLE runtime_deployment DROP CONSTRAINT runtime_deployment_setup_check; +ALTER TABLE runtime_deployment ADD CONSTRAINT runtime_deployment_setup_check CHECK ( + NOT web_managed OR (local_node_id IS NULL AND ( + (provider_kind='' AND mode='' AND generation=0 AND idle_seconds=0 AND retention_seconds=0) OR + (provider_kind='docker' AND mode='nodes' AND generation>0 AND idle_seconds=0 AND retention_seconds=0) OR + (provider_kind='microsandbox' AND mode='nodes' AND generation>0 AND idle_seconds>0 AND retention_seconds>0) OR + (provider_kind='e2b' AND mode='direct' AND generation>0 AND idle_seconds=0 AND retention_seconds=0) + )) +); diff --git a/services/agents-api/migrations/000080_archived_cancellation_receipts.sql b/services/agents-api/migrations/000080_archived_cancellation_receipts.sql new file mode 100644 index 000000000..97e0ff974 --- /dev/null +++ b/services/agents-api/migrations/000080_archived_cancellation_receipts.sql @@ -0,0 +1,17 @@ +-- +goose Up +-- Only the archive transaction that first revokes the device may retain the +-- existing delivery's cancellation receipt. Old revocations are not adopted. +ALTER TABLE devices + ADD COLUMN archive_cancel_turn_id uuid REFERENCES turns(id) ON DELETE SET NULL; + +-- +goose Down +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM devices d JOIN runtime_allocations a ON a.device_id = d.id + WHERE d.archive_cancel_turn_id IS NOT NULL AND a.state <> 'released') THEN + RAISE EXCEPTION 'Cannot remove archived cancellation receipts while cleanup is unsettled'; + END IF; +END $$; +-- +goose StatementEnd +ALTER TABLE devices DROP COLUMN archive_cancel_turn_id; diff --git a/services/agents-api/tests/official_client.py b/services/agents-api/tests/official_client.py index aaa781442..0c9dc730c 100644 --- a/services/agents-api/tests/official_client.py +++ b/services/agents-api/tests/official_client.py @@ -36,6 +36,7 @@ from official_session_metadata import verify_session_metadata, verify_active_session_metadata from official_session_creators import verify_session_creators, verify_creator_recovery from official_source_file_list import verify_source_file_list, verify_source_file_list_recovery +from official_diagnostics import finish_server from openai import AuthenticationError, BadRequestError, ConflictError, InternalServerError, NotFoundError, OpenAI import yaml @@ -114,7 +115,8 @@ def project_bindings(directory): core_key_digests.write_text(json.dumps([hashlib.sha256(admin_token.encode()).hexdigest()])) credential_key = Path(directory) / "credential-key.txt" credential_key.touch(mode=0o600) - credential_key.write_text(base64.b64encode(secrets.token_bytes(32)).decode() + "\n") + credential_key_value = base64.b64encode(secrets.token_bytes(32)).decode() + credential_key.write_text(credential_key_value + "\n") env = dict(os.environ, OAC_DATABASE_URL=dsn, OAC_CORE_KEY_DIGESTS_FILE=str(core_key_digests), OAC_ADDR=f"127.0.0.1:{port}", OAC_DEFAULT_HARNESS="codex") env["OAC_CREDENTIAL_KEY_FILE"] = str(credential_key) # Enable the real Worker/gateway admission path without connecting a daemon. @@ -122,24 +124,22 @@ def project_bindings(directory): env["OAC_PUBLIC_URL"] = f"http://127.0.0.1:{port}" with (Path(directory) / "server.log").open("w+") as log: def start(): + nonlocal process child = subprocess.Popen([binary], env=env, stdout=log, stderr=log) - try: - deadline = time.monotonic() + 20 - with httpx2.Client(trust_env=False, timeout=1) as probe: - while time.monotonic() < deadline: - if child.poll() is not None: - raise AssertionError("Agents API exited during startup") - try: - if probe.get(base + "/healthz").status_code == 200: - return child - except httpx2.TransportError: - pass - time.sleep(0.1) - raise AssertionError("Agents API did not become healthy") - except BaseException: - child.terminate() - child.wait(timeout=15) - raise + # The outer finally also owns failed-start cleanup and diagnostics. + process = child + deadline = time.monotonic() + 20 + with httpx2.Client(trust_env=False, timeout=1) as probe: + while time.monotonic() < deadline: + if child.poll() is not None: + raise AssertionError("Agents API exited during startup") + try: + if probe.get(base + "/healthz").status_code == 200: + return child + except httpx2.TransportError: + pass + time.sleep(0.1) + raise AssertionError("Agents API did not become healthy") def client(token, **scope): return OpenAI(api_key=token, **scope, base_url=base + "/v1", max_retries=0, _strict_response_validation=True, http_client=httpx2.Client(trust_env=False, timeout=10, event_hooks={"response": [validate_response]})) @@ -377,16 +377,8 @@ def issue_key(project_id, name): print("Official Go client: creation/retries, retrieval, bidirectional pagination and tenant isolation passed.") print("Official client: upstream and generated response schemas, persistence/restart, retries, pagination, tenant isolation and explicit unsupported options passed.") finally: - if process and process.poll() is None: - process.terminate() - process.wait(timeout=15) - log.flush() - log.seek(0) - output = log.read() - assert admin_token not in output, "Administrator token leaked into the service log" - assert all(token not in output for token in tokens), "API key leaked into the service log" - assert credential_canary not in output, "Credential token leaked into the service log" - assert credential_key.read_text().strip() not in output, "Credential key leaked into the service log" + finish_server(process, log, [admin_token, *tokens, credential_canary, + credential_key_value], sys.exc_info()[1]) if __name__ == "__main__": diff --git a/services/agents-api/tests/official_diagnostics.py b/services/agents-api/tests/official_diagnostics.py new file mode 100644 index 000000000..31a794f74 --- /dev/null +++ b/services/agents-api/tests/official_diagnostics.py @@ -0,0 +1,86 @@ +"""Bounded failure facts for the real-service official-client fixture.""" + +import json +import re +import subprocess +import sys + + +EVENTS = { + "Core process configuration loaded from the process environment": "service_start", + "oac-core startup failed": "service_exit", +} +ERRORS = { + "context canceled": "context_canceled", + "context deadline exceeded": "deadline_exceeded", + "conn closed": "connection_closed", + "invalid input": "invalid_input", +} +SQLSTATES = { + "08006": "connection_failure", "22P02": "invalid_text_representation", + "23503": "foreign_key_violation", "23505": "unique_violation", + "23514": "check_violation", "40001": "serialization_failure", + "40P01": "deadlock_detected", "57014": "query_canceled", + "57P01": "admin_shutdown", "XX000": "database_internal_error", +} + + +def failure_facts(output, sensitive): + """Project fixed labels only; arbitrary messages, fields and stack lines stay private.""" + events = [] + omitted = 0 + for line in output.splitlines()[-32:]: + if len(line) > 16384 or any(value and value in line for value in sensitive): + omitted += 1 + continue + try: + entry = json.loads(line) + except (ValueError, RecursionError): + omitted += 1 + continue + if not isinstance(entry, dict) or not isinstance(entry.get("msg"), str) or entry["msg"] not in EVENTS: + omitted += 1 + continue + event = {"event": EVENTS[entry["msg"]]} + error = entry.get("error") + if isinstance(error, str): + event["error"] = ERRORS.get(error, "unclassified") + match = re.search(r"\(SQLSTATE ([0-9A-Z]{5})\)$", error) + if match and match[1] in SQLSTATES: + event.update(sqlstate=match[1], error=SQLSTATES[match[1]]) + events.append(event) + return {"events": events, "omitted_lines": omitted} + + +def finish_server(process, log, sensitive, primary_error, stream=None): + """Clean up without replacing the primary test failure with a cleanup failure.""" + exit_code = None + cleanup_failed = False + try: + if process is not None: + exit_code = process.poll() + if exit_code is None: + process.terminate() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=15) + raise + log.flush() + log.seek(0) + output = log.read() + assert not any(value and value in output for value in sensitive), "Fixture secret leaked into the service log" + except BaseException: + if primary_error is None: + raise + cleanup_failed = True + if primary_error is not None: + try: + facts = failure_facts(output, sensitive) if "output" in locals() else {"events": [], "log_unavailable": True} + # null means the service was alive when failure cleanup began; do not + # misreport our own SIGTERM as the cause of the test failure. + facts.update(exit_code=exit_code, cleanup_failed=cleanup_failed) + print("Official-client service failure: " + json.dumps(facts), file=stream or sys.stderr) + except BaseException: + pass # Even a broken diagnostic stream must retain the primary failure. diff --git a/services/agents-api/tests/official_diagnostics_test.py b/services/agents-api/tests/official_diagnostics_test.py new file mode 100644 index 000000000..a8fe1b351 --- /dev/null +++ b/services/agents-api/tests/official_diagnostics_test.py @@ -0,0 +1,75 @@ +"""Failure diagnostics must preserve both the primary exception and fixture secrets.""" + +import io +import json +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +from official_diagnostics import failure_facts, finish_server + + +class DiagnosticsTests(unittest.TestCase): + def test_failed_owned_process_reports_exit_and_safe_error(self): + with tempfile.TemporaryFile(mode="w+") as log: + code = 'import json; print(json.dumps({"msg":"oac-core startup failed","error":"ERROR: deadlock detected (SQLSTATE 40P01)"})); raise SystemExit(7)' + process = subprocess.Popen([sys.executable, "-c", code], stdout=log, stderr=log) + self.assertEqual(process.wait(timeout=10), 7) + diagnostics = io.StringIO() + primary = RuntimeError("primary fixture failure") + with self.assertRaises(RuntimeError) as caught: + try: + raise primary + finally: + finish_server(process, log, [], sys.exc_info()[1], diagnostics) + self.assertIs(caught.exception, primary) + facts = json.loads(diagnostics.getvalue().split(": ", 1)[1]) + self.assertEqual(facts["exit_code"], 7) + self.assertEqual(facts["events"], [{"event": "service_exit", "error": "deadlock_detected", "sqlstate": "40P01"}]) + + def test_secrets_and_untrusted_fields_never_enter_diagnostics(self): + secret = "dynamic-issued-project-key" + lines = [json.dumps({"msg": "oac-core startup failed", "error": secret}), + json.dumps({"msg": "oac-core startup failed", "error": "postgres://user:unknown-password@host/db", "Authorization": "Bearer unknown-key", "body": "private request"}), + "panic: private request", json.dumps({"msg": ["invalid log"]})] + diagnostics = io.StringIO() + primary = RuntimeError("primary") + with self.assertRaises(RuntimeError) as caught: + try: + raise primary + finally: + finish_server(None, io.StringIO("\n".join(lines)), [secret], sys.exc_info()[1], diagnostics) + self.assertIs(caught.exception, primary) + facts = json.loads(diagnostics.getvalue().split(": ", 1)[1]) + self.assertTrue(facts["cleanup_failed"]) + self.assertEqual(facts["events"], [{"event": "service_exit", "error": "unclassified"}]) + self.assertEqual(facts["omitted_lines"], 3) + self.assertNotIn(secret, diagnostics.getvalue()) + self.assertNotIn("private", diagnostics.getvalue()) + self.assertNotIn("unknown", diagnostics.getvalue()) + + def test_timeout_kills_child_without_masking_primary(self): + process = mock.Mock() + process.poll.return_value = None + process.wait.side_effect = [subprocess.TimeoutExpired("private command", 15), 0] + output = io.StringIO() + finish_server(process, io.StringIO(""), [], RuntimeError("primary"), output) + process.kill.assert_called_once() + self.assertEqual(process.wait.call_count, 2) + facts = json.loads(output.getvalue().split(": ", 1)[1]) + self.assertIsNone(facts["exit_code"]) + self.assertTrue(facts["cleanup_failed"]) + + def test_success_keeps_secret_assertion_and_has_no_diagnostic(self): + with self.assertRaisesRegex(AssertionError, "Fixture secret"): + finish_server(None, io.StringIO("secret"), ["secret"], None) + output = io.StringIO() + finish_server(None, io.StringIO(""), [], None, output) + self.assertEqual(output.getvalue(), "") + self.assertEqual(len(failure_facts('\n'.join([json.dumps({"msg": "oac-core startup failed"})] * 100), [])['events']), 32) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/core-console/node_installation_test.go b/services/core-console/node_installation_test.go index 0979c46ea..f7983a2df 100644 --- a/services/core-console/node_installation_test.go +++ b/services/core-console/node_installation_test.go @@ -21,6 +21,11 @@ func TestPairedConsoleProxiesOnlyAdministration(t *testing.T) { if r.Header.Get("Authorization") != "Bearer server-admin" { t.Errorf("incorrect upstream authority for %s", r.URL.Path) } + if r.URL.Path == "/core/v1/sandbox/deployment/maintenance" { + w.WriteHeader(http.StatusNotFound) + } else if r.Method == "DELETE" && r.URL.RawQuery != "expected_generation=7" { + t.Error("reset cancellation query was lost") + } _, _ = io.WriteString(w, "{}") })) defer upstream.Close() @@ -45,8 +50,11 @@ func TestPairedConsoleProxiesOnlyAdministration(t *testing.T) { {"POST", "/core/v1/sandbox/deployment", "none", 401}, {"PUT", "/core/v1/sandbox/deployment", "session", 200}, {"PUT", "/core/v1/sandbox/deployment", "none", 401}, - {"PATCH", "/core/v1/sandbox/deployment/maintenance", "session", 200}, + {"PATCH", "/core/v1/sandbox/deployment/maintenance", "session", 404}, {"PATCH", "/core/v1/sandbox/deployment/maintenance", "none", 401}, + {"POST", "/core/v1/sandbox/deployment/reset", "session", 200}, + {"DELETE", "/core/v1/sandbox/deployment/reset?expected_generation=7", "session", 200}, + {"DELETE", "/core/v1/sandbox/deployment/reset?expected_generation=7", "none", 401}, {"GET", "/core/v1/sandbox/nodes", "node", 401}, {"GET", "/core/v1/projects", "session", 200}, {"POST", "/api/v1/sandbox-node/enroll", "node", 404}, @@ -81,7 +89,7 @@ func TestPairedConsoleProxiesOnlyAdministration(t *testing.T) { t.Fatalf("console configuration = %s", body) } } - if calls.Load() != 4 { + if calls.Load() != 6 { t.Fatalf("unexpected upstream requests: %d", calls.Load()) } r := consoleRequest(t, server, "POST", "/core/v1/sandbox/deployment")