From 2722f356bcecce41926d13625f5b79f126d84ec1 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 20:44:40 +0800 Subject: [PATCH 1/7] refactor(sandbox): unify declared compute suspension protocol --- .../agents-api/node-generation-protocol.md | 9 +- docs/sandbox-provider.md | 22 +-- .../server/managed_generation_operations.go | 42 +++-- services/core/cmd/server/managed_setup.go | 2 +- .../db/queries/runtime_suspension.sql | 7 + .../db/sqlc/runtime_suspension.sql.go | 15 ++ .../archive_cancellation_cleanup_test.go | 4 +- .../provider_operations_fixture_test.go | 13 +- .../internal/execution/runtime_compute.go | 85 ++++++---- .../execution/runtime_compute_wake.go | 18 +-- .../internal/execution/runtime_lifecycle.go | 8 +- services/core/internal/execution/worker.go | 5 + .../internal/sandbox/docker/operations.go | 15 +- .../core/internal/sandbox/e2b/operations.go | 15 +- .../sandbox/microsandbox/operations.go | 5 +- .../internal/sandbox/microsandbox/provider.go | 34 ++-- .../sandbox/microsandbox/provider_test.go | 14 +- .../sandbox/microsandbox/suspension.go | 146 ++++++++++++++++++ .../microsandbox/suspension_contract_test.go | 57 +++++++ .../internal/sandbox/microsandbox/types.go | 52 ++++++- services/core/internal/sandbox/node/agent.go | 2 +- .../internal/sandbox/node/generation_json.go | 2 +- .../core/internal/sandbox/node/node_test.go | 2 +- .../core/internal/sandbox/node/operations.go | 3 +- .../internal/sandbox/node/operations_test.go | 2 +- .../node/provider_operations_fixture_test.go | 16 +- services/core/internal/sandbox/node/proxy.go | 14 +- services/core/internal/sandbox/node/wire.go | 43 +++--- services/core/internal/sandbox/operations.go | 10 +- .../core/internal/sandbox/operations_test.go | 2 +- .../sandbox/providers/registration.go | 9 +- .../sandbox/providers/registration_test.go | 3 +- .../internal/sandbox/providers/registry.go | 4 +- .../sandbox/providers/registry_test.go | 4 +- .../core/internal/sandbox/sandbox_provider.go | 108 ++++++++++++- services/core/internal/sandbox/suspension.go | 57 ------- .../core/internal/sandbox/suspension_test.go | 55 +++++++ .../admin_session_archive_worker_http_test.go | 2 +- .../store/provider_operations_fixture_test.go | 22 ++- .../store/runtime_compute_lifecycle_test.go | 95 +++++++++--- .../runtime_node_lifecycle_fixture_test.go | 12 +- .../core/internal/store/runtime_suspension.go | 16 ++ .../runtime_wake_hint_integration_test.go | 6 +- .../store/sandbox_deployment_setup.go | 2 +- .../tools/microsandbox-provider/README.md | 10 +- 45 files changed, 791 insertions(+), 278 deletions(-) create mode 100644 services/core/internal/sandbox/microsandbox/suspension.go create mode 100644 services/core/internal/sandbox/microsandbox/suspension_contract_test.go delete mode 100644 services/core/internal/sandbox/suspension.go create mode 100644 services/core/internal/sandbox/suspension_test.go diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index 09d84fd2a..195bfa711 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -42,20 +42,21 @@ Each operation carries its own arguments and returns the following result on suc | `command` | `RunCommand` | `command` | `command` | | `observe` | `Observe` | `observation` | `sample` | | `initial` | `Initial` | None | `compute` | -| `new_compute` | `NewCompute` | Positive compute `generation` and optional `snapshot` | `compute` | +| `new_compute` | `NewCompute` | Positive `generation` and optional `retained` | `compute` | | `compute` | `GetCompute` | `compute` | `state` | +| `renew_compute` | `RenewCompute` | Exact current `compute` | `state` | | `kill_compute` | `KillCompute` | `compute` | None | | `resume_compute` | `ResumeCompute` | `compute` | `state` | | `command_compute` | `RunCommandCompute` | `compute` and `command` | `command` | | `suspend` | `Suspend` | `suspend` | `state` | | `resume` | `Resume` | `resume` | `state` | -| `delete_snapshot` | `DeleteSnapshot` | `snapshot` | None | +| `delete_retained` | `DeleteRetained` | `retained` | None | -A request whose `connection_id`, `owner_epoch` or `sequence` does not match closes the connection. A malformed request gets an `invalid` response. A node without generation management accepts only its enrolled `deployment_generation`; a generation-managing node runs the request on that generation's provider and answers `unconfirmed` when it cannot. Core sends `create` and a `resume` that is not observe-only only to a generation that is ready on that node, and keeps at most 32 requests pending per connection. +A request whose `connection_id`, `owner_epoch` or `sequence` does not match closes the connection. A malformed request gets an `invalid` response. A node without generation management accepts only its enrolled `deployment_generation`; a generation-managing node runs the request on that generation's provider and answers `unconfirmed` when it cannot. Core sends `create` and a `resume` that is not reconciliation-only only to a generation that is ready on that node, and keeps at most 32 requests pending per connection. The budget is relative: the node anchors `timeout_ms` to its own clock on receipt and consumes it while the request waits in its queue, so the hosts' clocks need not agree. Core still bounds its own wait. A full node queue closes the connection. -The `response` frame carries `id` and `connection_id`. A successful response carries the result named in the operation table, with no result field for `kill`, `kill_compute` or `delete_snapshot`. A failed response carries an `error_code`: +The `response` frame carries `id` and `connection_id`. A successful response carries the result named in the operation table, with no result field for `kill`, `kill_compute` or `delete_retained`. A failed response carries an `error_code`: | `error_code` | Meaning | | --- | --- | diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index 1cf4b37d8..0bae2290b 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -42,13 +42,13 @@ Every provider implements the methods of each interface below and returns a comp | Contract | Requirement | Responsibility | | --- | --- | --- | | `sandbox.SandboxProvider` | All five operations supported | Allocation lifecycle and bounded commands | -| `sandbox.CheckpointProvider` | Explicit decision for every method, the same for all of them | Exact compute incarnations, capture and restore, retained-source resume and cleanup | +| `sandbox.SuspensionProvider` | Explicit decision for every method, the same for all of them | Exact incarnations, renewal, suspension, retained-state recovery and cleanup | | `runtimeobs.Source` | Explicit decision | Ownership-checked read-only observations | | `runtimeobs.BatchSource` | Explicit decision; requires `Source` | Bounded observations in input order, with per-target errors | | `sandbox.SelectionDiscoverer` | Explicit decision | Read-only native configuration discovery before commit | | `sandbox.CredentialVerifier` | Explicit decision | Verify access to owned resources without mutation | -`CheckpointProvider` adds `Initial` and `NewCompute` (construct compute references without allocating), `GetCompute`, `Suspend`, `Resume`, `ResumeCompute` (thaw only the same resident instance after an aborted pause), `KillCompute`, `DeleteSnapshot` and `RunCommandCompute`, which runs a bounded command in one exact compute incarnation. Core uses `RunCommandCompute` to wake a parked daemon after a restore ([`runtime_compute_wake.go`](../services/core/internal/execution/runtime_compute_wake.go)). +`SuspensionProvider` adds `Initial` and `NewCompute` (construct compute references without allocating), `GetCompute`, `RenewCompute`, `Suspend`, `Resume`, `ResumeCompute` (thaw only the same resident instance after an aborted pause), `KillCompute`, `DeleteRetained` and `RunCommandCompute`, which runs a bounded command in one exact compute incarnation. Core uses `RunCommandCompute` to wake a parked daemon after a restore ([`runtime_compute_wake.go`](../services/core/internal/execution/runtime_compute_wake.go)). Each declaration entry is `state: supported` with no reason, or `state: unsupported` with an authored reason code. Missing, zero, unknown or unsafe entries and missing methods fail validation. Adding a method to an interface requires an explicit decision and implementation in every adapter; never supply a base type or generate blanket unsupported implementations. @@ -88,7 +88,7 @@ Every call receives a bounded context. Expiry or cancellation ends the caller's `ErrInvalid`, `ErrOwnership`, `ErrExists`, `ErrNotFound`, `ErrComputeUnconfirmed` and `ErrCommandUnconfirmed` keep their defined meanings. An unclassified native or transport error is unknown, never permission to retry a mutation. Core never reads provider diagnostics as lifecycle truth or exposes native error text or credentials; the node transport maps errors to fixed codes, and direct SDK details stay private. -Checkpoint support adds `Compute` generation, name and ID and `SnapshotIdentity`; persist operation IDs and the provider's snapshot provenance unchanged. `ObserveOnly` on suspend or resume observes the previous attempt and never starts another capture or restore. `ResumeCompute` only thaws the retained source and never cold-starts a stopped one. Cleanup targets the exact compute incarnation and snapshot, not whatever instance now has the same name. Read [`runtime_compute.go`](../services/core/internal/execution/runtime_compute.go) and its failure tests before declaring checkpoint support. +Suspension support adds exact `Compute` generations and opaque `RetainedState` receipts. Persist their identities and operation IDs unchanged. `ReconcileOnly` observes the original attempt and may finish its owned cleanup, but never starts another capture or restore. `ResumeCompute` only thaws the retained source and never cold-starts a stopped one. Cleanup targets the exact compute incarnation and retained state. Read the [shared suspension contract](#suspension) before declaring support. ### Four distinct readiness facts @@ -110,7 +110,7 @@ A new provider takes these steps: 1. Implement the operation contracts in the adapter package, with native contract tests. 2. Add its specification and resource validators and, for native resource discovery before commit, an optional read-only `SelectionDiscoverer`. Put native credential verification behind `CredentialVerifier`. 3. Implement `sandbox.ConfigurationAdapter` over a typed native configuration. `DecodeInput` strictly parses the separate public `configuration` and write-only `credential` objects of a request. `Encode` produces whitelisted public selectors, read-only observations and separate secret bytes, and never passes request JSON through. `Decode` restores stored selectors, and keeps access to owned resources, without remote admission or new template validation. `Normalize` copies its input before changing it. `ResolveChange`, `Equal` and `WithCredential` own inheritance, identity and credential composition. `Requirements` declares whether a credential and a public Core origin are required and whether configuration discovery is supported. Also implement `ConfigurationDiscoverer`, even when discovery is unsupported: it validates the query and returns a safe catalog, never a mutation or an admission decision, while Core keeps authorization, input limits and deadlines. A node provider accepts only an empty public object, rejects credentials and returns Unsupported for discovery and credential replacement. -4. Register its constructor, policies, configuration adapter, operation declaration and defaults in `providers/registry.go`. Node proxy identity and checkpoint support read this entry. The installer's projection combines the registered policies with the shared field bounds in `sandbox/deployment_contract.go`; regenerate it with `go run ./services/core/cmd/specification-contract -write`. +4. Register its constructor, policies, configuration adapter, operation declaration and defaults in `providers/registry.go`. Node proxy identity and suspension support read this entry. The installer's projection combines the registered policies with the shared field bounds in `sandbox/deployment_contract.go`; regenerate it with `go run ./services/core/cmd/specification-contract -write`. 5. Supply the distribution artifacts for the adapter and its helper, and offer the provider to operators through the registered configuration contract. **Known design gap:** the installer's `--sandbox` choices and Web's setup views carry provider-specific options, such as E2B's installer flags and Web views. Exposing another provider through these surfaces currently requires shared installer and Web edits. This coupling does not meet [Complexity stays in the adapter](../AGENTS.md#complexity-stays-in-the-adapter); new integrations must express their configuration through the protocol and keep vendor-specific behavior in the adapter. Never add a Session or Turn scheduling path, a vendor column or API field, or a vendor switch in the store. @@ -124,7 +124,7 @@ A new provider takes these steps: - A `nodes` registration has only `BuildLocal`, and a `direct` registration only `BuildDirect`; missing, mixed or unknown modes are rejected. - The specification and resource validators, the configuration adapter and a complete operation declaration are mandatory, so an incomplete registration cannot publish a partial installer projection. - The Runtime input policy either accepts the pinned Runtime or gives the adapter's fixed reason for rejecting it, never both. -- Checkpoint support requires node mode and positive idle and retention defaults that fit Runtime durations; a provider without checkpoint support configures no suspension defaults. +- Suspension support requires positive idle and retention defaults that fit Runtime durations in either direct or node mode; a provider without suspension support configures no suspension defaults. The configuration adapter must be non-nil, including its concrete value. Every `ConfigurationRequirements` field needs an explicit valid decision: `Credential` and `PublicOrigin` are `Required` or `NotRequired`, and `Discovery` uses the shared supported or unsupported declaration with a safe reason. A new requirement field or discovery method needs an explicit validation update and never inherits an existing decision. Configuration discovery is distinct from resource selection discovery, and requiring a credential does not promise the `VerifyCredential` operation. These checks establish complete registration, not correct native SDK behavior; constructor and adapter contract tests still apply. @@ -134,7 +134,7 @@ Preview and persistence use `providers.Normalize` and `providers.Describe`. `Sel A direct adapter with a credential verifies all retained generations and allocation references before a key is replaced. The common `sandbox.CallFence` excludes native calls and waits for helper completion, including calls whose callers timed out; execution invokes the prepared verification and fencing callbacks without branching on a vendor. -Vendor deployment validation and SDK setup stay at the construction boundary, and construction never creates an Environment. For node-local adapters `providers.Built` returns the provider, probe, installation identity, backend fingerprint and specification digest, and the factory also returns its close function. `execution.RuntimeProvider` binds the adapter to its kind, installation ID, backend fingerprint, generation, mode and node ownership; the database owns the selection, and the in-memory copy is never another authority. Docker and microsandbox run on nodes, and E2B is constructed directly. The node proxy exposes checkpoint operations only for a backend whose registered declaration supports them, and common lifecycle code admits suspension through `CheckpointProvider`, never through a provider name. +Vendor deployment validation and SDK setup stay at the construction boundary, and construction never creates an Environment. For node-local adapters `providers.Built` returns the provider, probe, installation identity, backend fingerprint and specification digest, and the factory also returns its close function. `execution.RuntimeProvider` binds the adapter to its kind, installation ID, backend fingerprint, generation, mode and node ownership; the database owns the selection, and the in-memory copy is never another authority. Docker and microsandbox run on nodes, and E2B is constructed directly. The node proxy exposes suspension operations only for a backend whose registered declaration supports them, and common lifecycle code admits suspension through `SuspensionProvider`, never through a provider name. The backend fingerprint identifies a native resource namespace, not capacity. Core keeps deployment generations so that owned allocations keep resolving to their original backend; never repoint retained allocations at a replacement backend. @@ -182,11 +182,15 @@ The deployment's CPU, memory and disk settings, `max_active`, `max_retained` and ### Suspension -A provider with checkpoint support can suspend idle work; the deployment's [`suspension`](../contracts/agents-api/sandbox-deployment.md#safe-response) policy sets the idle time and snapshot retention. Core suspends only after at least one Turn is terminal, when no root or Subagent Turn is queued, in progress or waiting, no input, file operation or initialization is pending, and real activity has been idle for the configured interval. For node allocations Core records the first root or child terminal transition with the database clock in the same transaction. Candidate filtering and the Session-locked recheck compare elapsed database time with the idle duration, and the initial snapshot retention deadline is anchored to the same database observation, so Core and database host clocks need not agree. Native completion timestamps stay unchanged in public history but never drive idle admission, and heartbeats never reset activity. Before acknowledging a planned suspension, the daemon closes admission and drains native cleanup, output receipts and file work. +Core suspends only after at least one terminal Turn, when no root or Subagent Turn, input, file operation or initialization is pending and real activity has exceeded the deployment's idle duration. The shared database clock, Session lock and lifecycle lease serialize admission; heartbeats never reset idle activity. Queued work and live Files access request wake; history and published artifacts do not. The daemon drains native cleanup, receipts and file work before acknowledging quiescence. A lost acknowledgement authorizes only exact-source rollback, never a new capture. -The Worker lease, the Session lock and the per-node gates own suspension for every provider. New Turn claims, file-write intents and capture admission serialize under the Session lock and share one compute-phase check; new pending work cancels a capture and wakes the same source. Normal preparation waits for the compute phase to be running, after the authenticated resume handshake, and pending input stays pending when its promotion conflicts with a lifecycle transition. Compute phases and revision-checked receipts live on the allocation. Core persists quiesce, capture and restore intent before the effect, only a fresh receipt performs a capture or restore, and recovery observes the exact attempt without retrying an unknown creation, capture or restore. A consumed snapshot never rolls a running generation back. Deletion, revocation and retention expiry win over wake, up to the final database compare-and-swap, and unknown cleanup identities are kept until owned resources are confirmed absent. Consumed artifacts and old compute are deleted, so suspension cycles never build a chain of writable disks. +The complete declared SuspensionProvider group uses one lifecycle for direct and node placements. A RetainedState is an allocation/source/operation-bound opaque adapter receipt, limited to 64 KiB of native Data; it does not claim a snapshot. Core never interprets native Data. Initial and NewCompute plan exact logical incarnations without allocating resources; native IDs may remain equal across logical generations. GetCompute only observes. RenewCompute extends the exact running incarnation without waking it; the common running path observes and renews before keeping ownership or clearing a wake. -Queued work and live Environment file access wake a suspended Environment; history and published Artifact reads do not. Planned suspension uses an Environment and suspension token on the daemon connection. A PID and start-time fenced local control signal (`RunCommandCompute`) wakes the parked daemon, which authenticates again before admitting work. A transient disconnect before confirmation retries the same armed suspension with bounded attempts and backoff; a permanent authentication or protocol rejection closes it. Core owns the snapshot's retention deadline, and the daemon has no timer for it. A lost quiesce acknowledgement may thaw the same source through explicit rollback but never authorizes capturing it. +Suspend owns native resource release and returns a bound retained handle, suspended status, ResourcesReleased and SuspendSettled before Core releases active capacity. ReconcileOnly forbids replay of the original capture or pause while permitting adapter-owned cleanup proved safe by a durable retained artifact. A retained-less result permits rollback only with SuspendSettled and a recoverable running or paused source. Every other uncertain outcome retains ownership and closes admission. Core never unconditionally destroys the source after Suspend. + +Resume consumes the retained state into the precommitted target exactly once. Recovery observes the same attempt. Core persists waking, authenticates and resumes the daemon, deletes the consumed retained resource, then commits running and admits work. DeleteRetained is idempotent artifact cleanup and preserves running compute. Failed cleanup keeps the waking phase and cannot trigger another restore. KillCompute remains genuinely destructive; cleanup of an old generation must not kill a newer live incarnation sharing its native ID. + +The existing Session lock, lifecycle lease, idle rule, capacity queries and cleanup order remain authoritative. Every unreleased allocation consumes max_retained, including running allocations. This pre-release retained-state shape requires ordinary cleanup of old live compute state before activation; deployments must refuse activation with unreleased checkpoint or resident receipts. Session history is preserved. ### Reset and archive diff --git a/services/core/cmd/server/managed_generation_operations.go b/services/core/cmd/server/managed_generation_operations.go index 8c39850fa..f56280186 100644 --- a/services/core/cmd/server/managed_generation_operations.go +++ b/services/core/cmd/server/managed_generation_operations.go @@ -15,13 +15,13 @@ func (p *generationRouter) Initial(ctx context.Context, r sandbox.Reference) (sa return sandbox.Compute{}, err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return sandbox.Compute{}, err } return cp.Initial(ctx, r) } -func (p *generationRouter) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, snapshot *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *generationRouter) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, snapshot *sandbox.RetainedState) (sandbox.Compute, error) { if err := providercontract.Require(p, "NewCompute"); err != nil { return sandbox.Compute{}, err } @@ -30,7 +30,7 @@ func (p *generationRouter) NewCompute(ctx context.Context, r sandbox.Reference, return sandbox.Compute{}, err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return sandbox.Compute{}, err } @@ -45,7 +45,7 @@ func (p *generationRouter) GetCompute(ctx context.Context, r sandbox.Reference, return sandbox.ComputeState{}, err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return sandbox.ComputeState{}, err } @@ -60,7 +60,7 @@ func (p *generationRouter) Suspend(ctx context.Context, q sandbox.SuspendRequest return sandbox.ComputeState{}, err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return sandbox.ComputeState{}, err } @@ -75,7 +75,7 @@ func (p *generationRouter) Resume(ctx context.Context, q sandbox.ResumeRequest) return sandbox.ComputeState{}, err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return sandbox.ComputeState{}, err } @@ -90,14 +90,14 @@ func (p *generationRouter) KillCompute(ctx context.Context, r sandbox.Reference, return err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return err } return cp.KillCompute(ctx, r, c) } -func (p *generationRouter) DeleteSnapshot(ctx context.Context, r sandbox.Reference, snapshot sandbox.SnapshotIdentity) error { - if err := providercontract.Require(p, "DeleteSnapshot"); err != nil { +func (p *generationRouter) DeleteRetained(ctx context.Context, r sandbox.Reference, snapshot sandbox.RetainedState) error { + if err := providercontract.Require(p, "DeleteRetained"); err != nil { return err } v, done, err := p.route(ctx, r) @@ -105,11 +105,11 @@ func (p *generationRouter) DeleteSnapshot(ctx context.Context, r sandbox.Referen return err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return err } - return cp.DeleteSnapshot(ctx, r, snapshot) + return cp.DeleteRetained(ctx, r, snapshot) } func (p *generationRouter) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { if err := providercontract.Require(p, "RunCommandCompute"); err != nil { @@ -120,7 +120,7 @@ func (p *generationRouter) RunCommandCompute(ctx context.Context, r sandbox.Refe return sandbox.CommandResult{}, err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return sandbox.CommandResult{}, err } @@ -135,7 +135,7 @@ func (p *generationRouter) ResumeCompute(ctx context.Context, r sandbox.Referenc return sandbox.ComputeState{}, err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return sandbox.ComputeState{}, err } @@ -147,3 +147,19 @@ func (*generationRouter) DiscoverSelection(context.Context, sandbox.Selection) ( func (*generationRouter) VerifyCredential(context.Context, []sandbox.Reference) error { return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: "generation_router_does_not_verify_configuration"} } + +func (p *generationRouter) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + if err := providercontract.Require(p, "RenewCompute"); err != nil { + return sandbox.ComputeState{}, err + } + v, done, err := p.route(ctx, r) + if err != nil { + return sandbox.ComputeState{}, err + } + defer done() + cp, err := sandbox.Suspension(v) + if err != nil { + return sandbox.ComputeState{}, err + } + return cp.RenewCompute(ctx, r, c) +} diff --git a/services/core/cmd/server/managed_setup.go b/services/core/cmd/server/managed_setup.go index 3ec771c93..614a8d28b 100644 --- a/services/core/cmd/server/managed_setup.go +++ b/services/core/cmd/server/managed_setup.go @@ -126,7 +126,7 @@ func (s *managedSetup) configuration(setup store.SandboxSetup) (execution.Prepar } selected := &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: s.publicURL + "/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider} - if sandbox.SupportsCheckpoint(provider) { + if sandbox.SupportsSuspension(provider) { selected.Suspension = &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Duration(setup.IdleSeconds) * time.Second, Retention: time.Duration(setup.RetentionSeconds) * time.Second, MaxActive: 4, MaxRetained: 16} } diff --git a/services/core/internal/db/queries/runtime_suspension.sql b/services/core/internal/db/queries/runtime_suspension.sql index d4b6a6e21..d1faed453 100644 --- a/services/core/internal/db/queries/runtime_suspension.sql +++ b/services/core/internal/db/queries/runtime_suspension.sql @@ -64,3 +64,10 @@ SELECT EXISTS ( SELECT 1 FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id WHERE e.session_id = $1 AND a.node_id IS NOT NULL )::boolean; + +-- name: HasIncompatibleRuntimeComputeState :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations + WHERE state <> 'released' AND compute_phase <> 'disabled' + AND (compute_state->>'protocol_version') IS DISTINCT FROM sqlc.arg(protocol_version)::text +)::boolean; diff --git a/services/core/internal/db/sqlc/runtime_suspension.sql.go b/services/core/internal/db/sqlc/runtime_suspension.sql.go index 5c4867839..1c7170cb1 100644 --- a/services/core/internal/db/sqlc/runtime_suspension.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension.sql.go @@ -88,6 +88,21 @@ func (q *Queries) GetRuntimeActivity(ctx context.Context, id pgtype.UUID) (GetRu return i, err } +const hasIncompatibleRuntimeComputeState = `-- name: HasIncompatibleRuntimeComputeState :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations + WHERE state <> 'released' AND compute_phase <> 'disabled' + AND (compute_state->>'protocol_version') IS DISTINCT FROM $1::text +)::boolean +` + +func (q *Queries) HasIncompatibleRuntimeComputeState(ctx context.Context, protocolVersion string) (bool, error) { + row := q.db.QueryRow(ctx, hasIncompatibleRuntimeComputeState, protocolVersion) + var column_1 bool + err := row.Scan(&column_1) + return column_1, err +} + const recordRuntimeTerminalActivity = `-- name: RecordRuntimeTerminalActivity :exec UPDATE runtime_allocations a SET compute_activity_at = clock_timestamp() FROM environments e diff --git a/services/core/internal/execution/archive_cancellation_cleanup_test.go b/services/core/internal/execution/archive_cancellation_cleanup_test.go index d1412a553..97f8493b8 100644 --- a/services/core/internal/execution/archive_cancellation_cleanup_test.go +++ b/services/core/internal/execution/archive_cancellation_cleanup_test.go @@ -46,7 +46,7 @@ func (waitingCleanupCheckpoint) ProviderOperations() providercontract.Operations } type waitingCleanupCheckpoint struct { - sandbox.CheckpointProvider + sandbox.SuspensionProvider beforeKill func() } @@ -106,7 +106,7 @@ func TestArchiveWaitingCleanupReceiptBarrier(t *testing.T) { } currentCompute := sandbox.Compute{ID: uuid.NewString(), Name: owner.ID + "-g0"} if checkpoint { - state, _ := json.Marshal(runtimeCompute{Current: currentCompute}) + state, _ := json.Marshal(runtimeCompute{Version: sandbox.SuspensionStateVersion, Current: currentCompute}) owner, err = writer.SetRuntimeCompute(t.Context(), owner, "running", state, nil, 0) if err != nil { t.Fatal(err) diff --git a/services/core/internal/execution/provider_operations_fixture_test.go b/services/core/internal/execution/provider_operations_fixture_test.go index 4b568704b..d536f7b90 100644 --- a/services/core/internal/execution/provider_operations_fixture_test.go +++ b/services/core/internal/execution/provider_operations_fixture_test.go @@ -16,11 +16,12 @@ func (*lifecycleOnlySandbox) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ObservationProviderType": {State: providercontract.Supported}, @@ -34,7 +35,7 @@ func (*lifecycleOnlySandbox) ProviderOperations() providercontract.Operations { func (*lifecycleOnlySandbox) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: "fixture_operation_not_supported"} } -func (*lifecycleOnlySandbox) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (*lifecycleOnlySandbox) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: "fixture_operation_not_supported"} } func (*lifecycleOnlySandbox) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -49,8 +50,8 @@ func (*lifecycleOnlySandbox) Resume(context.Context, sandbox.ResumeRequest) (san func (*lifecycleOnlySandbox) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: "fixture_operation_not_supported"} } -func (*lifecycleOnlySandbox) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: "fixture_operation_not_supported"} +func (*lifecycleOnlySandbox) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: "fixture_operation_not_supported"} } func (*lifecycleOnlySandbox) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: "fixture_operation_not_supported"} @@ -75,3 +76,7 @@ func (*lifecycleOnlySandbox) ObservationProviderType() string { return "fixture" func (p *lifecycleOnlySandbox) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { return p, nil } + +func (p *lifecycleOnlySandbox) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index 055d0e2ba..905cc815e 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -1,9 +1,11 @@ package execution import ( + "bytes" "context" "encoding/json" "errors" + "io" "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" @@ -22,12 +24,13 @@ type RuntimeSuspensionPolicy struct { } type runtimeCompute struct { - Current sandbox.Compute `json:"current"` - Target *sandbox.Compute `json:"target,omitempty"` - Snapshot *sandbox.SnapshotIdentity `json:"snapshot,omitempty"` - SuspendID string `json:"suspend_id,omitempty"` - RestoreID string `json:"restore_id,omitempty"` - Rollback bool `json:"rollback,omitempty"` + Version string `json:"protocol_version"` + Current sandbox.Compute `json:"current"` + Target *sandbox.Compute `json:"target,omitempty"` + Retained *sandbox.RetainedState `json:"retained,omitempty"` + SuspendID string `json:"suspend_id,omitempty"` + RestoreID string `json:"restore_id,omitempty"` + Rollback bool `json:"rollback,omitempty"` } func (r *runtimeLifecycle) computeCapacity(ctx context.Context, key string) error { @@ -48,6 +51,7 @@ func (r *runtimeLifecycle) computeCapacity(ctx context.Context, key string) erro return nil } func (r *runtimeLifecycle) saveCompute(ctx context.Context, owner store.RuntimeAllocation, phase string, state runtimeCompute, until *time.Time) (store.RuntimeAllocation, error) { + state.Version = sandbox.SuspensionStateVersion raw, err := json.Marshal(state) if err != nil { return owner, err @@ -63,7 +67,7 @@ func (r *runtimeLifecycle) saveCompute(ctx context.Context, owner store.RuntimeA return r.store.SetRuntimeCompute(ctx, owner, phase, raw, until, idleTimeout) } func (r *runtimeLifecycle) enableCompute(ctx context.Context, owner store.RuntimeAllocation) error { - p, capabilityErr := sandbox.Checkpoint(r.config.Provider) + p, capabilityErr := sandbox.Suspension(r.config.Provider) if capabilityErr != nil { return capabilityErr } @@ -83,12 +87,12 @@ func (r *runtimeLifecycle) enableCompute(ctx context.Context, owner store.Runtim } func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner store.RuntimeAllocation) error { - p, capabilityErr := sandbox.Checkpoint(r.config.Provider) + p, capabilityErr := sandbox.Suspension(r.config.Provider) if capabilityErr != nil { return capabilityErr } var state runtimeCompute - if json.Unmarshal(owner.ComputeState, &state) != nil || state.Current.ID == "" { + if decodeRuntimeCompute(owner.ComputeState, &state) != nil || state.Current.ID == "" { return sandbox.ErrOwnership } if owner.SessionDeleted || owner.Expired || owner.State == "cleanup_pending" { @@ -122,7 +126,7 @@ func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner store.Runti } } -func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute) error { +func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SuspensionProvider, owner store.RuntimeAllocation, state runtimeCompute) error { compute, err := p.GetCompute(ctx, runtimeReference(owner), state.Current) if err != nil { return err @@ -130,6 +134,13 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.Checkpoint if compute.Status != "running" || !compute.BootstrapComplete { return sandbox.ErrComputeUnconfirmed } + renewed, err := p.RenewCompute(ctx, runtimeReference(owner), state.Current) + if err != nil { + return err + } + if sandbox.ValidateComputeResult(state.Current, renewed.Compute) != nil || renewed.Status != "running" || !renewed.BootstrapComplete { + return sandbox.ErrComputeUnconfirmed + } peer, err := authorizedRuntimePeer(ctx, r.store, r.registry, owner.DeviceID) if err != nil { return err @@ -189,16 +200,16 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.Checkpoint return r.captureCompute(ctx, p, suspending, state, false) } -func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute, observeOnly bool) error { - result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Snapshot: state.Snapshot, ObserveOnly: observeOnly}) +func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.SuspensionProvider, owner store.RuntimeAllocation, state runtimeCompute, observeOnly bool) error { + result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Retained: state.Retained, ReconcileOnly: observeOnly}) if err != nil { return err } - if result.Compute.ID != state.Current.ID { - return sandbox.ErrOwnership + if err := sandbox.ValidateSuspendResult(sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Retained: state.Retained, ReconcileOnly: observeOnly}, result); err != nil { + return err } - if result.Snapshot == nil { - if !observeOnly || result.SourceStopped || (result.Status != "running" && result.Status != "paused") { + if result.Retained == nil { + if !observeOnly || !result.SuspendSettled || result.ResourcesReleased || (result.Status != "running" && result.Status != "paused") { return sandbox.ErrComputeUnconfirmed } state.Rollback = true @@ -208,21 +219,12 @@ func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.Checkpo } return r.wakeCompute(ctx, p, next, state) } - state.Snapshot = result.Snapshot - // Store the verified artifact before any recovery-path kill. Snapshot failure - // or an unknown result cannot silently fall back to a cold Environment. - next, err := r.saveCompute(ctx, owner, "suspending", state, owner.ComputeRetainedUntil) - if err != nil { - return err - } - if err := ignoreComputeAbsent(p.KillCompute(ctx, runtimeReference(owner), state.Current)); err != nil { - return err - } - _, err = r.saveCompute(ctx, next, "suspended", state, next.ComputeRetainedUntil) + state.Retained = result.Retained + _, err = r.saveCompute(ctx, owner, "suspended", state, owner.ComputeRetainedUntil) return err } -func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute) error { +func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.SuspensionProvider, owner store.RuntimeAllocation, state runtimeCompute) error { activity, err := r.store.RuntimeActivity(ctx, owner) if err != nil { return err @@ -233,13 +235,16 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.Che if err := r.computeCapacityForAllocation(ctx, owner); err != nil { return err } - if state.Snapshot == nil || state.Target != nil { + if state.Retained == nil || state.Target != nil { return sandbox.ErrOwnership } - target, err := p.NewCompute(ctx, runtimeReference(owner), state.Current.Generation+1, state.Snapshot) + target, err := p.NewCompute(ctx, runtimeReference(owner), state.Current.Generation+1, state.Retained) if err != nil { return err } + if target.Name == "" || target.Generation != state.Current.Generation+1 || target.RestoredFrom == nil || *target.RestoredFrom != *state.Retained { + return sandbox.ErrOwnership + } state.Target, state.RestoreID = &target, uuid.NewString() next, err := r.saveCompute(ctx, owner, "restoring", state, owner.ComputeRetainedUntil) if err != nil { @@ -247,15 +252,15 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.Che } return r.restoreCompute(ctx, p, next, state, false) } -func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute, observeOnly bool) error { - if state.Target == nil || state.Snapshot == nil || state.Rollback { +func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.SuspensionProvider, owner store.RuntimeAllocation, state runtimeCompute, observeOnly bool) error { + if state.Target == nil || state.Retained == nil || state.Rollback { return sandbox.ErrOwnership } - result, err := p.Resume(ctx, sandbox.ResumeRequest{Reference: runtimeReference(owner), OperationID: state.RestoreID, Snapshot: *state.Snapshot, Target: *state.Target, ObserveOnly: observeOnly}) + result, err := p.Resume(ctx, sandbox.ResumeRequest{Reference: runtimeReference(owner), OperationID: state.RestoreID, Retained: *state.Retained, Target: *state.Target, ReconcileOnly: observeOnly}) if err != nil { return err } - if result.Status != "running" || result.Compute.ID == "" { + if result.Status != "running" || !result.BootstrapComplete || sandbox.ValidateComputeResult(*state.Target, result.Compute) != nil { return sandbox.ErrComputeUnconfirmed } state.Current, state.Target = result.Compute, nil @@ -282,3 +287,15 @@ func (r *runtimeLifecycle) computeCapacityForAllocation(ctx context.Context, own } return r.computeCapacity(ctx, owner.ProviderKey) } + +func decodeRuntimeCompute(raw []byte, state *runtimeCompute) error { + d := json.NewDecoder(bytes.NewReader(raw)) + d.DisallowUnknownFields() + if err := d.Decode(state); err != nil { + return err + } + if d.Decode(new(any)) != io.EOF || state.Version != sandbox.SuspensionStateVersion { + return sandbox.ErrOwnership + } + return nil +} diff --git a/services/core/internal/execution/runtime_compute_wake.go b/services/core/internal/execution/runtime_compute_wake.go index 9eae10eb9..fc560bc23 100644 --- a/services/core/internal/execution/runtime_compute_wake.go +++ b/services/core/internal/execution/runtime_compute_wake.go @@ -11,7 +11,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) -func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute) error { +func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SuspensionProvider, owner store.RuntimeAllocation, state runtimeCompute) error { if state.Rollback { if _, err := p.ResumeCompute(ctx, runtimeReference(owner), state.Current); err != nil { return err @@ -54,8 +54,8 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.Checkpoint } // The artifact has been consumed. Never restore it after this generation // admits work, even if garbage collection or the final database commit fails. - if state.Snapshot != nil { - if err := ignoreComputeAbsent(p.DeleteSnapshot(ctx, runtimeReference(owner), *state.Snapshot)); err != nil { + if state.Retained != nil { + if err := ignoreComputeAbsent(p.DeleteRetained(ctx, runtimeReference(owner), *state.Retained)); err != nil { return err } } @@ -69,19 +69,19 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.Checkpoint return r.observeConnection(ctx, next) } -func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute) error { +func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.SuspensionProvider, owner store.RuntimeAllocation, state runtimeCompute) error { if err := r.store.CheckExecutionOwnership(ctx); err != nil { return err } // An uncommitted artifact is found by its persisted attempt, never a directory // glob. The helper's allocation lock also waits for an earlier unknown call. - if owner.ComputePhase == "suspending" && state.Snapshot == nil { - result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, ObserveOnly: true}) + if owner.ComputePhase == "suspending" && state.Retained == nil { + result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, ReconcileOnly: true}) if err != nil && !errors.Is(err, sandbox.ErrNotFound) { return err } if err == nil { - state.Snapshot = result.Snapshot + state.Retained = result.Retained } } if state.Target != nil { @@ -92,8 +92,8 @@ func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.Checkpo if err := ignoreComputeAbsent(p.KillCompute(ctx, runtimeReference(owner), state.Current)); err != nil { return err } - if state.Snapshot != nil { - if err := ignoreComputeAbsent(p.DeleteSnapshot(ctx, runtimeReference(owner), *state.Snapshot)); err != nil { + if state.Retained != nil { + if err := ignoreComputeAbsent(p.DeleteRetained(ctx, runtimeReference(owner), *state.Retained)); err != nil { return err } } diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index b09dda19f..d80ac7140 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -96,12 +96,12 @@ func validatedRuntimeProvider(config *RuntimeProvider, registry *runtimegateway. if copied.Mode != "" && copied.Mode != "nodes" && copied.Mode != "direct" { return RuntimeProvider{}, sandbox.ErrInvalid } - if copied.Mode == "direct" && (copied.ProviderKind == "" || copied.LocalNodeID != "" || copied.Suspension != nil) { + if copied.Mode == "direct" && (copied.ProviderKind == "" || copied.LocalNodeID != "") { return RuntimeProvider{}, sandbox.ErrInvalid } if config.Suspension != nil { policy := *config.Suspension - if !sandbox.SupportsCheckpoint(config.Provider) || policy.IdleTimeout < time.Second || policy.Retention < time.Second || policy.MaxActive < 1 || policy.MaxRetained < policy.MaxActive { + if !sandbox.SupportsSuspension(config.Provider) || policy.IdleTimeout < time.Second || policy.Retention < time.Second || policy.MaxActive < 1 || policy.MaxRetained < policy.MaxActive { return RuntimeProvider{}, sandbox.ErrInvalid } copied.Suspension = &policy @@ -185,7 +185,7 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p if err := r.computeFreshCapacity(ctx, providerKey); err != nil { return store.RuntimeAllocation{}, err } - if policy := r.config.Suspension; policy != nil && r.config.ProviderKind == "" { + if policy := r.config.Suspension; policy != nil && (r.config.ProviderKind == "" || r.config.Mode == "direct") { count, err := r.store.CountRuntimeRetainedAllocations(ctx, providerKey) if err != nil { return store.RuntimeAllocation{}, err @@ -429,7 +429,7 @@ func (w *Worker) runManagedRuntimes(ctx context.Context) error { // Manager deployments reserve capacity with Session placement before provisioning. func (r *runtimeLifecycle) computeFreshCapacity(ctx context.Context, key string) error { - if r.config.ProviderKind != "" { + if r.config.ProviderKind != "" && r.config.Mode != "direct" { return nil } return r.computeCapacity(ctx, key) diff --git a/services/core/internal/execution/worker.go b/services/core/internal/execution/worker.go index 03e42f4e1..d63e29328 100644 --- a/services/core/internal/execution/worker.go +++ b/services/core/internal/execution/worker.go @@ -9,6 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) @@ -40,6 +41,10 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher) (*Worker, error) { } owned := *dispatcher owned.Store = lease.Store() + if err := owned.Store.CheckRuntimeComputeProtocol(ctx, sandbox.SuspensionStateVersion); err != nil { + _ = lease.Close(context.Background()) + return nil, err + } owned.notifications = &executionNotifications{} worker := &Worker{concurrency: dispatcher.MaxConcurrentExecutions, dispatcher: &owned, admission: dispatcher.Store, lease: lease, directoryReads: make(chan directoryReadRequest), fileWrites: make(chan fileWriteRequest), stopped: make(chan struct{}), scheduleWake: make(chan struct{}, 1), enrolledConnections: make(map[string]*runtimeConnection)} worker.runtimes, err = newRuntimeManager(owned.Store, owned.Registry, owned.ManagedRuntimes) diff --git a/services/core/internal/sandbox/docker/operations.go b/services/core/internal/sandbox/docker/operations.go index c2758008d..94f0bb64d 100644 --- a/services/core/internal/sandbox/docker/operations.go +++ b/services/core/internal/sandbox/docker/operations.go @@ -7,7 +7,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -var _ sandbox.CheckpointProvider = (*Provider)(nil) +var _ sandbox.SuspensionProvider = (*Provider)(nil) var _ sandbox.SelectionDiscoverer = (*Provider)(nil) var _ sandbox.CredentialVerifier = (*Provider)(nil) var _ runtimeobs.BatchSource = (*Provider)(nil) @@ -22,11 +22,12 @@ func Operations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "NewCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "GetCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "Suspend": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "Resume": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "KillCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "ObservationProviderType": {State: providercontract.Supported}, @@ -41,7 +42,7 @@ func (*Provider) ProviderOperations() providercontract.Operations { return Opera func (p *Provider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: Operations()["Initial"].Reason} } -func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: Operations()["NewCompute"].Reason} } func (p *Provider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -56,8 +57,8 @@ func (p *Provider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.Compu func (p *Provider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: Operations()["KillCompute"].Reason} } -func (p *Provider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: Operations()["DeleteSnapshot"].Reason} +func (p *Provider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: Operations()["DeleteRetained"].Reason} } func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: Operations()["RunCommandCompute"].Reason} @@ -74,3 +75,7 @@ func (p *Provider) DiscoverSelection(context.Context, sandbox.Selection) (sandbo func (p *Provider) VerifyCredential(context.Context, []sandbox.Reference) error { return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: Operations()["VerifyCredential"].Reason} } + +func (p *Provider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: Operations()["RenewCompute"].Reason} +} diff --git a/services/core/internal/sandbox/e2b/operations.go b/services/core/internal/sandbox/e2b/operations.go index 14530ebf9..dab82e81c 100644 --- a/services/core/internal/sandbox/e2b/operations.go +++ b/services/core/internal/sandbox/e2b/operations.go @@ -7,7 +7,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -var _ sandbox.CheckpointProvider = (*Provider)(nil) +var _ sandbox.SuspensionProvider = (*Provider)(nil) var _ sandbox.SelectionDiscoverer = (*Provider)(nil) var _ sandbox.CredentialVerifier = (*Provider)(nil) var _ runtimeobs.BatchSource = (*Provider)(nil) @@ -22,11 +22,12 @@ func Operations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "NewCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "GetCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "Suspend": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "Resume": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "KillCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "ObservationProviderType": {State: providercontract.Supported}, @@ -41,7 +42,7 @@ func (*Provider) ProviderOperations() providercontract.Operations { return Opera func (p *Provider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: Operations()["Initial"].Reason} } -func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: Operations()["NewCompute"].Reason} } func (p *Provider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -56,8 +57,8 @@ func (p *Provider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.Compu func (p *Provider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: Operations()["KillCompute"].Reason} } -func (p *Provider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: Operations()["DeleteSnapshot"].Reason} +func (p *Provider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: Operations()["DeleteRetained"].Reason} } func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: Operations()["RunCommandCompute"].Reason} @@ -65,3 +66,7 @@ func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox func (p *Provider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: Operations()["ResumeCompute"].Reason} } + +func (p *Provider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: Operations()["RenewCompute"].Reason} +} diff --git a/services/core/internal/sandbox/microsandbox/operations.go b/services/core/internal/sandbox/microsandbox/operations.go index f59c9ed02..4737a94a1 100644 --- a/services/core/internal/sandbox/microsandbox/operations.go +++ b/services/core/internal/sandbox/microsandbox/operations.go @@ -7,7 +7,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -var _ sandbox.CheckpointProvider = (*Provider)(nil) +var _ sandbox.SuspensionProvider = (*Provider)(nil) var _ sandbox.SelectionDiscoverer = (*Provider)(nil) var _ sandbox.CredentialVerifier = (*Provider)(nil) var _ runtimeobs.BatchSource = (*Provider)(nil) @@ -22,11 +22,12 @@ func Operations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Supported}, "NewCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, "GetCompute": {State: providercontract.Supported}, "Suspend": {State: providercontract.Supported}, "Resume": {State: providercontract.Supported}, "KillCompute": {State: providercontract.Supported}, - "DeleteSnapshot": {State: providercontract.Supported}, + "DeleteRetained": {State: providercontract.Supported}, "RunCommandCompute": {State: providercontract.Supported}, "ResumeCompute": {State: providercontract.Supported}, "ObservationProviderType": {State: providercontract.Supported}, diff --git a/services/core/internal/sandbox/microsandbox/provider.go b/services/core/internal/sandbox/microsandbox/provider.go index d6df19ec7..a22dd898f 100644 --- a/services/core/internal/sandbox/microsandbox/provider.go +++ b/services/core/internal/sandbox/microsandbox/provider.go @@ -16,7 +16,7 @@ type Provider struct { } var _ sandbox.SandboxProvider = (*Provider)(nil) -var _ sandbox.CheckpointProvider = (*Provider)(nil) +var _ sandbox.SuspensionProvider = (*Provider)(nil) func New(c Config) (*Provider, error) { return NewWithCaller(c, &ProcessCaller{}) } func NewWithCaller(c Config, caller Caller) (*Provider, error) { @@ -26,7 +26,7 @@ func NewWithCaller(c Config, caller Caller) (*Provider, error) { c.Network.Rules = append([]NetworkRule(nil), c.Network.Rules...) return &Provider{config: c, caller: caller}, nil } -func (p *Provider) Initial(ctx context.Context, r sandbox.Reference) (Compute, error) { +func (p *Provider) nativeInitial(ctx context.Context, r sandbox.Reference) (Compute, error) { if err := ctx.Err(); err != nil { return Compute{}, err } @@ -88,7 +88,7 @@ func (p *Provider) responseState(ctx context.Context, q Request, out Response) ( } want := q.Compute if q.Operation == "create" { - want, e = p.Initial(ctx, q.Reference) + want, e = p.nativeInitial(ctx, q.Reference) if e != nil { return State{}, e } @@ -142,48 +142,48 @@ func (p *Provider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Inf return result, err } func (p *Provider) GetInfo(ctx context.Context, r sandbox.Reference) (sandbox.Info, error) { - c, e := p.Initial(ctx, r) + c, e := p.nativeInitial(ctx, r) if e != nil { return sandbox.Info{}, e } - s, e := p.GetCompute(ctx, r, c) + s, e := p.nativeGetCompute(ctx, r, c) return info(r, s), e } func (p *Provider) Renew(ctx context.Context, r sandbox.Reference) (sandbox.Info, error) { return p.GetInfo(ctx, r) } func (p *Provider) Kill(ctx context.Context, r sandbox.Reference) error { - c, e := p.Initial(ctx, r) + c, e := p.nativeInitial(ctx, r) if e != nil { return e } - return p.KillCompute(ctx, r, c) + return p.nativeKillCompute(ctx, r, c) } func (p *Provider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { - compute, e := p.Initial(ctx, r) + compute, e := p.nativeInitial(ctx, r) if e != nil { return sandbox.CommandResult{}, e } - return p.RunCommandCompute(ctx, r, compute, c) + return p.nativeRunCommandCompute(ctx, r, compute, c) } -func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { +func (p *Provider) nativeGetCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { return p.state(ctx, Request{Operation: "inspect", Reference: r, Compute: c}) } -func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c Compute) error { +func (p *Provider) nativeKillCompute(ctx context.Context, r sandbox.Reference, c Compute) error { _, e := p.call(ctx, Request{Operation: "kill", Reference: r, Compute: c}) return e } -func (p *Provider) DeleteSnapshot(ctx context.Context, r sandbox.Reference, s SnapshotIdentity) error { +func (p *Provider) nativeDeleteSnapshot(ctx context.Context, r sandbox.Reference, s SnapshotIdentity) error { _, e := p.call(ctx, Request{Operation: "delete_snapshot", Reference: r, Snapshot: &s}) return e } -func (p *Provider) Suspend(ctx context.Context, q SuspendRequest) (State, error) { +func (p *Provider) nativeSuspend(ctx context.Context, q SuspendRequest) (State, error) { return p.state(ctx, Request{Operation: "suspend", Reference: q.Reference, Suspend: &q}) } -func (p *Provider) Resume(ctx context.Context, q ResumeRequest) (State, error) { +func (p *Provider) nativeResume(ctx context.Context, q ResumeRequest) (State, error) { return p.state(ctx, Request{Operation: "resume", Reference: q.Reference, Resume: &q}) } -func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c Compute, command sandbox.Command) (sandbox.CommandResult, error) { +func (p *Provider) nativeRunCommandCompute(ctx context.Context, r sandbox.Reference, c Compute, command sandbox.Command) (sandbox.CommandResult, error) { out, e := p.call(ctx, Request{Operation: "command", Reference: r, Compute: c, Command: &command}) if e != nil { return sandbox.CommandResult{}, e @@ -196,11 +196,11 @@ func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c // ResumeCompute thaws the exact resident source after an aborted suspension. // It never starts stopped compute or restores a checkpoint. -func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { +func (p *Provider) nativeResumeCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { return p.state(ctx, Request{Operation: "resume_compute", Reference: r, Compute: c}) } -func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, generation uint64, snapshot *SnapshotIdentity) (Compute, error) { +func (p *Provider) nativeNewCompute(ctx context.Context, r sandbox.Reference, generation uint64, snapshot *SnapshotIdentity) (Compute, error) { if err := ctx.Err(); err != nil { return Compute{}, err } diff --git a/services/core/internal/sandbox/microsandbox/provider_test.go b/services/core/internal/sandbox/microsandbox/provider_test.go index 94bfa9881..42f955dc1 100644 --- a/services/core/internal/sandbox/microsandbox/provider_test.go +++ b/services/core/internal/sandbox/microsandbox/provider_test.go @@ -44,7 +44,7 @@ func TestRejectsChangedComputeIdentity(t *testing.T) { if e != nil { t.Fatal(e) } - _, e = p.GetCompute(deadline(t), r, Compute{Name: Name(c, r, 0), ID: "local:4"}) + _, e = p.nativeGetCompute(deadline(t), r, Compute{Name: Name(c, r, 0), ID: "local:4"}) if !errors.Is(e, sandbox.ErrOwnership) { t.Fatalf("foreign identity accepted: %v", e) } @@ -60,7 +60,7 @@ func TestRestoreMustRetainExactProvenance(t *testing.T) { got.RestoredFrom = &v return Response{Version: ProtocolVersion, State: &State{Compute: got, Status: "running", BootstrapComplete: true}}, nil })) - _, e := p.Resume(deadline(t), ResumeRequest{Reference: r, OperationID: "66666666-6666-4666-8666-666666666666", Snapshot: s, Target: target}) + _, e := p.nativeResume(deadline(t), ResumeRequest{Reference: r, OperationID: "66666666-6666-4666-8666-666666666666", Snapshot: s, Target: target}) if !errors.Is(e, sandbox.ErrOwnership) { t.Fatalf("different snapshot accepted: %v", e) } @@ -70,7 +70,7 @@ func TestRejectsSnapshotPathBeforeHelper(t *testing.T) { s.Reference = "/foreign/checkpoint" calls := 0 p, _ := NewWithCaller(c, callerFunc(func(context.Context, Request) (Response, error) { calls++; return Response{}, nil })) - if e := p.DeleteSnapshot(deadline(t), r, s); !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { + if e := p.nativeDeleteSnapshot(deadline(t), r, s); !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { t.Fatalf("e=%v calls=%d", e, calls) } } @@ -83,7 +83,7 @@ func TestObserveOnlyPreservesCapturedButResidentState(t *testing.T) { } return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: "paused", Snapshot: &s}}, nil })) - got, e := p.Suspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) + got, e := p.nativeSuspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) if e != nil || got.SourceStopped || got.Status != "paused" { t.Fatalf("state=%+v error=%v", got, e) } @@ -117,11 +117,11 @@ func TestNoDeadlineOrForeignAllocationNeverCallsHelper(t *testing.T) { r := testRef() foreign := r foreign.EnvironmentID = "77777777-7777-4777-8777-777777777777" - initial, initialErr := p.Initial(deadline(t), r) + initial, initialErr := p.nativeInitial(deadline(t), r) if initialErr != nil { t.Fatal(initialErr) } - _, e = p.GetCompute(deadline(t), foreign, initial) + _, e = p.nativeGetCompute(deadline(t), foreign, initial) if !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { t.Fatalf("e=%v calls=%d", e, calls) } @@ -146,7 +146,7 @@ func TestMissingSnapshotObservationAllowsOnlyIntactSourceRollback(t *testing.T) p, _ := NewWithCaller(c, callerFunc(func(context.Context, Request) (Response, error) { return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: status, BootstrapComplete: true}}, nil })) - _, e := p.Suspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) + _, e := p.nativeSuspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) if status == "running" || status == "paused" { if e != nil { t.Fatal(e) diff --git a/services/core/internal/sandbox/microsandbox/suspension.go b/services/core/internal/sandbox/microsandbox/suspension.go new file mode 100644 index 000000000..5e1a22e16 --- /dev/null +++ b/services/core/internal/sandbox/microsandbox/suspension.go @@ -0,0 +1,146 @@ +package microsandbox + +import ( + "context" + "encoding/json" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// retained translates a verified native snapshot without exposing its schema to Core. +func retained(s SnapshotIdentity) sandbox.RetainedState { + raw, _ := json.Marshal(s) + return sandbox.RetainedState{Reference: s.Reference, ID: s.ID, OperationID: s.OperationID, SourceGeneration: s.SourceGeneration, SourceName: s.SourceName, SourceID: s.SourceID, Data: string(raw)} +} +func (p *Provider) snapshot(r sandbox.Reference, s sandbox.RetainedState) (SnapshotIdentity, error) { + var native SnapshotIdentity + if sandbox.ValidateRetained(s) != nil || json.Unmarshal([]byte(s.Data), &native) != nil || retained(native) != s || ValidateSnapshot(p.config, r, native) != nil { + return native, sandbox.ErrOwnership + } + return native, nil +} +func compute(c Compute) sandbox.Compute { + out := sandbox.Compute{Generation: c.Generation, Name: c.Name, ID: c.ID} + if c.RestoredFrom != nil { + s := retained(*c.RestoredFrom) + out.RestoredFrom = &s + } + return out +} +func (p *Provider) nativeCompute(r sandbox.Reference, c sandbox.Compute) (Compute, error) { + out := Compute{Generation: c.Generation, Name: c.Name, ID: c.ID} + if c.RestoredFrom != nil { + s, e := p.snapshot(r, *c.RestoredFrom) + if e != nil { + return out, e + } + out.RestoredFrom = &s + } + if ValidateCompute(p.config, r, out) != nil { + return out, sandbox.ErrOwnership + } + return out, nil +} +func state(s State) sandbox.ComputeState { + out := sandbox.ComputeState{Compute: compute(s.Compute), Status: s.Status, BootstrapComplete: s.BootstrapComplete, ResourcesReleased: s.SourceStopped} + if s.Snapshot != nil { + v := retained(*s.Snapshot) + out.Retained = &v + } + return out +} +func (p *Provider) Initial(ctx context.Context, r sandbox.Reference) (sandbox.Compute, error) { + c, e := p.nativeInitial(ctx, r) + return compute(c), e +} +func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, s *sandbox.RetainedState) (sandbox.Compute, error) { + var snap *SnapshotIdentity + if s != nil { + v, e := p.snapshot(r, *s) + if e != nil { + return sandbox.Compute{}, e + } + snap = &v + } + c, e := p.nativeNewCompute(ctx, r, g, snap) + return compute(c), e +} +func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + n, e := p.nativeCompute(r, c) + if e != nil { + return sandbox.ComputeState{}, e + } + s, e := p.nativeGetCompute(ctx, r, n) + return state(s), e +} +func (p *Provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.GetCompute(ctx, r, c) +} +func (p *Provider) Suspend(ctx context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { + c, e := p.nativeCompute(q.Reference, q.Source) + if e != nil { + return sandbox.ComputeState{}, e + } + n := SuspendRequest{Reference: q.Reference, OperationID: q.OperationID, Source: c, ObserveOnly: q.ReconcileOnly} + if q.Retained != nil { + s, e := p.snapshot(q.Reference, *q.Retained) + if e != nil { + return sandbox.ComputeState{}, e + } + n.Snapshot = &s + } + s, e := p.nativeSuspend(ctx, n) + if e == nil && s.Snapshot != nil { + // Complete the same exact-source cleanup previously performed by Core. + // Native KillCompute still verifies ownership and removes its disks. + e = p.nativeKillCompute(ctx, q.Reference, c) + if e == nil { + s.SourceStopped = true + s.Status = "suspended" + } + } + out := state(s) + // The helper holds the allocation lock until the original native work settles. + out.SuspendSettled = e == nil + return out, e +} +func (p *Provider) Resume(ctx context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { + c, e := p.nativeCompute(q.Reference, q.Target) + if e != nil { + return sandbox.ComputeState{}, e + } + s, e := p.snapshot(q.Reference, q.Retained) + if e != nil { + return sandbox.ComputeState{}, e + } + v, e := p.nativeResume(ctx, ResumeRequest{Reference: q.Reference, OperationID: q.OperationID, Snapshot: s, Target: c, ObserveOnly: q.ReconcileOnly}) + return state(v), e +} +func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) error { + n, e := p.nativeCompute(r, c) + if e != nil { + return e + } + return p.nativeKillCompute(ctx, r, n) +} +func (p *Provider) DeleteRetained(ctx context.Context, r sandbox.Reference, s sandbox.RetainedState) error { + n, e := p.snapshot(r, s) + if e != nil { + return e + } + return p.nativeDeleteSnapshot(ctx, r, n) +} +func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, q sandbox.Command) (sandbox.CommandResult, error) { + n, e := p.nativeCompute(r, c) + if e != nil { + return sandbox.CommandResult{}, e + } + return p.nativeRunCommandCompute(ctx, r, n, q) +} +func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + n, e := p.nativeCompute(r, c) + if e != nil { + return sandbox.ComputeState{}, e + } + v, e := p.nativeResumeCompute(ctx, r, n) + return state(v), e +} diff --git a/services/core/internal/sandbox/microsandbox/suspension_contract_test.go b/services/core/internal/sandbox/microsandbox/suspension_contract_test.go new file mode 100644 index 000000000..284bc2159 --- /dev/null +++ b/services/core/internal/sandbox/microsandbox/suspension_contract_test.go @@ -0,0 +1,57 @@ +package microsandbox + +import ( + "context" + "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "testing" +) + +func TestSharedSuspendSettlesExactSourceCleanupInsideAdapter(t *testing.T) { + for _, reconcile := range []bool{false, true} { + for _, cleanupFails := range []bool{false, true} { + c, r, snap := testConfig(), testRef(), testSnapshot() + source := Compute{Name: snap.SourceName, ID: snap.SourceID} + calls := []string{} + p, e := NewWithCaller(c, callerFunc(func(_ context.Context, q Request) (Response, error) { + calls = append(calls, q.Operation) + if q.Operation == "suspend" { + if q.Suspend.ObserveOnly != reconcile { + t.Fatal("reconcile intent changed") + } + return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: "suspended", BootstrapComplete: true, Snapshot: &snap, SourceStopped: !reconcile}}, nil + } + if q.Operation != "kill" || q.Compute.ID != source.ID { + t.Fatal("wrong source cleanup", q.Operation) + } + if cleanupFails { + return Response{}, errors.New("lost cleanup") + } + return Response{Version: ProtocolVersion}, nil + })) + if e != nil { + t.Fatal(e) + } + q := sandbox.SuspendRequest{Reference: r, OperationID: snap.OperationID, Source: compute(source), ReconcileOnly: reconcile} + got, e := p.Suspend(deadline(t), q) + if len(calls) != 2 || calls[0] != "suspend" || calls[1] != "kill" { + t.Fatal(calls) + } + if cleanupFails { + if e == nil || got.SuspendSettled { + t.Fatal("failed cleanup released capacity") + } + continue + } + if e != nil { + t.Fatal(e) + } + if e = sandbox.ValidateSuspendResult(q, got); e != nil { + t.Fatal(e) + } + if got.Retained == nil || got.Retained.Data == "" { + t.Fatal("missing native proof") + } + } + } +} diff --git a/services/core/internal/sandbox/microsandbox/types.go b/services/core/internal/sandbox/microsandbox/types.go index 12ddc1f69..e9b990f18 100644 --- a/services/core/internal/sandbox/microsandbox/types.go +++ b/services/core/internal/sandbox/microsandbox/types.go @@ -40,12 +40,52 @@ type NetworkPolicy struct { } type NetworkRule struct{ Action, Direction, Destination, Protocol, Port string } -// These aliases keep the helper wire private while Core uses provider-neutral types. -type Compute = sandbox.Compute -type SnapshotIdentity = sandbox.SnapshotIdentity -type State = sandbox.ComputeState -type SuspendRequest = sandbox.SuspendRequest -type ResumeRequest = sandbox.ResumeRequest +// Native snapshot wire types belong to this adapter. +type Compute struct { + Generation uint64 + Name string + ID string + RestoredFrom *SnapshotIdentity +} + +// SnapshotIdentity is provider evidence from a verified full snapshot. Core +// persists it unchanged and records consumption separately; it never invents +// paths, checksums, native checkpoint fields, or source identity. +type SnapshotIdentity struct { + Reference string + ID string + Digest string + CheckpointID string + CheckpointRoot string + OperationID string + SourceGeneration uint64 + SourceName string + SourceID string +} + +type State struct { + Compute Compute + Status string + BootstrapComplete bool + Snapshot *SnapshotIdentity + SourceStopped bool +} +type SuspendRequest struct { + Reference sandbox.Reference + OperationID string + Source Compute + Snapshot *SnapshotIdentity + // Recovery observes the previous attempt and never starts a new capture. + ObserveOnly bool +} +type ResumeRequest struct { + Reference sandbox.Reference + OperationID string + Snapshot SnapshotIdentity + Target Compute + // Recovery observes the previous target and never starts a new restore. + ObserveOnly bool +} // Request and Response are the finite, private helper boundary. Confidential // Bootstrap and Command bytes travel only through stdin and are never logged. diff --git a/services/core/internal/sandbox/node/agent.go b/services/core/internal/sandbox/node/agent.go index 3d31e488b..5daf06eb6 100644 --- a/services/core/internal/sandbox/node/agent.go +++ b/services/core/internal/sandbox/node/agent.go @@ -84,7 +84,7 @@ func Run(ctx context.Context, config AgentConfig) error { if err := sandbox.ValidateProvider(config.Provider); err != nil { return err } - if sandbox.SupportsCheckpoint(config.Provider) != providers.SupportsCheckpoint(config.Identity.Provider) { + if sandbox.SupportsSuspension(config.Provider) != providers.SupportsSuspension(config.Identity.Provider) { return sandbox.ErrInvalid } } diff --git a/services/core/internal/sandbox/node/generation_json.go b/services/core/internal/sandbox/node/generation_json.go index c57e0ae07..e5ac8337c 100644 --- a/services/core/internal/sandbox/node/generation_json.go +++ b/services/core/internal/sandbox/node/generation_json.go @@ -130,7 +130,7 @@ func validateGenerationJSON(raw []byte, kind string) error { } } if value := values["request"]; value != nil { - if _, err := generationObject(value, "deployment_generation id sequence connection_id owner_epoch operation timeout_ms reference", "bootstrap compute generation command suspend resume snapshot observation", ""); err != nil { + if _, err := generationObject(value, "deployment_generation id sequence connection_id owner_epoch operation timeout_ms reference", "bootstrap compute generation command suspend resume retained observation", ""); err != nil { return err } } diff --git a/services/core/internal/sandbox/node/node_test.go b/services/core/internal/sandbox/node/node_test.go index 792bc3f48..a9893ac7a 100644 --- a/services/core/internal/sandbox/node/node_test.go +++ b/services/core/internal/sandbox/node/node_test.go @@ -166,7 +166,7 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing func TestOfflineIsUnknownAndDockerDoesNotAdvertiseCheckpoint(t *testing.T) { h := NewHub(HubOptions{OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) p := h.Proxy(uuid.NewString(), "docker", 1) - if sandbox.SupportsCheckpoint(p) { + if sandbox.SupportsSuspension(p) { t.Fatal("docker advertised checkpoint") } _, err := p.GetInfo(context.Background(), reference()) diff --git a/services/core/internal/sandbox/node/operations.go b/services/core/internal/sandbox/node/operations.go index 8f016664b..aca02dcb5 100644 --- a/services/core/internal/sandbox/node/operations.go +++ b/services/core/internal/sandbox/node/operations.go @@ -10,10 +10,11 @@ var operationMethods = map[string]string{ "initial": "Initial", "new_compute": "NewCompute", "compute": "GetCompute", + "renew_compute": "RenewCompute", "suspend": "Suspend", "resume": "Resume", "kill_compute": "KillCompute", - "delete_snapshot": "DeleteSnapshot", + "delete_retained": "DeleteRetained", "command_compute": "RunCommandCompute", "resume_compute": "ResumeCompute", "observe": "Observe", diff --git a/services/core/internal/sandbox/node/operations_test.go b/services/core/internal/sandbox/node/operations_test.go index dfafaf4c0..894b5f090 100644 --- a/services/core/internal/sandbox/node/operations_test.go +++ b/services/core/internal/sandbox/node/operations_test.go @@ -50,7 +50,7 @@ func TestUnsupportedProxyRejectsBeforeNodeResolution(t *testing.T) { } } func TestNodeOperationMappingCoversForwardedMethods(t *testing.T) { - for _, method := range []string{"Create", "GetInfo", "Renew", "Kill", "RunCommand", "Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteSnapshot", "RunCommandCompute", "ResumeCompute", "Observe"} { + for _, method := range []string{"Create", "GetInfo", "Renew", "Kill", "RunCommand", "Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteRetained", "RunCommandCompute", "ResumeCompute", "Observe"} { if wire := operationWire(method); wire == "" || operationMethod(wire) != method { t.Fatal(method) } diff --git a/services/core/internal/sandbox/node/provider_operations_fixture_test.go b/services/core/internal/sandbox/node/provider_operations_fixture_test.go index 8d34c1511..acb4cc126 100644 --- a/services/core/internal/sandbox/node/provider_operations_fixture_test.go +++ b/services/core/internal/sandbox/node/provider_operations_fixture_test.go @@ -16,11 +16,12 @@ func (*fakeProvider) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ObservationProviderType": {State: providercontract.Supported}, @@ -34,7 +35,7 @@ func (*fakeProvider) ProviderOperations() providercontract.Operations { func (*fakeProvider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: "fixture_operation_not_supported"} } -func (*fakeProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (*fakeProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: "fixture_operation_not_supported"} } func (*fakeProvider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -49,8 +50,8 @@ func (*fakeProvider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.Com func (*fakeProvider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: "fixture_operation_not_supported"} } -func (*fakeProvider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: "fixture_operation_not_supported"} +func (*fakeProvider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: "fixture_operation_not_supported"} } func (*fakeProvider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: "fixture_operation_not_supported"} @@ -79,11 +80,12 @@ func (*observationProvider) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ObservationProviderType": {State: providercontract.Supported}, @@ -104,3 +106,7 @@ func (*observationProvider) ObservationProviderType() string { return "fixture" func (p *observationProvider) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { return p, nil } + +func (p *fakeProvider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} diff --git a/services/core/internal/sandbox/node/proxy.go b/services/core/internal/sandbox/node/proxy.go index cf30880ea..caf5e7dbc 100644 --- a/services/core/internal/sandbox/node/proxy.go +++ b/services/core/internal/sandbox/node/proxy.go @@ -17,7 +17,7 @@ type provider struct { } var _ sandbox.SandboxProvider = (*provider)(nil) -var _ sandbox.CheckpointProvider = (*provider)(nil) +var _ sandbox.SuspensionProvider = (*provider)(nil) // Proxy binds a fixed node and deployment generation explicitly. func (h *Hub) Proxy(id, kind string, generation uint64) sandbox.SandboxProvider { @@ -125,8 +125,8 @@ func (p *provider) Initial(ctx context.Context, r sandbox.Reference) (sandbox.Co } return *out.Compute, nil } -func (p *provider) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, s *sandbox.SnapshotIdentity) (sandbox.Compute, error) { - out, e := p.call(ctx, request{Operation: "new_compute", Reference: r, Generation: g, Snapshot: s}) +func (p *provider) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, s *sandbox.RetainedState) (sandbox.Compute, error) { + out, e := p.call(ctx, request{Operation: "new_compute", Reference: r, Generation: g, Retained: s}) if e != nil { return sandbox.Compute{}, e } @@ -158,8 +158,8 @@ func (p *provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandb _, e := p.call(ctx, request{Operation: "kill_compute", Reference: r, Compute: &c}) return e } -func (p *provider) DeleteSnapshot(ctx context.Context, r sandbox.Reference, s sandbox.SnapshotIdentity) error { - _, e := p.call(ctx, request{Operation: "delete_snapshot", Reference: r, Snapshot: &s}) +func (p *provider) DeleteRetained(ctx context.Context, r sandbox.Reference, s sandbox.RetainedState) error { + _, e := p.call(ctx, request{Operation: "delete_retained", Reference: r, Retained: &s}) return e } func (p *provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, v sandbox.Command) (sandbox.CommandResult, error) { @@ -175,3 +175,7 @@ func (h *Hub) GenerationProvider(kind string, resolve func(context.Context, sand p := &provider{hub: h, kind: kind, resolveGeneration: resolve} return p } + +func (p *provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.state(ctx, request{Operation: "renew_compute", Reference: r, Compute: &c}) +} diff --git a/services/core/internal/sandbox/node/wire.go b/services/core/internal/sandbox/node/wire.go index 0194e3c82..1c2f43bd5 100644 --- a/services/core/internal/sandbox/node/wire.go +++ b/services/core/internal/sandbox/node/wire.go @@ -17,7 +17,7 @@ import ( "github.com/gorilla/websocket" ) -const ProtocolVersion = 4 +const ProtocolVersion = 5 const MaxControlFrameBytes = 32 * 1024 const MaxFrameBytes = 72 * 1024 * 1024 const maxPending = 32 @@ -85,15 +85,15 @@ type request struct { TimeoutMillis int64 `json:"timeout_ms"` // deadline is anchored to the receiving host and never crosses the wire. deadline time.Time - Reference sandbox.Reference `json:"reference"` - Bootstrap *sandbox.Bootstrap `json:"bootstrap,omitempty"` - Compute *sandbox.Compute `json:"compute,omitempty"` - Generation uint64 `json:"generation,omitempty"` - Command *sandbox.Command `json:"command,omitempty"` - Suspend *sandbox.SuspendRequest `json:"suspend,omitempty"` - Resume *sandbox.ResumeRequest `json:"resume,omitempty"` - Snapshot *sandbox.SnapshotIdentity `json:"snapshot,omitempty"` - Observation *runtimeobs.Target `json:"observation,omitempty"` + Reference sandbox.Reference `json:"reference"` + Bootstrap *sandbox.Bootstrap `json:"bootstrap,omitempty"` + Compute *sandbox.Compute `json:"compute,omitempty"` + Generation uint64 `json:"generation,omitempty"` + Command *sandbox.Command `json:"command,omitempty"` + Suspend *sandbox.SuspendRequest `json:"suspend,omitempty"` + Resume *sandbox.ResumeRequest `json:"resume,omitempty"` + Retained *sandbox.RetainedState `json:"retained,omitempty"` + Observation *runtimeobs.Target `json:"observation,omitempty"` } type response struct { @@ -238,7 +238,7 @@ func (q request) validate() error { return sandbox.ErrInvalid } count := 0 - for _, ok := range []bool{q.Bootstrap != nil, q.Compute != nil, q.Command != nil, q.Suspend != nil, q.Resume != nil, q.Snapshot != nil, q.Observation != nil} { + for _, ok := range []bool{q.Bootstrap != nil, q.Compute != nil, q.Command != nil, q.Suspend != nil, q.Resume != nil, q.Retained != nil, q.Observation != nil} { if ok { count++ } @@ -257,10 +257,10 @@ func (q request) validate() error { return nil } case "new_compute": - if count == 0 || count == 1 && q.Snapshot != nil { + if count == 0 || count == 1 && q.Retained != nil { return nil } - case "compute", "kill_compute", "resume_compute": + case "compute", "renew_compute", "kill_compute", "resume_compute": if count == 1 && q.Compute != nil { return nil } @@ -280,8 +280,8 @@ func (q request) validate() error { if count == 1 && q.Resume != nil && q.Resume.Reference == q.Reference { return nil } - case "delete_snapshot": - if count == 1 && q.Snapshot != nil { + case "delete_retained": + if count == 1 && q.Retained != nil { return nil } } @@ -324,7 +324,7 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response command, err = p.RunCommand(ctx, q.Reference, *q.Command) out.Command = &command default: - cp, checkpointErr := sandbox.Checkpoint(p) + cp, checkpointErr := sandbox.Suspension(p) if checkpointErr != nil { err = checkpointErr break @@ -336,11 +336,14 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response compute, err = cp.Initial(ctx, q.Reference) out.Compute = &compute case "new_compute": - compute, err = cp.NewCompute(ctx, q.Reference, q.Generation, q.Snapshot) + compute, err = cp.NewCompute(ctx, q.Reference, q.Generation, q.Retained) out.Compute = &compute case "compute": state, err = cp.GetCompute(ctx, q.Reference, *q.Compute) out.State = &state + case "renew_compute": + state, err = cp.RenewCompute(ctx, q.Reference, *q.Compute) + out.State = &state case "suspend": state, err = cp.Suspend(ctx, *q.Suspend) out.State = &state @@ -349,8 +352,8 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response out.State = &state case "kill_compute": err = cp.KillCompute(ctx, q.Reference, *q.Compute) - case "delete_snapshot": - err = cp.DeleteSnapshot(ctx, q.Reference, *q.Snapshot) + case "delete_retained": + err = cp.DeleteRetained(ctx, q.Reference, *q.Retained) case "resume_compute": state, err = cp.ResumeCompute(ctx, q.Reference, *q.Compute) out.State = &state @@ -379,7 +382,7 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response } func requiresReady(q request) bool { - return q.Operation == "create" || q.Operation == "resume" && q.Resume != nil && !q.Resume.ObserveOnly + return q.Operation == "create" || q.Operation == "resume" && q.Resume != nil && !q.Resume.ReconcileOnly } // setTimeout consumes sender queue time without comparing clocks across hosts. diff --git a/services/core/internal/sandbox/operations.go b/services/core/internal/sandbox/operations.go index 3faf4454b..487db4ee1 100644 --- a/services/core/internal/sandbox/operations.go +++ b/services/core/internal/sandbox/operations.go @@ -11,7 +11,7 @@ import ( // These existing interfaces are the canonical operation inventory. Declarations // must cover every method, including explicit unsupported implementations. var providerInterfaces = []reflect.Type{ - reflect.TypeFor[SandboxProvider](), reflect.TypeFor[CheckpointProvider](), + reflect.TypeFor[SandboxProvider](), reflect.TypeFor[SuspensionProvider](), reflect.TypeFor[SelectionDiscoverer](), reflect.TypeFor[CredentialVerifier](), reflect.TypeFor[runtimeobs.SourceResolver](), reflect.TypeFor[runtimeobs.Source](), reflect.TypeFor[runtimeobs.BatchSource](), } @@ -59,7 +59,7 @@ func ValidateOperations(operations providercontract.Operations) error { } // The checkpoint lifecycle is indivisible: partial cleanup or restore support // cannot safely own a compute incarnation. - checkpoint := reflect.TypeFor[CheckpointProvider]() + checkpoint := reflect.TypeFor[SuspensionProvider]() for i := 0; i < checkpoint.NumMethod(); i++ { name := checkpoint.Method(i).Name if _, required := reflect.TypeFor[SandboxProvider]().MethodByName(name); !required && operations[name].State != operations["Initial"].State { @@ -72,15 +72,15 @@ func ValidateOperations(operations providercontract.Operations) error { return nil } -func SupportsCheckpoint(p SandboxProvider) bool { +func SupportsSuspension(p SandboxProvider) bool { return providercontract.Require(p, "Initial") == nil } -func Checkpoint(p SandboxProvider) (CheckpointProvider, error) { +func Suspension(p SandboxProvider) (SuspensionProvider, error) { if err := providercontract.Require(p, "Initial"); err != nil { return nil, err } - cp, ok := p.(CheckpointProvider) + cp, ok := p.(SuspensionProvider) if !ok { return nil, providercontract.ErrContract } diff --git a/services/core/internal/sandbox/operations_test.go b/services/core/internal/sandbox/operations_test.go index f1cc091d1..8fc64acc6 100644 --- a/services/core/internal/sandbox/operations_test.go +++ b/services/core/internal/sandbox/operations_test.go @@ -34,7 +34,7 @@ func TestDeclarationsRejectMissingUnknownAndContradictoryOperations(t *testing.T {"resolver unsupported", func(o providercontract.Operations) { o["ResolveObservationSource"] = providercontract.Support{State: providercontract.Unsupported, Reason: "no_resolver"} }}, - {"omitted", func(o providercontract.Operations) { delete(o, "DeleteSnapshot") }}, + {"omitted", func(o providercontract.Operations) { delete(o, "DeleteRetained") }}, {"zero", func(o providercontract.Operations) { o["ObserveBatch"] = providercontract.Support{} }}, {"unknown", func(o providercontract.Operations) { o["FutureOperation"] = providercontract.Support{State: providercontract.Supported} diff --git a/services/core/internal/sandbox/providers/registration.go b/services/core/internal/sandbox/providers/registration.go index 712283e55..23d4c2731 100644 --- a/services/core/internal/sandbox/providers/registration.go +++ b/services/core/internal/sandbox/providers/registration.go @@ -51,16 +51,15 @@ func ValidateRegistration(a Adapter) error { if err := sandbox.ValidateOperations(operations); err != nil { return err } - // The current common lifecycle admits checkpoint suspension only on nodes, - // and creates its policy whenever checkpoint support is declared. + // One suspension policy applies to every declared lifecycle and placement. if operations["Initial"].State == providercontract.Supported { const maximumSeconds = int64((1<<63 - 1) / time.Second) - if a.Mode != "nodes" || a.IdleSeconds < 1 || a.RetentionSeconds < 1 || + if a.IdleSeconds < 1 || a.RetentionSeconds < 1 || a.IdleSeconds > maximumSeconds || a.RetentionSeconds > maximumSeconds { - return invalid("checkpoint policy") + return invalid("suspension policy") } } else if a.IdleSeconds != 0 || a.RetentionSeconds != 0 { - return invalid("non-checkpoint policy") + return invalid("unsupported suspension policy") } return nil } diff --git a/services/core/internal/sandbox/providers/registration_test.go b/services/core/internal/sandbox/providers/registration_test.go index 65fc3a355..867cf63c6 100644 --- a/services/core/internal/sandbox/providers/registration_test.go +++ b/services/core/internal/sandbox/providers/registration_test.go @@ -179,7 +179,7 @@ func TestRegistrationCheckpointPolicy(t *testing.T) { {"missing idle", "microsandbox", 0, 20, false, false}, {"missing retention", "microsandbox", 20, 0, false, false}, {"overflow", "microsandbox", 1<<63 - 1, 20, false, false}, - {"direct suspension", "microsandbox", 20, 20, true, false}, + {"direct suspension", "microsandbox", 20, 20, true, true}, {"unsupported suspension", "docker", 20, 20, false, false}, {"independent durations", "microsandbox", 300, 30, false, true}, {"no suspension", "docker", 0, 0, false, true}, @@ -189,6 +189,7 @@ func TestRegistrationCheckpointPolicy(t *testing.T) { a.IdleSeconds, a.RetentionSeconds = tc.idle, tc.retention if tc.direct { a.Mode, a.BuildLocal, a.BuildDirect = "direct", nil, adapters["e2b"].BuildDirect + a.NodeArtifacts = nil } err := ValidateRegistration(a) if (err == nil) != tc.valid || err != nil && !errors.Is(err, providercontract.ErrContract) { diff --git a/services/core/internal/sandbox/providers/registry.go b/services/core/internal/sandbox/providers/registry.go index f579441f7..5df43d642 100644 --- a/services/core/internal/sandbox/providers/registry.go +++ b/services/core/internal/sandbox/providers/registry.go @@ -66,7 +66,7 @@ func Lookup(kind string) (Adapter, error) { return a, nil } func IsNode(kind string) bool { a, e := Lookup(kind); return e == nil && a.Mode == "nodes" } -func SupportsCheckpoint(kind string) bool { +func SupportsSuspension(kind string) bool { a, e := Lookup(kind) return e == nil && a.Operations()["Initial"].State == providercontract.Supported } @@ -74,7 +74,7 @@ func SupportsCheckpoint(kind string) bool { // RetainedLimit keeps nodes without checkpoint support within their active capacity. func RetainedLimit(kind string, active, retained int) int { a, err := Lookup(kind) - if err == nil && a.Mode == "nodes" && !SupportsCheckpoint(kind) { + if err == nil && a.Mode == "nodes" && !SupportsSuspension(kind) { return active } return retained diff --git a/services/core/internal/sandbox/providers/registry_test.go b/services/core/internal/sandbox/providers/registry_test.go index 3d3639b13..d9f51144f 100644 --- a/services/core/internal/sandbox/providers/registry_test.go +++ b/services/core/internal/sandbox/providers/registry_test.go @@ -25,7 +25,7 @@ func TestRegistrationOwnsDeploymentPolicy(t *testing.T) { t.Fatalf("wrong namespace or defaults: %+v %v", d, err) } a, err := Lookup(tc.kind) - if err != nil || SupportsCheckpoint(tc.kind) != tc.checkpoint || IsNode(tc.kind) != (tc.mode == "nodes") || (a.BuildLocal != nil) != (tc.mode == "nodes") || (a.BuildDirect != nil) != (tc.mode == "direct") { + if err != nil || SupportsSuspension(tc.kind) != tc.checkpoint || IsNode(tc.kind) != (tc.mode == "nodes") || (a.BuildLocal != nil) != (tc.mode == "nodes") || (a.BuildDirect != nil) != (tc.mode == "direct") { t.Fatal("inconsistent construction/capability registration", err) } }) @@ -66,7 +66,7 @@ func TestNewRegistrationDoesNotNeedCoreDispatchChanges(t *testing.T) { adapters[kind] = adapters["docker"] defer delete(adapters, kind) s, err := Normalize(sandbox.Selection{Provider: kind, DeploymentSpec: validRegistrationSpec()}) - if err != nil || s.Provider != kind || !IsNode(kind) || SupportsCheckpoint(kind) { + if err != nil || s.Provider != kind || !IsNode(kind) || SupportsSuspension(kind) { t.Fatal("new entry did not follow shared boundary", err) } d, err := Describe(kind, uuid.NewString()) diff --git a/services/core/internal/sandbox/sandbox_provider.go b/services/core/internal/sandbox/sandbox_provider.go index 663526b8e..4336392a6 100644 --- a/services/core/internal/sandbox/sandbox_provider.go +++ b/services/core/internal/sandbox/sandbox_provider.go @@ -4,7 +4,7 @@ // SandboxProvider owns compute and bootstrap, Runtime owns capability preparation, // and Harness adapters own native execution. Compute running is not execution ready. // -// Required operations are on SandboxProvider. CheckpointProvider and runtimeobs +// Required operations are on SandboxProvider. SuspensionProvider and runtimeobs // observation remain separate small interfaces. Every registered adapter explicitly // declares and implements each operation, including safe Unsupported rejections. // Method-set presence never means an extension is supported. ValidateProvider and @@ -92,17 +92,18 @@ type SandboxProvider interface { RunCommand(context.Context, Reference, Command) (CommandResult, error) } -// CheckpointProvider is an explicitly declared extension. It supplies +// SuspensionProvider is an explicitly declared extension. It supplies // exact-incarnation operations; Worker and Store remain the lifecycle owner. -type CheckpointProvider interface { +type SuspensionProvider interface { SandboxProvider Initial(context.Context, Reference) (Compute, error) - NewCompute(context.Context, Reference, uint64, *SnapshotIdentity) (Compute, error) + NewCompute(context.Context, Reference, uint64, *RetainedState) (Compute, error) GetCompute(context.Context, Reference, Compute) (ComputeState, error) + RenewCompute(context.Context, Reference, Compute) (ComputeState, error) Suspend(context.Context, SuspendRequest) (ComputeState, error) Resume(context.Context, ResumeRequest) (ComputeState, error) KillCompute(context.Context, Reference, Compute) error - DeleteSnapshot(context.Context, Reference, SnapshotIdentity) error + DeleteRetained(context.Context, Reference, RetainedState) error RunCommandCompute(context.Context, Reference, Compute, Command) (CommandResult, error) // ResumeCompute thaws only the same resident instance after an aborted pause. ResumeCompute(context.Context, Reference, Compute) (ComputeState, error) @@ -114,3 +115,100 @@ type ProcessPaths struct { ArtifactRoot string StateRoot string } + +// ValidateRetained checks the shared envelope; only its adapter interprets Data. +func ValidateRetained(s RetainedState) error { + if s.Reference == "" || s.ID == "" || s.OperationID == "" || s.SourceID == "" || s.SourceName == "" || len(s.Data) == 0 || len(s.Data) > 64*1024 { + return ErrInvalid + } + return nil +} + +// ErrComputeUnconfirmed requires observation of the retained operation identity; +// it does not authorize another Create, capture, restore, or cold start. +var ErrComputeUnconfirmed = errors.New("sandbox lifecycle outcome unconfirmed") + +// Compute identifies one incarnation of an allocation. Name is provider-derived. +// ID is empty only until the original create or restore result is observed. +type Compute struct { + Generation uint64 + Name string + ID string + RestoredFrom *RetainedState +} + +// RetainedState is adapter-owned recoverable state. Data is opaque to Core. +// A retained state does not imply an independent snapshot. +type RetainedState struct { + Reference string + ID string + Data string + OperationID string + SourceGeneration uint64 + SourceName string + SourceID string +} + +type ComputeState struct { + Compute Compute + Status string + BootstrapComplete bool + Retained *RetainedState + ResourcesReleased bool + SuspendSettled bool +} +type SuspendRequest struct { + Reference Reference + OperationID string + Source Compute + Retained *RetainedState + // Recovery settles the previous attempt without another capture. + // Ownership-verified cleanup of a durable retained artifact may complete. + ReconcileOnly bool +} +type ResumeRequest struct { + Reference Reference + OperationID string + Retained RetainedState + Target Compute + // Recovery observes the previous target and never starts a new restore. + ReconcileOnly bool +} + +// ValidateComputeResult binds an observation to its precommitted incarnation. +func ValidateComputeResult(want, got Compute) error { + if got.ID == "" || got.Name != want.Name || got.Generation != want.Generation || (want.ID != "" && got.ID != want.ID) || (want.RestoredFrom == nil) != (got.RestoredFrom == nil) { + return ErrOwnership + } + if want.RestoredFrom != nil && *want.RestoredFrom != *got.RestoredFrom { + return ErrOwnership + } + return nil +} + +// ValidateSuspendResult distinguishes settled rollback from uncertain native work. +func ValidateSuspendResult(q SuspendRequest, s ComputeState) error { + if ValidateComputeResult(q.Source, s.Compute) != nil { + return ErrOwnership + } + if !s.SuspendSettled || !s.BootstrapComplete { + return ErrComputeUnconfirmed + } + if s.Retained == nil { + if !q.ReconcileOnly || s.ResourcesReleased || (s.Status != "running" && s.Status != "paused") { + return ErrComputeUnconfirmed + } + return nil + } + v := s.Retained + if ValidateRetained(*v) != nil || v.OperationID != q.OperationID || v.SourceID != q.Source.ID || v.SourceName != q.Source.Name || v.SourceGeneration != q.Source.Generation || (q.Retained != nil && *q.Retained != *v) { + return ErrOwnership + } + if !s.ResourcesReleased || s.Status != "suspended" { + return ErrComputeUnconfirmed + } + return nil +} + +// SuspensionStateVersion fences incompatible durable lifecycle shapes. +const SuspensionStateVersion = "1" diff --git a/services/core/internal/sandbox/suspension.go b/services/core/internal/sandbox/suspension.go deleted file mode 100644 index 5a600a877..000000000 --- a/services/core/internal/sandbox/suspension.go +++ /dev/null @@ -1,57 +0,0 @@ -package sandbox - -import ( - "errors" -) - -// ErrComputeUnconfirmed requires observation of the retained operation identity; -// it does not authorize another Create, capture, restore, or cold start. -var ErrComputeUnconfirmed = errors.New("sandbox lifecycle outcome unconfirmed") - -// Compute identifies one incarnation of an allocation. Name is provider-derived. -// ID is empty only until the original create or restore result is observed. -type Compute struct { - Generation uint64 - Name string - ID string - RestoredFrom *SnapshotIdentity -} - -// SnapshotIdentity is provider evidence from a verified full snapshot. Core -// persists it unchanged and records consumption separately; it never invents -// paths, checksums, native checkpoint fields, or source identity. -type SnapshotIdentity struct { - Reference string - ID string - Digest string - CheckpointID string - CheckpointRoot string - OperationID string - SourceGeneration uint64 - SourceName string - SourceID string -} - -type ComputeState struct { - Compute Compute - Status string - BootstrapComplete bool - Snapshot *SnapshotIdentity - SourceStopped bool -} -type SuspendRequest struct { - Reference Reference - OperationID string - Source Compute - Snapshot *SnapshotIdentity - // Recovery observes the previous attempt and never starts a new capture. - ObserveOnly bool -} -type ResumeRequest struct { - Reference Reference - OperationID string - Snapshot SnapshotIdentity - Target Compute - // Recovery observes the previous target and never starts a new restore. - ObserveOnly bool -} diff --git a/services/core/internal/sandbox/suspension_test.go b/services/core/internal/sandbox/suspension_test.go new file mode 100644 index 000000000..add0fb28b --- /dev/null +++ b/services/core/internal/sandbox/suspension_test.go @@ -0,0 +1,55 @@ +package sandbox + +import ( + "strings" + "testing" +) + +func TestSuspensionRequiresBoundSettledResourceRelease(t *testing.T) { + c := Compute{ID: "native", Name: "source", Generation: 2} + r := RetainedState{Reference: "allocation", ID: "retained", OperationID: "op", SourceGeneration: 2, SourceName: "source", SourceID: "native", Data: "private-proof"} + q := SuspendRequest{OperationID: "op", Source: c} + valid := ComputeState{Compute: c, Status: "suspended", BootstrapComplete: true, Retained: &r, ResourcesReleased: true, SuspendSettled: true} + if err := ValidateSuspendResult(q, valid); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + change func(*ComputeState) + }{ + {"unsettled", func(s *ComputeState) { s.SuspendSettled = false }}, + {"capacity still held", func(s *ComputeState) { s.ResourcesReleased = false }}, + {"running", func(s *ComputeState) { s.Status = "running" }}, + {"foreign source", func(s *ComputeState) { s.Compute.ID = "foreign" }}, + {"foreign handle", func(s *ComputeState) { v := *s.Retained; v.OperationID = "other"; s.Retained = &v }}, + {"oversized", func(s *ComputeState) { v := *s.Retained; v.Data = strings.Repeat("x", 65537); s.Retained = &v }}, + } { + t.Run(tc.name, func(t *testing.T) { + s := valid + tc.change(&s) + if ValidateSuspendResult(q, s) == nil { + t.Fatal("invalid suspension accepted") + } + }) + } + rollback := ComputeState{Compute: c, Status: "running", BootstrapComplete: true, SuspendSettled: true} + if ValidateSuspendResult(q, rollback) == nil { + t.Fatal("fresh dispatch accepted rollback") + } + q.ReconcileOnly = true + if err := ValidateSuspendResult(q, rollback); err != nil { + t.Fatal(err) + } + rollback.SuspendSettled = false + if ValidateSuspendResult(q, rollback) == nil { + t.Fatal("running observation inferred settlement") + } +} +func TestComputeResultFencesSameNativeIDAcrossGenerations(t *testing.T) { + old := Compute{ID: "same-native", Name: "allocation", Generation: 1} + next := old + next.Generation++ + if ValidateComputeResult(old, next) == nil { + t.Fatal("stale logical generation accepted") + } +} diff --git a/services/core/internal/store/admin_session_archive_worker_http_test.go b/services/core/internal/store/admin_session_archive_worker_http_test.go index 2f137749c..c9e91c474 100644 --- a/services/core/internal/store/admin_session_archive_worker_http_test.go +++ b/services/core/internal/store/admin_session_archive_worker_http_test.go @@ -129,7 +129,7 @@ func TestAdminSessionArchiveWorkerHTTPPostgres(t *testing.T) { t.Fatal("archive did not commit administrator audit", audits, err) } stop() - // Snapshot after shutdown: cancellation and lifecycle draining are complete. + // Retained after shutdown: cancellation and lifecycle draining are complete. // A stale handler must not fall back to the still-open admission Store. snapshot := func() string { t.Helper() diff --git a/services/core/internal/store/provider_operations_fixture_test.go b/services/core/internal/store/provider_operations_fixture_test.go index 68634162e..145e2507e 100644 --- a/services/core/internal/store/provider_operations_fixture_test.go +++ b/services/core/internal/store/provider_operations_fixture_test.go @@ -16,11 +16,12 @@ func (*lifecycleProvider) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ObservationProviderType": {State: providercontract.Supported}, @@ -34,7 +35,7 @@ func (*lifecycleProvider) ProviderOperations() providercontract.Operations { func (*lifecycleProvider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: "fixture_operation_not_supported"} } -func (*lifecycleProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (*lifecycleProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: "fixture_operation_not_supported"} } func (*lifecycleProvider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -49,8 +50,8 @@ func (*lifecycleProvider) Resume(context.Context, sandbox.ResumeRequest) (sandbo func (*lifecycleProvider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: "fixture_operation_not_supported"} } -func (*lifecycleProvider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: "fixture_operation_not_supported"} +func (*lifecycleProvider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: "fixture_operation_not_supported"} } func (*lifecycleProvider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: "fixture_operation_not_supported"} @@ -70,7 +71,7 @@ func (*lifecycleProvider) DiscoverSelection(context.Context, sandbox.Selection) func (*lifecycleProvider) VerifyCredential(context.Context, []sandbox.Reference) error { return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: "fixture_operation_not_supported"} } -func (*fakeCheckpointProvider) ProviderOperations() providercontract.Operations { +func (*fakeSuspensionProvider) ProviderOperations() providercontract.Operations { return providercontract.Operations{ "Create": {State: providercontract.Supported}, "GetInfo": {State: providercontract.Supported}, @@ -79,11 +80,12 @@ func (*fakeCheckpointProvider) ProviderOperations() providercontract.Operations "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Supported}, "NewCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, "GetCompute": {State: providercontract.Supported}, "Suspend": {State: providercontract.Supported}, "Resume": {State: providercontract.Supported}, "KillCompute": {State: providercontract.Supported}, - "DeleteSnapshot": {State: providercontract.Supported}, + "DeleteRetained": {State: providercontract.Supported}, "RunCommandCompute": {State: providercontract.Supported}, "ResumeCompute": {State: providercontract.Supported}, "ObservationProviderType": {State: providercontract.Supported}, @@ -100,7 +102,11 @@ func (p *lifecycleProvider) ResolveObservationSource(context.Context) (runtimeob return p, nil } -func (*fakeCheckpointProvider) ObservationProviderType() string { return "fixture" } -func (p *fakeCheckpointProvider) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { +func (*fakeSuspensionProvider) ObservationProviderType() string { return "fixture" } +func (p *fakeSuspensionProvider) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { return p, nil } + +func (p *lifecycleProvider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} diff --git a/services/core/internal/store/runtime_compute_lifecycle_test.go b/services/core/internal/store/runtime_compute_lifecycle_test.go index 0626d5f1e..9535794b1 100644 --- a/services/core/internal/store/runtime_compute_lifecycle_test.go +++ b/services/core/internal/store/runtime_compute_lifecycle_test.go @@ -26,11 +26,11 @@ import ( // The controlled provider records external effects independently of DB phases. // Lost replies retain those effects so recovery must use observation, not replay. -type fakeCheckpointProvider struct { +type fakeSuspensionProvider struct { preparation *initializationPeer lifecycleProvider computes map[string]sandbox.ComputeState - snapshots map[string]sandbox.SnapshotIdentity + snapshots map[string]sandbox.RetainedState bootstraps map[string]sandbox.Bootstrap peers map[string]*websocket.Conn registry *runtimegateway.Registry @@ -41,15 +41,16 @@ type fakeCheckpointProvider struct { quiesces, resumes atomic.Int32 loseCapture, loseRestore, rejectQuiesce bool beforeQuiesce func() + renewals atomic.Int32 } -func (p *fakeCheckpointProvider) Initial(_ context.Context, r sandbox.Reference) (sandbox.Compute, error) { +func (p *fakeSuspensionProvider) Initial(_ context.Context, r sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{Name: r.AllocationID + "-g0"}, nil } -func (p *fakeCheckpointProvider) NewCompute(_ context.Context, r sandbox.Reference, generation uint64, parent *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *fakeSuspensionProvider) NewCompute(_ context.Context, r sandbox.Reference, generation uint64, parent *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{Generation: generation, Name: fmt.Sprintf("%s-g%d", r.AllocationID, generation), RestoredFrom: parent}, nil } -func (p *fakeCheckpointProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { +func (p *fakeSuspensionProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { info, err := p.lifecycleProvider.Create(ctx, b) p.mu.Lock() defer p.mu.Unlock() @@ -59,7 +60,7 @@ func (p *fakeCheckpointProvider) Create(ctx context.Context, b sandbox.Bootstrap p.bootstraps[b.AllocationID] = b return info, err } -func (p *fakeCheckpointProvider) GetCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) GetCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[c.Name] @@ -71,24 +72,28 @@ func (p *fakeCheckpointProvider) GetCompute(_ context.Context, _ sandbox.Referen } return state, nil } -func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) Suspend(_ context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[q.Source.Name] if !ok { + if snapshot, exists := p.snapshots[q.OperationID]; exists && q.ReconcileOnly { + p.captureObservations++ + return sandbox.ComputeState{Compute: q.Source, Status: "suspended", Retained: &snapshot, BootstrapComplete: true, ResourcesReleased: true, SuspendSettled: true}, nil + } return sandbox.ComputeState{}, sandbox.ErrNotFound } if state.Compute.ID != q.Source.ID { return sandbox.ComputeState{}, sandbox.ErrOwnership } - if q.ObserveOnly { + if q.ReconcileOnly { p.captureObservations++ } else { p.captures++ if _, exists := p.snapshots[q.OperationID]; exists { return sandbox.ComputeState{}, errors.New("capture replayed") } - p.snapshots[q.OperationID] = sandbox.SnapshotIdentity{Reference: "snapshot-" + q.OperationID, ID: uuid.NewString(), Digest: "verified", CheckpointID: "checkpoint", CheckpointRoot: "private", OperationID: q.OperationID, SourceGeneration: q.Source.Generation, SourceName: q.Source.Name, SourceID: q.Source.ID} + p.snapshots[q.OperationID] = sandbox.RetainedState{Reference: "snapshot-" + q.OperationID, ID: uuid.NewString(), Data: "verified-native-state", OperationID: q.OperationID, SourceGeneration: q.Source.Generation, SourceName: q.Source.Name, SourceID: q.Source.ID} state.Status = "paused" p.computes[q.Source.Name] = state if p.loseCapture { @@ -97,14 +102,20 @@ func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendReq } } if snapshot, exists := p.snapshots[q.OperationID]; exists { - state.Snapshot = &snapshot - } + state.Retained = &snapshot + state.SuspendSettled = true + state.ResourcesReleased = true + state.Status = "suspended" + delete(p.computes, q.Source.Name) + p.computeKills++ + } + state.SuspendSettled = true return state, nil } -func (p *fakeCheckpointProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { p.mu.Lock() defer p.mu.Unlock() - if q.ObserveOnly { + if q.ReconcileOnly { p.restoreObservations++ state, ok := p.computes[q.Target.Name] if !ok { @@ -126,7 +137,7 @@ func (p *fakeCheckpointProvider) Resume(_ context.Context, q sandbox.ResumeReque } return state, nil } -func (p *fakeCheckpointProvider) KillCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) error { +func (p *fakeSuspensionProvider) KillCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) error { p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[c.Name] @@ -140,7 +151,7 @@ func (p *fakeCheckpointProvider) KillCompute(_ context.Context, _ sandbox.Refere delete(p.computes, c.Name) return nil } -func (p *fakeCheckpointProvider) DeleteSnapshot(_ context.Context, _ sandbox.Reference, s sandbox.SnapshotIdentity) error { +func (p *fakeSuspensionProvider) DeleteRetained(_ context.Context, _ sandbox.Reference, s sandbox.RetainedState) error { p.mu.Lock() defer p.mu.Unlock() old, ok := p.snapshots[s.OperationID] @@ -154,7 +165,7 @@ func (p *fakeCheckpointProvider) DeleteSnapshot(_ context.Context, _ sandbox.Ref delete(p.snapshots, s.OperationID) return nil } -func (p *fakeCheckpointProvider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { state, err := p.GetCompute(ctx, r, c) if err != nil { return state, err @@ -165,7 +176,7 @@ func (p *fakeCheckpointProvider) ResumeCompute(ctx context.Context, r sandbox.Re p.computes[c.Name] = state return state, nil } -func (p *fakeCheckpointProvider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { +func (p *fakeSuspensionProvider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { if _, err := p.GetCompute(ctx, r, c); err != nil { return sandbox.CommandResult{}, err } @@ -178,7 +189,7 @@ func (p *fakeCheckpointProvider) RunCommandCompute(ctx context.Context, r sandbo p.mu.Unlock() return sandbox.CommandResult{}, p.connect(ctx, b) } -func (p *fakeCheckpointProvider) connect(ctx context.Context, b sandbox.Bootstrap) error { +func (p *fakeSuspensionProvider) connect(ctx context.Context, b sandbox.Bootstrap) error { header := http.Header{"Authorization": []string{"Bearer " + b.Credential}} conn, _, err := websocket.DefaultDialer.DialContext(ctx, p.endpoint+"?device_id="+b.DeviceID+"&version="+proto.Version, header) if err != nil { @@ -260,7 +271,7 @@ type computeLifecycleFixture struct { t *testing.T store *store.Store pool *pgxpool.Pool - provider *fakeCheckpointProvider + provider *fakeSuspensionProvider worker *execution.Worker stop func() key string @@ -271,7 +282,7 @@ func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *compu t.Helper() s, pool := store.NewManagedTestStore(t) registry := runtimegateway.NewRegistry() - p := &fakeCheckpointProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.SnapshotIdentity{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} + p := &fakeSuspensionProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.RetainedState{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} handler := runtimegateway.NewHandler(runtimegateway.HandlerConfig{Authenticator: runtimegateway.NewAuthenticator(s), Registry: registry}) server := httptest.NewServer(http.HandlerFunc(handler.WS)) p.endpoint = "ws" + strings.TrimPrefix(server.URL, "http") @@ -556,3 +567,47 @@ func TestRuntimeComputeLifecycleCapacityBoundsActiveAndRetained(t *testing.T) { t.Fatal("retained limit created a third allocation") } } + +func (p *fakeSuspensionProvider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + p.renewals.Add(1) + return p.GetCompute(ctx, r, c) +} + +func TestRuntimeComputeProtocolUpgradeRefusesOldReceiptsBeforeCleanup(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, _, env, owner := f.create() + f.complete(owner) + retained := f.phase(tenant, env.ID, "suspended") + f.stop() + f.sql(`UPDATE runtime_allocations SET compute_state=(compute_state-'protocol_version'-'retained') || jsonb_build_object('snapshot',compute_state->'retained') WHERE id=$1`, owner.ID) + deletes := f.provider.snapshotDeletes + w, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: f.store, Registry: f.provider.registry, ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: f.key, BackendFingerprint: strings.Repeat("a", 64), Provider: f.provider, Suspension: &f.policy}}) + if err == nil || w != nil || !strings.Contains(err.Error(), "previous release") { + t.Fatalf("incompatible state activated: %v", err) + } + if f.provider.snapshotDeletes != deletes || len(f.provider.snapshots) != 1 { + t.Fatal("upgrade lost owned artifact") + } + var state []byte + if err := f.pool.QueryRow(t.Context(), `SELECT compute_state FROM runtime_allocations WHERE id=$1`, owner.ID).Scan(&state); err != nil { + t.Fatal(err) + } + if !strings.Contains(string(state), `"snapshot"`) { + t.Fatal("old receipt was rewritten") + } + f.sql(`UPDATE runtime_allocations SET compute_state=$2::jsonb WHERE id=$1`, owner.ID, retained.ComputeState) + f.start() +} + +func TestRuntimeComputeRepeatedWakeStillRenewsCurrentIncarnation(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, _, env, owner := f.create() + before := f.provider.renewals.Load() + for range 3 { + f.sql(`UPDATE runtime_allocations SET compute_wake_requested=true,compute_activity_at=clock_timestamp() WHERE id=$1`, owner.ID) + f.phase(tenant, env.ID, "running") + } + if f.provider.renewals.Load() < before+3 { + t.Fatal("wake requests bypassed native lease renewal") + } +} diff --git a/services/core/internal/store/runtime_node_lifecycle_fixture_test.go b/services/core/internal/store/runtime_node_lifecycle_fixture_test.go index 94352bc34..b50581d0c 100644 --- a/services/core/internal/store/runtime_node_lifecycle_fixture_test.go +++ b/services/core/internal/store/runtime_node_lifecycle_fixture_test.go @@ -26,7 +26,7 @@ import ( ) type nodeIsolationProvider struct { - *fakeCheckpointProvider + *fakeSuspensionProvider blockMu sync.Mutex blocked map[string]bool mode string @@ -50,7 +50,7 @@ func (p *nodeIsolationProvider) block(ctx context.Context, r sandbox.Reference, return ctx.Err() } func (p *nodeIsolationProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { - info, err := p.fakeCheckpointProvider.Create(ctx, b) + info, err := p.fakeSuspensionProvider.Create(ctx, b) if err == nil { err = p.connect(ctx, b) } @@ -60,13 +60,13 @@ func (p *nodeIsolationProvider) GetInfo(ctx context.Context, r sandbox.Reference if err := p.block(ctx, r, "observe"); err != nil { return sandbox.Info{}, err } - return p.fakeCheckpointProvider.GetInfo(ctx, r) + return p.fakeSuspensionProvider.GetInfo(ctx, r) } func (p *nodeIsolationProvider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { if err := p.block(ctx, r, "observe"); err != nil { return sandbox.ComputeState{}, err } - return p.fakeCheckpointProvider.GetCompute(ctx, r, c) + return p.fakeSuspensionProvider.GetCompute(ctx, r, c) } func (p *nodeIsolationProvider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { return p.preparation.RunCommand(ctx, r, c) @@ -95,8 +95,8 @@ func newNodeIsolationFixture(t *testing.T, mode string) *nodeIsolationFixture { } s := store.NewWithCredentialCipher(pool, cipher) registry := runtimegateway.NewRegistry() - cp := &fakeCheckpointProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.SnapshotIdentity{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} - p := &nodeIsolationProvider{fakeCheckpointProvider: cp, blocked: map[string]bool{}, mode: mode, entered: make(chan struct{})} + cp := &fakeSuspensionProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.RetainedState{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} + p := &nodeIsolationProvider{fakeSuspensionProvider: cp, blocked: map[string]bool{}, mode: mode, entered: make(chan struct{})} preparationContext, cancelPreparation := context.WithCancel(t.Context()) t.Cleanup(cancelPreparation) cp.preparation = &initializationPeer{t: t, apply: func(request proto.RuntimePreparePayload, data []byte) proto.RuntimePrepareResultPayload { diff --git a/services/core/internal/store/runtime_suspension.go b/services/core/internal/store/runtime_suspension.go index 4661292ef..f7eac0e0b 100644 --- a/services/core/internal/store/runtime_suspension.go +++ b/services/core/internal/store/runtime_suspension.go @@ -180,3 +180,19 @@ func checkRuntimeComputeAdmission(ctx context.Context, q *sqlc.Queries, session } return err } + +// CheckRuntimeComputeProtocol blocks activation before incompatible retained state +// could lose cleanup evidence. The previous executable must drain its resources. +func (s *Store) CheckRuntimeComputeProtocol(ctx context.Context, version string) error { + if err := s.CheckExecutionOwnership(ctx); err != nil { + return err + } + incompatible, err := s.queries.HasIncompatibleRuntimeComputeState(ctx, version) + if err != nil { + return err + } + if incompatible { + return errors.New("incompatible retained runtime state: use the previous release to archive allocations before upgrading; history is preserved") + } + return nil +} diff --git a/services/core/internal/store/runtime_wake_hint_integration_test.go b/services/core/internal/store/runtime_wake_hint_integration_test.go index e2c4c069d..c25480aef 100644 --- a/services/core/internal/store/runtime_wake_hint_integration_test.go +++ b/services/core/internal/store/runtime_wake_hint_integration_test.go @@ -15,7 +15,7 @@ import ( ) type wakeHintScanProvider struct { - *fakeCheckpointProvider + *fakeSuspensionProvider sentinel string release chan struct{} scans chan int @@ -34,7 +34,7 @@ func (p *wakeHintScanProvider) GetCompute(ctx context.Context, reference sandbox } } } - return p.fakeCheckpointProvider.GetCompute(ctx, reference, compute) + return p.fakeSuspensionProvider.GetCompute(ctx, reference, compute) } type wakeHintIntegrationTarget struct { @@ -69,7 +69,7 @@ func newWakeHintIntegration(t *testing.T) *wakeHintIntegration { target.owner = f.phase(target.tenant, target.environment.ID, "suspended") f.stop() provider := &wakeHintScanProvider{ - fakeCheckpointProvider: f.provider, sentinel: sentinel.owner.ID, + fakeSuspensionProvider: f.provider, sentinel: sentinel.owner.ID, release: make(chan struct{}), scans: make(chan int, 16), } worker, err := execution.StartWorker(t.Context(), &execution.Dispatcher{ diff --git a/services/core/internal/store/sandbox_deployment_setup.go b/services/core/internal/store/sandbox_deployment_setup.go index fc636b06b..57bb46f5e 100644 --- a/services/core/internal/store/sandbox_deployment_setup.go +++ b/services/core/internal/store/sandbox_deployment_setup.go @@ -187,7 +187,7 @@ func runtimeDeploymentView(d sqlc.RuntimeDeployment, publicURL string) (RuntimeD result.CredentialConfigured = len(d.ProviderCredential) > 0 } - if providers.SupportsCheckpoint(d.ProviderKind) { + if providers.SupportsSuspension(d.ProviderKind) { result.Suspension = &SandboxSuspensionView{IdleSeconds: d.IdleSeconds, RetentionSeconds: d.RetentionSeconds} } return result, nil diff --git a/services/core/tools/microsandbox-provider/README.md b/services/core/tools/microsandbox-provider/README.md index 554b4fad8..faca699e9 100644 --- a/services/core/tools/microsandbox-provider/README.md +++ b/services/core/tools/microsandbox-provider/README.md @@ -1,6 +1,6 @@ # microsandbox Sandbox Provider helper -microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The helper links the microsandbox Go SDK v0.7.2 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the provider-neutral `sandbox.CheckpointProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. +microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The helper links the microsandbox Go SDK v0.7.2 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the provider-neutral `sandbox.SuspensionProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. [Add a Sandbox Provider](../../../../docs/sandbox-provider.md) owns the provider contract. [Sandbox deployment](../../../../contracts/agents-api/sandbox-deployment.md) owns the resources, Runtime release and suspension policy; the [nodes guide](../../../../docs/getting-started/nodes.md) owns node installation, host requirements, the node's directories and its network policy. @@ -29,14 +29,14 @@ A helper response carries `CreateSettled` with a configuration rejection only af Core persists operation IDs, source and target generations, exact identities and snapshot evidence before it depends on them. `Initial` and `NewCompute` only construct references and allocate nothing. - **Suspend** pauses the exact VM, captures a full snapshot under the persisted operation's derived group and member, verifies the complete checkpoint closure, then force-stops the source. Pausing alone does not release memory. A completed matching artifact is inspected instead of captured again. A full snapshot records a resource proof only after the source's limits match. -- **KillCompute** checks the precise incarnation before it stops the VM and removes its writable disks. Core calls it after it has persisted the verified snapshot, even when Suspend already stopped the source, so no chain of old writable disks grows across suspension cycles. +- **KillCompute** checks the precise incarnation before it stops the VM and removes its writable disks. Suspend completes native source cleanup under the allocation lock before reporting resource release, including recovery of a captured artifact. Core uses KillCompute for allocation cleanup. - **Restore** verifies the exact artifact and creates the precommitted target name. An existing target is adopted only when its immutable ID, if known, and its persisted `snapshot_parent` agree. Upstream restore defaults to public networking, so restore passes the same explicit host policy as creation, and no undeclared host resource or mount is inherited. - Native restore leaves the managed root size unset because the target inherits the verified full snapshot. The helper accepts that only with a matching snapshot resource proof and the exact source and target identities, and it checks the target's CPU, memory and Environment disk before keeping the inherited proof. A missing root size never counts as unlimited capacity, and retained state is never resized. - Fresh restore and retry share one completion: verify the original artifact and resource proof, inspect the running target's resources and ancestry, persist its missing derived resource-proof label, then strictly reread the same native ID ([`restore_completion.go`](restore_completion.go)). A conflicting proof is an error. Native restore does not copy the source's ownership labels; ancestry supplies that evidence. There is no ordinary Start, replacement, disk-only restore or cold boot. - **ResumeCompute** thaws the same resident source after an aborted suspension. The pinned SDK handle method is name-based; the allocation lock and ID checks before and after the call fence every managed replacement. Manual lifecycle changes in the managed namespace are unsupported. -- **DeleteSnapshot** accepts only the derived operation selector and the matching full artifact identity, never an arbitrary path. Core owns retention, consumed snapshot generations and cleanup order. A checkpoint never rolls back work admitted after its first restore. +- **DeleteRetained** accepts only the derived operation selector and the matching full artifact identity, never an arbitrary path. Core owns retention, consumed snapshot generations and cleanup order. A checkpoint never rolls back work admitted after its first restore. -After a lost response Core uses `ObserveOnly`. It never starts a capture or restore, and observing a suspend operation never kills its source. Observation checks artifact integrity and source ownership independently of resource checks, so resource drift cannot hide a retained artifact from cleanup; Core can persist recovered snapshot evidence before KillCompute. If the artifact is absent but the exact source is still running or paused with a settled bootstrap, observation returns the source without a snapshot and Core can abort the suspension; thawing and further execution still require the resource checks. For an interrupted restore, `ObserveOnly` may finish the missing resource proof on the exact target but never restarts a stopped target, changes resources or restores again. Missing state never authorizes a replay. +After a lost response Core uses `ReconcileOnly`. The adapter observes the original capture or restore and never starts it again. When a complete matching snapshot exists, reconciliation finishes exact source cleanup before reporting settled suspension and released resources. Artifact integrity and source ownership are checked independently of resource qualification, so resource drift cannot prevent owned cleanup. If capture is settled without an artifact and the exact source is still running or paused with a settled bootstrap, the adapter reports that outcome so Core can abort suspension; thawing and further execution still require resource checks. For an interrupted restore, reconciliation may finish the missing resource proof on the exact target but never restarts a stopped target, changes resources or restores again. Missing state never authorizes a replay. GetCompute, commands, cleanup and the next suspension verify restored provenance from the persisted VM configuration after the consumed artifact is deleted. @@ -62,3 +62,5 @@ The helper returns only the native observation time, exact uptime, cumulative vC ## Tests `make check-microsandbox-provider`, part of `make check`, runs the pure-Go adapter tests everywhere and this module's tests on Linux; other hosts print an explicit skip for the Linux-only module. + +The native helper wire remains version 2. The Go adapter wraps native snapshot identity in the shared opaque retained-state handle and completes exact-source cleanup through the existing ownership-checked helper operations. Reconciliation never repeats snapshot capture. SuspendSettled is reported only after these serialized operations complete; captured-artifact cleanup does not require execution resource qualification. From 5c0b3047914c1c92865aa056b9ceaa06ddb73726 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 21:03:05 +0800 Subject: [PATCH 2/7] test(store): isolate synthetic suspension receipts from worker startup --- services/core/internal/store/runtime_suspension_test.go | 4 ++-- .../core/internal/store/runtime_worker_recovery_test.go | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/services/core/internal/store/runtime_suspension_test.go b/services/core/internal/store/runtime_suspension_test.go index 53fc609c1..97bd45471 100644 --- a/services/core/internal/store/runtime_suspension_test.go +++ b/services/core/internal/store/runtime_suspension_test.go @@ -15,7 +15,7 @@ import ( func runtimeSuspensionFixture(t *testing.T) (*Store, *Store, *pgxpool.Pool, RuntimeAllocation) { t.Helper() - s, pool := testStore(t) + s, pool := newManagedTestStore(t) w := executionLease(t, s).Store() tenant := uuid.NewString() _, environment := localEnvironment(t, s, tenant) @@ -260,7 +260,7 @@ func TestRuntimeSuspensionRetentionAndDeletedSession(t *testing.T) { } func TestRuntimeSuspensionCountsUncertainCapacityUntilReleased(t *testing.T) { - s, pool := testStore(t) + s, pool := newManagedTestStore(t) w := executionLease(t, s).Store() provider := uuid.NewString() cases := []struct { diff --git a/services/core/internal/store/runtime_worker_recovery_test.go b/services/core/internal/store/runtime_worker_recovery_test.go index 41676132a..f8edb69d5 100644 --- a/services/core/internal/store/runtime_worker_recovery_test.go +++ b/services/core/internal/store/runtime_worker_recovery_test.go @@ -21,6 +21,15 @@ func insertWorkerRuntimeAllocation(t *testing.T, pool *pgxpool.Pool, h *dispatch if err != nil { t.Fatal(err) } + // This synthetic allocation must not outlive the test in the shared fixture + // database, where the next worker validates every retained protocol receipt. + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := pool.Exec(ctx, "DELETE FROM runtime_allocations WHERE environment_id=$1", h.device.EnvironmentID); err != nil { + t.Error(err) + } + }) } func runtimeWorkerHarness(t *testing.T) (*dispatchHarness, *pgxpool.Pool) { From 4e0331a6b9f596262167fc000f36eefe7c916da1 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 21:03:31 +0800 Subject: [PATCH 3/7] test(store): use current protocol in worker recovery fixture --- .../core/internal/store/runtime_worker_recovery_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/services/core/internal/store/runtime_worker_recovery_test.go b/services/core/internal/store/runtime_worker_recovery_test.go index f8edb69d5..c888ae745 100644 --- a/services/core/internal/store/runtime_worker_recovery_test.go +++ b/services/core/internal/store/runtime_worker_recovery_test.go @@ -9,6 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" @@ -16,8 +17,12 @@ import ( func insertWorkerRuntimeAllocation(t *testing.T, pool *pgxpool.Pool, h *dispatchHarness, phase string) { t.Helper() - _, err := pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,compute_phase,compute_retained_until,deployment_generation) - VALUES($1,$2,$3,$4,'running',true,$5,clock_timestamp()+interval '1 hour',(SELECT generation FROM runtime_deployment))`, uuid.NewString(), h.device.EnvironmentID, h.device.ID, uuid.NewString(), phase) + state, err := json.Marshal(map[string]any{"protocol_version": sandbox.SuspensionStateVersion, "current": sandbox.Compute{Name: h.device.EnvironmentID, ID: h.device.EnvironmentID}}) + if err != nil { + t.Fatal(err) + } + _, err = pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,compute_phase,compute_retained_until,deployment_generation,compute_state) + VALUES($1,$2,$3,$4,'running',true,$5,clock_timestamp()+interval '1 hour',(SELECT generation FROM runtime_deployment),$6)`, uuid.NewString(), h.device.EnvironmentID, h.device.ID, uuid.NewString(), phase, state) if err != nil { t.Fatal(err) } From 9d4f833193c3e12f67b248b945c3f645bbb4ac39 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 21:31:28 +0800 Subject: [PATCH 4/7] fix(runtime): validate allocation protocol and use shared idle clock --- docs/sandbox-provider.md | 2 +- .../db/queries/runtime_allocations.sql | 4 ++-- .../db/queries/runtime_suspension.sql | 6 ++---- .../db/sqlc/runtime_allocations.sql.go | 6 ++++-- .../db/sqlc/runtime_suspension.sql.go | 6 ++---- .../internal/store/runtime_allocations.go | 2 ++ .../store/runtime_compute_lifecycle_test.go | 15 +++++++++++++ .../internal/store/runtime_idle_clock_test.go | 21 +++++++++++++++++++ .../internal/store/runtime_suspension_test.go | 6 ++++++ 9 files changed, 55 insertions(+), 13 deletions(-) diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index 0bae2290b..a2ce35fdc 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -190,7 +190,7 @@ Suspend owns native resource release and returns a bound retained handle, suspen Resume consumes the retained state into the precommitted target exactly once. Recovery observes the same attempt. Core persists waking, authenticates and resumes the daemon, deletes the consumed retained resource, then commits running and admits work. DeleteRetained is idempotent artifact cleanup and preserves running compute. Failed cleanup keeps the waking phase and cannot trigger another restore. KillCompute remains genuinely destructive; cleanup of an old generation must not kill a newer live incarnation sharing its native ID. -The existing Session lock, lifecycle lease, idle rule, capacity queries and cleanup order remain authoritative. Every unreleased allocation consumes max_retained, including running allocations. This pre-release retained-state shape requires ordinary cleanup of old live compute state before activation; deployments must refuse activation with unreleased checkpoint or resident receipts. Session history is preserved. +The existing Session lock, lifecycle lease, idle rule, capacity queries and cleanup order remain authoritative. Every unreleased allocation consumes max_retained, including running allocations. Every allocation is stamped with the shared compute protocol version at reservation, including allocations whose suspension phase is disabled. Activation refuses any unreleased allocation with a missing or different version; the previous release must complete ordinary cleanup before upgrading. Session history is preserved. ### Reset and archive diff --git a/services/core/internal/db/queries/runtime_allocations.sql b/services/core/internal/db/queries/runtime_allocations.sql index 0296b6249..5772d3233 100644 --- a/services/core/internal/db/queries/runtime_allocations.sql +++ b/services/core/internal/db/queries/runtime_allocations.sql @@ -1,6 +1,6 @@ -- name: CreateRuntimeAllocation :one -INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation) -VALUES ($1, $2, $3, $4, $5, $6) RETURNING *; +INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation, compute_state) +VALUES ($1, $2, $3, $4, $5, $6, jsonb_build_object('protocol_version', sqlc.arg(protocol_version)::text)) RETURNING *; -- name: GetRuntimeAllocation :one SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired diff --git a/services/core/internal/db/queries/runtime_suspension.sql b/services/core/internal/db/queries/runtime_suspension.sql index d1faed453..d38e03d87 100644 --- a/services/core/internal/db/queries/runtime_suspension.sql +++ b/services/core/internal/db/queries/runtime_suspension.sql @@ -27,8 +27,6 @@ WHERE id = $1 AND compute_phase = 'running' AND compute_activity_at <= $2; -- name: GetRuntimeActivity :one SELECT clock_timestamp()::timestamptz AS observed_at, GREATEST(a.compute_activity_at, - CASE WHEN a.node_id IS NULL THEN COALESCE((SELECT max(t.completed_at) FROM turns t WHERE t.session_id = e.session_id), a.created_at) END, - CASE WHEN a.node_id IS NULL THEN (SELECT max(t.completed_at) FROM subagent_turns t WHERE t.session_id = e.session_id) END, (SELECT max(f.settled_at) FROM environment_file_writes f WHERE f.environment_id = e.id))::timestamptz AS last_activity, (EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) @@ -57,7 +55,7 @@ SELECT EXISTS ( UPDATE runtime_allocations a SET compute_activity_at = clock_timestamp() FROM environments e WHERE a.environment_id = e.id AND e.session_id = $1 - AND a.node_id IS NOT NULL AND a.state = 'running'; + AND a.state = 'running'; -- name: SessionHasRuntimeNode :one SELECT EXISTS ( @@ -68,6 +66,6 @@ SELECT EXISTS ( -- name: HasIncompatibleRuntimeComputeState :one SELECT EXISTS ( SELECT 1 FROM runtime_allocations - WHERE state <> 'released' AND compute_phase <> 'disabled' + WHERE state <> 'released' AND (compute_state->>'protocol_version') IS DISTINCT FROM sqlc.arg(protocol_version)::text )::boolean; diff --git a/services/core/internal/db/sqlc/runtime_allocations.sql.go b/services/core/internal/db/sqlc/runtime_allocations.sql.go index 7e47aef04..c9cf1e813 100644 --- a/services/core/internal/db/sqlc/runtime_allocations.sql.go +++ b/services/core/internal/db/sqlc/runtime_allocations.sql.go @@ -12,8 +12,8 @@ import ( ) const createRuntimeAllocation = `-- name: CreateRuntimeAllocation :one -INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation) -VALUES ($1, $2, $3, $4, $5, $6) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation +INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation, compute_state) +VALUES ($1, $2, $3, $4, $5, $6, jsonb_build_object('protocol_version', $7::text)) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` type CreateRuntimeAllocationParams struct { @@ -23,6 +23,7 @@ type CreateRuntimeAllocationParams struct { ProviderKey pgtype.UUID `json:"provider_key"` NodeID pgtype.UUID `json:"node_id"` DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` + ProtocolVersion string `json:"protocol_version"` } func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntimeAllocationParams) (RuntimeAllocation, error) { @@ -33,6 +34,7 @@ func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntime arg.ProviderKey, arg.NodeID, arg.DeploymentGeneration, + arg.ProtocolVersion, ) var i RuntimeAllocation err := row.Scan( diff --git a/services/core/internal/db/sqlc/runtime_suspension.sql.go b/services/core/internal/db/sqlc/runtime_suspension.sql.go index 1c7170cb1..f23211019 100644 --- a/services/core/internal/db/sqlc/runtime_suspension.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension.sql.go @@ -54,8 +54,6 @@ func (q *Queries) CountRuntimeRetainedAllocations(ctx context.Context, providerK const getRuntimeActivity = `-- name: GetRuntimeActivity :one SELECT clock_timestamp()::timestamptz AS observed_at, GREATEST(a.compute_activity_at, - CASE WHEN a.node_id IS NULL THEN COALESCE((SELECT max(t.completed_at) FROM turns t WHERE t.session_id = e.session_id), a.created_at) END, - CASE WHEN a.node_id IS NULL THEN (SELECT max(t.completed_at) FROM subagent_turns t WHERE t.session_id = e.session_id) END, (SELECT max(f.settled_at) FROM environment_file_writes f WHERE f.environment_id = e.id))::timestamptz AS last_activity, (EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) @@ -91,7 +89,7 @@ func (q *Queries) GetRuntimeActivity(ctx context.Context, id pgtype.UUID) (GetRu const hasIncompatibleRuntimeComputeState = `-- name: HasIncompatibleRuntimeComputeState :one SELECT EXISTS ( SELECT 1 FROM runtime_allocations - WHERE state <> 'released' AND compute_phase <> 'disabled' + WHERE state <> 'released' AND (compute_state->>'protocol_version') IS DISTINCT FROM $1::text )::boolean ` @@ -107,7 +105,7 @@ const recordRuntimeTerminalActivity = `-- name: RecordRuntimeTerminalActivity :e UPDATE runtime_allocations a SET compute_activity_at = clock_timestamp() FROM environments e WHERE a.environment_id = e.id AND e.session_id = $1 - AND a.node_id IS NOT NULL AND a.state = 'running' + AND a.state = 'running' ` func (q *Queries) RecordRuntimeTerminalActivity(ctx context.Context, sessionID pgtype.UUID) error { diff --git a/services/core/internal/store/runtime_allocations.go b/services/core/internal/store/runtime_allocations.go index 18bccfcdf..5e8dc74a4 100644 --- a/services/core/internal/store/runtime_allocations.go +++ b/services/core/internal/store/runtime_allocations.go @@ -11,6 +11,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) // RuntimeAllocation retains compute ownership, not public readiness. It survives @@ -118,6 +119,7 @@ func (s *Store) ReserveRuntimeAllocation(ctx context.Context, tenant, environmen row, err := q.CreateRuntimeAllocation(ctx, sqlc.CreateRuntimeAllocationParams{ ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, EnvironmentID: lookup.ID, DeviceID: device.ID, ProviderKey: provider, NodeID: nodeID, DeploymentGeneration: generation, + ProtocolVersion: sandbox.SuspensionStateVersion, }) if err == nil { result = runtimeAllocationFromRow(row, session, lookup.TenantID, pgtype.Timestamptz{}, false) diff --git a/services/core/internal/store/runtime_compute_lifecycle_test.go b/services/core/internal/store/runtime_compute_lifecycle_test.go index 9535794b1..76f939cd6 100644 --- a/services/core/internal/store/runtime_compute_lifecycle_test.go +++ b/services/core/internal/store/runtime_compute_lifecycle_test.go @@ -611,3 +611,18 @@ func TestRuntimeComputeRepeatedWakeStillRenewsCurrentIncarnation(t *testing.T) { t.Fatal("wake requests bypassed native lease renewal") } } + +func TestRuntimeComputeProtocolRejectsOldDisabledAllocation(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + _, _, _, owner := f.create() + f.stop() + f.sql(`UPDATE runtime_allocations SET compute_phase='disabled',compute_state='{}'::jsonb WHERE id=$1`, owner.ID) + before := f.provider.renewals.Load() + w, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: f.store, Registry: f.provider.registry, ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: f.key, BackendFingerprint: strings.Repeat("a", 64), Provider: f.provider, Suspension: &f.policy}}) + if err == nil || w != nil || !strings.Contains(err.Error(), "previous release") { + t.Fatal("old disabled allocation activated", err) + } + if f.provider.renewals.Load() != before || len(f.provider.computes) != 1 { + t.Fatal("upgrade changed owned native compute") + } +} diff --git a/services/core/internal/store/runtime_idle_clock_test.go b/services/core/internal/store/runtime_idle_clock_test.go index 71890388d..67b8033a5 100644 --- a/services/core/internal/store/runtime_idle_clock_test.go +++ b/services/core/internal/store/runtime_idle_clock_test.go @@ -221,3 +221,24 @@ func TestManagedIdleClockReconnectPreservesReceipts(t *testing.T) { t.Fatal(err) } } + +func TestDirectManagedIdleClockIgnoresNativeClockSkew(t *testing.T) { + for _, skew := range []time.Duration{-269 * time.Second, 269 * time.Second} { + t.Run(skew.String(), func(t *testing.T) { + s, w, owner := managedIdleClockFixture(t) + runtimeSuspensionSQL(t, s.pool, "UPDATE runtime_allocations SET node_id=NULL WHERE id=$1", owner.ID) + owner.NodeID = "" + turn := uuid.NewString() + runtimeSuspensionSQL(t, s.pool, "INSERT INTO turns(id,session_id,status,started_at) VALUES($1,$2,'in_progress',clock_timestamp())", turn, owner.SessionID) + source := runtimeDatabaseTime(t, s).Add(skew).UnixMilli() + outcome := json.RawMessage(fmt.Sprintf(`{"done":{"source_completed_at_ms":%d}}`, source)) + before := runtimeDatabaseTime(t, s) + completed, err := w.CompleteExecution(t.Context(), owner.TenantID, owner.SessionID, turn, TurnCompleted, outcome, "", 0) + after := runtimeDatabaseTime(t, s) + if err != nil || completed.CompletedAt.UnixMilli() != source { + t.Fatal(completed, err) + } + verifyManagedIdleClock(t, s, w, owner, before, after) + }) + } +} diff --git a/services/core/internal/store/runtime_suspension_test.go b/services/core/internal/store/runtime_suspension_test.go index 97bd45471..aba844f89 100644 --- a/services/core/internal/store/runtime_suspension_test.go +++ b/services/core/internal/store/runtime_suspension_test.go @@ -405,6 +405,12 @@ func TestRuntimeSuspensionRechecksCompletionAgainstIdleTimeout(t *testing.T) { default: runtimeSuspensionSQL(t, pool, `INSERT INTO environment_file_writes(id,environment_id,device_id,request_sha256,state,created_at,settled_at) VALUES($1,$2,$3,$4,$5,clock_timestamp()-interval '10 minutes',clock_timestamp())`, uuid.NewString(), owner.EnvironmentID, owner.DeviceID, strings.Repeat("a", 64), strings.TrimPrefix(kind, "file_")) } + if kind == "root" || kind == "subagent" { + id, _ := parseConnectionGeneration(owner.SessionID) + if err := s.queries.RecordRuntimeTerminalActivity(t.Context(), id); err != nil { + t.Fatal(err) + } + } until := time.Now().Add(time.Hour) if _, err := w.SetRuntimeCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, idleTimeout); !errors.Is(err, ErrTurnConflict) { t.Fatal("completion after idle observation did not fence quiesce", err) From 3dbcd0bd32f868995adcb3d016ac8361471bf4f0 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 21:32:04 +0800 Subject: [PATCH 5/7] test(installer): isolate synthetic supplementary groups --- deploy/install/test_node_install.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/install/test_node_install.py b/deploy/install/test_node_install.py index afbea2a90..9d27055fd 100644 --- a/deploy/install/test_node_install.py +++ b/deploy/install/test_node_install.py @@ -645,6 +645,7 @@ def run_as(account, function, *arguments): service_account=lambda: self.account), mock.patch.object(installer.shutil, "which", side_effect=lambda tool: None if tool == "docker" and not self.docker_installed else "/usr/bin/" + tool), mock.patch.object(installer.grp, "getgrnam", side_effect=lambda name: SimpleNamespace(gr_mem=["oac-node"] if self.joined else [])), + mock.patch.object(installer.os, "getgrouplist", side_effect=lambda name, gid: [gid]), mock.patch.object(installer.grp, "getgrgid", side_effect=lambda gid: SimpleNamespace(gr_name=self.device_group))): patch.start() self.addCleanup(patch.stop) From ed9fb3b50056fb041ab1a6187379c6ad0cde553b Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 21:38:50 +0800 Subject: [PATCH 6/7] test(store): isolate suspension execution configuration fixture --- .../core/internal/store/session_execution_configuration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/core/internal/store/session_execution_configuration_test.go b/services/core/internal/store/session_execution_configuration_test.go index 7310abe46..94c3e5e7a 100644 --- a/services/core/internal/store/session_execution_configuration_test.go +++ b/services/core/internal/store/session_execution_configuration_test.go @@ -243,7 +243,7 @@ func TestSessionExecutionConfigurationConcurrentRetryKeepsWinner(t *testing.T) { } func TestSessionExecutionConfigurationSurvivesSuspendResume(t *testing.T) { - s, pool := testStore(t) + s, pool := newManagedTestStore(t) w := executionLease(t, s).Store() tenant := uuid.NewString() session, err := s.CreateSession(t.Context(), tenant, executionProjectionInput("agent")) From 9ac12031597518ba806dde0db34ab657b90e981a Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 21:57:10 +0800 Subject: [PATCH 7/7] test(runtime): assert ingestion clock and refresh activity fence --- .../internal/store/runtime_suspension_test.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/services/core/internal/store/runtime_suspension_test.go b/services/core/internal/store/runtime_suspension_test.go index aba844f89..d570594ea 100644 --- a/services/core/internal/store/runtime_suspension_test.go +++ b/services/core/internal/store/runtime_suspension_test.go @@ -301,16 +301,18 @@ func TestRuntimeSuspensionCountsUncertainCapacityUntilReleased(t *testing.T) { } func TestRuntimeSuspensionIdleStartsAfterLastCompletion(t *testing.T) { - _, w, pool, owner := runtimeSuspensionFixture(t) - turn := runtimeSuspensionCompleted(t, pool, owner) + s, w, pool, owner := runtimeSuspensionFixture(t) + runtimeSuspensionCompleted(t, pool, owner) runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 hours' WHERE id=$1`, owner.ID) - var completed time.Time - if err := pool.QueryRow(t.Context(), `SELECT completed_at FROM turns WHERE id=$1`, turn).Scan(&completed); err != nil { + before := runtimeDatabaseTime(t, s) + id, _ := parseConnectionGeneration(owner.SessionID) + if err := s.queries.RecordRuntimeTerminalActivity(t.Context(), id); err != nil { t.Fatal(err) } + after := runtimeDatabaseTime(t, s) activity, err := w.RuntimeActivity(t.Context(), owner) - if err != nil || !activity.LastActivity.Equal(completed) { - t.Fatal("long Turn completion did not restart idle interval", activity, completed, err) + if err != nil || activity.LastActivity.Before(before) || activity.LastActivity.After(after) || activity.ReadyToSuspend(time.Minute) { + t.Fatal("long Turn completion did not restart the ingestion idle interval", activity, before, after, err) } } @@ -422,6 +424,11 @@ func TestRuntimeSuspensionRechecksCompletionAgainstIdleTimeout(t *testing.T) { if _, err := w.SetRuntimeCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 0); !errors.Is(err, ErrInvalidInput) { t.Fatal("missing idle timeout accepted", err) } + // Re-observe the allocation after terminal ingestion advanced its activity fence. + owner, err = s.GetRuntimeAllocation(t.Context(), owner.TenantID, owner.EnvironmentID) + if err != nil { + t.Fatal(err) + } if _, err := w.SetRuntimeCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, time.Nanosecond); err != nil { t.Fatal("elapsed idle timeout rejected", err) }