From 2722f356bcecce41926d13625f5b79f126d84ec1 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 20:44:40 +0800 Subject: [PATCH 01/12] refactor(sandbox): unify declared compute suspension protocol --- .../agents-api/node-generation-protocol.md | 9 +- docs/sandbox-provider.md | 22 +-- .../server/managed_generation_operations.go | 42 +++-- services/core/cmd/server/managed_setup.go | 2 +- .../db/queries/runtime_suspension.sql | 7 + .../db/sqlc/runtime_suspension.sql.go | 15 ++ .../archive_cancellation_cleanup_test.go | 4 +- .../provider_operations_fixture_test.go | 13 +- .../internal/execution/runtime_compute.go | 85 ++++++---- .../execution/runtime_compute_wake.go | 18 +-- .../internal/execution/runtime_lifecycle.go | 8 +- services/core/internal/execution/worker.go | 5 + .../internal/sandbox/docker/operations.go | 15 +- .../core/internal/sandbox/e2b/operations.go | 15 +- .../sandbox/microsandbox/operations.go | 5 +- .../internal/sandbox/microsandbox/provider.go | 34 ++-- .../sandbox/microsandbox/provider_test.go | 14 +- .../sandbox/microsandbox/suspension.go | 146 ++++++++++++++++++ .../microsandbox/suspension_contract_test.go | 57 +++++++ .../internal/sandbox/microsandbox/types.go | 52 ++++++- services/core/internal/sandbox/node/agent.go | 2 +- .../internal/sandbox/node/generation_json.go | 2 +- .../core/internal/sandbox/node/node_test.go | 2 +- .../core/internal/sandbox/node/operations.go | 3 +- .../internal/sandbox/node/operations_test.go | 2 +- .../node/provider_operations_fixture_test.go | 16 +- services/core/internal/sandbox/node/proxy.go | 14 +- services/core/internal/sandbox/node/wire.go | 43 +++--- services/core/internal/sandbox/operations.go | 10 +- .../core/internal/sandbox/operations_test.go | 2 +- .../sandbox/providers/registration.go | 9 +- .../sandbox/providers/registration_test.go | 3 +- .../internal/sandbox/providers/registry.go | 4 +- .../sandbox/providers/registry_test.go | 4 +- .../core/internal/sandbox/sandbox_provider.go | 108 ++++++++++++- services/core/internal/sandbox/suspension.go | 57 ------- .../core/internal/sandbox/suspension_test.go | 55 +++++++ .../admin_session_archive_worker_http_test.go | 2 +- .../store/provider_operations_fixture_test.go | 22 ++- .../store/runtime_compute_lifecycle_test.go | 95 +++++++++--- .../runtime_node_lifecycle_fixture_test.go | 12 +- .../core/internal/store/runtime_suspension.go | 16 ++ .../runtime_wake_hint_integration_test.go | 6 +- .../store/sandbox_deployment_setup.go | 2 +- .../tools/microsandbox-provider/README.md | 10 +- 45 files changed, 791 insertions(+), 278 deletions(-) create mode 100644 services/core/internal/sandbox/microsandbox/suspension.go create mode 100644 services/core/internal/sandbox/microsandbox/suspension_contract_test.go delete mode 100644 services/core/internal/sandbox/suspension.go create mode 100644 services/core/internal/sandbox/suspension_test.go diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index 09d84fd2a..195bfa711 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -42,20 +42,21 @@ Each operation carries its own arguments and returns the following result on suc | `command` | `RunCommand` | `command` | `command` | | `observe` | `Observe` | `observation` | `sample` | | `initial` | `Initial` | None | `compute` | -| `new_compute` | `NewCompute` | Positive compute `generation` and optional `snapshot` | `compute` | +| `new_compute` | `NewCompute` | Positive `generation` and optional `retained` | `compute` | | `compute` | `GetCompute` | `compute` | `state` | +| `renew_compute` | `RenewCompute` | Exact current `compute` | `state` | | `kill_compute` | `KillCompute` | `compute` | None | | `resume_compute` | `ResumeCompute` | `compute` | `state` | | `command_compute` | `RunCommandCompute` | `compute` and `command` | `command` | | `suspend` | `Suspend` | `suspend` | `state` | | `resume` | `Resume` | `resume` | `state` | -| `delete_snapshot` | `DeleteSnapshot` | `snapshot` | None | +| `delete_retained` | `DeleteRetained` | `retained` | None | -A request whose `connection_id`, `owner_epoch` or `sequence` does not match closes the connection. A malformed request gets an `invalid` response. A node without generation management accepts only its enrolled `deployment_generation`; a generation-managing node runs the request on that generation's provider and answers `unconfirmed` when it cannot. Core sends `create` and a `resume` that is not observe-only only to a generation that is ready on that node, and keeps at most 32 requests pending per connection. +A request whose `connection_id`, `owner_epoch` or `sequence` does not match closes the connection. A malformed request gets an `invalid` response. A node without generation management accepts only its enrolled `deployment_generation`; a generation-managing node runs the request on that generation's provider and answers `unconfirmed` when it cannot. Core sends `create` and a `resume` that is not reconciliation-only only to a generation that is ready on that node, and keeps at most 32 requests pending per connection. The budget is relative: the node anchors `timeout_ms` to its own clock on receipt and consumes it while the request waits in its queue, so the hosts' clocks need not agree. Core still bounds its own wait. A full node queue closes the connection. -The `response` frame carries `id` and `connection_id`. A successful response carries the result named in the operation table, with no result field for `kill`, `kill_compute` or `delete_snapshot`. A failed response carries an `error_code`: +The `response` frame carries `id` and `connection_id`. A successful response carries the result named in the operation table, with no result field for `kill`, `kill_compute` or `delete_retained`. A failed response carries an `error_code`: | `error_code` | Meaning | | --- | --- | diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index 1cf4b37d8..0bae2290b 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -42,13 +42,13 @@ Every provider implements the methods of each interface below and returns a comp | Contract | Requirement | Responsibility | | --- | --- | --- | | `sandbox.SandboxProvider` | All five operations supported | Allocation lifecycle and bounded commands | -| `sandbox.CheckpointProvider` | Explicit decision for every method, the same for all of them | Exact compute incarnations, capture and restore, retained-source resume and cleanup | +| `sandbox.SuspensionProvider` | Explicit decision for every method, the same for all of them | Exact incarnations, renewal, suspension, retained-state recovery and cleanup | | `runtimeobs.Source` | Explicit decision | Ownership-checked read-only observations | | `runtimeobs.BatchSource` | Explicit decision; requires `Source` | Bounded observations in input order, with per-target errors | | `sandbox.SelectionDiscoverer` | Explicit decision | Read-only native configuration discovery before commit | | `sandbox.CredentialVerifier` | Explicit decision | Verify access to owned resources without mutation | -`CheckpointProvider` adds `Initial` and `NewCompute` (construct compute references without allocating), `GetCompute`, `Suspend`, `Resume`, `ResumeCompute` (thaw only the same resident instance after an aborted pause), `KillCompute`, `DeleteSnapshot` and `RunCommandCompute`, which runs a bounded command in one exact compute incarnation. Core uses `RunCommandCompute` to wake a parked daemon after a restore ([`runtime_compute_wake.go`](../services/core/internal/execution/runtime_compute_wake.go)). +`SuspensionProvider` adds `Initial` and `NewCompute` (construct compute references without allocating), `GetCompute`, `RenewCompute`, `Suspend`, `Resume`, `ResumeCompute` (thaw only the same resident instance after an aborted pause), `KillCompute`, `DeleteRetained` and `RunCommandCompute`, which runs a bounded command in one exact compute incarnation. Core uses `RunCommandCompute` to wake a parked daemon after a restore ([`runtime_compute_wake.go`](../services/core/internal/execution/runtime_compute_wake.go)). Each declaration entry is `state: supported` with no reason, or `state: unsupported` with an authored reason code. Missing, zero, unknown or unsafe entries and missing methods fail validation. Adding a method to an interface requires an explicit decision and implementation in every adapter; never supply a base type or generate blanket unsupported implementations. @@ -88,7 +88,7 @@ Every call receives a bounded context. Expiry or cancellation ends the caller's `ErrInvalid`, `ErrOwnership`, `ErrExists`, `ErrNotFound`, `ErrComputeUnconfirmed` and `ErrCommandUnconfirmed` keep their defined meanings. An unclassified native or transport error is unknown, never permission to retry a mutation. Core never reads provider diagnostics as lifecycle truth or exposes native error text or credentials; the node transport maps errors to fixed codes, and direct SDK details stay private. -Checkpoint support adds `Compute` generation, name and ID and `SnapshotIdentity`; persist operation IDs and the provider's snapshot provenance unchanged. `ObserveOnly` on suspend or resume observes the previous attempt and never starts another capture or restore. `ResumeCompute` only thaws the retained source and never cold-starts a stopped one. Cleanup targets the exact compute incarnation and snapshot, not whatever instance now has the same name. Read [`runtime_compute.go`](../services/core/internal/execution/runtime_compute.go) and its failure tests before declaring checkpoint support. +Suspension support adds exact `Compute` generations and opaque `RetainedState` receipts. Persist their identities and operation IDs unchanged. `ReconcileOnly` observes the original attempt and may finish its owned cleanup, but never starts another capture or restore. `ResumeCompute` only thaws the retained source and never cold-starts a stopped one. Cleanup targets the exact compute incarnation and retained state. Read the [shared suspension contract](#suspension) before declaring support. ### Four distinct readiness facts @@ -110,7 +110,7 @@ A new provider takes these steps: 1. Implement the operation contracts in the adapter package, with native contract tests. 2. Add its specification and resource validators and, for native resource discovery before commit, an optional read-only `SelectionDiscoverer`. Put native credential verification behind `CredentialVerifier`. 3. Implement `sandbox.ConfigurationAdapter` over a typed native configuration. `DecodeInput` strictly parses the separate public `configuration` and write-only `credential` objects of a request. `Encode` produces whitelisted public selectors, read-only observations and separate secret bytes, and never passes request JSON through. `Decode` restores stored selectors, and keeps access to owned resources, without remote admission or new template validation. `Normalize` copies its input before changing it. `ResolveChange`, `Equal` and `WithCredential` own inheritance, identity and credential composition. `Requirements` declares whether a credential and a public Core origin are required and whether configuration discovery is supported. Also implement `ConfigurationDiscoverer`, even when discovery is unsupported: it validates the query and returns a safe catalog, never a mutation or an admission decision, while Core keeps authorization, input limits and deadlines. A node provider accepts only an empty public object, rejects credentials and returns Unsupported for discovery and credential replacement. -4. Register its constructor, policies, configuration adapter, operation declaration and defaults in `providers/registry.go`. Node proxy identity and checkpoint support read this entry. The installer's projection combines the registered policies with the shared field bounds in `sandbox/deployment_contract.go`; regenerate it with `go run ./services/core/cmd/specification-contract -write`. +4. Register its constructor, policies, configuration adapter, operation declaration and defaults in `providers/registry.go`. Node proxy identity and suspension support read this entry. The installer's projection combines the registered policies with the shared field bounds in `sandbox/deployment_contract.go`; regenerate it with `go run ./services/core/cmd/specification-contract -write`. 5. Supply the distribution artifacts for the adapter and its helper, and offer the provider to operators through the registered configuration contract. **Known design gap:** the installer's `--sandbox` choices and Web's setup views carry provider-specific options, such as E2B's installer flags and Web views. Exposing another provider through these surfaces currently requires shared installer and Web edits. This coupling does not meet [Complexity stays in the adapter](../AGENTS.md#complexity-stays-in-the-adapter); new integrations must express their configuration through the protocol and keep vendor-specific behavior in the adapter. Never add a Session or Turn scheduling path, a vendor column or API field, or a vendor switch in the store. @@ -124,7 +124,7 @@ A new provider takes these steps: - A `nodes` registration has only `BuildLocal`, and a `direct` registration only `BuildDirect`; missing, mixed or unknown modes are rejected. - The specification and resource validators, the configuration adapter and a complete operation declaration are mandatory, so an incomplete registration cannot publish a partial installer projection. - The Runtime input policy either accepts the pinned Runtime or gives the adapter's fixed reason for rejecting it, never both. -- Checkpoint support requires node mode and positive idle and retention defaults that fit Runtime durations; a provider without checkpoint support configures no suspension defaults. +- Suspension support requires positive idle and retention defaults that fit Runtime durations in either direct or node mode; a provider without suspension support configures no suspension defaults. The configuration adapter must be non-nil, including its concrete value. Every `ConfigurationRequirements` field needs an explicit valid decision: `Credential` and `PublicOrigin` are `Required` or `NotRequired`, and `Discovery` uses the shared supported or unsupported declaration with a safe reason. A new requirement field or discovery method needs an explicit validation update and never inherits an existing decision. Configuration discovery is distinct from resource selection discovery, and requiring a credential does not promise the `VerifyCredential` operation. These checks establish complete registration, not correct native SDK behavior; constructor and adapter contract tests still apply. @@ -134,7 +134,7 @@ Preview and persistence use `providers.Normalize` and `providers.Describe`. `Sel A direct adapter with a credential verifies all retained generations and allocation references before a key is replaced. The common `sandbox.CallFence` excludes native calls and waits for helper completion, including calls whose callers timed out; execution invokes the prepared verification and fencing callbacks without branching on a vendor. -Vendor deployment validation and SDK setup stay at the construction boundary, and construction never creates an Environment. For node-local adapters `providers.Built` returns the provider, probe, installation identity, backend fingerprint and specification digest, and the factory also returns its close function. `execution.RuntimeProvider` binds the adapter to its kind, installation ID, backend fingerprint, generation, mode and node ownership; the database owns the selection, and the in-memory copy is never another authority. Docker and microsandbox run on nodes, and E2B is constructed directly. The node proxy exposes checkpoint operations only for a backend whose registered declaration supports them, and common lifecycle code admits suspension through `CheckpointProvider`, never through a provider name. +Vendor deployment validation and SDK setup stay at the construction boundary, and construction never creates an Environment. For node-local adapters `providers.Built` returns the provider, probe, installation identity, backend fingerprint and specification digest, and the factory also returns its close function. `execution.RuntimeProvider` binds the adapter to its kind, installation ID, backend fingerprint, generation, mode and node ownership; the database owns the selection, and the in-memory copy is never another authority. Docker and microsandbox run on nodes, and E2B is constructed directly. The node proxy exposes suspension operations only for a backend whose registered declaration supports them, and common lifecycle code admits suspension through `SuspensionProvider`, never through a provider name. The backend fingerprint identifies a native resource namespace, not capacity. Core keeps deployment generations so that owned allocations keep resolving to their original backend; never repoint retained allocations at a replacement backend. @@ -182,11 +182,15 @@ The deployment's CPU, memory and disk settings, `max_active`, `max_retained` and ### Suspension -A provider with checkpoint support can suspend idle work; the deployment's [`suspension`](../contracts/agents-api/sandbox-deployment.md#safe-response) policy sets the idle time and snapshot retention. Core suspends only after at least one Turn is terminal, when no root or Subagent Turn is queued, in progress or waiting, no input, file operation or initialization is pending, and real activity has been idle for the configured interval. For node allocations Core records the first root or child terminal transition with the database clock in the same transaction. Candidate filtering and the Session-locked recheck compare elapsed database time with the idle duration, and the initial snapshot retention deadline is anchored to the same database observation, so Core and database host clocks need not agree. Native completion timestamps stay unchanged in public history but never drive idle admission, and heartbeats never reset activity. Before acknowledging a planned suspension, the daemon closes admission and drains native cleanup, output receipts and file work. +Core suspends only after at least one terminal Turn, when no root or Subagent Turn, input, file operation or initialization is pending and real activity has exceeded the deployment's idle duration. The shared database clock, Session lock and lifecycle lease serialize admission; heartbeats never reset idle activity. Queued work and live Files access request wake; history and published artifacts do not. The daemon drains native cleanup, receipts and file work before acknowledging quiescence. A lost acknowledgement authorizes only exact-source rollback, never a new capture. -The Worker lease, the Session lock and the per-node gates own suspension for every provider. New Turn claims, file-write intents and capture admission serialize under the Session lock and share one compute-phase check; new pending work cancels a capture and wakes the same source. Normal preparation waits for the compute phase to be running, after the authenticated resume handshake, and pending input stays pending when its promotion conflicts with a lifecycle transition. Compute phases and revision-checked receipts live on the allocation. Core persists quiesce, capture and restore intent before the effect, only a fresh receipt performs a capture or restore, and recovery observes the exact attempt without retrying an unknown creation, capture or restore. A consumed snapshot never rolls a running generation back. Deletion, revocation and retention expiry win over wake, up to the final database compare-and-swap, and unknown cleanup identities are kept until owned resources are confirmed absent. Consumed artifacts and old compute are deleted, so suspension cycles never build a chain of writable disks. +The complete declared SuspensionProvider group uses one lifecycle for direct and node placements. A RetainedState is an allocation/source/operation-bound opaque adapter receipt, limited to 64 KiB of native Data; it does not claim a snapshot. Core never interprets native Data. Initial and NewCompute plan exact logical incarnations without allocating resources; native IDs may remain equal across logical generations. GetCompute only observes. RenewCompute extends the exact running incarnation without waking it; the common running path observes and renews before keeping ownership or clearing a wake. -Queued work and live Environment file access wake a suspended Environment; history and published Artifact reads do not. Planned suspension uses an Environment and suspension token on the daemon connection. A PID and start-time fenced local control signal (`RunCommandCompute`) wakes the parked daemon, which authenticates again before admitting work. A transient disconnect before confirmation retries the same armed suspension with bounded attempts and backoff; a permanent authentication or protocol rejection closes it. Core owns the snapshot's retention deadline, and the daemon has no timer for it. A lost quiesce acknowledgement may thaw the same source through explicit rollback but never authorizes capturing it. +Suspend owns native resource release and returns a bound retained handle, suspended status, ResourcesReleased and SuspendSettled before Core releases active capacity. ReconcileOnly forbids replay of the original capture or pause while permitting adapter-owned cleanup proved safe by a durable retained artifact. A retained-less result permits rollback only with SuspendSettled and a recoverable running or paused source. Every other uncertain outcome retains ownership and closes admission. Core never unconditionally destroys the source after Suspend. + +Resume consumes the retained state into the precommitted target exactly once. Recovery observes the same attempt. Core persists waking, authenticates and resumes the daemon, deletes the consumed retained resource, then commits running and admits work. DeleteRetained is idempotent artifact cleanup and preserves running compute. Failed cleanup keeps the waking phase and cannot trigger another restore. KillCompute remains genuinely destructive; cleanup of an old generation must not kill a newer live incarnation sharing its native ID. + +The existing Session lock, lifecycle lease, idle rule, capacity queries and cleanup order remain authoritative. Every unreleased allocation consumes max_retained, including running allocations. This pre-release retained-state shape requires ordinary cleanup of old live compute state before activation; deployments must refuse activation with unreleased checkpoint or resident receipts. Session history is preserved. ### Reset and archive diff --git a/services/core/cmd/server/managed_generation_operations.go b/services/core/cmd/server/managed_generation_operations.go index 8c39850fa..f56280186 100644 --- a/services/core/cmd/server/managed_generation_operations.go +++ b/services/core/cmd/server/managed_generation_operations.go @@ -15,13 +15,13 @@ func (p *generationRouter) Initial(ctx context.Context, r sandbox.Reference) (sa return sandbox.Compute{}, err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return sandbox.Compute{}, err } return cp.Initial(ctx, r) } -func (p *generationRouter) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, snapshot *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *generationRouter) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, snapshot *sandbox.RetainedState) (sandbox.Compute, error) { if err := providercontract.Require(p, "NewCompute"); err != nil { return sandbox.Compute{}, err } @@ -30,7 +30,7 @@ func (p *generationRouter) NewCompute(ctx context.Context, r sandbox.Reference, return sandbox.Compute{}, err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return sandbox.Compute{}, err } @@ -45,7 +45,7 @@ func (p *generationRouter) GetCompute(ctx context.Context, r sandbox.Reference, return sandbox.ComputeState{}, err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return sandbox.ComputeState{}, err } @@ -60,7 +60,7 @@ func (p *generationRouter) Suspend(ctx context.Context, q sandbox.SuspendRequest return sandbox.ComputeState{}, err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return sandbox.ComputeState{}, err } @@ -75,7 +75,7 @@ func (p *generationRouter) Resume(ctx context.Context, q sandbox.ResumeRequest) return sandbox.ComputeState{}, err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return sandbox.ComputeState{}, err } @@ -90,14 +90,14 @@ func (p *generationRouter) KillCompute(ctx context.Context, r sandbox.Reference, return err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return err } return cp.KillCompute(ctx, r, c) } -func (p *generationRouter) DeleteSnapshot(ctx context.Context, r sandbox.Reference, snapshot sandbox.SnapshotIdentity) error { - if err := providercontract.Require(p, "DeleteSnapshot"); err != nil { +func (p *generationRouter) DeleteRetained(ctx context.Context, r sandbox.Reference, snapshot sandbox.RetainedState) error { + if err := providercontract.Require(p, "DeleteRetained"); err != nil { return err } v, done, err := p.route(ctx, r) @@ -105,11 +105,11 @@ func (p *generationRouter) DeleteSnapshot(ctx context.Context, r sandbox.Referen return err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return err } - return cp.DeleteSnapshot(ctx, r, snapshot) + return cp.DeleteRetained(ctx, r, snapshot) } func (p *generationRouter) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { if err := providercontract.Require(p, "RunCommandCompute"); err != nil { @@ -120,7 +120,7 @@ func (p *generationRouter) RunCommandCompute(ctx context.Context, r sandbox.Refe return sandbox.CommandResult{}, err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return sandbox.CommandResult{}, err } @@ -135,7 +135,7 @@ func (p *generationRouter) ResumeCompute(ctx context.Context, r sandbox.Referenc return sandbox.ComputeState{}, err } defer done() - cp, err := sandbox.Checkpoint(v) + cp, err := sandbox.Suspension(v) if err != nil { return sandbox.ComputeState{}, err } @@ -147,3 +147,19 @@ func (*generationRouter) DiscoverSelection(context.Context, sandbox.Selection) ( func (*generationRouter) VerifyCredential(context.Context, []sandbox.Reference) error { return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: "generation_router_does_not_verify_configuration"} } + +func (p *generationRouter) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + if err := providercontract.Require(p, "RenewCompute"); err != nil { + return sandbox.ComputeState{}, err + } + v, done, err := p.route(ctx, r) + if err != nil { + return sandbox.ComputeState{}, err + } + defer done() + cp, err := sandbox.Suspension(v) + if err != nil { + return sandbox.ComputeState{}, err + } + return cp.RenewCompute(ctx, r, c) +} diff --git a/services/core/cmd/server/managed_setup.go b/services/core/cmd/server/managed_setup.go index 3ec771c93..614a8d28b 100644 --- a/services/core/cmd/server/managed_setup.go +++ b/services/core/cmd/server/managed_setup.go @@ -126,7 +126,7 @@ func (s *managedSetup) configuration(setup store.SandboxSetup) (execution.Prepar } selected := &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: s.publicURL + "/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider} - if sandbox.SupportsCheckpoint(provider) { + if sandbox.SupportsSuspension(provider) { selected.Suspension = &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Duration(setup.IdleSeconds) * time.Second, Retention: time.Duration(setup.RetentionSeconds) * time.Second, MaxActive: 4, MaxRetained: 16} } diff --git a/services/core/internal/db/queries/runtime_suspension.sql b/services/core/internal/db/queries/runtime_suspension.sql index d4b6a6e21..d1faed453 100644 --- a/services/core/internal/db/queries/runtime_suspension.sql +++ b/services/core/internal/db/queries/runtime_suspension.sql @@ -64,3 +64,10 @@ SELECT EXISTS ( SELECT 1 FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id WHERE e.session_id = $1 AND a.node_id IS NOT NULL )::boolean; + +-- name: HasIncompatibleRuntimeComputeState :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations + WHERE state <> 'released' AND compute_phase <> 'disabled' + AND (compute_state->>'protocol_version') IS DISTINCT FROM sqlc.arg(protocol_version)::text +)::boolean; diff --git a/services/core/internal/db/sqlc/runtime_suspension.sql.go b/services/core/internal/db/sqlc/runtime_suspension.sql.go index 5c4867839..1c7170cb1 100644 --- a/services/core/internal/db/sqlc/runtime_suspension.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension.sql.go @@ -88,6 +88,21 @@ func (q *Queries) GetRuntimeActivity(ctx context.Context, id pgtype.UUID) (GetRu return i, err } +const hasIncompatibleRuntimeComputeState = `-- name: HasIncompatibleRuntimeComputeState :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations + WHERE state <> 'released' AND compute_phase <> 'disabled' + AND (compute_state->>'protocol_version') IS DISTINCT FROM $1::text +)::boolean +` + +func (q *Queries) HasIncompatibleRuntimeComputeState(ctx context.Context, protocolVersion string) (bool, error) { + row := q.db.QueryRow(ctx, hasIncompatibleRuntimeComputeState, protocolVersion) + var column_1 bool + err := row.Scan(&column_1) + return column_1, err +} + const recordRuntimeTerminalActivity = `-- name: RecordRuntimeTerminalActivity :exec UPDATE runtime_allocations a SET compute_activity_at = clock_timestamp() FROM environments e diff --git a/services/core/internal/execution/archive_cancellation_cleanup_test.go b/services/core/internal/execution/archive_cancellation_cleanup_test.go index d1412a553..97f8493b8 100644 --- a/services/core/internal/execution/archive_cancellation_cleanup_test.go +++ b/services/core/internal/execution/archive_cancellation_cleanup_test.go @@ -46,7 +46,7 @@ func (waitingCleanupCheckpoint) ProviderOperations() providercontract.Operations } type waitingCleanupCheckpoint struct { - sandbox.CheckpointProvider + sandbox.SuspensionProvider beforeKill func() } @@ -106,7 +106,7 @@ func TestArchiveWaitingCleanupReceiptBarrier(t *testing.T) { } currentCompute := sandbox.Compute{ID: uuid.NewString(), Name: owner.ID + "-g0"} if checkpoint { - state, _ := json.Marshal(runtimeCompute{Current: currentCompute}) + state, _ := json.Marshal(runtimeCompute{Version: sandbox.SuspensionStateVersion, Current: currentCompute}) owner, err = writer.SetRuntimeCompute(t.Context(), owner, "running", state, nil, 0) if err != nil { t.Fatal(err) diff --git a/services/core/internal/execution/provider_operations_fixture_test.go b/services/core/internal/execution/provider_operations_fixture_test.go index 4b568704b..d536f7b90 100644 --- a/services/core/internal/execution/provider_operations_fixture_test.go +++ b/services/core/internal/execution/provider_operations_fixture_test.go @@ -16,11 +16,12 @@ func (*lifecycleOnlySandbox) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ObservationProviderType": {State: providercontract.Supported}, @@ -34,7 +35,7 @@ func (*lifecycleOnlySandbox) ProviderOperations() providercontract.Operations { func (*lifecycleOnlySandbox) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: "fixture_operation_not_supported"} } -func (*lifecycleOnlySandbox) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (*lifecycleOnlySandbox) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: "fixture_operation_not_supported"} } func (*lifecycleOnlySandbox) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -49,8 +50,8 @@ func (*lifecycleOnlySandbox) Resume(context.Context, sandbox.ResumeRequest) (san func (*lifecycleOnlySandbox) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: "fixture_operation_not_supported"} } -func (*lifecycleOnlySandbox) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: "fixture_operation_not_supported"} +func (*lifecycleOnlySandbox) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: "fixture_operation_not_supported"} } func (*lifecycleOnlySandbox) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: "fixture_operation_not_supported"} @@ -75,3 +76,7 @@ func (*lifecycleOnlySandbox) ObservationProviderType() string { return "fixture" func (p *lifecycleOnlySandbox) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { return p, nil } + +func (p *lifecycleOnlySandbox) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index 055d0e2ba..905cc815e 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -1,9 +1,11 @@ package execution import ( + "bytes" "context" "encoding/json" "errors" + "io" "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" @@ -22,12 +24,13 @@ type RuntimeSuspensionPolicy struct { } type runtimeCompute struct { - Current sandbox.Compute `json:"current"` - Target *sandbox.Compute `json:"target,omitempty"` - Snapshot *sandbox.SnapshotIdentity `json:"snapshot,omitempty"` - SuspendID string `json:"suspend_id,omitempty"` - RestoreID string `json:"restore_id,omitempty"` - Rollback bool `json:"rollback,omitempty"` + Version string `json:"protocol_version"` + Current sandbox.Compute `json:"current"` + Target *sandbox.Compute `json:"target,omitempty"` + Retained *sandbox.RetainedState `json:"retained,omitempty"` + SuspendID string `json:"suspend_id,omitempty"` + RestoreID string `json:"restore_id,omitempty"` + Rollback bool `json:"rollback,omitempty"` } func (r *runtimeLifecycle) computeCapacity(ctx context.Context, key string) error { @@ -48,6 +51,7 @@ func (r *runtimeLifecycle) computeCapacity(ctx context.Context, key string) erro return nil } func (r *runtimeLifecycle) saveCompute(ctx context.Context, owner store.RuntimeAllocation, phase string, state runtimeCompute, until *time.Time) (store.RuntimeAllocation, error) { + state.Version = sandbox.SuspensionStateVersion raw, err := json.Marshal(state) if err != nil { return owner, err @@ -63,7 +67,7 @@ func (r *runtimeLifecycle) saveCompute(ctx context.Context, owner store.RuntimeA return r.store.SetRuntimeCompute(ctx, owner, phase, raw, until, idleTimeout) } func (r *runtimeLifecycle) enableCompute(ctx context.Context, owner store.RuntimeAllocation) error { - p, capabilityErr := sandbox.Checkpoint(r.config.Provider) + p, capabilityErr := sandbox.Suspension(r.config.Provider) if capabilityErr != nil { return capabilityErr } @@ -83,12 +87,12 @@ func (r *runtimeLifecycle) enableCompute(ctx context.Context, owner store.Runtim } func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner store.RuntimeAllocation) error { - p, capabilityErr := sandbox.Checkpoint(r.config.Provider) + p, capabilityErr := sandbox.Suspension(r.config.Provider) if capabilityErr != nil { return capabilityErr } var state runtimeCompute - if json.Unmarshal(owner.ComputeState, &state) != nil || state.Current.ID == "" { + if decodeRuntimeCompute(owner.ComputeState, &state) != nil || state.Current.ID == "" { return sandbox.ErrOwnership } if owner.SessionDeleted || owner.Expired || owner.State == "cleanup_pending" { @@ -122,7 +126,7 @@ func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner store.Runti } } -func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute) error { +func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SuspensionProvider, owner store.RuntimeAllocation, state runtimeCompute) error { compute, err := p.GetCompute(ctx, runtimeReference(owner), state.Current) if err != nil { return err @@ -130,6 +134,13 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.Checkpoint if compute.Status != "running" || !compute.BootstrapComplete { return sandbox.ErrComputeUnconfirmed } + renewed, err := p.RenewCompute(ctx, runtimeReference(owner), state.Current) + if err != nil { + return err + } + if sandbox.ValidateComputeResult(state.Current, renewed.Compute) != nil || renewed.Status != "running" || !renewed.BootstrapComplete { + return sandbox.ErrComputeUnconfirmed + } peer, err := authorizedRuntimePeer(ctx, r.store, r.registry, owner.DeviceID) if err != nil { return err @@ -189,16 +200,16 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.Checkpoint return r.captureCompute(ctx, p, suspending, state, false) } -func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute, observeOnly bool) error { - result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Snapshot: state.Snapshot, ObserveOnly: observeOnly}) +func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.SuspensionProvider, owner store.RuntimeAllocation, state runtimeCompute, observeOnly bool) error { + result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Retained: state.Retained, ReconcileOnly: observeOnly}) if err != nil { return err } - if result.Compute.ID != state.Current.ID { - return sandbox.ErrOwnership + if err := sandbox.ValidateSuspendResult(sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Retained: state.Retained, ReconcileOnly: observeOnly}, result); err != nil { + return err } - if result.Snapshot == nil { - if !observeOnly || result.SourceStopped || (result.Status != "running" && result.Status != "paused") { + if result.Retained == nil { + if !observeOnly || !result.SuspendSettled || result.ResourcesReleased || (result.Status != "running" && result.Status != "paused") { return sandbox.ErrComputeUnconfirmed } state.Rollback = true @@ -208,21 +219,12 @@ func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.Checkpo } return r.wakeCompute(ctx, p, next, state) } - state.Snapshot = result.Snapshot - // Store the verified artifact before any recovery-path kill. Snapshot failure - // or an unknown result cannot silently fall back to a cold Environment. - next, err := r.saveCompute(ctx, owner, "suspending", state, owner.ComputeRetainedUntil) - if err != nil { - return err - } - if err := ignoreComputeAbsent(p.KillCompute(ctx, runtimeReference(owner), state.Current)); err != nil { - return err - } - _, err = r.saveCompute(ctx, next, "suspended", state, next.ComputeRetainedUntil) + state.Retained = result.Retained + _, err = r.saveCompute(ctx, owner, "suspended", state, owner.ComputeRetainedUntil) return err } -func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute) error { +func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.SuspensionProvider, owner store.RuntimeAllocation, state runtimeCompute) error { activity, err := r.store.RuntimeActivity(ctx, owner) if err != nil { return err @@ -233,13 +235,16 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.Che if err := r.computeCapacityForAllocation(ctx, owner); err != nil { return err } - if state.Snapshot == nil || state.Target != nil { + if state.Retained == nil || state.Target != nil { return sandbox.ErrOwnership } - target, err := p.NewCompute(ctx, runtimeReference(owner), state.Current.Generation+1, state.Snapshot) + target, err := p.NewCompute(ctx, runtimeReference(owner), state.Current.Generation+1, state.Retained) if err != nil { return err } + if target.Name == "" || target.Generation != state.Current.Generation+1 || target.RestoredFrom == nil || *target.RestoredFrom != *state.Retained { + return sandbox.ErrOwnership + } state.Target, state.RestoreID = &target, uuid.NewString() next, err := r.saveCompute(ctx, owner, "restoring", state, owner.ComputeRetainedUntil) if err != nil { @@ -247,15 +252,15 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.Che } return r.restoreCompute(ctx, p, next, state, false) } -func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute, observeOnly bool) error { - if state.Target == nil || state.Snapshot == nil || state.Rollback { +func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.SuspensionProvider, owner store.RuntimeAllocation, state runtimeCompute, observeOnly bool) error { + if state.Target == nil || state.Retained == nil || state.Rollback { return sandbox.ErrOwnership } - result, err := p.Resume(ctx, sandbox.ResumeRequest{Reference: runtimeReference(owner), OperationID: state.RestoreID, Snapshot: *state.Snapshot, Target: *state.Target, ObserveOnly: observeOnly}) + result, err := p.Resume(ctx, sandbox.ResumeRequest{Reference: runtimeReference(owner), OperationID: state.RestoreID, Retained: *state.Retained, Target: *state.Target, ReconcileOnly: observeOnly}) if err != nil { return err } - if result.Status != "running" || result.Compute.ID == "" { + if result.Status != "running" || !result.BootstrapComplete || sandbox.ValidateComputeResult(*state.Target, result.Compute) != nil { return sandbox.ErrComputeUnconfirmed } state.Current, state.Target = result.Compute, nil @@ -282,3 +287,15 @@ func (r *runtimeLifecycle) computeCapacityForAllocation(ctx context.Context, own } return r.computeCapacity(ctx, owner.ProviderKey) } + +func decodeRuntimeCompute(raw []byte, state *runtimeCompute) error { + d := json.NewDecoder(bytes.NewReader(raw)) + d.DisallowUnknownFields() + if err := d.Decode(state); err != nil { + return err + } + if d.Decode(new(any)) != io.EOF || state.Version != sandbox.SuspensionStateVersion { + return sandbox.ErrOwnership + } + return nil +} diff --git a/services/core/internal/execution/runtime_compute_wake.go b/services/core/internal/execution/runtime_compute_wake.go index 9eae10eb9..fc560bc23 100644 --- a/services/core/internal/execution/runtime_compute_wake.go +++ b/services/core/internal/execution/runtime_compute_wake.go @@ -11,7 +11,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) -func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute) error { +func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SuspensionProvider, owner store.RuntimeAllocation, state runtimeCompute) error { if state.Rollback { if _, err := p.ResumeCompute(ctx, runtimeReference(owner), state.Current); err != nil { return err @@ -54,8 +54,8 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.Checkpoint } // The artifact has been consumed. Never restore it after this generation // admits work, even if garbage collection or the final database commit fails. - if state.Snapshot != nil { - if err := ignoreComputeAbsent(p.DeleteSnapshot(ctx, runtimeReference(owner), *state.Snapshot)); err != nil { + if state.Retained != nil { + if err := ignoreComputeAbsent(p.DeleteRetained(ctx, runtimeReference(owner), *state.Retained)); err != nil { return err } } @@ -69,19 +69,19 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.Checkpoint return r.observeConnection(ctx, next) } -func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute) error { +func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.SuspensionProvider, owner store.RuntimeAllocation, state runtimeCompute) error { if err := r.store.CheckExecutionOwnership(ctx); err != nil { return err } // An uncommitted artifact is found by its persisted attempt, never a directory // glob. The helper's allocation lock also waits for an earlier unknown call. - if owner.ComputePhase == "suspending" && state.Snapshot == nil { - result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, ObserveOnly: true}) + if owner.ComputePhase == "suspending" && state.Retained == nil { + result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, ReconcileOnly: true}) if err != nil && !errors.Is(err, sandbox.ErrNotFound) { return err } if err == nil { - state.Snapshot = result.Snapshot + state.Retained = result.Retained } } if state.Target != nil { @@ -92,8 +92,8 @@ func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.Checkpo if err := ignoreComputeAbsent(p.KillCompute(ctx, runtimeReference(owner), state.Current)); err != nil { return err } - if state.Snapshot != nil { - if err := ignoreComputeAbsent(p.DeleteSnapshot(ctx, runtimeReference(owner), *state.Snapshot)); err != nil { + if state.Retained != nil { + if err := ignoreComputeAbsent(p.DeleteRetained(ctx, runtimeReference(owner), *state.Retained)); err != nil { return err } } diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index b09dda19f..d80ac7140 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -96,12 +96,12 @@ func validatedRuntimeProvider(config *RuntimeProvider, registry *runtimegateway. if copied.Mode != "" && copied.Mode != "nodes" && copied.Mode != "direct" { return RuntimeProvider{}, sandbox.ErrInvalid } - if copied.Mode == "direct" && (copied.ProviderKind == "" || copied.LocalNodeID != "" || copied.Suspension != nil) { + if copied.Mode == "direct" && (copied.ProviderKind == "" || copied.LocalNodeID != "") { return RuntimeProvider{}, sandbox.ErrInvalid } if config.Suspension != nil { policy := *config.Suspension - if !sandbox.SupportsCheckpoint(config.Provider) || policy.IdleTimeout < time.Second || policy.Retention < time.Second || policy.MaxActive < 1 || policy.MaxRetained < policy.MaxActive { + if !sandbox.SupportsSuspension(config.Provider) || policy.IdleTimeout < time.Second || policy.Retention < time.Second || policy.MaxActive < 1 || policy.MaxRetained < policy.MaxActive { return RuntimeProvider{}, sandbox.ErrInvalid } copied.Suspension = &policy @@ -185,7 +185,7 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p if err := r.computeFreshCapacity(ctx, providerKey); err != nil { return store.RuntimeAllocation{}, err } - if policy := r.config.Suspension; policy != nil && r.config.ProviderKind == "" { + if policy := r.config.Suspension; policy != nil && (r.config.ProviderKind == "" || r.config.Mode == "direct") { count, err := r.store.CountRuntimeRetainedAllocations(ctx, providerKey) if err != nil { return store.RuntimeAllocation{}, err @@ -429,7 +429,7 @@ func (w *Worker) runManagedRuntimes(ctx context.Context) error { // Manager deployments reserve capacity with Session placement before provisioning. func (r *runtimeLifecycle) computeFreshCapacity(ctx context.Context, key string) error { - if r.config.ProviderKind != "" { + if r.config.ProviderKind != "" && r.config.Mode != "direct" { return nil } return r.computeCapacity(ctx, key) diff --git a/services/core/internal/execution/worker.go b/services/core/internal/execution/worker.go index 03e42f4e1..d63e29328 100644 --- a/services/core/internal/execution/worker.go +++ b/services/core/internal/execution/worker.go @@ -9,6 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) @@ -40,6 +41,10 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher) (*Worker, error) { } owned := *dispatcher owned.Store = lease.Store() + if err := owned.Store.CheckRuntimeComputeProtocol(ctx, sandbox.SuspensionStateVersion); err != nil { + _ = lease.Close(context.Background()) + return nil, err + } owned.notifications = &executionNotifications{} worker := &Worker{concurrency: dispatcher.MaxConcurrentExecutions, dispatcher: &owned, admission: dispatcher.Store, lease: lease, directoryReads: make(chan directoryReadRequest), fileWrites: make(chan fileWriteRequest), stopped: make(chan struct{}), scheduleWake: make(chan struct{}, 1), enrolledConnections: make(map[string]*runtimeConnection)} worker.runtimes, err = newRuntimeManager(owned.Store, owned.Registry, owned.ManagedRuntimes) diff --git a/services/core/internal/sandbox/docker/operations.go b/services/core/internal/sandbox/docker/operations.go index c2758008d..94f0bb64d 100644 --- a/services/core/internal/sandbox/docker/operations.go +++ b/services/core/internal/sandbox/docker/operations.go @@ -7,7 +7,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -var _ sandbox.CheckpointProvider = (*Provider)(nil) +var _ sandbox.SuspensionProvider = (*Provider)(nil) var _ sandbox.SelectionDiscoverer = (*Provider)(nil) var _ sandbox.CredentialVerifier = (*Provider)(nil) var _ runtimeobs.BatchSource = (*Provider)(nil) @@ -22,11 +22,12 @@ func Operations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "NewCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "GetCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "Suspend": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "Resume": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "KillCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "ObservationProviderType": {State: providercontract.Supported}, @@ -41,7 +42,7 @@ func (*Provider) ProviderOperations() providercontract.Operations { return Opera func (p *Provider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: Operations()["Initial"].Reason} } -func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: Operations()["NewCompute"].Reason} } func (p *Provider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -56,8 +57,8 @@ func (p *Provider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.Compu func (p *Provider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: Operations()["KillCompute"].Reason} } -func (p *Provider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: Operations()["DeleteSnapshot"].Reason} +func (p *Provider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: Operations()["DeleteRetained"].Reason} } func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: Operations()["RunCommandCompute"].Reason} @@ -74,3 +75,7 @@ func (p *Provider) DiscoverSelection(context.Context, sandbox.Selection) (sandbo func (p *Provider) VerifyCredential(context.Context, []sandbox.Reference) error { return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: Operations()["VerifyCredential"].Reason} } + +func (p *Provider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: Operations()["RenewCompute"].Reason} +} diff --git a/services/core/internal/sandbox/e2b/operations.go b/services/core/internal/sandbox/e2b/operations.go index 14530ebf9..dab82e81c 100644 --- a/services/core/internal/sandbox/e2b/operations.go +++ b/services/core/internal/sandbox/e2b/operations.go @@ -7,7 +7,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -var _ sandbox.CheckpointProvider = (*Provider)(nil) +var _ sandbox.SuspensionProvider = (*Provider)(nil) var _ sandbox.SelectionDiscoverer = (*Provider)(nil) var _ sandbox.CredentialVerifier = (*Provider)(nil) var _ runtimeobs.BatchSource = (*Provider)(nil) @@ -22,11 +22,12 @@ func Operations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "NewCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "GetCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "Suspend": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "Resume": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "KillCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, "ObservationProviderType": {State: providercontract.Supported}, @@ -41,7 +42,7 @@ func (*Provider) ProviderOperations() providercontract.Operations { return Opera func (p *Provider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: Operations()["Initial"].Reason} } -func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: Operations()["NewCompute"].Reason} } func (p *Provider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -56,8 +57,8 @@ func (p *Provider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.Compu func (p *Provider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: Operations()["KillCompute"].Reason} } -func (p *Provider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: Operations()["DeleteSnapshot"].Reason} +func (p *Provider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: Operations()["DeleteRetained"].Reason} } func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: Operations()["RunCommandCompute"].Reason} @@ -65,3 +66,7 @@ func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox func (p *Provider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: Operations()["ResumeCompute"].Reason} } + +func (p *Provider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: Operations()["RenewCompute"].Reason} +} diff --git a/services/core/internal/sandbox/microsandbox/operations.go b/services/core/internal/sandbox/microsandbox/operations.go index f59c9ed02..4737a94a1 100644 --- a/services/core/internal/sandbox/microsandbox/operations.go +++ b/services/core/internal/sandbox/microsandbox/operations.go @@ -7,7 +7,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -var _ sandbox.CheckpointProvider = (*Provider)(nil) +var _ sandbox.SuspensionProvider = (*Provider)(nil) var _ sandbox.SelectionDiscoverer = (*Provider)(nil) var _ sandbox.CredentialVerifier = (*Provider)(nil) var _ runtimeobs.BatchSource = (*Provider)(nil) @@ -22,11 +22,12 @@ func Operations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Supported}, "NewCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, "GetCompute": {State: providercontract.Supported}, "Suspend": {State: providercontract.Supported}, "Resume": {State: providercontract.Supported}, "KillCompute": {State: providercontract.Supported}, - "DeleteSnapshot": {State: providercontract.Supported}, + "DeleteRetained": {State: providercontract.Supported}, "RunCommandCompute": {State: providercontract.Supported}, "ResumeCompute": {State: providercontract.Supported}, "ObservationProviderType": {State: providercontract.Supported}, diff --git a/services/core/internal/sandbox/microsandbox/provider.go b/services/core/internal/sandbox/microsandbox/provider.go index d6df19ec7..a22dd898f 100644 --- a/services/core/internal/sandbox/microsandbox/provider.go +++ b/services/core/internal/sandbox/microsandbox/provider.go @@ -16,7 +16,7 @@ type Provider struct { } var _ sandbox.SandboxProvider = (*Provider)(nil) -var _ sandbox.CheckpointProvider = (*Provider)(nil) +var _ sandbox.SuspensionProvider = (*Provider)(nil) func New(c Config) (*Provider, error) { return NewWithCaller(c, &ProcessCaller{}) } func NewWithCaller(c Config, caller Caller) (*Provider, error) { @@ -26,7 +26,7 @@ func NewWithCaller(c Config, caller Caller) (*Provider, error) { c.Network.Rules = append([]NetworkRule(nil), c.Network.Rules...) return &Provider{config: c, caller: caller}, nil } -func (p *Provider) Initial(ctx context.Context, r sandbox.Reference) (Compute, error) { +func (p *Provider) nativeInitial(ctx context.Context, r sandbox.Reference) (Compute, error) { if err := ctx.Err(); err != nil { return Compute{}, err } @@ -88,7 +88,7 @@ func (p *Provider) responseState(ctx context.Context, q Request, out Response) ( } want := q.Compute if q.Operation == "create" { - want, e = p.Initial(ctx, q.Reference) + want, e = p.nativeInitial(ctx, q.Reference) if e != nil { return State{}, e } @@ -142,48 +142,48 @@ func (p *Provider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Inf return result, err } func (p *Provider) GetInfo(ctx context.Context, r sandbox.Reference) (sandbox.Info, error) { - c, e := p.Initial(ctx, r) + c, e := p.nativeInitial(ctx, r) if e != nil { return sandbox.Info{}, e } - s, e := p.GetCompute(ctx, r, c) + s, e := p.nativeGetCompute(ctx, r, c) return info(r, s), e } func (p *Provider) Renew(ctx context.Context, r sandbox.Reference) (sandbox.Info, error) { return p.GetInfo(ctx, r) } func (p *Provider) Kill(ctx context.Context, r sandbox.Reference) error { - c, e := p.Initial(ctx, r) + c, e := p.nativeInitial(ctx, r) if e != nil { return e } - return p.KillCompute(ctx, r, c) + return p.nativeKillCompute(ctx, r, c) } func (p *Provider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { - compute, e := p.Initial(ctx, r) + compute, e := p.nativeInitial(ctx, r) if e != nil { return sandbox.CommandResult{}, e } - return p.RunCommandCompute(ctx, r, compute, c) + return p.nativeRunCommandCompute(ctx, r, compute, c) } -func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { +func (p *Provider) nativeGetCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { return p.state(ctx, Request{Operation: "inspect", Reference: r, Compute: c}) } -func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c Compute) error { +func (p *Provider) nativeKillCompute(ctx context.Context, r sandbox.Reference, c Compute) error { _, e := p.call(ctx, Request{Operation: "kill", Reference: r, Compute: c}) return e } -func (p *Provider) DeleteSnapshot(ctx context.Context, r sandbox.Reference, s SnapshotIdentity) error { +func (p *Provider) nativeDeleteSnapshot(ctx context.Context, r sandbox.Reference, s SnapshotIdentity) error { _, e := p.call(ctx, Request{Operation: "delete_snapshot", Reference: r, Snapshot: &s}) return e } -func (p *Provider) Suspend(ctx context.Context, q SuspendRequest) (State, error) { +func (p *Provider) nativeSuspend(ctx context.Context, q SuspendRequest) (State, error) { return p.state(ctx, Request{Operation: "suspend", Reference: q.Reference, Suspend: &q}) } -func (p *Provider) Resume(ctx context.Context, q ResumeRequest) (State, error) { +func (p *Provider) nativeResume(ctx context.Context, q ResumeRequest) (State, error) { return p.state(ctx, Request{Operation: "resume", Reference: q.Reference, Resume: &q}) } -func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c Compute, command sandbox.Command) (sandbox.CommandResult, error) { +func (p *Provider) nativeRunCommandCompute(ctx context.Context, r sandbox.Reference, c Compute, command sandbox.Command) (sandbox.CommandResult, error) { out, e := p.call(ctx, Request{Operation: "command", Reference: r, Compute: c, Command: &command}) if e != nil { return sandbox.CommandResult{}, e @@ -196,11 +196,11 @@ func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c // ResumeCompute thaws the exact resident source after an aborted suspension. // It never starts stopped compute or restores a checkpoint. -func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { +func (p *Provider) nativeResumeCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { return p.state(ctx, Request{Operation: "resume_compute", Reference: r, Compute: c}) } -func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, generation uint64, snapshot *SnapshotIdentity) (Compute, error) { +func (p *Provider) nativeNewCompute(ctx context.Context, r sandbox.Reference, generation uint64, snapshot *SnapshotIdentity) (Compute, error) { if err := ctx.Err(); err != nil { return Compute{}, err } diff --git a/services/core/internal/sandbox/microsandbox/provider_test.go b/services/core/internal/sandbox/microsandbox/provider_test.go index 94bfa9881..42f955dc1 100644 --- a/services/core/internal/sandbox/microsandbox/provider_test.go +++ b/services/core/internal/sandbox/microsandbox/provider_test.go @@ -44,7 +44,7 @@ func TestRejectsChangedComputeIdentity(t *testing.T) { if e != nil { t.Fatal(e) } - _, e = p.GetCompute(deadline(t), r, Compute{Name: Name(c, r, 0), ID: "local:4"}) + _, e = p.nativeGetCompute(deadline(t), r, Compute{Name: Name(c, r, 0), ID: "local:4"}) if !errors.Is(e, sandbox.ErrOwnership) { t.Fatalf("foreign identity accepted: %v", e) } @@ -60,7 +60,7 @@ func TestRestoreMustRetainExactProvenance(t *testing.T) { got.RestoredFrom = &v return Response{Version: ProtocolVersion, State: &State{Compute: got, Status: "running", BootstrapComplete: true}}, nil })) - _, e := p.Resume(deadline(t), ResumeRequest{Reference: r, OperationID: "66666666-6666-4666-8666-666666666666", Snapshot: s, Target: target}) + _, e := p.nativeResume(deadline(t), ResumeRequest{Reference: r, OperationID: "66666666-6666-4666-8666-666666666666", Snapshot: s, Target: target}) if !errors.Is(e, sandbox.ErrOwnership) { t.Fatalf("different snapshot accepted: %v", e) } @@ -70,7 +70,7 @@ func TestRejectsSnapshotPathBeforeHelper(t *testing.T) { s.Reference = "/foreign/checkpoint" calls := 0 p, _ := NewWithCaller(c, callerFunc(func(context.Context, Request) (Response, error) { calls++; return Response{}, nil })) - if e := p.DeleteSnapshot(deadline(t), r, s); !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { + if e := p.nativeDeleteSnapshot(deadline(t), r, s); !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { t.Fatalf("e=%v calls=%d", e, calls) } } @@ -83,7 +83,7 @@ func TestObserveOnlyPreservesCapturedButResidentState(t *testing.T) { } return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: "paused", Snapshot: &s}}, nil })) - got, e := p.Suspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) + got, e := p.nativeSuspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) if e != nil || got.SourceStopped || got.Status != "paused" { t.Fatalf("state=%+v error=%v", got, e) } @@ -117,11 +117,11 @@ func TestNoDeadlineOrForeignAllocationNeverCallsHelper(t *testing.T) { r := testRef() foreign := r foreign.EnvironmentID = "77777777-7777-4777-8777-777777777777" - initial, initialErr := p.Initial(deadline(t), r) + initial, initialErr := p.nativeInitial(deadline(t), r) if initialErr != nil { t.Fatal(initialErr) } - _, e = p.GetCompute(deadline(t), foreign, initial) + _, e = p.nativeGetCompute(deadline(t), foreign, initial) if !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { t.Fatalf("e=%v calls=%d", e, calls) } @@ -146,7 +146,7 @@ func TestMissingSnapshotObservationAllowsOnlyIntactSourceRollback(t *testing.T) p, _ := NewWithCaller(c, callerFunc(func(context.Context, Request) (Response, error) { return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: status, BootstrapComplete: true}}, nil })) - _, e := p.Suspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) + _, e := p.nativeSuspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) if status == "running" || status == "paused" { if e != nil { t.Fatal(e) diff --git a/services/core/internal/sandbox/microsandbox/suspension.go b/services/core/internal/sandbox/microsandbox/suspension.go new file mode 100644 index 000000000..5e1a22e16 --- /dev/null +++ b/services/core/internal/sandbox/microsandbox/suspension.go @@ -0,0 +1,146 @@ +package microsandbox + +import ( + "context" + "encoding/json" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// retained translates a verified native snapshot without exposing its schema to Core. +func retained(s SnapshotIdentity) sandbox.RetainedState { + raw, _ := json.Marshal(s) + return sandbox.RetainedState{Reference: s.Reference, ID: s.ID, OperationID: s.OperationID, SourceGeneration: s.SourceGeneration, SourceName: s.SourceName, SourceID: s.SourceID, Data: string(raw)} +} +func (p *Provider) snapshot(r sandbox.Reference, s sandbox.RetainedState) (SnapshotIdentity, error) { + var native SnapshotIdentity + if sandbox.ValidateRetained(s) != nil || json.Unmarshal([]byte(s.Data), &native) != nil || retained(native) != s || ValidateSnapshot(p.config, r, native) != nil { + return native, sandbox.ErrOwnership + } + return native, nil +} +func compute(c Compute) sandbox.Compute { + out := sandbox.Compute{Generation: c.Generation, Name: c.Name, ID: c.ID} + if c.RestoredFrom != nil { + s := retained(*c.RestoredFrom) + out.RestoredFrom = &s + } + return out +} +func (p *Provider) nativeCompute(r sandbox.Reference, c sandbox.Compute) (Compute, error) { + out := Compute{Generation: c.Generation, Name: c.Name, ID: c.ID} + if c.RestoredFrom != nil { + s, e := p.snapshot(r, *c.RestoredFrom) + if e != nil { + return out, e + } + out.RestoredFrom = &s + } + if ValidateCompute(p.config, r, out) != nil { + return out, sandbox.ErrOwnership + } + return out, nil +} +func state(s State) sandbox.ComputeState { + out := sandbox.ComputeState{Compute: compute(s.Compute), Status: s.Status, BootstrapComplete: s.BootstrapComplete, ResourcesReleased: s.SourceStopped} + if s.Snapshot != nil { + v := retained(*s.Snapshot) + out.Retained = &v + } + return out +} +func (p *Provider) Initial(ctx context.Context, r sandbox.Reference) (sandbox.Compute, error) { + c, e := p.nativeInitial(ctx, r) + return compute(c), e +} +func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, s *sandbox.RetainedState) (sandbox.Compute, error) { + var snap *SnapshotIdentity + if s != nil { + v, e := p.snapshot(r, *s) + if e != nil { + return sandbox.Compute{}, e + } + snap = &v + } + c, e := p.nativeNewCompute(ctx, r, g, snap) + return compute(c), e +} +func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + n, e := p.nativeCompute(r, c) + if e != nil { + return sandbox.ComputeState{}, e + } + s, e := p.nativeGetCompute(ctx, r, n) + return state(s), e +} +func (p *Provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.GetCompute(ctx, r, c) +} +func (p *Provider) Suspend(ctx context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { + c, e := p.nativeCompute(q.Reference, q.Source) + if e != nil { + return sandbox.ComputeState{}, e + } + n := SuspendRequest{Reference: q.Reference, OperationID: q.OperationID, Source: c, ObserveOnly: q.ReconcileOnly} + if q.Retained != nil { + s, e := p.snapshot(q.Reference, *q.Retained) + if e != nil { + return sandbox.ComputeState{}, e + } + n.Snapshot = &s + } + s, e := p.nativeSuspend(ctx, n) + if e == nil && s.Snapshot != nil { + // Complete the same exact-source cleanup previously performed by Core. + // Native KillCompute still verifies ownership and removes its disks. + e = p.nativeKillCompute(ctx, q.Reference, c) + if e == nil { + s.SourceStopped = true + s.Status = "suspended" + } + } + out := state(s) + // The helper holds the allocation lock until the original native work settles. + out.SuspendSettled = e == nil + return out, e +} +func (p *Provider) Resume(ctx context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { + c, e := p.nativeCompute(q.Reference, q.Target) + if e != nil { + return sandbox.ComputeState{}, e + } + s, e := p.snapshot(q.Reference, q.Retained) + if e != nil { + return sandbox.ComputeState{}, e + } + v, e := p.nativeResume(ctx, ResumeRequest{Reference: q.Reference, OperationID: q.OperationID, Snapshot: s, Target: c, ObserveOnly: q.ReconcileOnly}) + return state(v), e +} +func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) error { + n, e := p.nativeCompute(r, c) + if e != nil { + return e + } + return p.nativeKillCompute(ctx, r, n) +} +func (p *Provider) DeleteRetained(ctx context.Context, r sandbox.Reference, s sandbox.RetainedState) error { + n, e := p.snapshot(r, s) + if e != nil { + return e + } + return p.nativeDeleteSnapshot(ctx, r, n) +} +func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, q sandbox.Command) (sandbox.CommandResult, error) { + n, e := p.nativeCompute(r, c) + if e != nil { + return sandbox.CommandResult{}, e + } + return p.nativeRunCommandCompute(ctx, r, n, q) +} +func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + n, e := p.nativeCompute(r, c) + if e != nil { + return sandbox.ComputeState{}, e + } + v, e := p.nativeResumeCompute(ctx, r, n) + return state(v), e +} diff --git a/services/core/internal/sandbox/microsandbox/suspension_contract_test.go b/services/core/internal/sandbox/microsandbox/suspension_contract_test.go new file mode 100644 index 000000000..284bc2159 --- /dev/null +++ b/services/core/internal/sandbox/microsandbox/suspension_contract_test.go @@ -0,0 +1,57 @@ +package microsandbox + +import ( + "context" + "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "testing" +) + +func TestSharedSuspendSettlesExactSourceCleanupInsideAdapter(t *testing.T) { + for _, reconcile := range []bool{false, true} { + for _, cleanupFails := range []bool{false, true} { + c, r, snap := testConfig(), testRef(), testSnapshot() + source := Compute{Name: snap.SourceName, ID: snap.SourceID} + calls := []string{} + p, e := NewWithCaller(c, callerFunc(func(_ context.Context, q Request) (Response, error) { + calls = append(calls, q.Operation) + if q.Operation == "suspend" { + if q.Suspend.ObserveOnly != reconcile { + t.Fatal("reconcile intent changed") + } + return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: "suspended", BootstrapComplete: true, Snapshot: &snap, SourceStopped: !reconcile}}, nil + } + if q.Operation != "kill" || q.Compute.ID != source.ID { + t.Fatal("wrong source cleanup", q.Operation) + } + if cleanupFails { + return Response{}, errors.New("lost cleanup") + } + return Response{Version: ProtocolVersion}, nil + })) + if e != nil { + t.Fatal(e) + } + q := sandbox.SuspendRequest{Reference: r, OperationID: snap.OperationID, Source: compute(source), ReconcileOnly: reconcile} + got, e := p.Suspend(deadline(t), q) + if len(calls) != 2 || calls[0] != "suspend" || calls[1] != "kill" { + t.Fatal(calls) + } + if cleanupFails { + if e == nil || got.SuspendSettled { + t.Fatal("failed cleanup released capacity") + } + continue + } + if e != nil { + t.Fatal(e) + } + if e = sandbox.ValidateSuspendResult(q, got); e != nil { + t.Fatal(e) + } + if got.Retained == nil || got.Retained.Data == "" { + t.Fatal("missing native proof") + } + } + } +} diff --git a/services/core/internal/sandbox/microsandbox/types.go b/services/core/internal/sandbox/microsandbox/types.go index 12ddc1f69..e9b990f18 100644 --- a/services/core/internal/sandbox/microsandbox/types.go +++ b/services/core/internal/sandbox/microsandbox/types.go @@ -40,12 +40,52 @@ type NetworkPolicy struct { } type NetworkRule struct{ Action, Direction, Destination, Protocol, Port string } -// These aliases keep the helper wire private while Core uses provider-neutral types. -type Compute = sandbox.Compute -type SnapshotIdentity = sandbox.SnapshotIdentity -type State = sandbox.ComputeState -type SuspendRequest = sandbox.SuspendRequest -type ResumeRequest = sandbox.ResumeRequest +// Native snapshot wire types belong to this adapter. +type Compute struct { + Generation uint64 + Name string + ID string + RestoredFrom *SnapshotIdentity +} + +// SnapshotIdentity is provider evidence from a verified full snapshot. Core +// persists it unchanged and records consumption separately; it never invents +// paths, checksums, native checkpoint fields, or source identity. +type SnapshotIdentity struct { + Reference string + ID string + Digest string + CheckpointID string + CheckpointRoot string + OperationID string + SourceGeneration uint64 + SourceName string + SourceID string +} + +type State struct { + Compute Compute + Status string + BootstrapComplete bool + Snapshot *SnapshotIdentity + SourceStopped bool +} +type SuspendRequest struct { + Reference sandbox.Reference + OperationID string + Source Compute + Snapshot *SnapshotIdentity + // Recovery observes the previous attempt and never starts a new capture. + ObserveOnly bool +} +type ResumeRequest struct { + Reference sandbox.Reference + OperationID string + Snapshot SnapshotIdentity + Target Compute + // Recovery observes the previous target and never starts a new restore. + ObserveOnly bool +} // Request and Response are the finite, private helper boundary. Confidential // Bootstrap and Command bytes travel only through stdin and are never logged. diff --git a/services/core/internal/sandbox/node/agent.go b/services/core/internal/sandbox/node/agent.go index 3d31e488b..5daf06eb6 100644 --- a/services/core/internal/sandbox/node/agent.go +++ b/services/core/internal/sandbox/node/agent.go @@ -84,7 +84,7 @@ func Run(ctx context.Context, config AgentConfig) error { if err := sandbox.ValidateProvider(config.Provider); err != nil { return err } - if sandbox.SupportsCheckpoint(config.Provider) != providers.SupportsCheckpoint(config.Identity.Provider) { + if sandbox.SupportsSuspension(config.Provider) != providers.SupportsSuspension(config.Identity.Provider) { return sandbox.ErrInvalid } } diff --git a/services/core/internal/sandbox/node/generation_json.go b/services/core/internal/sandbox/node/generation_json.go index c57e0ae07..e5ac8337c 100644 --- a/services/core/internal/sandbox/node/generation_json.go +++ b/services/core/internal/sandbox/node/generation_json.go @@ -130,7 +130,7 @@ func validateGenerationJSON(raw []byte, kind string) error { } } if value := values["request"]; value != nil { - if _, err := generationObject(value, "deployment_generation id sequence connection_id owner_epoch operation timeout_ms reference", "bootstrap compute generation command suspend resume snapshot observation", ""); err != nil { + if _, err := generationObject(value, "deployment_generation id sequence connection_id owner_epoch operation timeout_ms reference", "bootstrap compute generation command suspend resume retained observation", ""); err != nil { return err } } diff --git a/services/core/internal/sandbox/node/node_test.go b/services/core/internal/sandbox/node/node_test.go index 792bc3f48..a9893ac7a 100644 --- a/services/core/internal/sandbox/node/node_test.go +++ b/services/core/internal/sandbox/node/node_test.go @@ -166,7 +166,7 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing func TestOfflineIsUnknownAndDockerDoesNotAdvertiseCheckpoint(t *testing.T) { h := NewHub(HubOptions{OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) p := h.Proxy(uuid.NewString(), "docker", 1) - if sandbox.SupportsCheckpoint(p) { + if sandbox.SupportsSuspension(p) { t.Fatal("docker advertised checkpoint") } _, err := p.GetInfo(context.Background(), reference()) diff --git a/services/core/internal/sandbox/node/operations.go b/services/core/internal/sandbox/node/operations.go index 8f016664b..aca02dcb5 100644 --- a/services/core/internal/sandbox/node/operations.go +++ b/services/core/internal/sandbox/node/operations.go @@ -10,10 +10,11 @@ var operationMethods = map[string]string{ "initial": "Initial", "new_compute": "NewCompute", "compute": "GetCompute", + "renew_compute": "RenewCompute", "suspend": "Suspend", "resume": "Resume", "kill_compute": "KillCompute", - "delete_snapshot": "DeleteSnapshot", + "delete_retained": "DeleteRetained", "command_compute": "RunCommandCompute", "resume_compute": "ResumeCompute", "observe": "Observe", diff --git a/services/core/internal/sandbox/node/operations_test.go b/services/core/internal/sandbox/node/operations_test.go index dfafaf4c0..894b5f090 100644 --- a/services/core/internal/sandbox/node/operations_test.go +++ b/services/core/internal/sandbox/node/operations_test.go @@ -50,7 +50,7 @@ func TestUnsupportedProxyRejectsBeforeNodeResolution(t *testing.T) { } } func TestNodeOperationMappingCoversForwardedMethods(t *testing.T) { - for _, method := range []string{"Create", "GetInfo", "Renew", "Kill", "RunCommand", "Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteSnapshot", "RunCommandCompute", "ResumeCompute", "Observe"} { + for _, method := range []string{"Create", "GetInfo", "Renew", "Kill", "RunCommand", "Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteRetained", "RunCommandCompute", "ResumeCompute", "Observe"} { if wire := operationWire(method); wire == "" || operationMethod(wire) != method { t.Fatal(method) } diff --git a/services/core/internal/sandbox/node/provider_operations_fixture_test.go b/services/core/internal/sandbox/node/provider_operations_fixture_test.go index 8d34c1511..acb4cc126 100644 --- a/services/core/internal/sandbox/node/provider_operations_fixture_test.go +++ b/services/core/internal/sandbox/node/provider_operations_fixture_test.go @@ -16,11 +16,12 @@ func (*fakeProvider) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ObservationProviderType": {State: providercontract.Supported}, @@ -34,7 +35,7 @@ func (*fakeProvider) ProviderOperations() providercontract.Operations { func (*fakeProvider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: "fixture_operation_not_supported"} } -func (*fakeProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (*fakeProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: "fixture_operation_not_supported"} } func (*fakeProvider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -49,8 +50,8 @@ func (*fakeProvider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.Com func (*fakeProvider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: "fixture_operation_not_supported"} } -func (*fakeProvider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: "fixture_operation_not_supported"} +func (*fakeProvider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: "fixture_operation_not_supported"} } func (*fakeProvider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: "fixture_operation_not_supported"} @@ -79,11 +80,12 @@ func (*observationProvider) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ObservationProviderType": {State: providercontract.Supported}, @@ -104,3 +106,7 @@ func (*observationProvider) ObservationProviderType() string { return "fixture" func (p *observationProvider) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { return p, nil } + +func (p *fakeProvider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} diff --git a/services/core/internal/sandbox/node/proxy.go b/services/core/internal/sandbox/node/proxy.go index cf30880ea..caf5e7dbc 100644 --- a/services/core/internal/sandbox/node/proxy.go +++ b/services/core/internal/sandbox/node/proxy.go @@ -17,7 +17,7 @@ type provider struct { } var _ sandbox.SandboxProvider = (*provider)(nil) -var _ sandbox.CheckpointProvider = (*provider)(nil) +var _ sandbox.SuspensionProvider = (*provider)(nil) // Proxy binds a fixed node and deployment generation explicitly. func (h *Hub) Proxy(id, kind string, generation uint64) sandbox.SandboxProvider { @@ -125,8 +125,8 @@ func (p *provider) Initial(ctx context.Context, r sandbox.Reference) (sandbox.Co } return *out.Compute, nil } -func (p *provider) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, s *sandbox.SnapshotIdentity) (sandbox.Compute, error) { - out, e := p.call(ctx, request{Operation: "new_compute", Reference: r, Generation: g, Snapshot: s}) +func (p *provider) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, s *sandbox.RetainedState) (sandbox.Compute, error) { + out, e := p.call(ctx, request{Operation: "new_compute", Reference: r, Generation: g, Retained: s}) if e != nil { return sandbox.Compute{}, e } @@ -158,8 +158,8 @@ func (p *provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandb _, e := p.call(ctx, request{Operation: "kill_compute", Reference: r, Compute: &c}) return e } -func (p *provider) DeleteSnapshot(ctx context.Context, r sandbox.Reference, s sandbox.SnapshotIdentity) error { - _, e := p.call(ctx, request{Operation: "delete_snapshot", Reference: r, Snapshot: &s}) +func (p *provider) DeleteRetained(ctx context.Context, r sandbox.Reference, s sandbox.RetainedState) error { + _, e := p.call(ctx, request{Operation: "delete_retained", Reference: r, Retained: &s}) return e } func (p *provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, v sandbox.Command) (sandbox.CommandResult, error) { @@ -175,3 +175,7 @@ func (h *Hub) GenerationProvider(kind string, resolve func(context.Context, sand p := &provider{hub: h, kind: kind, resolveGeneration: resolve} return p } + +func (p *provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.state(ctx, request{Operation: "renew_compute", Reference: r, Compute: &c}) +} diff --git a/services/core/internal/sandbox/node/wire.go b/services/core/internal/sandbox/node/wire.go index 0194e3c82..1c2f43bd5 100644 --- a/services/core/internal/sandbox/node/wire.go +++ b/services/core/internal/sandbox/node/wire.go @@ -17,7 +17,7 @@ import ( "github.com/gorilla/websocket" ) -const ProtocolVersion = 4 +const ProtocolVersion = 5 const MaxControlFrameBytes = 32 * 1024 const MaxFrameBytes = 72 * 1024 * 1024 const maxPending = 32 @@ -85,15 +85,15 @@ type request struct { TimeoutMillis int64 `json:"timeout_ms"` // deadline is anchored to the receiving host and never crosses the wire. deadline time.Time - Reference sandbox.Reference `json:"reference"` - Bootstrap *sandbox.Bootstrap `json:"bootstrap,omitempty"` - Compute *sandbox.Compute `json:"compute,omitempty"` - Generation uint64 `json:"generation,omitempty"` - Command *sandbox.Command `json:"command,omitempty"` - Suspend *sandbox.SuspendRequest `json:"suspend,omitempty"` - Resume *sandbox.ResumeRequest `json:"resume,omitempty"` - Snapshot *sandbox.SnapshotIdentity `json:"snapshot,omitempty"` - Observation *runtimeobs.Target `json:"observation,omitempty"` + Reference sandbox.Reference `json:"reference"` + Bootstrap *sandbox.Bootstrap `json:"bootstrap,omitempty"` + Compute *sandbox.Compute `json:"compute,omitempty"` + Generation uint64 `json:"generation,omitempty"` + Command *sandbox.Command `json:"command,omitempty"` + Suspend *sandbox.SuspendRequest `json:"suspend,omitempty"` + Resume *sandbox.ResumeRequest `json:"resume,omitempty"` + Retained *sandbox.RetainedState `json:"retained,omitempty"` + Observation *runtimeobs.Target `json:"observation,omitempty"` } type response struct { @@ -238,7 +238,7 @@ func (q request) validate() error { return sandbox.ErrInvalid } count := 0 - for _, ok := range []bool{q.Bootstrap != nil, q.Compute != nil, q.Command != nil, q.Suspend != nil, q.Resume != nil, q.Snapshot != nil, q.Observation != nil} { + for _, ok := range []bool{q.Bootstrap != nil, q.Compute != nil, q.Command != nil, q.Suspend != nil, q.Resume != nil, q.Retained != nil, q.Observation != nil} { if ok { count++ } @@ -257,10 +257,10 @@ func (q request) validate() error { return nil } case "new_compute": - if count == 0 || count == 1 && q.Snapshot != nil { + if count == 0 || count == 1 && q.Retained != nil { return nil } - case "compute", "kill_compute", "resume_compute": + case "compute", "renew_compute", "kill_compute", "resume_compute": if count == 1 && q.Compute != nil { return nil } @@ -280,8 +280,8 @@ func (q request) validate() error { if count == 1 && q.Resume != nil && q.Resume.Reference == q.Reference { return nil } - case "delete_snapshot": - if count == 1 && q.Snapshot != nil { + case "delete_retained": + if count == 1 && q.Retained != nil { return nil } } @@ -324,7 +324,7 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response command, err = p.RunCommand(ctx, q.Reference, *q.Command) out.Command = &command default: - cp, checkpointErr := sandbox.Checkpoint(p) + cp, checkpointErr := sandbox.Suspension(p) if checkpointErr != nil { err = checkpointErr break @@ -336,11 +336,14 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response compute, err = cp.Initial(ctx, q.Reference) out.Compute = &compute case "new_compute": - compute, err = cp.NewCompute(ctx, q.Reference, q.Generation, q.Snapshot) + compute, err = cp.NewCompute(ctx, q.Reference, q.Generation, q.Retained) out.Compute = &compute case "compute": state, err = cp.GetCompute(ctx, q.Reference, *q.Compute) out.State = &state + case "renew_compute": + state, err = cp.RenewCompute(ctx, q.Reference, *q.Compute) + out.State = &state case "suspend": state, err = cp.Suspend(ctx, *q.Suspend) out.State = &state @@ -349,8 +352,8 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response out.State = &state case "kill_compute": err = cp.KillCompute(ctx, q.Reference, *q.Compute) - case "delete_snapshot": - err = cp.DeleteSnapshot(ctx, q.Reference, *q.Snapshot) + case "delete_retained": + err = cp.DeleteRetained(ctx, q.Reference, *q.Retained) case "resume_compute": state, err = cp.ResumeCompute(ctx, q.Reference, *q.Compute) out.State = &state @@ -379,7 +382,7 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response } func requiresReady(q request) bool { - return q.Operation == "create" || q.Operation == "resume" && q.Resume != nil && !q.Resume.ObserveOnly + return q.Operation == "create" || q.Operation == "resume" && q.Resume != nil && !q.Resume.ReconcileOnly } // setTimeout consumes sender queue time without comparing clocks across hosts. diff --git a/services/core/internal/sandbox/operations.go b/services/core/internal/sandbox/operations.go index 3faf4454b..487db4ee1 100644 --- a/services/core/internal/sandbox/operations.go +++ b/services/core/internal/sandbox/operations.go @@ -11,7 +11,7 @@ import ( // These existing interfaces are the canonical operation inventory. Declarations // must cover every method, including explicit unsupported implementations. var providerInterfaces = []reflect.Type{ - reflect.TypeFor[SandboxProvider](), reflect.TypeFor[CheckpointProvider](), + reflect.TypeFor[SandboxProvider](), reflect.TypeFor[SuspensionProvider](), reflect.TypeFor[SelectionDiscoverer](), reflect.TypeFor[CredentialVerifier](), reflect.TypeFor[runtimeobs.SourceResolver](), reflect.TypeFor[runtimeobs.Source](), reflect.TypeFor[runtimeobs.BatchSource](), } @@ -59,7 +59,7 @@ func ValidateOperations(operations providercontract.Operations) error { } // The checkpoint lifecycle is indivisible: partial cleanup or restore support // cannot safely own a compute incarnation. - checkpoint := reflect.TypeFor[CheckpointProvider]() + checkpoint := reflect.TypeFor[SuspensionProvider]() for i := 0; i < checkpoint.NumMethod(); i++ { name := checkpoint.Method(i).Name if _, required := reflect.TypeFor[SandboxProvider]().MethodByName(name); !required && operations[name].State != operations["Initial"].State { @@ -72,15 +72,15 @@ func ValidateOperations(operations providercontract.Operations) error { return nil } -func SupportsCheckpoint(p SandboxProvider) bool { +func SupportsSuspension(p SandboxProvider) bool { return providercontract.Require(p, "Initial") == nil } -func Checkpoint(p SandboxProvider) (CheckpointProvider, error) { +func Suspension(p SandboxProvider) (SuspensionProvider, error) { if err := providercontract.Require(p, "Initial"); err != nil { return nil, err } - cp, ok := p.(CheckpointProvider) + cp, ok := p.(SuspensionProvider) if !ok { return nil, providercontract.ErrContract } diff --git a/services/core/internal/sandbox/operations_test.go b/services/core/internal/sandbox/operations_test.go index f1cc091d1..8fc64acc6 100644 --- a/services/core/internal/sandbox/operations_test.go +++ b/services/core/internal/sandbox/operations_test.go @@ -34,7 +34,7 @@ func TestDeclarationsRejectMissingUnknownAndContradictoryOperations(t *testing.T {"resolver unsupported", func(o providercontract.Operations) { o["ResolveObservationSource"] = providercontract.Support{State: providercontract.Unsupported, Reason: "no_resolver"} }}, - {"omitted", func(o providercontract.Operations) { delete(o, "DeleteSnapshot") }}, + {"omitted", func(o providercontract.Operations) { delete(o, "DeleteRetained") }}, {"zero", func(o providercontract.Operations) { o["ObserveBatch"] = providercontract.Support{} }}, {"unknown", func(o providercontract.Operations) { o["FutureOperation"] = providercontract.Support{State: providercontract.Supported} diff --git a/services/core/internal/sandbox/providers/registration.go b/services/core/internal/sandbox/providers/registration.go index 712283e55..23d4c2731 100644 --- a/services/core/internal/sandbox/providers/registration.go +++ b/services/core/internal/sandbox/providers/registration.go @@ -51,16 +51,15 @@ func ValidateRegistration(a Adapter) error { if err := sandbox.ValidateOperations(operations); err != nil { return err } - // The current common lifecycle admits checkpoint suspension only on nodes, - // and creates its policy whenever checkpoint support is declared. + // One suspension policy applies to every declared lifecycle and placement. if operations["Initial"].State == providercontract.Supported { const maximumSeconds = int64((1<<63 - 1) / time.Second) - if a.Mode != "nodes" || a.IdleSeconds < 1 || a.RetentionSeconds < 1 || + if a.IdleSeconds < 1 || a.RetentionSeconds < 1 || a.IdleSeconds > maximumSeconds || a.RetentionSeconds > maximumSeconds { - return invalid("checkpoint policy") + return invalid("suspension policy") } } else if a.IdleSeconds != 0 || a.RetentionSeconds != 0 { - return invalid("non-checkpoint policy") + return invalid("unsupported suspension policy") } return nil } diff --git a/services/core/internal/sandbox/providers/registration_test.go b/services/core/internal/sandbox/providers/registration_test.go index 65fc3a355..867cf63c6 100644 --- a/services/core/internal/sandbox/providers/registration_test.go +++ b/services/core/internal/sandbox/providers/registration_test.go @@ -179,7 +179,7 @@ func TestRegistrationCheckpointPolicy(t *testing.T) { {"missing idle", "microsandbox", 0, 20, false, false}, {"missing retention", "microsandbox", 20, 0, false, false}, {"overflow", "microsandbox", 1<<63 - 1, 20, false, false}, - {"direct suspension", "microsandbox", 20, 20, true, false}, + {"direct suspension", "microsandbox", 20, 20, true, true}, {"unsupported suspension", "docker", 20, 20, false, false}, {"independent durations", "microsandbox", 300, 30, false, true}, {"no suspension", "docker", 0, 0, false, true}, @@ -189,6 +189,7 @@ func TestRegistrationCheckpointPolicy(t *testing.T) { a.IdleSeconds, a.RetentionSeconds = tc.idle, tc.retention if tc.direct { a.Mode, a.BuildLocal, a.BuildDirect = "direct", nil, adapters["e2b"].BuildDirect + a.NodeArtifacts = nil } err := ValidateRegistration(a) if (err == nil) != tc.valid || err != nil && !errors.Is(err, providercontract.ErrContract) { diff --git a/services/core/internal/sandbox/providers/registry.go b/services/core/internal/sandbox/providers/registry.go index f579441f7..5df43d642 100644 --- a/services/core/internal/sandbox/providers/registry.go +++ b/services/core/internal/sandbox/providers/registry.go @@ -66,7 +66,7 @@ func Lookup(kind string) (Adapter, error) { return a, nil } func IsNode(kind string) bool { a, e := Lookup(kind); return e == nil && a.Mode == "nodes" } -func SupportsCheckpoint(kind string) bool { +func SupportsSuspension(kind string) bool { a, e := Lookup(kind) return e == nil && a.Operations()["Initial"].State == providercontract.Supported } @@ -74,7 +74,7 @@ func SupportsCheckpoint(kind string) bool { // RetainedLimit keeps nodes without checkpoint support within their active capacity. func RetainedLimit(kind string, active, retained int) int { a, err := Lookup(kind) - if err == nil && a.Mode == "nodes" && !SupportsCheckpoint(kind) { + if err == nil && a.Mode == "nodes" && !SupportsSuspension(kind) { return active } return retained diff --git a/services/core/internal/sandbox/providers/registry_test.go b/services/core/internal/sandbox/providers/registry_test.go index 3d3639b13..d9f51144f 100644 --- a/services/core/internal/sandbox/providers/registry_test.go +++ b/services/core/internal/sandbox/providers/registry_test.go @@ -25,7 +25,7 @@ func TestRegistrationOwnsDeploymentPolicy(t *testing.T) { t.Fatalf("wrong namespace or defaults: %+v %v", d, err) } a, err := Lookup(tc.kind) - if err != nil || SupportsCheckpoint(tc.kind) != tc.checkpoint || IsNode(tc.kind) != (tc.mode == "nodes") || (a.BuildLocal != nil) != (tc.mode == "nodes") || (a.BuildDirect != nil) != (tc.mode == "direct") { + if err != nil || SupportsSuspension(tc.kind) != tc.checkpoint || IsNode(tc.kind) != (tc.mode == "nodes") || (a.BuildLocal != nil) != (tc.mode == "nodes") || (a.BuildDirect != nil) != (tc.mode == "direct") { t.Fatal("inconsistent construction/capability registration", err) } }) @@ -66,7 +66,7 @@ func TestNewRegistrationDoesNotNeedCoreDispatchChanges(t *testing.T) { adapters[kind] = adapters["docker"] defer delete(adapters, kind) s, err := Normalize(sandbox.Selection{Provider: kind, DeploymentSpec: validRegistrationSpec()}) - if err != nil || s.Provider != kind || !IsNode(kind) || SupportsCheckpoint(kind) { + if err != nil || s.Provider != kind || !IsNode(kind) || SupportsSuspension(kind) { t.Fatal("new entry did not follow shared boundary", err) } d, err := Describe(kind, uuid.NewString()) diff --git a/services/core/internal/sandbox/sandbox_provider.go b/services/core/internal/sandbox/sandbox_provider.go index 663526b8e..4336392a6 100644 --- a/services/core/internal/sandbox/sandbox_provider.go +++ b/services/core/internal/sandbox/sandbox_provider.go @@ -4,7 +4,7 @@ // SandboxProvider owns compute and bootstrap, Runtime owns capability preparation, // and Harness adapters own native execution. Compute running is not execution ready. // -// Required operations are on SandboxProvider. CheckpointProvider and runtimeobs +// Required operations are on SandboxProvider. SuspensionProvider and runtimeobs // observation remain separate small interfaces. Every registered adapter explicitly // declares and implements each operation, including safe Unsupported rejections. // Method-set presence never means an extension is supported. ValidateProvider and @@ -92,17 +92,18 @@ type SandboxProvider interface { RunCommand(context.Context, Reference, Command) (CommandResult, error) } -// CheckpointProvider is an explicitly declared extension. It supplies +// SuspensionProvider is an explicitly declared extension. It supplies // exact-incarnation operations; Worker and Store remain the lifecycle owner. -type CheckpointProvider interface { +type SuspensionProvider interface { SandboxProvider Initial(context.Context, Reference) (Compute, error) - NewCompute(context.Context, Reference, uint64, *SnapshotIdentity) (Compute, error) + NewCompute(context.Context, Reference, uint64, *RetainedState) (Compute, error) GetCompute(context.Context, Reference, Compute) (ComputeState, error) + RenewCompute(context.Context, Reference, Compute) (ComputeState, error) Suspend(context.Context, SuspendRequest) (ComputeState, error) Resume(context.Context, ResumeRequest) (ComputeState, error) KillCompute(context.Context, Reference, Compute) error - DeleteSnapshot(context.Context, Reference, SnapshotIdentity) error + DeleteRetained(context.Context, Reference, RetainedState) error RunCommandCompute(context.Context, Reference, Compute, Command) (CommandResult, error) // ResumeCompute thaws only the same resident instance after an aborted pause. ResumeCompute(context.Context, Reference, Compute) (ComputeState, error) @@ -114,3 +115,100 @@ type ProcessPaths struct { ArtifactRoot string StateRoot string } + +// ValidateRetained checks the shared envelope; only its adapter interprets Data. +func ValidateRetained(s RetainedState) error { + if s.Reference == "" || s.ID == "" || s.OperationID == "" || s.SourceID == "" || s.SourceName == "" || len(s.Data) == 0 || len(s.Data) > 64*1024 { + return ErrInvalid + } + return nil +} + +// ErrComputeUnconfirmed requires observation of the retained operation identity; +// it does not authorize another Create, capture, restore, or cold start. +var ErrComputeUnconfirmed = errors.New("sandbox lifecycle outcome unconfirmed") + +// Compute identifies one incarnation of an allocation. Name is provider-derived. +// ID is empty only until the original create or restore result is observed. +type Compute struct { + Generation uint64 + Name string + ID string + RestoredFrom *RetainedState +} + +// RetainedState is adapter-owned recoverable state. Data is opaque to Core. +// A retained state does not imply an independent snapshot. +type RetainedState struct { + Reference string + ID string + Data string + OperationID string + SourceGeneration uint64 + SourceName string + SourceID string +} + +type ComputeState struct { + Compute Compute + Status string + BootstrapComplete bool + Retained *RetainedState + ResourcesReleased bool + SuspendSettled bool +} +type SuspendRequest struct { + Reference Reference + OperationID string + Source Compute + Retained *RetainedState + // Recovery settles the previous attempt without another capture. + // Ownership-verified cleanup of a durable retained artifact may complete. + ReconcileOnly bool +} +type ResumeRequest struct { + Reference Reference + OperationID string + Retained RetainedState + Target Compute + // Recovery observes the previous target and never starts a new restore. + ReconcileOnly bool +} + +// ValidateComputeResult binds an observation to its precommitted incarnation. +func ValidateComputeResult(want, got Compute) error { + if got.ID == "" || got.Name != want.Name || got.Generation != want.Generation || (want.ID != "" && got.ID != want.ID) || (want.RestoredFrom == nil) != (got.RestoredFrom == nil) { + return ErrOwnership + } + if want.RestoredFrom != nil && *want.RestoredFrom != *got.RestoredFrom { + return ErrOwnership + } + return nil +} + +// ValidateSuspendResult distinguishes settled rollback from uncertain native work. +func ValidateSuspendResult(q SuspendRequest, s ComputeState) error { + if ValidateComputeResult(q.Source, s.Compute) != nil { + return ErrOwnership + } + if !s.SuspendSettled || !s.BootstrapComplete { + return ErrComputeUnconfirmed + } + if s.Retained == nil { + if !q.ReconcileOnly || s.ResourcesReleased || (s.Status != "running" && s.Status != "paused") { + return ErrComputeUnconfirmed + } + return nil + } + v := s.Retained + if ValidateRetained(*v) != nil || v.OperationID != q.OperationID || v.SourceID != q.Source.ID || v.SourceName != q.Source.Name || v.SourceGeneration != q.Source.Generation || (q.Retained != nil && *q.Retained != *v) { + return ErrOwnership + } + if !s.ResourcesReleased || s.Status != "suspended" { + return ErrComputeUnconfirmed + } + return nil +} + +// SuspensionStateVersion fences incompatible durable lifecycle shapes. +const SuspensionStateVersion = "1" diff --git a/services/core/internal/sandbox/suspension.go b/services/core/internal/sandbox/suspension.go deleted file mode 100644 index 5a600a877..000000000 --- a/services/core/internal/sandbox/suspension.go +++ /dev/null @@ -1,57 +0,0 @@ -package sandbox - -import ( - "errors" -) - -// ErrComputeUnconfirmed requires observation of the retained operation identity; -// it does not authorize another Create, capture, restore, or cold start. -var ErrComputeUnconfirmed = errors.New("sandbox lifecycle outcome unconfirmed") - -// Compute identifies one incarnation of an allocation. Name is provider-derived. -// ID is empty only until the original create or restore result is observed. -type Compute struct { - Generation uint64 - Name string - ID string - RestoredFrom *SnapshotIdentity -} - -// SnapshotIdentity is provider evidence from a verified full snapshot. Core -// persists it unchanged and records consumption separately; it never invents -// paths, checksums, native checkpoint fields, or source identity. -type SnapshotIdentity struct { - Reference string - ID string - Digest string - CheckpointID string - CheckpointRoot string - OperationID string - SourceGeneration uint64 - SourceName string - SourceID string -} - -type ComputeState struct { - Compute Compute - Status string - BootstrapComplete bool - Snapshot *SnapshotIdentity - SourceStopped bool -} -type SuspendRequest struct { - Reference Reference - OperationID string - Source Compute - Snapshot *SnapshotIdentity - // Recovery observes the previous attempt and never starts a new capture. - ObserveOnly bool -} -type ResumeRequest struct { - Reference Reference - OperationID string - Snapshot SnapshotIdentity - Target Compute - // Recovery observes the previous target and never starts a new restore. - ObserveOnly bool -} diff --git a/services/core/internal/sandbox/suspension_test.go b/services/core/internal/sandbox/suspension_test.go new file mode 100644 index 000000000..add0fb28b --- /dev/null +++ b/services/core/internal/sandbox/suspension_test.go @@ -0,0 +1,55 @@ +package sandbox + +import ( + "strings" + "testing" +) + +func TestSuspensionRequiresBoundSettledResourceRelease(t *testing.T) { + c := Compute{ID: "native", Name: "source", Generation: 2} + r := RetainedState{Reference: "allocation", ID: "retained", OperationID: "op", SourceGeneration: 2, SourceName: "source", SourceID: "native", Data: "private-proof"} + q := SuspendRequest{OperationID: "op", Source: c} + valid := ComputeState{Compute: c, Status: "suspended", BootstrapComplete: true, Retained: &r, ResourcesReleased: true, SuspendSettled: true} + if err := ValidateSuspendResult(q, valid); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + change func(*ComputeState) + }{ + {"unsettled", func(s *ComputeState) { s.SuspendSettled = false }}, + {"capacity still held", func(s *ComputeState) { s.ResourcesReleased = false }}, + {"running", func(s *ComputeState) { s.Status = "running" }}, + {"foreign source", func(s *ComputeState) { s.Compute.ID = "foreign" }}, + {"foreign handle", func(s *ComputeState) { v := *s.Retained; v.OperationID = "other"; s.Retained = &v }}, + {"oversized", func(s *ComputeState) { v := *s.Retained; v.Data = strings.Repeat("x", 65537); s.Retained = &v }}, + } { + t.Run(tc.name, func(t *testing.T) { + s := valid + tc.change(&s) + if ValidateSuspendResult(q, s) == nil { + t.Fatal("invalid suspension accepted") + } + }) + } + rollback := ComputeState{Compute: c, Status: "running", BootstrapComplete: true, SuspendSettled: true} + if ValidateSuspendResult(q, rollback) == nil { + t.Fatal("fresh dispatch accepted rollback") + } + q.ReconcileOnly = true + if err := ValidateSuspendResult(q, rollback); err != nil { + t.Fatal(err) + } + rollback.SuspendSettled = false + if ValidateSuspendResult(q, rollback) == nil { + t.Fatal("running observation inferred settlement") + } +} +func TestComputeResultFencesSameNativeIDAcrossGenerations(t *testing.T) { + old := Compute{ID: "same-native", Name: "allocation", Generation: 1} + next := old + next.Generation++ + if ValidateComputeResult(old, next) == nil { + t.Fatal("stale logical generation accepted") + } +} diff --git a/services/core/internal/store/admin_session_archive_worker_http_test.go b/services/core/internal/store/admin_session_archive_worker_http_test.go index 2f137749c..c9e91c474 100644 --- a/services/core/internal/store/admin_session_archive_worker_http_test.go +++ b/services/core/internal/store/admin_session_archive_worker_http_test.go @@ -129,7 +129,7 @@ func TestAdminSessionArchiveWorkerHTTPPostgres(t *testing.T) { t.Fatal("archive did not commit administrator audit", audits, err) } stop() - // Snapshot after shutdown: cancellation and lifecycle draining are complete. + // Retained after shutdown: cancellation and lifecycle draining are complete. // A stale handler must not fall back to the still-open admission Store. snapshot := func() string { t.Helper() diff --git a/services/core/internal/store/provider_operations_fixture_test.go b/services/core/internal/store/provider_operations_fixture_test.go index 68634162e..145e2507e 100644 --- a/services/core/internal/store/provider_operations_fixture_test.go +++ b/services/core/internal/store/provider_operations_fixture_test.go @@ -16,11 +16,12 @@ func (*lifecycleProvider) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ObservationProviderType": {State: providercontract.Supported}, @@ -34,7 +35,7 @@ func (*lifecycleProvider) ProviderOperations() providercontract.Operations { func (*lifecycleProvider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: "fixture_operation_not_supported"} } -func (*lifecycleProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (*lifecycleProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: "fixture_operation_not_supported"} } func (*lifecycleProvider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -49,8 +50,8 @@ func (*lifecycleProvider) Resume(context.Context, sandbox.ResumeRequest) (sandbo func (*lifecycleProvider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: "fixture_operation_not_supported"} } -func (*lifecycleProvider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: "fixture_operation_not_supported"} +func (*lifecycleProvider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: "fixture_operation_not_supported"} } func (*lifecycleProvider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: "fixture_operation_not_supported"} @@ -70,7 +71,7 @@ func (*lifecycleProvider) DiscoverSelection(context.Context, sandbox.Selection) func (*lifecycleProvider) VerifyCredential(context.Context, []sandbox.Reference) error { return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: "fixture_operation_not_supported"} } -func (*fakeCheckpointProvider) ProviderOperations() providercontract.Operations { +func (*fakeSuspensionProvider) ProviderOperations() providercontract.Operations { return providercontract.Operations{ "Create": {State: providercontract.Supported}, "GetInfo": {State: providercontract.Supported}, @@ -79,11 +80,12 @@ func (*fakeCheckpointProvider) ProviderOperations() providercontract.Operations "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Supported}, "NewCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, "GetCompute": {State: providercontract.Supported}, "Suspend": {State: providercontract.Supported}, "Resume": {State: providercontract.Supported}, "KillCompute": {State: providercontract.Supported}, - "DeleteSnapshot": {State: providercontract.Supported}, + "DeleteRetained": {State: providercontract.Supported}, "RunCommandCompute": {State: providercontract.Supported}, "ResumeCompute": {State: providercontract.Supported}, "ObservationProviderType": {State: providercontract.Supported}, @@ -100,7 +102,11 @@ func (p *lifecycleProvider) ResolveObservationSource(context.Context) (runtimeob return p, nil } -func (*fakeCheckpointProvider) ObservationProviderType() string { return "fixture" } -func (p *fakeCheckpointProvider) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { +func (*fakeSuspensionProvider) ObservationProviderType() string { return "fixture" } +func (p *fakeSuspensionProvider) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { return p, nil } + +func (p *lifecycleProvider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} diff --git a/services/core/internal/store/runtime_compute_lifecycle_test.go b/services/core/internal/store/runtime_compute_lifecycle_test.go index 0626d5f1e..9535794b1 100644 --- a/services/core/internal/store/runtime_compute_lifecycle_test.go +++ b/services/core/internal/store/runtime_compute_lifecycle_test.go @@ -26,11 +26,11 @@ import ( // The controlled provider records external effects independently of DB phases. // Lost replies retain those effects so recovery must use observation, not replay. -type fakeCheckpointProvider struct { +type fakeSuspensionProvider struct { preparation *initializationPeer lifecycleProvider computes map[string]sandbox.ComputeState - snapshots map[string]sandbox.SnapshotIdentity + snapshots map[string]sandbox.RetainedState bootstraps map[string]sandbox.Bootstrap peers map[string]*websocket.Conn registry *runtimegateway.Registry @@ -41,15 +41,16 @@ type fakeCheckpointProvider struct { quiesces, resumes atomic.Int32 loseCapture, loseRestore, rejectQuiesce bool beforeQuiesce func() + renewals atomic.Int32 } -func (p *fakeCheckpointProvider) Initial(_ context.Context, r sandbox.Reference) (sandbox.Compute, error) { +func (p *fakeSuspensionProvider) Initial(_ context.Context, r sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{Name: r.AllocationID + "-g0"}, nil } -func (p *fakeCheckpointProvider) NewCompute(_ context.Context, r sandbox.Reference, generation uint64, parent *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *fakeSuspensionProvider) NewCompute(_ context.Context, r sandbox.Reference, generation uint64, parent *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{Generation: generation, Name: fmt.Sprintf("%s-g%d", r.AllocationID, generation), RestoredFrom: parent}, nil } -func (p *fakeCheckpointProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { +func (p *fakeSuspensionProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { info, err := p.lifecycleProvider.Create(ctx, b) p.mu.Lock() defer p.mu.Unlock() @@ -59,7 +60,7 @@ func (p *fakeCheckpointProvider) Create(ctx context.Context, b sandbox.Bootstrap p.bootstraps[b.AllocationID] = b return info, err } -func (p *fakeCheckpointProvider) GetCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) GetCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[c.Name] @@ -71,24 +72,28 @@ func (p *fakeCheckpointProvider) GetCompute(_ context.Context, _ sandbox.Referen } return state, nil } -func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) Suspend(_ context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[q.Source.Name] if !ok { + if snapshot, exists := p.snapshots[q.OperationID]; exists && q.ReconcileOnly { + p.captureObservations++ + return sandbox.ComputeState{Compute: q.Source, Status: "suspended", Retained: &snapshot, BootstrapComplete: true, ResourcesReleased: true, SuspendSettled: true}, nil + } return sandbox.ComputeState{}, sandbox.ErrNotFound } if state.Compute.ID != q.Source.ID { return sandbox.ComputeState{}, sandbox.ErrOwnership } - if q.ObserveOnly { + if q.ReconcileOnly { p.captureObservations++ } else { p.captures++ if _, exists := p.snapshots[q.OperationID]; exists { return sandbox.ComputeState{}, errors.New("capture replayed") } - p.snapshots[q.OperationID] = sandbox.SnapshotIdentity{Reference: "snapshot-" + q.OperationID, ID: uuid.NewString(), Digest: "verified", CheckpointID: "checkpoint", CheckpointRoot: "private", OperationID: q.OperationID, SourceGeneration: q.Source.Generation, SourceName: q.Source.Name, SourceID: q.Source.ID} + p.snapshots[q.OperationID] = sandbox.RetainedState{Reference: "snapshot-" + q.OperationID, ID: uuid.NewString(), Data: "verified-native-state", OperationID: q.OperationID, SourceGeneration: q.Source.Generation, SourceName: q.Source.Name, SourceID: q.Source.ID} state.Status = "paused" p.computes[q.Source.Name] = state if p.loseCapture { @@ -97,14 +102,20 @@ func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendReq } } if snapshot, exists := p.snapshots[q.OperationID]; exists { - state.Snapshot = &snapshot - } + state.Retained = &snapshot + state.SuspendSettled = true + state.ResourcesReleased = true + state.Status = "suspended" + delete(p.computes, q.Source.Name) + p.computeKills++ + } + state.SuspendSettled = true return state, nil } -func (p *fakeCheckpointProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { p.mu.Lock() defer p.mu.Unlock() - if q.ObserveOnly { + if q.ReconcileOnly { p.restoreObservations++ state, ok := p.computes[q.Target.Name] if !ok { @@ -126,7 +137,7 @@ func (p *fakeCheckpointProvider) Resume(_ context.Context, q sandbox.ResumeReque } return state, nil } -func (p *fakeCheckpointProvider) KillCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) error { +func (p *fakeSuspensionProvider) KillCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) error { p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[c.Name] @@ -140,7 +151,7 @@ func (p *fakeCheckpointProvider) KillCompute(_ context.Context, _ sandbox.Refere delete(p.computes, c.Name) return nil } -func (p *fakeCheckpointProvider) DeleteSnapshot(_ context.Context, _ sandbox.Reference, s sandbox.SnapshotIdentity) error { +func (p *fakeSuspensionProvider) DeleteRetained(_ context.Context, _ sandbox.Reference, s sandbox.RetainedState) error { p.mu.Lock() defer p.mu.Unlock() old, ok := p.snapshots[s.OperationID] @@ -154,7 +165,7 @@ func (p *fakeCheckpointProvider) DeleteSnapshot(_ context.Context, _ sandbox.Ref delete(p.snapshots, s.OperationID) return nil } -func (p *fakeCheckpointProvider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { state, err := p.GetCompute(ctx, r, c) if err != nil { return state, err @@ -165,7 +176,7 @@ func (p *fakeCheckpointProvider) ResumeCompute(ctx context.Context, r sandbox.Re p.computes[c.Name] = state return state, nil } -func (p *fakeCheckpointProvider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { +func (p *fakeSuspensionProvider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { if _, err := p.GetCompute(ctx, r, c); err != nil { return sandbox.CommandResult{}, err } @@ -178,7 +189,7 @@ func (p *fakeCheckpointProvider) RunCommandCompute(ctx context.Context, r sandbo p.mu.Unlock() return sandbox.CommandResult{}, p.connect(ctx, b) } -func (p *fakeCheckpointProvider) connect(ctx context.Context, b sandbox.Bootstrap) error { +func (p *fakeSuspensionProvider) connect(ctx context.Context, b sandbox.Bootstrap) error { header := http.Header{"Authorization": []string{"Bearer " + b.Credential}} conn, _, err := websocket.DefaultDialer.DialContext(ctx, p.endpoint+"?device_id="+b.DeviceID+"&version="+proto.Version, header) if err != nil { @@ -260,7 +271,7 @@ type computeLifecycleFixture struct { t *testing.T store *store.Store pool *pgxpool.Pool - provider *fakeCheckpointProvider + provider *fakeSuspensionProvider worker *execution.Worker stop func() key string @@ -271,7 +282,7 @@ func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *compu t.Helper() s, pool := store.NewManagedTestStore(t) registry := runtimegateway.NewRegistry() - p := &fakeCheckpointProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.SnapshotIdentity{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} + p := &fakeSuspensionProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.RetainedState{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} handler := runtimegateway.NewHandler(runtimegateway.HandlerConfig{Authenticator: runtimegateway.NewAuthenticator(s), Registry: registry}) server := httptest.NewServer(http.HandlerFunc(handler.WS)) p.endpoint = "ws" + strings.TrimPrefix(server.URL, "http") @@ -556,3 +567,47 @@ func TestRuntimeComputeLifecycleCapacityBoundsActiveAndRetained(t *testing.T) { t.Fatal("retained limit created a third allocation") } } + +func (p *fakeSuspensionProvider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + p.renewals.Add(1) + return p.GetCompute(ctx, r, c) +} + +func TestRuntimeComputeProtocolUpgradeRefusesOldReceiptsBeforeCleanup(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, _, env, owner := f.create() + f.complete(owner) + retained := f.phase(tenant, env.ID, "suspended") + f.stop() + f.sql(`UPDATE runtime_allocations SET compute_state=(compute_state-'protocol_version'-'retained') || jsonb_build_object('snapshot',compute_state->'retained') WHERE id=$1`, owner.ID) + deletes := f.provider.snapshotDeletes + w, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: f.store, Registry: f.provider.registry, ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: f.key, BackendFingerprint: strings.Repeat("a", 64), Provider: f.provider, Suspension: &f.policy}}) + if err == nil || w != nil || !strings.Contains(err.Error(), "previous release") { + t.Fatalf("incompatible state activated: %v", err) + } + if f.provider.snapshotDeletes != deletes || len(f.provider.snapshots) != 1 { + t.Fatal("upgrade lost owned artifact") + } + var state []byte + if err := f.pool.QueryRow(t.Context(), `SELECT compute_state FROM runtime_allocations WHERE id=$1`, owner.ID).Scan(&state); err != nil { + t.Fatal(err) + } + if !strings.Contains(string(state), `"snapshot"`) { + t.Fatal("old receipt was rewritten") + } + f.sql(`UPDATE runtime_allocations SET compute_state=$2::jsonb WHERE id=$1`, owner.ID, retained.ComputeState) + f.start() +} + +func TestRuntimeComputeRepeatedWakeStillRenewsCurrentIncarnation(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, _, env, owner := f.create() + before := f.provider.renewals.Load() + for range 3 { + f.sql(`UPDATE runtime_allocations SET compute_wake_requested=true,compute_activity_at=clock_timestamp() WHERE id=$1`, owner.ID) + f.phase(tenant, env.ID, "running") + } + if f.provider.renewals.Load() < before+3 { + t.Fatal("wake requests bypassed native lease renewal") + } +} diff --git a/services/core/internal/store/runtime_node_lifecycle_fixture_test.go b/services/core/internal/store/runtime_node_lifecycle_fixture_test.go index 94352bc34..b50581d0c 100644 --- a/services/core/internal/store/runtime_node_lifecycle_fixture_test.go +++ b/services/core/internal/store/runtime_node_lifecycle_fixture_test.go @@ -26,7 +26,7 @@ import ( ) type nodeIsolationProvider struct { - *fakeCheckpointProvider + *fakeSuspensionProvider blockMu sync.Mutex blocked map[string]bool mode string @@ -50,7 +50,7 @@ func (p *nodeIsolationProvider) block(ctx context.Context, r sandbox.Reference, return ctx.Err() } func (p *nodeIsolationProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { - info, err := p.fakeCheckpointProvider.Create(ctx, b) + info, err := p.fakeSuspensionProvider.Create(ctx, b) if err == nil { err = p.connect(ctx, b) } @@ -60,13 +60,13 @@ func (p *nodeIsolationProvider) GetInfo(ctx context.Context, r sandbox.Reference if err := p.block(ctx, r, "observe"); err != nil { return sandbox.Info{}, err } - return p.fakeCheckpointProvider.GetInfo(ctx, r) + return p.fakeSuspensionProvider.GetInfo(ctx, r) } func (p *nodeIsolationProvider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { if err := p.block(ctx, r, "observe"); err != nil { return sandbox.ComputeState{}, err } - return p.fakeCheckpointProvider.GetCompute(ctx, r, c) + return p.fakeSuspensionProvider.GetCompute(ctx, r, c) } func (p *nodeIsolationProvider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { return p.preparation.RunCommand(ctx, r, c) @@ -95,8 +95,8 @@ func newNodeIsolationFixture(t *testing.T, mode string) *nodeIsolationFixture { } s := store.NewWithCredentialCipher(pool, cipher) registry := runtimegateway.NewRegistry() - cp := &fakeCheckpointProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.SnapshotIdentity{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} - p := &nodeIsolationProvider{fakeCheckpointProvider: cp, blocked: map[string]bool{}, mode: mode, entered: make(chan struct{})} + cp := &fakeSuspensionProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.RetainedState{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} + p := &nodeIsolationProvider{fakeSuspensionProvider: cp, blocked: map[string]bool{}, mode: mode, entered: make(chan struct{})} preparationContext, cancelPreparation := context.WithCancel(t.Context()) t.Cleanup(cancelPreparation) cp.preparation = &initializationPeer{t: t, apply: func(request proto.RuntimePreparePayload, data []byte) proto.RuntimePrepareResultPayload { diff --git a/services/core/internal/store/runtime_suspension.go b/services/core/internal/store/runtime_suspension.go index 4661292ef..f7eac0e0b 100644 --- a/services/core/internal/store/runtime_suspension.go +++ b/services/core/internal/store/runtime_suspension.go @@ -180,3 +180,19 @@ func checkRuntimeComputeAdmission(ctx context.Context, q *sqlc.Queries, session } return err } + +// CheckRuntimeComputeProtocol blocks activation before incompatible retained state +// could lose cleanup evidence. The previous executable must drain its resources. +func (s *Store) CheckRuntimeComputeProtocol(ctx context.Context, version string) error { + if err := s.CheckExecutionOwnership(ctx); err != nil { + return err + } + incompatible, err := s.queries.HasIncompatibleRuntimeComputeState(ctx, version) + if err != nil { + return err + } + if incompatible { + return errors.New("incompatible retained runtime state: use the previous release to archive allocations before upgrading; history is preserved") + } + return nil +} diff --git a/services/core/internal/store/runtime_wake_hint_integration_test.go b/services/core/internal/store/runtime_wake_hint_integration_test.go index e2c4c069d..c25480aef 100644 --- a/services/core/internal/store/runtime_wake_hint_integration_test.go +++ b/services/core/internal/store/runtime_wake_hint_integration_test.go @@ -15,7 +15,7 @@ import ( ) type wakeHintScanProvider struct { - *fakeCheckpointProvider + *fakeSuspensionProvider sentinel string release chan struct{} scans chan int @@ -34,7 +34,7 @@ func (p *wakeHintScanProvider) GetCompute(ctx context.Context, reference sandbox } } } - return p.fakeCheckpointProvider.GetCompute(ctx, reference, compute) + return p.fakeSuspensionProvider.GetCompute(ctx, reference, compute) } type wakeHintIntegrationTarget struct { @@ -69,7 +69,7 @@ func newWakeHintIntegration(t *testing.T) *wakeHintIntegration { target.owner = f.phase(target.tenant, target.environment.ID, "suspended") f.stop() provider := &wakeHintScanProvider{ - fakeCheckpointProvider: f.provider, sentinel: sentinel.owner.ID, + fakeSuspensionProvider: f.provider, sentinel: sentinel.owner.ID, release: make(chan struct{}), scans: make(chan int, 16), } worker, err := execution.StartWorker(t.Context(), &execution.Dispatcher{ diff --git a/services/core/internal/store/sandbox_deployment_setup.go b/services/core/internal/store/sandbox_deployment_setup.go index fc636b06b..57bb46f5e 100644 --- a/services/core/internal/store/sandbox_deployment_setup.go +++ b/services/core/internal/store/sandbox_deployment_setup.go @@ -187,7 +187,7 @@ func runtimeDeploymentView(d sqlc.RuntimeDeployment, publicURL string) (RuntimeD result.CredentialConfigured = len(d.ProviderCredential) > 0 } - if providers.SupportsCheckpoint(d.ProviderKind) { + if providers.SupportsSuspension(d.ProviderKind) { result.Suspension = &SandboxSuspensionView{IdleSeconds: d.IdleSeconds, RetentionSeconds: d.RetentionSeconds} } return result, nil diff --git a/services/core/tools/microsandbox-provider/README.md b/services/core/tools/microsandbox-provider/README.md index 554b4fad8..faca699e9 100644 --- a/services/core/tools/microsandbox-provider/README.md +++ b/services/core/tools/microsandbox-provider/README.md @@ -1,6 +1,6 @@ # microsandbox Sandbox Provider helper -microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The helper links the microsandbox Go SDK v0.7.2 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the provider-neutral `sandbox.CheckpointProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. +microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The helper links the microsandbox Go SDK v0.7.2 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the provider-neutral `sandbox.SuspensionProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. [Add a Sandbox Provider](../../../../docs/sandbox-provider.md) owns the provider contract. [Sandbox deployment](../../../../contracts/agents-api/sandbox-deployment.md) owns the resources, Runtime release and suspension policy; the [nodes guide](../../../../docs/getting-started/nodes.md) owns node installation, host requirements, the node's directories and its network policy. @@ -29,14 +29,14 @@ A helper response carries `CreateSettled` with a configuration rejection only af Core persists operation IDs, source and target generations, exact identities and snapshot evidence before it depends on them. `Initial` and `NewCompute` only construct references and allocate nothing. - **Suspend** pauses the exact VM, captures a full snapshot under the persisted operation's derived group and member, verifies the complete checkpoint closure, then force-stops the source. Pausing alone does not release memory. A completed matching artifact is inspected instead of captured again. A full snapshot records a resource proof only after the source's limits match. -- **KillCompute** checks the precise incarnation before it stops the VM and removes its writable disks. Core calls it after it has persisted the verified snapshot, even when Suspend already stopped the source, so no chain of old writable disks grows across suspension cycles. +- **KillCompute** checks the precise incarnation before it stops the VM and removes its writable disks. Suspend completes native source cleanup under the allocation lock before reporting resource release, including recovery of a captured artifact. Core uses KillCompute for allocation cleanup. - **Restore** verifies the exact artifact and creates the precommitted target name. An existing target is adopted only when its immutable ID, if known, and its persisted `snapshot_parent` agree. Upstream restore defaults to public networking, so restore passes the same explicit host policy as creation, and no undeclared host resource or mount is inherited. - Native restore leaves the managed root size unset because the target inherits the verified full snapshot. The helper accepts that only with a matching snapshot resource proof and the exact source and target identities, and it checks the target's CPU, memory and Environment disk before keeping the inherited proof. A missing root size never counts as unlimited capacity, and retained state is never resized. - Fresh restore and retry share one completion: verify the original artifact and resource proof, inspect the running target's resources and ancestry, persist its missing derived resource-proof label, then strictly reread the same native ID ([`restore_completion.go`](restore_completion.go)). A conflicting proof is an error. Native restore does not copy the source's ownership labels; ancestry supplies that evidence. There is no ordinary Start, replacement, disk-only restore or cold boot. - **ResumeCompute** thaws the same resident source after an aborted suspension. The pinned SDK handle method is name-based; the allocation lock and ID checks before and after the call fence every managed replacement. Manual lifecycle changes in the managed namespace are unsupported. -- **DeleteSnapshot** accepts only the derived operation selector and the matching full artifact identity, never an arbitrary path. Core owns retention, consumed snapshot generations and cleanup order. A checkpoint never rolls back work admitted after its first restore. +- **DeleteRetained** accepts only the derived operation selector and the matching full artifact identity, never an arbitrary path. Core owns retention, consumed snapshot generations and cleanup order. A checkpoint never rolls back work admitted after its first restore. -After a lost response Core uses `ObserveOnly`. It never starts a capture or restore, and observing a suspend operation never kills its source. Observation checks artifact integrity and source ownership independently of resource checks, so resource drift cannot hide a retained artifact from cleanup; Core can persist recovered snapshot evidence before KillCompute. If the artifact is absent but the exact source is still running or paused with a settled bootstrap, observation returns the source without a snapshot and Core can abort the suspension; thawing and further execution still require the resource checks. For an interrupted restore, `ObserveOnly` may finish the missing resource proof on the exact target but never restarts a stopped target, changes resources or restores again. Missing state never authorizes a replay. +After a lost response Core uses `ReconcileOnly`. The adapter observes the original capture or restore and never starts it again. When a complete matching snapshot exists, reconciliation finishes exact source cleanup before reporting settled suspension and released resources. Artifact integrity and source ownership are checked independently of resource qualification, so resource drift cannot prevent owned cleanup. If capture is settled without an artifact and the exact source is still running or paused with a settled bootstrap, the adapter reports that outcome so Core can abort suspension; thawing and further execution still require resource checks. For an interrupted restore, reconciliation may finish the missing resource proof on the exact target but never restarts a stopped target, changes resources or restores again. Missing state never authorizes a replay. GetCompute, commands, cleanup and the next suspension verify restored provenance from the persisted VM configuration after the consumed artifact is deleted. @@ -62,3 +62,5 @@ The helper returns only the native observation time, exact uptime, cumulative vC ## Tests `make check-microsandbox-provider`, part of `make check`, runs the pure-Go adapter tests everywhere and this module's tests on Linux; other hosts print an explicit skip for the Linux-only module. + +The native helper wire remains version 2. The Go adapter wraps native snapshot identity in the shared opaque retained-state handle and completes exact-source cleanup through the existing ownership-checked helper operations. Reconciliation never repeats snapshot capture. SuspendSettled is reported only after these serialized operations complete; captured-artifact cleanup does not require execution resource qualification. From 5c0b3047914c1c92865aa056b9ceaa06ddb73726 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 21:03:05 +0800 Subject: [PATCH 02/12] test(store): isolate synthetic suspension receipts from worker startup --- services/core/internal/store/runtime_suspension_test.go | 4 ++-- .../core/internal/store/runtime_worker_recovery_test.go | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/services/core/internal/store/runtime_suspension_test.go b/services/core/internal/store/runtime_suspension_test.go index 53fc609c1..97bd45471 100644 --- a/services/core/internal/store/runtime_suspension_test.go +++ b/services/core/internal/store/runtime_suspension_test.go @@ -15,7 +15,7 @@ import ( func runtimeSuspensionFixture(t *testing.T) (*Store, *Store, *pgxpool.Pool, RuntimeAllocation) { t.Helper() - s, pool := testStore(t) + s, pool := newManagedTestStore(t) w := executionLease(t, s).Store() tenant := uuid.NewString() _, environment := localEnvironment(t, s, tenant) @@ -260,7 +260,7 @@ func TestRuntimeSuspensionRetentionAndDeletedSession(t *testing.T) { } func TestRuntimeSuspensionCountsUncertainCapacityUntilReleased(t *testing.T) { - s, pool := testStore(t) + s, pool := newManagedTestStore(t) w := executionLease(t, s).Store() provider := uuid.NewString() cases := []struct { diff --git a/services/core/internal/store/runtime_worker_recovery_test.go b/services/core/internal/store/runtime_worker_recovery_test.go index 41676132a..f8edb69d5 100644 --- a/services/core/internal/store/runtime_worker_recovery_test.go +++ b/services/core/internal/store/runtime_worker_recovery_test.go @@ -21,6 +21,15 @@ func insertWorkerRuntimeAllocation(t *testing.T, pool *pgxpool.Pool, h *dispatch if err != nil { t.Fatal(err) } + // This synthetic allocation must not outlive the test in the shared fixture + // database, where the next worker validates every retained protocol receipt. + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := pool.Exec(ctx, "DELETE FROM runtime_allocations WHERE environment_id=$1", h.device.EnvironmentID); err != nil { + t.Error(err) + } + }) } func runtimeWorkerHarness(t *testing.T) (*dispatchHarness, *pgxpool.Pool) { From 4e0331a6b9f596262167fc000f36eefe7c916da1 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 21:03:31 +0800 Subject: [PATCH 03/12] test(store): use current protocol in worker recovery fixture --- .../core/internal/store/runtime_worker_recovery_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/services/core/internal/store/runtime_worker_recovery_test.go b/services/core/internal/store/runtime_worker_recovery_test.go index f8edb69d5..c888ae745 100644 --- a/services/core/internal/store/runtime_worker_recovery_test.go +++ b/services/core/internal/store/runtime_worker_recovery_test.go @@ -9,6 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" @@ -16,8 +17,12 @@ import ( func insertWorkerRuntimeAllocation(t *testing.T, pool *pgxpool.Pool, h *dispatchHarness, phase string) { t.Helper() - _, err := pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,compute_phase,compute_retained_until,deployment_generation) - VALUES($1,$2,$3,$4,'running',true,$5,clock_timestamp()+interval '1 hour',(SELECT generation FROM runtime_deployment))`, uuid.NewString(), h.device.EnvironmentID, h.device.ID, uuid.NewString(), phase) + state, err := json.Marshal(map[string]any{"protocol_version": sandbox.SuspensionStateVersion, "current": sandbox.Compute{Name: h.device.EnvironmentID, ID: h.device.EnvironmentID}}) + if err != nil { + t.Fatal(err) + } + _, err = pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,compute_phase,compute_retained_until,deployment_generation,compute_state) + VALUES($1,$2,$3,$4,'running',true,$5,clock_timestamp()+interval '1 hour',(SELECT generation FROM runtime_deployment),$6)`, uuid.NewString(), h.device.EnvironmentID, h.device.ID, uuid.NewString(), phase, state) if err != nil { t.Fatal(err) } From 9d4f833193c3e12f67b248b945c3f645bbb4ac39 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 21:31:28 +0800 Subject: [PATCH 04/12] fix(runtime): validate allocation protocol and use shared idle clock --- docs/sandbox-provider.md | 2 +- .../db/queries/runtime_allocations.sql | 4 ++-- .../db/queries/runtime_suspension.sql | 6 ++---- .../db/sqlc/runtime_allocations.sql.go | 6 ++++-- .../db/sqlc/runtime_suspension.sql.go | 6 ++---- .../internal/store/runtime_allocations.go | 2 ++ .../store/runtime_compute_lifecycle_test.go | 15 +++++++++++++ .../internal/store/runtime_idle_clock_test.go | 21 +++++++++++++++++++ .../internal/store/runtime_suspension_test.go | 6 ++++++ 9 files changed, 55 insertions(+), 13 deletions(-) diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index 0bae2290b..a2ce35fdc 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -190,7 +190,7 @@ Suspend owns native resource release and returns a bound retained handle, suspen Resume consumes the retained state into the precommitted target exactly once. Recovery observes the same attempt. Core persists waking, authenticates and resumes the daemon, deletes the consumed retained resource, then commits running and admits work. DeleteRetained is idempotent artifact cleanup and preserves running compute. Failed cleanup keeps the waking phase and cannot trigger another restore. KillCompute remains genuinely destructive; cleanup of an old generation must not kill a newer live incarnation sharing its native ID. -The existing Session lock, lifecycle lease, idle rule, capacity queries and cleanup order remain authoritative. Every unreleased allocation consumes max_retained, including running allocations. This pre-release retained-state shape requires ordinary cleanup of old live compute state before activation; deployments must refuse activation with unreleased checkpoint or resident receipts. Session history is preserved. +The existing Session lock, lifecycle lease, idle rule, capacity queries and cleanup order remain authoritative. Every unreleased allocation consumes max_retained, including running allocations. Every allocation is stamped with the shared compute protocol version at reservation, including allocations whose suspension phase is disabled. Activation refuses any unreleased allocation with a missing or different version; the previous release must complete ordinary cleanup before upgrading. Session history is preserved. ### Reset and archive diff --git a/services/core/internal/db/queries/runtime_allocations.sql b/services/core/internal/db/queries/runtime_allocations.sql index 0296b6249..5772d3233 100644 --- a/services/core/internal/db/queries/runtime_allocations.sql +++ b/services/core/internal/db/queries/runtime_allocations.sql @@ -1,6 +1,6 @@ -- name: CreateRuntimeAllocation :one -INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation) -VALUES ($1, $2, $3, $4, $5, $6) RETURNING *; +INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation, compute_state) +VALUES ($1, $2, $3, $4, $5, $6, jsonb_build_object('protocol_version', sqlc.arg(protocol_version)::text)) RETURNING *; -- name: GetRuntimeAllocation :one SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired diff --git a/services/core/internal/db/queries/runtime_suspension.sql b/services/core/internal/db/queries/runtime_suspension.sql index d1faed453..d38e03d87 100644 --- a/services/core/internal/db/queries/runtime_suspension.sql +++ b/services/core/internal/db/queries/runtime_suspension.sql @@ -27,8 +27,6 @@ WHERE id = $1 AND compute_phase = 'running' AND compute_activity_at <= $2; -- name: GetRuntimeActivity :one SELECT clock_timestamp()::timestamptz AS observed_at, GREATEST(a.compute_activity_at, - CASE WHEN a.node_id IS NULL THEN COALESCE((SELECT max(t.completed_at) FROM turns t WHERE t.session_id = e.session_id), a.created_at) END, - CASE WHEN a.node_id IS NULL THEN (SELECT max(t.completed_at) FROM subagent_turns t WHERE t.session_id = e.session_id) END, (SELECT max(f.settled_at) FROM environment_file_writes f WHERE f.environment_id = e.id))::timestamptz AS last_activity, (EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) @@ -57,7 +55,7 @@ SELECT EXISTS ( UPDATE runtime_allocations a SET compute_activity_at = clock_timestamp() FROM environments e WHERE a.environment_id = e.id AND e.session_id = $1 - AND a.node_id IS NOT NULL AND a.state = 'running'; + AND a.state = 'running'; -- name: SessionHasRuntimeNode :one SELECT EXISTS ( @@ -68,6 +66,6 @@ SELECT EXISTS ( -- name: HasIncompatibleRuntimeComputeState :one SELECT EXISTS ( SELECT 1 FROM runtime_allocations - WHERE state <> 'released' AND compute_phase <> 'disabled' + WHERE state <> 'released' AND (compute_state->>'protocol_version') IS DISTINCT FROM sqlc.arg(protocol_version)::text )::boolean; diff --git a/services/core/internal/db/sqlc/runtime_allocations.sql.go b/services/core/internal/db/sqlc/runtime_allocations.sql.go index 7e47aef04..c9cf1e813 100644 --- a/services/core/internal/db/sqlc/runtime_allocations.sql.go +++ b/services/core/internal/db/sqlc/runtime_allocations.sql.go @@ -12,8 +12,8 @@ import ( ) const createRuntimeAllocation = `-- name: CreateRuntimeAllocation :one -INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation) -VALUES ($1, $2, $3, $4, $5, $6) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation +INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation, compute_state) +VALUES ($1, $2, $3, $4, $5, $6, jsonb_build_object('protocol_version', $7::text)) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` type CreateRuntimeAllocationParams struct { @@ -23,6 +23,7 @@ type CreateRuntimeAllocationParams struct { ProviderKey pgtype.UUID `json:"provider_key"` NodeID pgtype.UUID `json:"node_id"` DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` + ProtocolVersion string `json:"protocol_version"` } func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntimeAllocationParams) (RuntimeAllocation, error) { @@ -33,6 +34,7 @@ func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntime arg.ProviderKey, arg.NodeID, arg.DeploymentGeneration, + arg.ProtocolVersion, ) var i RuntimeAllocation err := row.Scan( diff --git a/services/core/internal/db/sqlc/runtime_suspension.sql.go b/services/core/internal/db/sqlc/runtime_suspension.sql.go index 1c7170cb1..f23211019 100644 --- a/services/core/internal/db/sqlc/runtime_suspension.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension.sql.go @@ -54,8 +54,6 @@ func (q *Queries) CountRuntimeRetainedAllocations(ctx context.Context, providerK const getRuntimeActivity = `-- name: GetRuntimeActivity :one SELECT clock_timestamp()::timestamptz AS observed_at, GREATEST(a.compute_activity_at, - CASE WHEN a.node_id IS NULL THEN COALESCE((SELECT max(t.completed_at) FROM turns t WHERE t.session_id = e.session_id), a.created_at) END, - CASE WHEN a.node_id IS NULL THEN (SELECT max(t.completed_at) FROM subagent_turns t WHERE t.session_id = e.session_id) END, (SELECT max(f.settled_at) FROM environment_file_writes f WHERE f.environment_id = e.id))::timestamptz AS last_activity, (EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) @@ -91,7 +89,7 @@ func (q *Queries) GetRuntimeActivity(ctx context.Context, id pgtype.UUID) (GetRu const hasIncompatibleRuntimeComputeState = `-- name: HasIncompatibleRuntimeComputeState :one SELECT EXISTS ( SELECT 1 FROM runtime_allocations - WHERE state <> 'released' AND compute_phase <> 'disabled' + WHERE state <> 'released' AND (compute_state->>'protocol_version') IS DISTINCT FROM $1::text )::boolean ` @@ -107,7 +105,7 @@ const recordRuntimeTerminalActivity = `-- name: RecordRuntimeTerminalActivity :e UPDATE runtime_allocations a SET compute_activity_at = clock_timestamp() FROM environments e WHERE a.environment_id = e.id AND e.session_id = $1 - AND a.node_id IS NOT NULL AND a.state = 'running' + AND a.state = 'running' ` func (q *Queries) RecordRuntimeTerminalActivity(ctx context.Context, sessionID pgtype.UUID) error { diff --git a/services/core/internal/store/runtime_allocations.go b/services/core/internal/store/runtime_allocations.go index 18bccfcdf..5e8dc74a4 100644 --- a/services/core/internal/store/runtime_allocations.go +++ b/services/core/internal/store/runtime_allocations.go @@ -11,6 +11,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) // RuntimeAllocation retains compute ownership, not public readiness. It survives @@ -118,6 +119,7 @@ func (s *Store) ReserveRuntimeAllocation(ctx context.Context, tenant, environmen row, err := q.CreateRuntimeAllocation(ctx, sqlc.CreateRuntimeAllocationParams{ ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, EnvironmentID: lookup.ID, DeviceID: device.ID, ProviderKey: provider, NodeID: nodeID, DeploymentGeneration: generation, + ProtocolVersion: sandbox.SuspensionStateVersion, }) if err == nil { result = runtimeAllocationFromRow(row, session, lookup.TenantID, pgtype.Timestamptz{}, false) diff --git a/services/core/internal/store/runtime_compute_lifecycle_test.go b/services/core/internal/store/runtime_compute_lifecycle_test.go index 9535794b1..76f939cd6 100644 --- a/services/core/internal/store/runtime_compute_lifecycle_test.go +++ b/services/core/internal/store/runtime_compute_lifecycle_test.go @@ -611,3 +611,18 @@ func TestRuntimeComputeRepeatedWakeStillRenewsCurrentIncarnation(t *testing.T) { t.Fatal("wake requests bypassed native lease renewal") } } + +func TestRuntimeComputeProtocolRejectsOldDisabledAllocation(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + _, _, _, owner := f.create() + f.stop() + f.sql(`UPDATE runtime_allocations SET compute_phase='disabled',compute_state='{}'::jsonb WHERE id=$1`, owner.ID) + before := f.provider.renewals.Load() + w, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: f.store, Registry: f.provider.registry, ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: f.key, BackendFingerprint: strings.Repeat("a", 64), Provider: f.provider, Suspension: &f.policy}}) + if err == nil || w != nil || !strings.Contains(err.Error(), "previous release") { + t.Fatal("old disabled allocation activated", err) + } + if f.provider.renewals.Load() != before || len(f.provider.computes) != 1 { + t.Fatal("upgrade changed owned native compute") + } +} diff --git a/services/core/internal/store/runtime_idle_clock_test.go b/services/core/internal/store/runtime_idle_clock_test.go index 71890388d..67b8033a5 100644 --- a/services/core/internal/store/runtime_idle_clock_test.go +++ b/services/core/internal/store/runtime_idle_clock_test.go @@ -221,3 +221,24 @@ func TestManagedIdleClockReconnectPreservesReceipts(t *testing.T) { t.Fatal(err) } } + +func TestDirectManagedIdleClockIgnoresNativeClockSkew(t *testing.T) { + for _, skew := range []time.Duration{-269 * time.Second, 269 * time.Second} { + t.Run(skew.String(), func(t *testing.T) { + s, w, owner := managedIdleClockFixture(t) + runtimeSuspensionSQL(t, s.pool, "UPDATE runtime_allocations SET node_id=NULL WHERE id=$1", owner.ID) + owner.NodeID = "" + turn := uuid.NewString() + runtimeSuspensionSQL(t, s.pool, "INSERT INTO turns(id,session_id,status,started_at) VALUES($1,$2,'in_progress',clock_timestamp())", turn, owner.SessionID) + source := runtimeDatabaseTime(t, s).Add(skew).UnixMilli() + outcome := json.RawMessage(fmt.Sprintf(`{"done":{"source_completed_at_ms":%d}}`, source)) + before := runtimeDatabaseTime(t, s) + completed, err := w.CompleteExecution(t.Context(), owner.TenantID, owner.SessionID, turn, TurnCompleted, outcome, "", 0) + after := runtimeDatabaseTime(t, s) + if err != nil || completed.CompletedAt.UnixMilli() != source { + t.Fatal(completed, err) + } + verifyManagedIdleClock(t, s, w, owner, before, after) + }) + } +} diff --git a/services/core/internal/store/runtime_suspension_test.go b/services/core/internal/store/runtime_suspension_test.go index 97bd45471..aba844f89 100644 --- a/services/core/internal/store/runtime_suspension_test.go +++ b/services/core/internal/store/runtime_suspension_test.go @@ -405,6 +405,12 @@ func TestRuntimeSuspensionRechecksCompletionAgainstIdleTimeout(t *testing.T) { default: runtimeSuspensionSQL(t, pool, `INSERT INTO environment_file_writes(id,environment_id,device_id,request_sha256,state,created_at,settled_at) VALUES($1,$2,$3,$4,$5,clock_timestamp()-interval '10 minutes',clock_timestamp())`, uuid.NewString(), owner.EnvironmentID, owner.DeviceID, strings.Repeat("a", 64), strings.TrimPrefix(kind, "file_")) } + if kind == "root" || kind == "subagent" { + id, _ := parseConnectionGeneration(owner.SessionID) + if err := s.queries.RecordRuntimeTerminalActivity(t.Context(), id); err != nil { + t.Fatal(err) + } + } until := time.Now().Add(time.Hour) if _, err := w.SetRuntimeCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, idleTimeout); !errors.Is(err, ErrTurnConflict) { t.Fatal("completion after idle observation did not fence quiesce", err) From 3dbcd0bd32f868995adcb3d016ac8361471bf4f0 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 21:32:04 +0800 Subject: [PATCH 05/12] test(installer): isolate synthetic supplementary groups --- deploy/install/test_node_install.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/install/test_node_install.py b/deploy/install/test_node_install.py index afbea2a90..9d27055fd 100644 --- a/deploy/install/test_node_install.py +++ b/deploy/install/test_node_install.py @@ -645,6 +645,7 @@ def run_as(account, function, *arguments): service_account=lambda: self.account), mock.patch.object(installer.shutil, "which", side_effect=lambda tool: None if tool == "docker" and not self.docker_installed else "/usr/bin/" + tool), mock.patch.object(installer.grp, "getgrnam", side_effect=lambda name: SimpleNamespace(gr_mem=["oac-node"] if self.joined else [])), + mock.patch.object(installer.os, "getgrouplist", side_effect=lambda name, gid: [gid]), mock.patch.object(installer.grp, "getgrgid", side_effect=lambda gid: SimpleNamespace(gr_name=self.device_group))): patch.start() self.addCleanup(patch.stop) From ed9fb3b50056fb041ab1a6187379c6ad0cde553b Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 21:38:50 +0800 Subject: [PATCH 06/12] test(store): isolate suspension execution configuration fixture --- .../core/internal/store/session_execution_configuration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/core/internal/store/session_execution_configuration_test.go b/services/core/internal/store/session_execution_configuration_test.go index 7310abe46..94c3e5e7a 100644 --- a/services/core/internal/store/session_execution_configuration_test.go +++ b/services/core/internal/store/session_execution_configuration_test.go @@ -243,7 +243,7 @@ func TestSessionExecutionConfigurationConcurrentRetryKeepsWinner(t *testing.T) { } func TestSessionExecutionConfigurationSurvivesSuspendResume(t *testing.T) { - s, pool := testStore(t) + s, pool := newManagedTestStore(t) w := executionLease(t, s).Store() tenant := uuid.NewString() session, err := s.CreateSession(t.Context(), tenant, executionProjectionInput("agent")) From 6656a47f91f2d5fde5bf73947b2c7e3c77114564 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 21:40:52 +0800 Subject: [PATCH 07/12] feat(e2b): implement idle suspension through shared provider lifecycle --- docs/runtime-bootstrap.md | 4 + internal/runtimebootstrap/bootstrap.go | 3 + .../cmd/server/managed_generations_test.go | 4 +- .../server/managed_setup_preflight_test.go | 11 +- services/core/deploy/e2b/build-template.py | 2 + .../core/deploy/e2b/build_template_test.py | 3 + .../deploy/e2b/helper_contract_generated.py | 13 +- services/core/deploy/e2b/init.py | 2 + services/core/deploy/e2b/init_test.py | 4 +- services/core/deploy/e2b/managed_init.py | 7 + services/core/deploy/e2b/managed_init_test.py | 8 +- .../execution/runtime_compute_wake.go | 3 +- .../internal/sandbox/e2b/helper_contract.go | 31 ++- .../sandbox/e2b/installed_paths_test.go | 3 +- .../sandbox/e2b/internal/contractgen/main.go | 24 +- .../core/internal/sandbox/e2b/operations.go | 52 +--- .../core/internal/sandbox/e2b/process_test.go | 7 +- .../core/internal/sandbox/e2b/provider.go | 12 +- .../core/internal/sandbox/e2b/suspension.go | 111 +++++++++ .../internal/sandbox/e2b/suspension_test.go | 48 ++++ .../internal/sandbox/providers/registry.go | 1 + .../sandbox/providers/registry_test.go | 2 +- services/core/tools/e2b-provider/README.md | 12 +- .../e2b-provider/helper_contract_generated.py | 13 +- services/core/tools/e2b-provider/provider.py | 8 +- .../core/tools/e2b-provider/provider_test.py | 3 +- .../core/tools/e2b-provider/state_test.py | 3 +- .../core/tools/e2b-provider/suspension.py | 225 ++++++++++++++++++ .../tools/e2b-provider/suspension_test.py | 157 ++++++++++++ .../tools/e2b-provider/testdata/contract.json | 94 ++++---- .../tools/microsandbox-provider/bootstrap.go | 5 +- 31 files changed, 755 insertions(+), 120 deletions(-) create mode 100644 services/core/internal/sandbox/e2b/suspension.go create mode 100644 services/core/internal/sandbox/e2b/suspension_test.go create mode 100644 services/core/tools/e2b-provider/suspension.py create mode 100644 services/core/tools/e2b-provider/suspension_test.py diff --git a/docs/runtime-bootstrap.md b/docs/runtime-bootstrap.md index 9a1ca5e96..1f72bcdc1 100644 --- a/docs/runtime-bootstrap.md +++ b/docs/runtime-bootstrap.md @@ -29,6 +29,10 @@ The Runtime validates the input and owns authentication and connection. A succes Self-hosted executors and operator-provisioned devices get their daemon identity in other ways; the [machine connection API](../contracts/agents-api/machine-api.md#credentials) lists every credential source. All of them enter the same Runtime execution loop. +## Hosted suspension control + +The private hosted park/wake control-file path is authored as `SuspendControlFile` in `internal/runtimebootstrap/bootstrap.go`. Core recovery and native Go adapters read that value; the E2B helper contract generator projects it into the template builder. Managed startup prepares its private directory and supplies `OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE` to enable Runtime suspension. This is a packaged protocol setting. The shared Sandbox Provider registration owns idle and retention defaults; the adapter owns its native lease timeout. + ## Verification `go test ./internal/runtimebootstrap ./apps/daemon/internal/cli` covers the input contract, the exclusivity of credential sources and restart behavior. Provider tests verify delivery and file permissions without relying on the Runtime's private storage. diff --git a/internal/runtimebootstrap/bootstrap.go b/internal/runtimebootstrap/bootstrap.go index bab5e7b16..9567d75e7 100644 --- a/internal/runtimebootstrap/bootstrap.go +++ b/internal/runtimebootstrap/bootstrap.go @@ -14,6 +14,9 @@ import ( "github.com/google/uuid" ) +// SuspendControlFile is the packaged private hosted Runtime park/wake location. +const SuspendControlFile = "/run/oac/daemon-suspend.json" + const Version = 1 const MaxBytes = 16 * 1024 diff --git a/services/core/cmd/server/managed_generations_test.go b/services/core/cmd/server/managed_generations_test.go index aca3167eb..927827cbf 100644 --- a/services/core/cmd/server/managed_generations_test.go +++ b/services/core/cmd/server/managed_generations_test.go @@ -44,7 +44,7 @@ q=json.load(sys.stdin) with (pathlib.Path(q['Config']['StateDir'])/'requests').open('a') as f: f.write(json.dumps(q)+'\n') info=dict(q['Reference'],State='running',ProviderID='owned',CreateSettled=True) if q['Operation']=='kill': info['State']='absent' -print(json.dumps({'Version':1,'Info':info})) +print(json.dumps({'Version':q['Version'],'Info':info})) ` if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) @@ -136,7 +136,7 @@ code = '' if k == 'revoked': code = 'unauthorized' elif k == 'unconfirmed': code = 'unconfirmed' elif not (k.startswith('team-a') and template in ('owned-a', 'new-a') or k == 'team-b' and template == 'public-b'): code = 'team_mismatch' -result = {'Version': 1, 'ErrorCode': code} +result = {'Version': q['Version'], 'ErrorCode': code} if not code: result['DeploymentValid'] = True if q['Operation'] == 'validate_deployment': diff --git a/services/core/cmd/server/managed_setup_preflight_test.go b/services/core/cmd/server/managed_setup_preflight_test.go index ceee02d7f..30ac02d38 100644 --- a/services/core/cmd/server/managed_setup_preflight_test.go +++ b/services/core/cmd/server/managed_setup_preflight_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -18,7 +19,7 @@ import ( func TestE2BRejectedSpecificationHasSafeActionableDiagnostic(t *testing.T) { helper := filepath.Join(t.TempDir(), "provider") - if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":1,\"ErrorCode\":\"invalid\"}'\n"), 0700); err != nil { + if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":"+strconv.Itoa(e2b.ProtocolVersion)+",\"ErrorCode\":\"invalid\"}'\n"), 0700); err != nil { t.Fatal(err) } state := filepath.Join(t.TempDir(), "e2b") @@ -48,12 +49,12 @@ op = q['Operation'] with (pathlib.Path(q['Config']['StateDir']) / 'operations').open('a') as log: log.write(op + '\n') if op == 'validate_deployment': - print(json.dumps({'Version': 1, 'ErrorCode': 'invalid'})) + print(json.dumps({'Version': q['Version'], 'ErrorCode': 'invalid'})) else: info = dict(q['Reference'], ProviderID='owned-compute', State='stopped', CreateSettled=True) if op == 'kill': info.update(ProviderID='', State='absent') - print(json.dumps({'Version': 1, 'Info': info})) + print(json.dumps({'Version': q['Version'], 'Info': info})) ` if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) @@ -98,7 +99,7 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { } helper := filepath.Join(t.TempDir(), "provider") requests := filepath.Join(state, "requests") - script := "#!/bin/sh\ncat >>" + requests + "\necho >>" + requests + "\nprintf '%s' '{\"Version\":1,\"ErrorCode\":\"\",\"DeploymentValid\":true,\"TemplateBuild\":{\"Status\":\"ready\",\"CPUs\":4,\"MemoryMiB\":4096,\"RootDiskMiB\":24063}}'\n" + script := "#!/bin/sh\ncat >>" + requests + "\necho >>" + requests + "\nprintf '%s' '{\"Version\":" + strconv.Itoa(e2b.ProtocolVersion) + ",\"ErrorCode\":\"\",\"DeploymentValid\":true,\"TemplateBuild\":{\"Status\":\"ready\",\"CPUs\":4,\"MemoryMiB\":4096,\"RootDiskMiB\":24063}}'\n" if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) } @@ -130,7 +131,7 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { func TestInitialE2BPublicTemplateOutsideTeamIsRejected(t *testing.T) { helper := filepath.Join(t.TempDir(), "provider") - if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":1,\"ErrorCode\":\"team_mismatch\"}'\n"), 0700); err != nil { + if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":"+strconv.Itoa(e2b.ProtocolVersion)+",\"ErrorCode\":\"team_mismatch\"}'\n"), 0700); err != nil { t.Fatal(err) } state := filepath.Join(t.TempDir(), "e2b") diff --git a/services/core/deploy/e2b/build-template.py b/services/core/deploy/e2b/build-template.py index d78926213..30616a94d 100644 --- a/services/core/deploy/e2b/build-template.py +++ b/services/core/deploy/e2b/build-template.py @@ -10,6 +10,7 @@ import tempfile from e2b import Template +from helper_contract_generated import SUSPEND_CONTROL_FILE BASE = 'node:22.23.1-bookworm-slim@sha256:8607a9064d4a571140998ae9e52a3b3fcf9cff361d04642d5971e6cd76d39e27' parser = argparse.ArgumentParser() @@ -27,6 +28,7 @@ if value.startswith(('HOME=', 'OAC_'))) if environment.get('OAC_RUNTIME_WORKSPACE') != '/environment/workspace': parser.error('Image does not use the colocated Runtime layout') +environment['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'] = SUSPEND_CONTROL_FILE dev_home = Path(os.environ.get('OAC_DEV_HOME') or Path.home() / '.oac') if not dev_home.is_absolute(): parser.error('OAC_DEV_HOME must be absolute') diff --git a/services/core/deploy/e2b/build_template_test.py b/services/core/deploy/e2b/build_template_test.py index 9a3b3034f..3e4f90508 100644 --- a/services/core/deploy/e2b/build_template_test.py +++ b/services/core/deploy/e2b/build_template_test.py @@ -64,6 +64,9 @@ def build(instance, **kwargs): self.assertIs(instance, template) context = Path(factory.call_args.kwargs['file_context_path']) self.assertEqual(stat.S_IMODE(context.stat().st_mode), 0o700) + from helper_contract_generated import SUSPEND_CONTROL_FILE + runtime_env = json.loads((context / 'runtime-env.json').read_text()) + self.assertEqual(runtime_env['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'], SUSPEND_CONTROL_FILE) with tarfile.open(context / 'runtime.tar.gz') as archive: modes = {m.name: stat.S_IMODE(m.mode) for m in archive.getmembers()} for parent in ('usr', 'usr/local', 'etc'): diff --git a/services/core/deploy/e2b/helper_contract_generated.py b/services/core/deploy/e2b/helper_contract_generated.py index ebd7fadf2..588bd0aca 100644 --- a/services/core/deploy/e2b/helper_contract_generated.py +++ b/services/core/deploy/e2b/helper_contract_generated.py @@ -1,5 +1,6 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" +COMPUTE_FIELDS = ["Generation","Name","ID","RestoredFrom"] ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized"] MANAGED_BOOTSTRAP_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","NetworkAccess","AllowedDomains","InstallationID","RuntimeBootstrap"] MANAGED_IDENTITY_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID"] @@ -10,9 +11,13 @@ MAX_REQUEST = 75497472 MAX_RESPONSE = 16777216 NETWORK_ACCESS = ["enabled","disabled","restricted"] -OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] -PROTOCOL_VERSION = 1 +OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential","compute_info","compute_renew","suspend","resume","compute_kill","delete_retained","compute_command","resume_compute"] +PROTOCOL_VERSION = 2 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] -REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] -RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observations"] +REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Compute","Suspend","Resume","Retained","Deadline"] +RESPONSE_FIELDS = ["Version","State","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observations"] +RESUME_FIELDS = ["Reference","OperationID","Retained","Target","ReconcileOnly"] +RETAINED_FIELDS = ["Reference","ID","Data","OperationID","SourceGeneration","SourceName","SourceID"] SDK_VERSION = "2.51.0" +SUSPEND_CONTROL_FILE = "/run/oac/daemon-suspend.json" +SUSPEND_FIELDS = ["Reference","OperationID","Source","Retained","ReconcileOnly"] diff --git a/services/core/deploy/e2b/init.py b/services/core/deploy/e2b/init.py index 75d8e1f07..45a590175 100644 --- a/services/core/deploy/e2b/init.py +++ b/services/core/deploy/e2b/init.py @@ -106,6 +106,8 @@ def initialize(): # Claim before any side effect. An interrupted attempt must never start twice. write_private(ROOT / 'launch.json', identity) environment = prepare_runtime() + # Application-owned startup does not participate in Core-managed suspension. + environment.pop("OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE", None) credential = PROFILE.parent / 'executor-key.json' write_private(credential, payload['executor_key'], owner=1000) source.unlink() diff --git a/services/core/deploy/e2b/init_test.py b/services/core/deploy/e2b/init_test.py index 3c416c13f..31a1489d3 100644 --- a/services/core/deploy/e2b/init_test.py +++ b/services/core/deploy/e2b/init_test.py @@ -37,7 +37,8 @@ def test_launch_handoff_is_private_and_cannot_replay(self): profile = Path(temporary) / 'private/default' environment_file = Path(temporary) / 'image.json' environment_file.write_text(json.dumps({'HOME': '/home/runtime', - 'OAC_RUNTIME_HOME': '/home/runtime/.oac', 'OAC_RUNTIME_WORKSPACE': '/environment/workspace'})) + 'OAC_RUNTIME_HOME': '/home/runtime/.oac', 'OAC_RUNTIME_WORKSPACE': '/environment/workspace', + 'OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE': '/private/control/fixture.json'})) (root / 'bootstrap.json').write_text(json.dumps(PAYLOAD)) real_chmod = Path.chmod @@ -57,6 +58,7 @@ def chmod(path, mode): self.assertEqual(argv[argv.index('--remote') + 1], PAYLOAD['remote_url']) self.assertNotIn('test-private-key', repr(popen.call_args)) self.assertNotIn('OAC_RUNTIME_SESSION_ID', options['env']) + self.assertNotIn('OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE', options['env']) self.assertEqual((options['user'], options['group'], options['extra_groups']), (1000, 1000, [])) self.assertEqual(options['umask'], 0o077) key = profile.parent / 'executor-key.json' diff --git a/services/core/deploy/e2b/managed_init.py b/services/core/deploy/e2b/managed_init.py index a64931f14..a332e08ce 100644 --- a/services/core/deploy/e2b/managed_init.py +++ b/services/core/deploy/e2b/managed_init.py @@ -47,6 +47,13 @@ def initialize(): binding = identity(payload) shared.write_private(root / 'managed-launch.json', binding) environment = shared.prepare_runtime() + control_file = Path(environment['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE']) + if not control_file.is_absolute() or '..' in control_file.parts: + raise ValueError('Absolute private suspend control file required') + control_directory = control_file.parent + control_directory.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chown(control_directory, 1000, 1000) + control_directory.chmod(0o700) environment.update(OAC_RUNTIME_ENVIRONMENT_ID=payload['EnvironmentID'], OAC_RUNTIME_SESSION_ID=payload['SessionID'], OAC_RUNTIME_NETWORK_ACCESS=payload['NetworkAccess'], diff --git a/services/core/deploy/e2b/managed_init_test.py b/services/core/deploy/e2b/managed_init_test.py index d9017ba82..53eb3c9ad 100644 --- a/services/core/deploy/e2b/managed_init_test.py +++ b/services/core/deploy/e2b/managed_init_test.py @@ -63,9 +63,11 @@ def exercise(self, failed=False): if failed: process.side_effect = RuntimeError('private process diagnostic') image_env = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'OAC_RUNTIME_HOME': str(Path(temporary) / '.oac'), - 'OAC_RUNTIME_WORKSPACE': '/environment/workspace'} + 'OAC_RUNTIME_WORKSPACE': '/environment/workspace', + 'OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE': str(Path(temporary) / 'control' / 'custom-suspend.json')} with patch.object(managed_init.shared, 'ROOT', root), patch.object(managed_init.shared, 'PROFILE', profile), \ patch.object(managed_init.shared, 'prepare_runtime', return_value=image_env), \ + patch.object(managed_init.os, 'chown') as chown, \ patch.object(managed_init.os, 'fchown'), patch.object(managed_init.subprocess, 'Popen', process): if failed: with self.assertRaises(RuntimeError): @@ -83,6 +85,10 @@ def exercise(self, failed=False): self.assertNotIn(data['RuntimeBootstrap']['credential'], json.dumps(process.call_args.kwargs['env'])) self.assertEqual(process.call_args.kwargs['env']['OAC_RUNTIME_ENVIRONMENT_ID'], data['EnvironmentID']) self.assertEqual(process.call_args.kwargs['user'], 1000) + control = Path(temporary) / 'control' + self.assertEqual(control.stat().st_mode & 0o777, 0o700) + chown.assert_called_once_with(control, 1000, 1000) + self.assertEqual(process.call_args.kwargs['env']['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'], str(control / 'custom-suspend.json')) if failed: self.assertFalse((root / 'managed-ready.json').exists()) else: diff --git a/services/core/internal/execution/runtime_compute_wake.go b/services/core/internal/execution/runtime_compute_wake.go index fc560bc23..c4490752d 100644 --- a/services/core/internal/execution/runtime_compute_wake.go +++ b/services/core/internal/execution/runtime_compute_wake.go @@ -6,6 +6,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" @@ -24,7 +25,7 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.Suspension } // This idempotent control signal is fenced by guest PID/start time and the // suspension token. It cannot execute or replay an agent request. - result, err := p.RunCommandCompute(ctx, runtimeReference(owner), state.Current, sandbox.Command{Args: []string{"oac-daemon", "resume", "--control-file", "/run/oac/daemon-suspend.json", "--environment-id", owner.EnvironmentID, "--suspend-id", state.SuspendID}}) + result, err := p.RunCommandCompute(ctx, runtimeReference(owner), state.Current, sandbox.Command{Args: []string{"oac-daemon", "resume", "--control-file", runtimebootstrap.SuspendControlFile, "--environment-id", owner.EnvironmentID, "--suspend-id", state.SuspendID}}) if err != nil { return err } diff --git a/services/core/internal/sandbox/e2b/helper_contract.go b/services/core/internal/sandbox/e2b/helper_contract.go index 7e9a96f25..2f8e2f50b 100644 --- a/services/core/internal/sandbox/e2b/helper_contract.go +++ b/services/core/internal/sandbox/e2b/helper_contract.go @@ -12,7 +12,7 @@ import ( //go:generate go run ./internal/contractgen // This adapter-private boundary is documented in tools/e2b-provider/README.md. -const ProtocolVersion = 1 +const ProtocolVersion = 2 const MaxOutputBytes = 1024 * 1024 const MaxRequestBytes = 72 * 1024 * 1024 const MaxResponseBytes = 16 * 1024 * 1024 @@ -22,7 +22,7 @@ const MaxCommandInputBytes = sandbox.MaxCommandInputBytes // HelperOperations declares the complete set of one-shot helper operations. func HelperOperations() []string { - return []string{"create", "inspect", "renew", "kill", "command", "validate_deployment", "observe", "list_templates", "list_builds", "verify_credential"} + return []string{"create", "inspect", "renew", "kill", "command", "validate_deployment", "observe", "list_templates", "list_builds", "verify_credential", "compute_info", "compute_renew", "suspend", "resume", "compute_kill", "delete_retained", "compute_command", "resume_compute"} } // HelperErrors are sanitized wire outcomes; an empty code denotes success. @@ -66,6 +66,28 @@ func (q Request) Validate() error { return sandbox.ErrInvalid } } + + switch q.Operation { + case "compute_info", "compute_renew", "compute_kill", "compute_command", "resume_compute": + if q.Compute == nil || q.Compute.Name != q.Reference.AllocationID || (q.Operation != "compute_info" && q.Compute.ID == "") { + return sandbox.ErrInvalid + } + if q.Operation == "compute_command" && q.Command == nil { + return sandbox.ErrInvalid + } + case "suspend": + if q.Suspend == nil || q.Suspend.Reference != q.Reference || !validID(q.Suspend.OperationID) || q.Suspend.Source.Name != q.Reference.AllocationID || q.Suspend.Source.ID == "" { + return sandbox.ErrInvalid + } + case "resume": + if q.Resume == nil || q.Resume.Reference != q.Reference || !validID(q.Resume.OperationID) || sandbox.ValidateRetained(q.Resume.Retained) != nil || q.Resume.Retained.Reference != q.Reference.AllocationID { + return sandbox.ErrInvalid + } + case "delete_retained": + if q.Retained == nil || sandbox.ValidateRetained(*q.Retained) != nil || q.Retained.Reference != q.Reference.AllocationID { + return sandbox.ErrInvalid + } + } return nil } @@ -79,10 +101,15 @@ type Request struct { Bootstrap *sandbox.Bootstrap `json:",omitempty"` RuntimeBootstrap *runtimebootstrap.Connection `json:",omitempty"` Command *sandbox.Command `json:",omitempty"` + Compute *sandbox.Compute `json:",omitempty"` + Suspend *sandbox.SuspendRequest `json:",omitempty"` + Resume *sandbox.ResumeRequest `json:",omitempty"` + Retained *sandbox.RetainedState `json:",omitempty"` Deadline time.Time } type Response struct { Version int + State *sandbox.ComputeState `json:",omitempty"` Info *sandbox.Info `json:",omitempty"` Command *sandbox.CommandResult `json:",omitempty"` ErrorCode string diff --git a/services/core/internal/sandbox/e2b/installed_paths_test.go b/services/core/internal/sandbox/e2b/installed_paths_test.go index 5012e0d97..35a813ef9 100644 --- a/services/core/internal/sandbox/e2b/installed_paths_test.go +++ b/services/core/internal/sandbox/e2b/installed_paths_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "strconv" "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -31,7 +32,7 @@ func TestConfigurationDiscoveryUsesSuppliedProcessPaths(t *testing.T) { if err := os.MkdirAll(filepath.Dir(binary), 0700); err != nil { t.Fatal(err) } - if err := os.WriteFile(binary, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":1,\"Templates\":[]}'\n"), 0700); err != nil { + if err := os.WriteFile(binary, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":"+strconv.Itoa(ProtocolVersion)+",\"Templates\":[]}'\n"), 0700); err != nil { t.Fatal(err) } input := sandbox.ConfigurationDiscoveryInput{Credential: json.RawMessage(`{"api_key":"synthetic-key"}`)} diff --git a/services/core/internal/sandbox/e2b/internal/contractgen/main.go b/services/core/internal/sandbox/e2b/internal/contractgen/main.go index 6d79d9544..7bc41c332 100644 --- a/services/core/internal/sandbox/e2b/internal/contractgen/main.go +++ b/services/core/internal/sandbox/e2b/internal/contractgen/main.go @@ -82,12 +82,17 @@ func main() { identity = append(identity, "InstallationID") values := map[string]any{ "PROTOCOL_VERSION": e2b.ProtocolVersion, "SDK_VERSION": sdk, - "MAX_REQUEST": e2b.MaxRequestBytes, "MAX_RESPONSE": e2b.MaxResponseBytes, + "SUSPEND_CONTROL_FILE": runtimebootstrap.SuspendControlFile, + "MAX_REQUEST": e2b.MaxRequestBytes, "MAX_RESPONSE": e2b.MaxResponseBytes, "MAX_OUTPUT": e2b.MaxOutputBytes, "MAX_COMMAND_INPUT": e2b.MaxCommandInputBytes, "MAX_OBSERVATION_REFERENCES": e2b.MaxObservationReferences, "MAX_CREDENTIAL_REFERENCES": e2b.MaxCredentialReferences, "OPERATIONS": e2b.HelperOperations(), "ERROR_CODES": e2b.HelperErrors(), "REQUEST_FIELDS": fields(reflect.TypeFor[e2b.Request]()), "RESPONSE_FIELDS": fields(reflect.TypeFor[e2b.Response]()), "REFERENCE_FIELDS": fields(reflect.TypeFor[sandbox.Reference]()), + "COMPUTE_FIELDS": fields(reflect.TypeFor[sandbox.Compute]()), + "RETAINED_FIELDS": fields(reflect.TypeFor[sandbox.RetainedState]()), + "SUSPEND_FIELDS": fields(reflect.TypeFor[sandbox.SuspendRequest]()), + "RESUME_FIELDS": fields(reflect.TypeFor[sandbox.ResumeRequest]()), "MANAGED_BOOTSTRAP_FIELDS": bootstrap, "MANAGED_IDENTITY_FIELDS": identity, "NETWORK_ACCESS": networkValues(filepath.Join(root, "internal/agentnetwork/policy.go")), } @@ -130,7 +135,24 @@ func fixtures() []byte { } for _, operation := range e2b.HelperOperations() { copy := q + copy.Operation = operation + c := sandbox.Compute{Name: r.AllocationID, ID: "native-fixture"} + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: installation, OperationID: installation, SourceName: c.Name, SourceID: c.ID, Data: "opaque"} + switch operation { + case "compute_info", "compute_renew", "compute_kill", "compute_command", "resume_compute": + copy.Compute = &c + if operation == "compute_command" { + copy.Command = &sandbox.Command{Args: []string{"true"}} + } + case "suspend": + copy.Suspend = &sandbox.SuspendRequest{Reference: r, OperationID: installation, Source: c} + case "resume": + target := sandbox.Compute{Generation: 1, Name: c.Name, ID: c.ID, RestoredFrom: &retained} + copy.Resume = &sandbox.ResumeRequest{Reference: r, OperationID: installation, Retained: retained, Target: target} + case "delete_retained": + copy.Retained = &retained + } if operation == "observe" || operation == "verify_credential" { copy.References = []sandbox.Reference{r} } diff --git a/services/core/internal/sandbox/e2b/operations.go b/services/core/internal/sandbox/e2b/operations.go index dab82e81c..69115f829 100644 --- a/services/core/internal/sandbox/e2b/operations.go +++ b/services/core/internal/sandbox/e2b/operations.go @@ -1,7 +1,6 @@ package e2b import ( - "context" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -20,16 +19,16 @@ func Operations() providercontract.Operations { "Renew": {State: providercontract.Supported}, "Kill": {State: providercontract.Supported}, "RunCommand": {State: providercontract.Supported}, - "Initial": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "NewCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "RenewCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "GetCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "Suspend": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "Resume": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "KillCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "DeleteRetained": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "RunCommandCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "ResumeCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "Initial": {State: providercontract.Supported}, + "NewCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, + "GetCompute": {State: providercontract.Supported}, + "Suspend": {State: providercontract.Supported}, + "Resume": {State: providercontract.Supported}, + "KillCompute": {State: providercontract.Supported}, + "DeleteRetained": {State: providercontract.Supported}, + "RunCommandCompute": {State: providercontract.Supported}, + "ResumeCompute": {State: providercontract.Supported}, "ObservationProviderType": {State: providercontract.Supported}, "ResolveObservationSource": {State: providercontract.Supported}, "Observe": {State: providercontract.Supported}, @@ -39,34 +38,3 @@ func Operations() providercontract.Operations { } } func (*Provider) ProviderOperations() providercontract.Operations { return Operations() } -func (p *Provider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { - return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: Operations()["Initial"].Reason} -} -func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { - return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: Operations()["NewCompute"].Reason} -} -func (p *Provider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "GetCompute", Reason: Operations()["GetCompute"].Reason} -} -func (p *Provider) Suspend(context.Context, sandbox.SuspendRequest) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "Suspend", Reason: Operations()["Suspend"].Reason} -} -func (p *Provider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "Resume", Reason: Operations()["Resume"].Reason} -} -func (p *Provider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { - return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: Operations()["KillCompute"].Reason} -} -func (p *Provider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { - return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: Operations()["DeleteRetained"].Reason} -} -func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { - return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: Operations()["RunCommandCompute"].Reason} -} -func (p *Provider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: Operations()["ResumeCompute"].Reason} -} - -func (p *Provider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: Operations()["RenewCompute"].Reason} -} diff --git a/services/core/internal/sandbox/e2b/process_test.go b/services/core/internal/sandbox/e2b/process_test.go index 2d15997f4..695333885 100644 --- a/services/core/internal/sandbox/e2b/process_test.go +++ b/services/core/internal/sandbox/e2b/process_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -29,7 +30,7 @@ func TestTimeoutDoesNotTerminateOwnedHelper(t *testing.T) { marker := filepath.Join(root, "settled") // The private test path contains no shell syntax; the real adapter never puts // credentials or request contents in argv or inherited environment. - script := "#!/bin/sh\nsleep 0.15\ntouch '" + marker + "'\nprintf '%s' '{\"Version\":1}'\n" + script := "#!/bin/sh\nsleep 0.15\ntouch '" + marker + "'\nprintf '%s' '{\"Version\":" + strconv.Itoa(ProtocolVersion) + "}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } @@ -50,7 +51,7 @@ func TestTimeoutDoesNotTerminateOwnedHelper(t *testing.T) { func TestEnvironmentDropsProviderSelectorsAndCredentials(t *testing.T) { root := t.TempDir() binary := filepath.Join(root, "helper") - script := "#!/bin/sh\nif test -n \"${E2B_API_KEY:-}${E2B_API_URL:-}${PRIVATE_MODEL_KEY:-}\"; then exit 1; fi\nprintf '%s' '{\"Version\":1}'\n" + script := "#!/bin/sh\nif test -n \"${E2B_API_KEY:-}${E2B_API_URL:-}${PRIVATE_MODEL_KEY:-}\"; then exit 1; fi\nprintf '%s' '{\"Version\":" + strconv.Itoa(ProtocolVersion) + "}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } @@ -67,7 +68,7 @@ func TestCredentialFenceWaitsForActualHelperExitAfterCancellation(t *testing.T) binary := filepath.Join(root, "helper") marker := filepath.Join(root, "started") finish := filepath.Join(root, "finish") - script := "#!/bin/sh\ntouch '" + marker + "'\nwhile ! test -f '" + finish + "'; do sleep 0.01; done\nprintf '%s' '{\"Version\":1}'\n" + script := "#!/bin/sh\ntouch '" + marker + "'\nwhile ! test -f '" + finish + "'; do sleep 0.01; done\nprintf '%s' '{\"Version\":" + strconv.Itoa(ProtocolVersion) + "}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } diff --git a/services/core/internal/sandbox/e2b/provider.go b/services/core/internal/sandbox/e2b/provider.go index c368d8866..17f92b58b 100644 --- a/services/core/internal/sandbox/e2b/provider.go +++ b/services/core/internal/sandbox/e2b/provider.go @@ -168,12 +168,20 @@ func (p *Provider) call(ctx context.Context, operation string, r sandbox.Referen } connection = &value } - out, err := p.caller.Call(ctx, Request{Version: ProtocolVersion, Operation: operation, Config: p.config, Reference: r, Bootstrap: b, RuntimeBootstrap: connection, Command: command, Deadline: deadline}) + return p.callRequest(ctx, Request{Version: ProtocolVersion, Operation: operation, Config: p.config, Reference: r, Bootstrap: b, RuntimeBootstrap: connection, Command: command, Deadline: deadline}) +} + +func (p *Provider) callRequest(ctx context.Context, q Request) (Response, error) { + operation, r := q.Operation, q.Reference + if q.Validate() != nil { + return Response{}, sandbox.ErrInvalid + } + out, err := p.caller.Call(ctx, q) if errors.Is(err, errHelperNotStarted) { return unstarted(operation, r), sandbox.ErrComputeUnconfirmed } if err != nil || out.Version != ProtocolVersion { - if operation == "command" { + if operation == "command" || operation == "compute_command" { return Response{}, sandbox.ErrCommandUnconfirmed } return Response{}, sandbox.ErrComputeUnconfirmed diff --git a/services/core/internal/sandbox/e2b/suspension.go b/services/core/internal/sandbox/e2b/suspension.go new file mode 100644 index 000000000..da3b4911f --- /dev/null +++ b/services/core/internal/sandbox/e2b/suspension.go @@ -0,0 +1,111 @@ +package e2b + +import ( + "context" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +func (p *Provider) Initial(_ context.Context, r sandbox.Reference) (sandbox.Compute, error) { + if !validReference(r) { + return sandbox.Compute{}, sandbox.ErrInvalid + } + return sandbox.Compute{Name: r.AllocationID}, nil +} +func (p *Provider) NewCompute(_ context.Context, r sandbox.Reference, generation uint64, retained *sandbox.RetainedState) (sandbox.Compute, error) { + if !validReference(r) || retained == nil || sandbox.ValidateRetained(*retained) != nil || retained.Reference != r.AllocationID || retained.SourceName != r.AllocationID || generation == 0 || generation != retained.SourceGeneration+1 { + return sandbox.Compute{}, sandbox.ErrInvalid + } + value := *retained + return sandbox.Compute{Generation: generation, Name: r.AllocationID, ID: retained.SourceID, RestoredFrom: &value}, nil +} +func (p *Provider) computeCall(ctx context.Context, q Request) (Response, error) { + deadline, ok := ctx.Deadline() + if !ok || ctx.Err() != nil { + return Response{}, sandbox.ErrInvalid + } + q.Version, q.Config, q.Deadline = ProtocolVersion, p.config, deadline + return p.callRequest(ctx, q) +} +func (p *Provider) computeState(ctx context.Context, operation string, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + if c.Name != r.AllocationID { + return sandbox.ComputeState{}, sandbox.ErrOwnership + } + out, err := p.computeCall(ctx, Request{Operation: operation, Reference: r, Compute: &c}) + if err != nil { + return sandbox.ComputeState{}, err + } + if out.State == nil || sandbox.ValidateComputeResult(c, out.State.Compute) != nil { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + return *out.State, nil +} +func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.computeState(ctx, "compute_info", r, c) +} +func (p *Provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.computeState(ctx, "compute_renew", r, c) +} +func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.computeState(ctx, "resume_compute", r, c) +} +func (p *Provider) Suspend(ctx context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { + if !validID(q.OperationID) || q.Source.Name != q.Reference.AllocationID || q.Source.ID == "" { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + out, err := p.computeCall(ctx, Request{Operation: "suspend", Reference: q.Reference, Suspend: &q}) + if err != nil { + return sandbox.ComputeState{}, err + } + if out.State == nil { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + if err = sandbox.ValidateSuspendResult(q, *out.State); err != nil { + return sandbox.ComputeState{}, err + } + if out.State.Retained != nil && out.State.Retained.Reference != q.Reference.AllocationID { + return sandbox.ComputeState{}, sandbox.ErrOwnership + } + return *out.State, nil +} +func (p *Provider) Resume(ctx context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { + planned, err := p.NewCompute(ctx, q.Reference, q.Target.Generation, &q.Retained) + if err != nil || sandbox.ValidateComputeResult(planned, q.Target) != nil || !validID(q.OperationID) { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + out, err := p.computeCall(ctx, Request{Operation: "resume", Reference: q.Reference, Resume: &q}) + if err != nil { + return sandbox.ComputeState{}, err + } + if out.State == nil || sandbox.ValidateComputeResult(q.Target, out.State.Compute) != nil || out.State.Status != "running" || !out.State.BootstrapComplete { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + return *out.State, nil +} +func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) error { + if c.Name != r.AllocationID || c.ID == "" { + return sandbox.ErrInvalid + } + _, err := p.computeCall(ctx, Request{Operation: "compute_kill", Reference: r, Compute: &c}) + return err +} +func (p *Provider) DeleteRetained(ctx context.Context, r sandbox.Reference, s sandbox.RetainedState) error { + if sandbox.ValidateRetained(s) != nil || s.Reference != r.AllocationID { + return sandbox.ErrInvalid + } + _, err := p.computeCall(ctx, Request{Operation: "delete_retained", Reference: r, Retained: &s}) + return err +} +func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { + if c.Name != r.AllocationID || c.ID == "" { + return sandbox.CommandResult{}, sandbox.ErrInvalid + } + out, err := p.computeCall(ctx, Request{Operation: "compute_command", Reference: r, Compute: &c, Command: &command}) + if err != nil { + return sandbox.CommandResult{}, err + } + if out.Command == nil { + return sandbox.CommandResult{}, sandbox.ErrCommandUnconfirmed + } + return *out.Command, nil +} diff --git a/services/core/internal/sandbox/e2b/suspension_test.go b/services/core/internal/sandbox/e2b/suspension_test.go new file mode 100644 index 000000000..d4a0c0c06 --- /dev/null +++ b/services/core/internal/sandbox/e2b/suspension_test.go @@ -0,0 +1,48 @@ +package e2b + +import ( + "errors" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/google/uuid" +) + +func TestSuspensionPlansSameNativeIDWithoutCallingHelper(t *testing.T) { + p, f, r := fixture(t) + initial, err := p.Initial(bounded(t), r) + if err != nil || initial.Name != r.AllocationID || initial.ID != "" { + t.Fatal(initial, err) + } + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: uuid.NewString(), OperationID: uuid.NewString(), SourceName: r.AllocationID, SourceID: "native-id", Data: "opaque"} + target, err := p.NewCompute(bounded(t), r, 1, &retained) + if err != nil || target.ID != retained.SourceID || target.Generation != 1 || *target.RestoredFrom != retained || len(f.requests) != 0 { + t.Fatal(target, err) + } + if _, err = p.NewCompute(bounded(t), r, 2, &retained); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("generation jump accepted", err) + } + retained.Reference = uuid.NewString() + if _, err = p.NewCompute(bounded(t), r, 1, &retained); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("foreign allocation accepted", err) + } +} +func TestSuspensionRejectsUnsettledAndForeignHelperOutcomes(t *testing.T) { + p, f, r := fixture(t) + current := sandbox.Compute{Name: r.AllocationID, ID: "native-id"} + q := sandbox.SuspendRequest{Reference: r, Source: current, OperationID: uuid.NewString()} + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: q.OperationID, OperationID: q.OperationID, SourceName: current.Name, SourceID: current.ID, Data: "opaque"} + f.response.State = &sandbox.ComputeState{Compute: current, Status: "suspended", BootstrapComplete: true, Retained: &retained, ResourcesReleased: true, SuspendSettled: true} + if _, err := p.Suspend(bounded(t), q); err != nil { + t.Fatal(err) + } + f.response.State.SuspendSettled = false + if _, err := p.Suspend(bounded(t), q); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { + t.Fatal(err) + } + f.response.State.SuspendSettled = true + f.response.State.Compute.Generation = 1 + if _, err := p.Suspend(bounded(t), q); !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal(err) + } +} diff --git a/services/core/internal/sandbox/providers/registry.go b/services/core/internal/sandbox/providers/registry.go index 5df43d642..be02ed619 100644 --- a/services/core/internal/sandbox/providers/registry.go +++ b/services/core/internal/sandbox/providers/registry.go @@ -50,6 +50,7 @@ var adapters = map[string]Adapter{ }, "e2b": { Policy: e2b.Policy(), Operations: e2b.Operations, Mode: "direct", BuildDirect: buildE2B, + IdleSeconds: 300, RetentionSeconds: 86400, Configuration: e2b.ConfigurationAdapter{}, ValidateSpecification: e2b.ValidateSpecification, ValidateResources: e2b.ValidateResources, }, diff --git a/services/core/internal/sandbox/providers/registry_test.go b/services/core/internal/sandbox/providers/registry_test.go index d9f51144f..ca0892e44 100644 --- a/services/core/internal/sandbox/providers/registry_test.go +++ b/services/core/internal/sandbox/providers/registry_test.go @@ -17,7 +17,7 @@ func TestRegistrationOwnsDeploymentPolicy(t *testing.T) { }{ {"docker", "nodes", "nodes", 0, 0, false}, {"microsandbox", "nodes", "nodes", 300, 86400, true}, - {"e2b", "direct", "e2b", 0, 0, false}, + {"e2b", "direct", "e2b", 300, 86400, true}, } { t.Run(tc.kind, func(t *testing.T) { d, err := Describe(tc.kind, installation) diff --git a/services/core/tools/e2b-provider/README.md b/services/core/tools/e2b-provider/README.md index aff992d1b..a345ae381 100644 --- a/services/core/tools/e2b-provider/README.md +++ b/services/core/tools/e2b-provider/README.md @@ -30,7 +30,7 @@ Compatible endpoints must return the SDK 2.51.0 template-list and template-build Run `go generate ./services/core/internal/sandbox/e2b` from the repository root after changing these declarations. `make check-e2b-provider` and the Go adapter tests reject stale projections; both languages consume generated valid and invalid exchanges covering wire types, extra fields, operation/reference bounds and managed-bootstrap fields. The helper build copies those fixtures with its source before running the pinned-SDK suite. -The boundary has version 1. The request and credentials arrive on standard input; standard output carries one bounded response with a sanitized error code. The helper removes ambient `E2B_*` and `PYTHON*` variables and calls the SDK only with the request's explicit API origin and sandbox domain. Each receipt is bound to those selectors, and a receipt without them belongs to the official endpoints (`https://api.e2b.app`, `e2b.app`). An endpoint change keeps earlier generations on their original API and sandbox domain; the candidate key must verify all retained ownership before an online switch. +The boundary has version 2. The request and credentials arrive on standard input; standard output carries one bounded response with a sanitized error code. The helper removes ambient `E2B_*` and `PYTHON*` variables and calls the SDK only with the request's explicit API origin and sandbox domain. Each receipt is bound to those selectors, and a receipt without them belongs to the official endpoints (`https://api.e2b.app`, `e2b.app`). An endpoint change keeps earlier generations on their original API and sandbox domain; the candidate key must verify all retained ownership before an online switch. ## Receipts and state directory @@ -50,7 +50,7 @@ An unknown Create is never repeated. A Create whose connection material was lost ## Inspection and cleanup -Inspection uses SDK metadata and ID reads only. SDK `connect` is never used because it can resume paused compute. The version-pinned constructor that restores a client from saved connection material is confined to [`sdk.py`](sdk.py) and covered by a no-connect, no-create test. Resource drift fails inspection but still permits ownership-based cleanup. +Inspection uses SDK metadata and ID reads only. Inspection never uses SDK `connect`, which is reserved for the explicit managed Resume operation. The version-pinned constructor that restores a client from saved connection material is confined to [`sdk.py`](sdk.py) and covered by a no-connect, no-create test. Resource drift fails inspection but still permits ownership-based cleanup. `CreateSettled` proves that the original Create and bootstrap can no longer mutate; it is independent of `BootstrapComplete`. An empty lookup never settles an unknown Create. Kill destroys every matching sandbox, confirms that none remains and only then records a settled tombstone; it returns `State=absent` with `CreateSettled`. A settled rejected Create with no sandbox IDs proves absence without a cloud request, so GetInfo and Kill still succeed when the key is invalid. Ordinary missing compute has no such proof. @@ -71,3 +71,11 @@ make check-e2b-provider ``` With `OAC_TEST_E2B_SDK_PYTHON` pointing at the pinned SDK environment, this runs this directory's tests and the [template scripts' tests](../../deploy/e2b/README.md#tests). The helper build runs this directory's suite and checks the relocated helper's `--check` report. These tests create no cloud resources. + +## Managed suspension + +The adapter implements the complete shared suspension group. Core's common activity rule quiesces the Runtime, then the adapter calls the pinned SDK's memory-preserving `Sandbox.pause`. The private allocation receipt commits the operation before native I/O and reports resource release only after observing the exact sandbox paused. + +Resume calls `Sandbox.connect` once with `on_resume=restore` and the existing native timeout. It preserves the native sandbox ID while advancing the shared logical generation. Fresh connection material is persisted before returning running. Unknown pause and connect outcomes are observed without replay; a missing connect receipt keeps execution unavailable. The adapter fences stale generations before commands and deletion. Consumed pause receipt cleanup preserves running compute. + +The shared registration owns the 300-second idle duration and 86400-second retention default. Native timeout remains 3600 seconds and automatic native resume remains disabled. SDK calls use the configured official-compatible endpoint. Generated declarations and fixtures own the private helper shape. diff --git a/services/core/tools/e2b-provider/helper_contract_generated.py b/services/core/tools/e2b-provider/helper_contract_generated.py index ebd7fadf2..588bd0aca 100644 --- a/services/core/tools/e2b-provider/helper_contract_generated.py +++ b/services/core/tools/e2b-provider/helper_contract_generated.py @@ -1,5 +1,6 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" +COMPUTE_FIELDS = ["Generation","Name","ID","RestoredFrom"] ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized"] MANAGED_BOOTSTRAP_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","NetworkAccess","AllowedDomains","InstallationID","RuntimeBootstrap"] MANAGED_IDENTITY_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID"] @@ -10,9 +11,13 @@ MAX_REQUEST = 75497472 MAX_RESPONSE = 16777216 NETWORK_ACCESS = ["enabled","disabled","restricted"] -OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] -PROTOCOL_VERSION = 1 +OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential","compute_info","compute_renew","suspend","resume","compute_kill","delete_retained","compute_command","resume_compute"] +PROTOCOL_VERSION = 2 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] -REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] -RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observations"] +REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Compute","Suspend","Resume","Retained","Deadline"] +RESPONSE_FIELDS = ["Version","State","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observations"] +RESUME_FIELDS = ["Reference","OperationID","Retained","Target","ReconcileOnly"] +RETAINED_FIELDS = ["Reference","ID","Data","OperationID","SourceGeneration","SourceName","SourceID"] SDK_VERSION = "2.51.0" +SUSPEND_CONTROL_FILE = "/run/oac/daemon-suspend.json" +SUSPEND_FIELDS = ["Reference","OperationID","Source","Retained","ReconcileOnly"] diff --git a/services/core/tools/e2b-provider/provider.py b/services/core/tools/e2b-provider/provider.py index dbf3111f0..f8333b503 100644 --- a/services/core/tools/e2b-provider/provider.py +++ b/services/core/tools/e2b-provider/provider.py @@ -221,7 +221,9 @@ def create(self): if definitely_rejected(error): self.receipt.save(status='rejected', settled=True) raise Failure('unconfirmed') from None - self.receipt.save(status='created', ids=[cloud.sandbox_id], connection=connection_material(cloud)) + self.receipt.save(status='created', ids=[cloud.sandbox_id], connection=connection_material(cloud), + compute={'Generation': 0, 'Name': self.reference['AllocationID'], + 'ID': cloud.sandbox_id, 'RestoredFrom': None}) # A create response must not steer envd traffic to an unrelated host. self.check_domain(cloud) # SDK Create returns connection material, but no metadata or resources. @@ -410,6 +412,10 @@ def execute(self): with Receipt(self.q, self.remaining) as self.receipt: try: operation = self.q['Operation'] + if operation in ('compute_info', 'compute_renew', 'suspend', 'resume', 'compute_kill', + 'delete_retained', 'compute_command', 'resume_compute'): + from suspension import Suspension + return Suspension(self, Sandbox, connection_material, run).execute() if operation == 'kill': self.kill() return {'Version': PROTOCOL_VERSION, 'Info': self.info(absent=True), 'ErrorCode': ''} diff --git a/services/core/tools/e2b-provider/provider_test.py b/services/core/tools/e2b-provider/provider_test.py index 74d708547..41dd34382 100644 --- a/services/core/tools/e2b-provider/provider_test.py +++ b/services/core/tools/e2b-provider/provider_test.py @@ -13,6 +13,7 @@ from e2b.exceptions import AuthenticationException, SandboxNotFoundException from provider import Provider +from helper_contract_generated import PROTOCOL_VERSION from sdk import restore, run from state import Failure, Receipt @@ -24,7 +25,7 @@ def setUp(self): self.reference = {key: str(uuid4()) for key in ['TenantID', 'EnvironmentID', 'AllocationID']} self.config = {'StateDir': self.temporary.name, 'InstallationID': str(uuid4()), 'APIKey': 'private-account-secret', 'Template': 'test:' + str(uuid4()), 'TimeoutSeconds': 120} - self.request = {'Version': 1, 'Operation': 'create', 'Config': self.config, + self.request = {'Version': PROTOCOL_VERSION, 'Operation': 'create', 'Config': self.config, 'Reference': self.reference, 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=30)).isoformat(), 'Bootstrap': dict(self.reference, SessionID=str(uuid4()), DeviceID=str(uuid4()), CoreURL='https://core.example/api/v1', Credential='private-runtime-secret', diff --git a/services/core/tools/e2b-provider/state_test.py b/services/core/tools/e2b-provider/state_test.py index ae573370a..7de903990 100644 --- a/services/core/tools/e2b-provider/state_test.py +++ b/services/core/tools/e2b-provider/state_test.py @@ -11,6 +11,7 @@ from unittest.mock import patch from provider import Provider +from helper_contract_generated import PROTOCOL_VERSION from state import Failure, Receipt @@ -18,7 +19,7 @@ class StateTest(unittest.TestCase): def setUp(self): self.root = tempfile.TemporaryDirectory() self.addCleanup(self.root.cleanup) - self.request = {'Version': 1, 'Operation': 'kill', + self.request = {'Version': PROTOCOL_VERSION, 'Operation': 'kill', 'Config': {'StateDir': self.root.name, 'InstallationID': str(uuid4()), 'APIKey': 'test'}, 'Reference': {key: str(uuid4()) for key in ['TenantID', 'EnvironmentID', 'AllocationID']}, 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=2)).isoformat()} diff --git a/services/core/tools/e2b-provider/suspension.py b/services/core/tools/e2b-provider/suspension.py new file mode 100644 index 000000000..313e10002 --- /dev/null +++ b/services/core/tools/e2b-provider/suspension.py @@ -0,0 +1,225 @@ +"""E2B's implementation of exact-incarnation suspension under the allocation lock.""" +import json + +from helper_contract_generated import PROTOCOL_VERSION, COMPUTE_FIELDS, SUSPEND_FIELDS, RESUME_FIELDS +from state import Failure + + +class Suspension: + def __init__(self, provider, sdk, material, command): + self.p, self.sdk, self.material, self.command = provider, sdk, material, command + + @property + def record(self): + record = self.p.receipt.data + if not record or record.get('settled') is not True: + raise Failure('unconfirmed') + return record + + def current(self): + current = self.record.get('compute') + if not isinstance(current, dict) or not current.get('ID'): + raise Failure('unconfirmed') + return current + + def matches(self, wanted, actual, unresolved=False): + if (not isinstance(wanted, dict) or set(wanted) != set(COMPUTE_FIELDS) or + type(wanted['Generation']) is not int or wanted['Generation'] < 0 or + wanted['Name'] != self.p.reference['AllocationID'] or + any(wanted[k] != actual[k] for k in ('Generation', 'Name', 'RestoredFrom')) or + (wanted['ID'] != actual['ID'] and not (unresolved and wanted['ID'] == ''))): + raise Failure('ownership') + + def cloud(self, execution=True): + found = self.p.discover() + if len(found) != 1: + raise Failure('unconfirmed') + cloud = found[0] + if cloud.sandbox_id != self.current()['ID']: + raise Failure('ownership') + if execution: + self.p.qualified(cloud) + return cloud + + def state(self, current, cloud): + return {'Compute': current, 'Status': cloud.state, + 'BootstrapComplete': self.record.get('bootstrap_complete') is True, + 'Retained': None, 'ResourcesReleased': False, 'SuspendSettled': False} + + def retained(self, operation, current): + return {'Reference': self.p.reference['AllocationID'], 'ID': operation, 'OperationID': operation, + 'SourceGeneration': current['Generation'], 'SourceName': current['Name'], + 'SourceID': current['ID'], + 'Data': json.dumps({'version': 1, 'sandbox_id': current['ID']}, sort_keys=True, separators=(',', ':'))} + + def request(self, field): + q = self.p.q.get(field) + if (not isinstance(q, dict) or set(q) != set(SUSPEND_FIELDS if field == 'Suspend' else RESUME_FIELDS) or + q.get('Reference') != self.p.reference): + raise Failure('invalid') + # UUID parsing is shared with the allocation envelope validator. + from provider import valid_id + if not valid_id(q.get('OperationID')) or type(q.get('ReconcileOnly')) is not bool: + raise Failure('invalid') + return q + + def inspect(self, operation): + current = self.current() + self.matches(self.p.q.get('Compute'), current, unresolved=operation == 'compute_info') + if self.record.get('status') == 'killed': + raise Failure('not_found') + cloud = self.cloud() + if operation != 'compute_info': + if cloud.state != 'running' or not self.record.get('bootstrap_complete'): + raise Failure('unconfirmed') + pending = self.record.get('suspension') + if pending and pending['phase'] not in ('resumed', 'consumed'): + raise Failure('unconfirmed') + if operation == 'compute_renew': + self.sdk.set_timeout(cloud.sandbox_id, self.p.config['TimeoutSeconds'], **self.p.options()) + cloud = self.cloud() + if operation == 'compute_command': + return {'Command': self.command(self.p.client(cloud), self.p.q['Command'], self.p.remaining)} + return {'State': self.state(current, cloud)} + + def suspend(self): + q = self.request('Suspend') + current = self.current() + self.matches(q.get('Source'), current) + handle = self.retained(q['OperationID'], current) + if q.get('Retained') not in (None, handle): + raise Failure('ownership') + entry = self.record.get('suspension') + if entry and entry['retained'] == handle: + if entry['phase'] not in ('pause_pending', 'paused'): + raise Failure('ownership') + elif q['ReconcileOnly']: + # No durable native intent: the allocation lock proves this helper + # never issued pause. Only a qualified running source can roll back. + if entry and entry['phase'] != 'consumed': + raise Failure('ownership') + cloud = self.cloud() + if cloud.state != 'running': + raise Failure('unconfirmed') + state = self.state(current, cloud) + state['SuspendSettled'] = True + return {'State': state} + else: + if entry and entry['phase'] != 'consumed': + raise Failure('ownership') + cloud = self.cloud() + if cloud.state != 'running' or not self.record.get('bootstrap_complete'): + raise Failure('unconfirmed') + entry = {'retained': handle, 'phase': 'pause_pending'} + self.p.receipt.save(suspension=entry) + # An error remains pending. A later observation may prove paused; + # a running observation never proves a timed-out pause cannot land. + self.sdk.pause(current['ID'], keep_memory=True, **self.p.options()) + cloud = self.cloud() + if cloud.state != 'paused': + raise Failure('unconfirmed') + self.p.receipt.save(suspension=dict(entry, phase='paused')) + state = self.state(current, cloud) + state.update(Status='suspended', Retained=handle, ResourcesReleased=True, SuspendSettled=True) + return {'State': state} + + def resume(self): + q = self.request('Resume') + entry = self.record.get('suspension') + if not entry or entry['retained'] != q.get('Retained'): + raise Failure('ownership') + retained = entry['retained'] + target = {'Generation': retained['SourceGeneration'] + 1, 'Name': retained['SourceName'], + 'ID': retained['SourceID'], 'RestoredFrom': retained} + self.matches(q.get('Target'), target) + if entry['phase'] in ('resume_pending', 'resumed', 'consumed'): + if entry.get('resume_id') != q['OperationID'] or entry.get('target') != target: + raise Failure('ownership') + elif entry['phase'] == 'paused' and not q['ReconcileOnly']: + cloud = self.cloud() + if cloud.state != 'paused': + raise Failure('unconfirmed') + entry = dict(entry, phase='resume_pending', resume_id=q['OperationID'], target=target, connected=False) + self.p.receipt.save(suspension=entry) + connected = self.sdk.connect(target['ID'], timeout=self.p.config['TimeoutSeconds'], + on_resume='restore', **self.p.options()) + if connected.sandbox_id != target['ID']: + raise Failure('ownership') + self.p.check_domain(connected) + entry = dict(entry, connected=True) + self.p.receipt.save(suspension=entry, connection=self.material(connected)) + else: + raise Failure('unconfirmed') + # Never repeat connect after an uncertain reply. Fresh connection material + # must have been durably received before execution can resume. + if not entry.get('connected'): + raise Failure('unconfirmed') + cloud = self.cloud() + if cloud.state != 'running' or not self.record.get('bootstrap_complete'): + raise Failure('unconfirmed') + if entry['phase'] != 'consumed': + self.p.receipt.save(compute=target, suspension=dict(entry, phase='resumed')) + return {'State': self.state(target, cloud)} + + def kill(self): + wanted = self.p.q.get('Compute') + current = self.current() + entry = self.record.get('suspension') + target = (entry or {}).get('target') + # Target-first cleanup may arrive before Resume was ever dispatched. + planned = None + if entry: + r = entry['retained'] + planned = {'Generation': r['SourceGeneration'] + 1, 'Name': r['SourceName'], + 'ID': r['SourceID'], 'RestoredFrom': r} + if wanted == planned and target is None and entry['phase'] == 'paused': + return {} + if wanted != current and wanted != target: + # Once destruction is confirmed, stale source cleanup is harmless. + if self.record.get('status') == 'killed' and isinstance(wanted, dict): + if wanted['ID'] == current['ID'] and wanted['Name'] == current['Name'] and wanted['Generation'] < current['Generation']: + return {} + raise Failure('ownership') + if entry and entry['phase'] == 'resume_pending' and entry.get('connected'): + # The SDK reply already settled the original connect. Cleanup owns + # recovery once Core leaves restoring, so reconcile that saved target + # here without another connect or a resource-qualification gate. + if wanted != target: + raise Failure('ownership') + found = self.p.discover() + if len(found) > 1 or any(cloud.sandbox_id != target['ID'] for cloud in found): + raise Failure('ownership') + entry = dict(entry, phase='resumed') + self.p.receipt.save(compute=target, suspension=entry) + if entry and entry['phase'] in ('pause_pending', 'resume_pending'): + # No successful native reply or settled pause has been observed. + raise Failure('unconfirmed') + self.p.kill() + return {} + + def delete(self): + wanted = self.p.q.get('Retained') + entry = self.record.get('suspension') + if not entry or wanted != entry['retained']: + raise Failure('ownership') + if entry['phase'] == 'consumed': + return {} + if self.record.get('status') != 'killed': + if entry['phase'] != 'resumed': + raise Failure('unconfirmed') + self.matches(entry['target'], self.current()) + if self.cloud().state != 'running': + raise Failure('unconfirmed') + # E2B consumed the pause image on restore. Keep its minimal generation + # receipt to reject delayed resume/cleanup; never kill running compute. + self.p.receipt.save(suspension=dict(entry, phase='consumed')) + return {} + + def execute(self): + operation = self.p.q['Operation'] + if operation in ('compute_info', 'compute_renew', 'compute_command', 'resume_compute'): + result = self.inspect(operation) + else: + result = {'suspend': self.suspend, 'resume': self.resume, + 'compute_kill': self.kill, 'delete_retained': self.delete}[operation]() + return dict(result, Version=PROTOCOL_VERSION, ErrorCode='') diff --git a/services/core/tools/e2b-provider/suspension_test.py b/services/core/tools/e2b-provider/suspension_test.py new file mode 100644 index 000000000..6d08e5d16 --- /dev/null +++ b/services/core/tools/e2b-provider/suspension_test.py @@ -0,0 +1,157 @@ +"""Native lifecycle effects and durable recovery are tested through the helper.""" +import unittest +from uuid import uuid4 + +from provider import Provider +import provider_test + + +class SuspensionTest(unittest.TestCase): + setUp = provider_test.ProviderTest.setUp + call = provider_test.ProviderTest.call + record = provider_test.ProviderTest.record + + def prepare(self): + self.assertEqual(self.call('create')['ErrorCode'], '') + self.api.pause.side_effect = lambda *a, **k: setattr(self.cloud, 'state', 'paused') + def connect(*args, **kwargs): + self.cloud.state = 'running' + return self.cloud + self.api.connect.side_effect = connect + return self.record()['compute'] + + def invoke(self, operation, **payload): + return Provider(dict(self.request, Operation=operation, **payload)).execute() + + def pause(self, source): + q = {'Reference': self.reference, 'OperationID': str(uuid4()), 'Source': source, + 'Retained': None, 'ReconcileOnly': False} + response = self.invoke('suspend', Suspend=q) + return q, response + + def resume_request(self, retained): + target = {'Generation': retained['SourceGeneration'] + 1, 'Name': retained['SourceName'], + 'ID': retained['SourceID'], 'RestoredFrom': retained} + return {'Reference': self.reference, 'OperationID': str(uuid4()), 'Target': target, + 'Retained': retained, 'ReconcileOnly': False} + + def test_two_cycles_keep_native_id_and_fence_old_generation(self): + current = self.prepare() + for generation in range(2): + old = current + q, paused = self.pause(current) + self.assertEqual(paused['ErrorCode'], '') + self.assertTrue(paused['State']['ResourcesReleased']) + retained = paused['State']['Retained'] + request = self.resume_request(retained) + result = self.invoke('resume', Resume=request) + self.assertEqual(result['ErrorCode'], '') + current = result['State']['Compute'] + self.assertEqual(current['ID'], old['ID']) + self.assertEqual(current['Generation'], generation + 1) + self.assertEqual(self.invoke('compute_kill', Compute=old)['ErrorCode'], 'ownership') + self.api.kill.assert_not_called() + # Core wakes the parked daemon before deleting the consumed receipt. + self.assertEqual(self.invoke('compute_command', Compute=current, + Command={'Args': ['oac-daemon', 'resume']})['ErrorCode'], '') + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], '') + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], '') + self.assertEqual(self.invoke('compute_renew', Compute=current)['ErrorCode'], '') + self.assertEqual(self.api.pause.call_count, 2) + self.assertEqual(self.api.connect.call_count, 2) + self.assertEqual(self.api.connect.call_args.kwargs['on_resume'], 'restore') + self.assertEqual(self.api.connect.call_args.kwargs['timeout'], self.config['TimeoutSeconds']) + self.assertTrue(self.api.pause.call_args.kwargs['keep_memory']) + + def test_lost_pause_reply_observes_without_replay(self): + current = self.prepare() + def lost(*args, **kwargs): + self.cloud.state = 'paused' + raise TimeoutError() + self.api.pause.side_effect = lost + q, result = self.pause(current) + self.assertEqual(result['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('suspend', Suspend=dict(q, ReconcileOnly=True))['ErrorCode'], '') + self.api.pause.assert_called_once() + + def test_pending_pause_running_cannot_roll_back_or_delete(self): + current = self.prepare() + self.api.pause.side_effect = TimeoutError() + q, result = self.pause(current) + self.assertEqual(result['ErrorCode'], 'unconfirmed') + for _ in range(2): + self.assertEqual(self.invoke('suspend', Suspend=dict(q, ReconcileOnly=True))['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('compute_kill', Compute=current)['ErrorCode'], 'unconfirmed') + self.api.pause.assert_called_once() + self.api.kill.assert_not_called() + + def test_missing_native_pause_intent_can_settle_running_rollback(self): + current = self.prepare() + q = {'Reference': self.reference, 'OperationID': str(uuid4()), 'Source': current, + 'Retained': None, 'ReconcileOnly': True} + state = self.invoke('suspend', Suspend=q)['State'] + self.assertTrue(state['SuspendSettled']) + self.assertFalse(state['ResourcesReleased']) + self.assertIsNone(state['Retained']) + self.api.pause.assert_not_called() + + def test_lost_resume_reply_never_replays_connect(self): + current = self.prepare() + _, paused = self.pause(current) + q = self.resume_request(paused['State']['Retained']) + def lost(*a, **k): + self.cloud.state = 'running' + raise TimeoutError() + self.api.connect.side_effect = lost + self.assertEqual(self.invoke('resume', Resume=q)['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('resume', Resume=dict(q, ReconcileOnly=True))['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('compute_kill', Compute=q['Target'])['ErrorCode'], 'unconfirmed') + self.api.connect.assert_called_once() + self.api.kill.assert_not_called() + + def test_lost_core_resume_reply_adopts_saved_target_without_connect(self): + current = self.prepare() + _, paused = self.pause(current) + q = self.resume_request(paused['State']['Retained']) + first = self.invoke('resume', Resume=q) + self.assertEqual(first['ErrorCode'], '') + second = self.invoke('resume', Resume=dict(q, ReconcileOnly=True)) + self.assertEqual(first, second) + self.api.connect.assert_called_once() + + def test_foreign_retained_and_incarnation_cannot_mutate(self): + current = self.prepare() + _, paused = self.pause(current) + retained = paused['State']['Retained'] + q = self.resume_request(dict(retained, SourceID='foreign-id')) + self.assertEqual(self.invoke('resume', Resume=q)['ErrorCode'], 'ownership') + self.assertEqual(self.invoke('delete_retained', Retained=dict(retained, ID='foreign'))['ErrorCode'], 'ownership') + self.api.connect.assert_not_called() + self.api.kill.assert_not_called() + + def test_archive_recovers_saved_connect_reply_before_cleanup(self): + from e2b.exceptions import SandboxNotFoundException + current = self.prepare() + _, paused = self.pause(current) + retained = paused['State']['Retained'] + q = self.resume_request(retained) + def connected_then_observation_lost(*args, **kwargs): + self.cloud.state = 'running' + self.api.get_info.side_effect = TimeoutError() + return self.cloud + self.api.connect.side_effect = connected_then_observation_lost + self.assertEqual(self.invoke('resume', Resume=q)['ErrorCode'], 'unconfirmed') + self.assertTrue(self.record()['suspension']['connected']) + self.assertEqual(self.record()['suspension']['phase'], 'resume_pending') + self.api.get_info.side_effect = None + # The old incarnation remains fenced until exact target cleanup settles. + self.assertEqual(self.invoke('compute_kill', Compute=current)['ErrorCode'], 'ownership') + def killed(*args, **kwargs): + self.api.get_info.side_effect = SandboxNotFoundException('gone') + self.api.kill.side_effect = killed + self.assertEqual(self.invoke('compute_kill', Compute=q['Target'])['ErrorCode'], '') + self.assertEqual(self.invoke('compute_kill', Compute=current)['ErrorCode'], '') + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], '') + self.api.connect.assert_called_once() + self.api.kill.assert_called_once() + self.assertEqual(self.record()['status'], 'killed') diff --git a/services/core/tools/e2b-provider/testdata/contract.json b/services/core/tools/e2b-provider/testdata/contract.json index aa79da367..11b1a9c0c 100644 --- a/services/core/tools/e2b-provider/testdata/contract.json +++ b/services/core/tools/e2b-provider/testdata/contract.json @@ -19,51 +19,59 @@ {"kind":"managed","name":"missing-SessionID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, {"kind":"managed","name":"missing-TenantID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444"},"valid":false}, {"kind":"managed","name":"restricted","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":["example.com"],"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"restricted","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, -{"kind":"request","name":"command","payload":{"Version":1,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"create","payload":{"Version":1,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"duplicate-observation","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, -{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"inspect","payload":{"Version":1,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"kill","payload":{"Version":1,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_builds","payload":{"Version":1,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_templates","payload":{"Version":1,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"observe","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"observe-count-0","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, -{"kind":"request","name":"observe-count-100","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000022-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000023-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000024-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000025-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000026-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000027-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000028-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000029-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000030-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000031-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000032-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000033-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000034-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000035-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000036-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000037-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000038-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000039-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000040-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000041-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000042-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000043-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000044-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000045-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000046-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000047-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000048-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000049-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000050-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000051-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000052-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000053-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000054-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000055-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000056-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000057-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000058-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000059-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000060-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000061-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000062-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000063-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000064-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"observe-count-101","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000022-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000023-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000024-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000025-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000026-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000027-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000028-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000029-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000030-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000031-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000032-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000033-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000034-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000035-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000036-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000037-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000038-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000039-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000040-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000041-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000042-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000043-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000044-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000045-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000046-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000047-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000048-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000049-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000050-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000051-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000052-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000053-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000054-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000055-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000056-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000057-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000058-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000059-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000060-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000061-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000062-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000063-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000064-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000065-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, -{"kind":"request","name":"observe-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"observe","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"observe-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"observe","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"observe-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"observe","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"renew","payload":{"Version":1,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"validate_deployment","payload":{"Version":1,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-0","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-32","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-33","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, +{"kind":"request","name":"command","payload":{"Version":2,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_command","payload":{"Version":2,"Operation":"compute_command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Command":{"Args":["true"],"Directory":"","Stdin":null},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_info","payload":{"Version":2,"Operation":"compute_info","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_kill","payload":{"Version":2,"Operation":"compute_kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_renew","payload":{"Version":2,"Operation":"compute_renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, +{"kind":"request","name":"create","payload":{"Version":2,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"delete_retained","payload":{"Version":2,"Operation":"delete_retained","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Retained":{"Reference":"33333333-3333-4333-8333-333333333333","ID":"66666666-6666-4666-8666-666666666666","Data":"opaque","OperationID":"66666666-6666-4666-8666-666666666666","SourceGeneration":0,"SourceName":"33333333-3333-4333-8333-333333333333","SourceID":"native-fixture"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"duplicate-observation","payload":{"Version":2,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, +{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, +{"kind":"request","name":"inspect","payload":{"Version":2,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"kill","payload":{"Version":2,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_builds","payload":{"Version":2,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_templates","payload":{"Version":2,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"observe","payload":{"Version":2,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"observe-count-0","payload":{"Version":2,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, +{"kind":"request","name":"observe-count-100","payload":{"Version":2,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000022-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000023-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000024-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000025-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000026-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000027-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000028-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000029-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000030-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000031-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000032-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000033-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000034-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000035-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000036-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000037-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000038-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000039-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000040-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000041-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000042-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000043-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000044-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000045-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000046-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000047-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000048-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000049-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000050-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000051-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000052-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000053-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000054-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000055-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000056-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000057-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000058-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000059-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000060-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000061-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000062-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000063-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000064-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"observe-count-101","payload":{"Version":2,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000022-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000023-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000024-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000025-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000026-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000027-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000028-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000029-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000030-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000031-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000032-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000033-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000034-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000035-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000036-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000037-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000038-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000039-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000040-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000041-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000042-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000043-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000044-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000045-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000046-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000047-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000048-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000049-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000050-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000051-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000052-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000053-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000054-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000055-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000056-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000057-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000058-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000059-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000060-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000061-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000062-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000063-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000064-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000065-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, +{"kind":"request","name":"observe-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"observe","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, +{"kind":"request","name":"observe-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"observe","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, +{"kind":"request","name":"observe-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"observe","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, +{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, +{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, +{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, +{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, +{"kind":"request","name":"renew","payload":{"Version":2,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"resume","payload":{"Version":2,"Operation":"resume","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Resume":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"66666666-6666-4666-8666-666666666666","Retained":{"Reference":"33333333-3333-4333-8333-333333333333","ID":"66666666-6666-4666-8666-666666666666","Data":"opaque","OperationID":"66666666-6666-4666-8666-666666666666","SourceGeneration":0,"SourceName":"33333333-3333-4333-8333-333333333333","SourceID":"native-fixture"},"Target":{"Generation":1,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":{"Reference":"33333333-3333-4333-8333-333333333333","ID":"66666666-6666-4666-8666-666666666666","Data":"opaque","OperationID":"66666666-6666-4666-8666-666666666666","SourceGeneration":0,"SourceName":"33333333-3333-4333-8333-333333333333","SourceID":"native-fixture"}},"ReconcileOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"resume_compute","payload":{"Version":2,"Operation":"resume_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"suspend","payload":{"Version":2,"Operation":"suspend","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Suspend":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"66666666-6666-4666-8666-666666666666","Source":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Retained":null,"ReconcileOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"validate_deployment","payload":{"Version":2,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-0","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-32","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-33","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, {"kind":"request","name":"version-bool","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":true},"valid":false}, {"kind":"request","name":"version-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":null},"valid":false}, {"kind":"request","name":"version-string","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":"1"},"valid":false}, -{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, -{"kind":"response","name":"error-","payload":{"Version":1,"ErrorCode":""},"valid":true}, -{"kind":"response","name":"error-command_unconfirmed","payload":{"Version":1,"ErrorCode":"command_unconfirmed"},"valid":true}, -{"kind":"response","name":"error-exists","payload":{"Version":1,"ErrorCode":"exists"},"valid":true}, -{"kind":"response","name":"error-invalid","payload":{"Version":1,"ErrorCode":"invalid"},"valid":true}, -{"kind":"response","name":"error-not_found","payload":{"Version":1,"ErrorCode":"not_found"},"valid":true}, -{"kind":"response","name":"error-ownership","payload":{"Version":1,"ErrorCode":"ownership"},"valid":true}, -{"kind":"response","name":"error-team_mismatch","payload":{"Version":1,"ErrorCode":"team_mismatch"},"valid":true}, -{"kind":"response","name":"error-template_invalid","payload":{"Version":1,"ErrorCode":"template_invalid"},"valid":true}, -{"kind":"response","name":"error-unauthorized","payload":{"Version":1,"ErrorCode":"unauthorized"},"valid":true}, -{"kind":"response","name":"error-unconfirmed","payload":{"Version":1,"ErrorCode":"unconfirmed"},"valid":true}, -{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":"unknown","Version":1},"valid":false}, -{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":1,"Version":1},"valid":false}, -{"kind":"response","name":"invalid-Extra","payload":{"ErrorCode":"","Extra":true,"Version":1},"valid":false}, +{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":3},"valid":false}, +{"kind":"response","name":"error-","payload":{"Version":2,"ErrorCode":""},"valid":true}, +{"kind":"response","name":"error-command_unconfirmed","payload":{"Version":2,"ErrorCode":"command_unconfirmed"},"valid":true}, +{"kind":"response","name":"error-exists","payload":{"Version":2,"ErrorCode":"exists"},"valid":true}, +{"kind":"response","name":"error-invalid","payload":{"Version":2,"ErrorCode":"invalid"},"valid":true}, +{"kind":"response","name":"error-not_found","payload":{"Version":2,"ErrorCode":"not_found"},"valid":true}, +{"kind":"response","name":"error-ownership","payload":{"Version":2,"ErrorCode":"ownership"},"valid":true}, +{"kind":"response","name":"error-team_mismatch","payload":{"Version":2,"ErrorCode":"team_mismatch"},"valid":true}, +{"kind":"response","name":"error-template_invalid","payload":{"Version":2,"ErrorCode":"template_invalid"},"valid":true}, +{"kind":"response","name":"error-unauthorized","payload":{"Version":2,"ErrorCode":"unauthorized"},"valid":true}, +{"kind":"response","name":"error-unconfirmed","payload":{"Version":2,"ErrorCode":"unconfirmed"},"valid":true}, +{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":"unknown","Version":2},"valid":false}, +{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":1,"Version":2},"valid":false}, +{"kind":"response","name":"invalid-Extra","payload":{"ErrorCode":"","Extra":true,"Version":2},"valid":false}, {"kind":"response","name":"invalid-Version","payload":{"ErrorCode":"","Version":true},"valid":false} ] diff --git a/services/core/tools/microsandbox-provider/bootstrap.go b/services/core/tools/microsandbox-provider/bootstrap.go index ad1af890a..a2ab1f2a2 100644 --- a/services/core/tools/microsandbox-provider/bootstrap.go +++ b/services/core/tools/microsandbox-provider/bootstrap.go @@ -8,6 +8,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentnetwork" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" wire "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/microsandbox" sdk "github.com/superradcompany/microsandbox/sdk/go" @@ -18,7 +19,7 @@ import ( const bootstrapScript = ` import ctypes,json,os,stat,subprocess,sys b=json.load(sys.stdin) -for p in ['/home/runtime','/home/runtime/.oac','/environment','/environment/workspace','/environment/staging','/environment/initialization','/environment/packages','/run/oac']: +for p in ['/home/runtime','/home/runtime/.oac','/environment','/environment/workspace','/environment/staging','/environment/initialization','/environment/packages',os.path.dirname(os.environ['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'])]: os.makedirs(p,mode=0o700,exist_ok=True) if not stat.S_ISDIR(os.lstat(p).st_mode): raise RuntimeError('invalid bootstrap directory') os.chmod(p,0o700);os.chown(p,1000,1000) @@ -64,7 +65,7 @@ func (b backend) create(ctx context.Context) (wire.Response, error) { "HOME": "/home/runtime", "OAC_RUNTIME_HOME": "/home/runtime/.oac", "OAC_RUNTIME_ENVIRONMENT_ID": bootstrap.EnvironmentID, "OAC_RUNTIME_SESSION_ID": bootstrap.SessionID, "OAC_RUNTIME_NETWORK_ACCESS": policy.Access, "OAC_RUNTIME_ALLOWED_DOMAINS": string(domains), - "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE": "/run/oac/daemon-suspend.json", + "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE": runtimebootstrap.SuspendControlFile, })) if e != nil { return wire.Response{}, e From 9ac12031597518ba806dde0db34ab657b90e981a Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 21:57:10 +0800 Subject: [PATCH 08/12] test(runtime): assert ingestion clock and refresh activity fence --- .../internal/store/runtime_suspension_test.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/services/core/internal/store/runtime_suspension_test.go b/services/core/internal/store/runtime_suspension_test.go index aba844f89..d570594ea 100644 --- a/services/core/internal/store/runtime_suspension_test.go +++ b/services/core/internal/store/runtime_suspension_test.go @@ -301,16 +301,18 @@ func TestRuntimeSuspensionCountsUncertainCapacityUntilReleased(t *testing.T) { } func TestRuntimeSuspensionIdleStartsAfterLastCompletion(t *testing.T) { - _, w, pool, owner := runtimeSuspensionFixture(t) - turn := runtimeSuspensionCompleted(t, pool, owner) + s, w, pool, owner := runtimeSuspensionFixture(t) + runtimeSuspensionCompleted(t, pool, owner) runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 hours' WHERE id=$1`, owner.ID) - var completed time.Time - if err := pool.QueryRow(t.Context(), `SELECT completed_at FROM turns WHERE id=$1`, turn).Scan(&completed); err != nil { + before := runtimeDatabaseTime(t, s) + id, _ := parseConnectionGeneration(owner.SessionID) + if err := s.queries.RecordRuntimeTerminalActivity(t.Context(), id); err != nil { t.Fatal(err) } + after := runtimeDatabaseTime(t, s) activity, err := w.RuntimeActivity(t.Context(), owner) - if err != nil || !activity.LastActivity.Equal(completed) { - t.Fatal("long Turn completion did not restart idle interval", activity, completed, err) + if err != nil || activity.LastActivity.Before(before) || activity.LastActivity.After(after) || activity.ReadyToSuspend(time.Minute) { + t.Fatal("long Turn completion did not restart the ingestion idle interval", activity, before, after, err) } } @@ -422,6 +424,11 @@ func TestRuntimeSuspensionRechecksCompletionAgainstIdleTimeout(t *testing.T) { if _, err := w.SetRuntimeCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 0); !errors.Is(err, ErrInvalidInput) { t.Fatal("missing idle timeout accepted", err) } + // Re-observe the allocation after terminal ingestion advanced its activity fence. + owner, err = s.GetRuntimeAllocation(t.Context(), owner.TenantID, owner.EnvironmentID) + if err != nil { + t.Fatal(err) + } if _, err := w.SetRuntimeCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, time.Nanosecond); err != nil { t.Fatal("elapsed idle timeout rejected", err) } From fba894c6687f1056cf4a11bac37f1ee285857b46 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 21:58:20 +0800 Subject: [PATCH 09/12] test(e2b): qualify suspension policy and isolate generation downgrade guard --- .../internal/store/sandbox_deployment_view_test.go | 2 +- .../core/internal/store/sandbox_generations_test.go | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/services/core/internal/store/sandbox_deployment_view_test.go b/services/core/internal/store/sandbox_deployment_view_test.go index 21da06627..04210869e 100644 --- a/services/core/internal/store/sandbox_deployment_view_test.go +++ b/services/core/internal/store/sandbox_deployment_view_test.go @@ -42,7 +42,7 @@ func TestSandboxDeploymentViewRecordsTemplateBuildAndSuspension(t *testing.T) { for _, want := range []string{ `"specification":{"resources":{"cpus":2,"memory_mib":2048}}`, `"template_build":{"status":"ready","resources":{"cpus":2,"memory_mib":2048,"root_disk_mib":24063}}`, - `"suspension":null`, + `"suspension":{"idle_seconds":300,"retention_seconds":86400}`, } { if !bytes.Contains(raw, []byte(want)) { t.Fatalf("E2B view lacks %s: %s", want, raw) diff --git a/services/core/internal/store/sandbox_generations_test.go b/services/core/internal/store/sandbox_generations_test.go index f3a0e8714..38d9137fa 100644 --- a/services/core/internal/store/sandbox_generations_test.go +++ b/services/core/internal/store/sandbox_generations_test.go @@ -4,6 +4,7 @@ import ( "database/sql" "errors" "os" + "strings" "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" @@ -288,14 +289,19 @@ func TestGenerationDowngradeRefusesOldAllocation(t *testing.T) { if _, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}); err != nil { t.Fatal(err) } + // Isolate the generation downgrade guard using the target schema's disabled + // suspension policy. Active suspension itself is not representable there. + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_deployment SET idle_seconds=0,retention_seconds=0"); err != nil { + t.Fatal(err) + } db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) defer db.Close() migrations, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) if err != nil { t.Fatal(err) } - if _, err = migrations.DownTo(t.Context(), 80); err == nil { - t.Fatal("downgrade erased old owned allocation") + if _, err = migrations.DownTo(t.Context(), 80); err == nil || !strings.Contains(err.Error(), "Cannot downgrade while retained ownership") { + t.Fatal("generation ownership guard did not reject downgrade", err) } // Earlier down migrations can commit before the generation guard vetoes // downgrade. Restore the current schema before invoking current Store code. From 022875c0fb5358576058ed31668fa8d4b8bc48a5 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 30 Sep 2026 22:15:50 +0800 Subject: [PATCH 10/12] test(migrations): pin historical suspension policy in downgrade fixtures --- .../internal/store/archived_cancellation_migration_test.go | 5 +++++ .../internal/store/provider_registration_migration_test.go | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/services/core/internal/store/archived_cancellation_migration_test.go b/services/core/internal/store/archived_cancellation_migration_test.go index 0ffa52a2d..15fdd701b 100644 --- a/services/core/internal/store/archived_cancellation_migration_test.go +++ b/services/core/internal/store/archived_cancellation_migration_test.go @@ -21,6 +21,11 @@ func TestArchivedCancellationMigrationDoesNotAdoptOldRevocations(t *testing.T) { if _, err := w.ArchiveManagedSession(adminDeleteContext(t.Context(), tenant, uuid.NewString()), tenant, session.ID, 1); err != nil { t.Fatal(err) } + // Exercise the historical migration with the target schema's disabled + // suspension policy, independently of the current provider defaults. + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_deployment SET idle_seconds=0,retention_seconds=0"); err != nil { + t.Fatal(err) + } db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) defer db.Close() provider, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) diff --git a/services/core/internal/store/provider_registration_migration_test.go b/services/core/internal/store/provider_registration_migration_test.go index 06a6074c8..b96dfc8b8 100644 --- a/services/core/internal/store/provider_registration_migration_test.go +++ b/services/core/internal/store/provider_registration_migration_test.go @@ -27,6 +27,11 @@ func TestProviderRegistrationDowngradePreservesCustomEndpoints(t *testing.T) { if _, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 2}); err != nil { t.Fatal(err) } + // Exercise the historical migration with the target schema's disabled + // suspension policy, independently of the current provider defaults. + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_deployment SET idle_seconds=0,retention_seconds=0"); err != nil { + t.Fatal(err) + } db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) defer db.Close() migrations, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) From 173d120d6bce84cde514be4ef2248f46350d2e94 Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 1 Oct 2026 14:05:09 +0800 Subject: [PATCH 11/12] test(modelconfiguration): use valid caller-owned observation fixture --- .../postgres/modelconfigurationpg/observation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/core/internal/persistence/postgres/modelconfigurationpg/observation_test.go b/services/core/internal/persistence/postgres/modelconfigurationpg/observation_test.go index 3e9047b7c..c52e6e9fb 100644 --- a/services/core/internal/persistence/postgres/modelconfigurationpg/observation_test.go +++ b/services/core/internal/persistence/postgres/modelconfigurationpg/observation_test.go @@ -84,7 +84,7 @@ func TestObservationExcludesOtherSourcesAndHistoricalSessions(t *testing.T) { if source == "deployment" { input.DeploymentProviderRevision = uuid.Nil } else { - input.Configuration = json.RawMessage(`{"agent":{"model":"frozen-model"},"environment":{"type":"openai_hosted"}}`) + input.Configuration = json.RawMessage(`{"agent":{"model":"frozen-model"},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`) if source == "unknown" { input.ModelProviderSource = "session" } From 03d36bc86d8dabf752d2ab3fc4e48825197e85d8 Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 3 Oct 2026 10:19:22 +0800 Subject: [PATCH 12/12] Raise direct runtime capacity to 100 active sandboxes --- docs/configuration.md | 4 ++++ services/core/cmd/server/managed_setup.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 114e7f667..b71dcee4d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -100,6 +100,10 @@ Runtime settings live in Core's database. Change them in Web; scripts use the sa Which harnesses are enabled, and the default one, are process settings (`core.harnesses`, `core.default_harness`); System shows them read-only. The [Core administration API](../contracts/agents-api/admin-api.md) lists every Core API route, and the [deployment contract](../contracts/agents-api/sandbox-deployment.md) defines the sandbox fields, limits and change rules. +### Direct-provider capacity + +Core allows up to 100 active sandboxes and 400 retained sandboxes for a direct Provider with suspension enabled. Retained sandboxes include active and suspended allocations and unconfirmed cleanup. These limits are independent of `core.execution_concurrency`; the latter limits concurrent execution work in Core. + ### Node capacity Core approves a node's capacity when you generate its Add node command: **Sandboxes at once** (`max_active`, default 2) and, for microsandbox only, **Retained sandboxes** (`max_retained`, default 8), with `max_retained >= max_active >= 1`. Docker never suspends sandboxes, so Web doesn't ask for it and Core keeps `max_retained` equal to `max_active`. Change them later with **Edit node**. Reservations and cleanup that is not confirmed count against capacity; lowering a limit stops no running sandbox. A node's own files can't change its capacity, size or Runtime. diff --git a/services/core/cmd/server/managed_setup.go b/services/core/cmd/server/managed_setup.go index dfce8cdc6..bb1cedab4 100644 --- a/services/core/cmd/server/managed_setup.go +++ b/services/core/cmd/server/managed_setup.go @@ -148,7 +148,7 @@ func (s *managedSetup) configuration(setup deployment.Setup) (execution.Prepared CoreURL: s.publicURL + "/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider} if setup.Suspension != nil { selected.Suspension = &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Duration(setup.Suspension.IdleSeconds) * time.Second, - Retention: time.Duration(setup.Suspension.RetentionSeconds) * time.Second, MaxActive: 4, MaxRetained: 16} + Retention: time.Duration(setup.Suspension.RetentionSeconds) * time.Second, MaxActive: 100, MaxRetained: 400} } return execution.PreparedRuntimeDeployment{Config: selected, Publish: s.publish}, nil }